PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::SafeSyslog;
  38 use PVE::Storage;
  39 use PVE::SysFSTools;
  40 use PVE::Systemd;
  41 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  42
  43 use PVE::QMPClient;
  44 use PVE::QemuConfig;
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Memory;
  47 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  48 use PVE::QemuServer::USB qw(parse_usb_device);
  49
  50 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  51 my $OVMF = {
  52     x86_64 => [
  53         "$EDK2_FW_BASE/OVMF_CODE.fd",
  54         "$EDK2_FW_BASE/OVMF_VARS.fd"
  55     ],
  56     aarch64 => [
  57         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  58         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  59     ],
  60 };
  61
  62 my $qemu_snap_storage = { rbd => 1 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename = PVE::INotify::nodename();
 112
 113 mkdir "/etc/pve/nodes/$nodename";
 114 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 115 mkdir $confdir;
 116
 117 my $var_run_tmpdir = "/var/run/qemu-server";
 118 mkdir $var_run_tmpdir;
 119
 120 my $lock_dir = "/var/lock/qemu-server";
 121 mkdir $lock_dir;
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153     'Cascadelake-Server' => 'GenuineIntel',
 154     KnightsMill => 'GenuineIntel',
 155
 156
 157     # AMD CPUs
 158     athlon => 'AuthenticAMD',
 159     phenom  => 'AuthenticAMD',
 160     Opteron_G1  => 'AuthenticAMD',
 161     Opteron_G2  => 'AuthenticAMD',
 162     Opteron_G3  => 'AuthenticAMD',
 163     Opteron_G4  => 'AuthenticAMD',
 164     Opteron_G5  => 'AuthenticAMD',
 165     EPYC => 'AuthenticAMD',
 166     'EPYC-IBPB' => 'AuthenticAMD',
 167
 168     # generic types, use vendor from host node
 169     host => 'default',
 170     kvm32 => 'default',
 171     kvm64 => 'default',
 172     qemu32 => 'default',
 173     qemu64 => 'default',
 174     max => 'default',
 175 };
 176
 177 my @supported_cpu_flags = (
 178     'pcid',
 179     'spec-ctrl',
 180     'ibpb',
 181     'ssbd',
 182     'virt-ssbd',
 183     'amd-ssbd',
 184     'amd-no-ssb',
 185     'pdpe1gb',
 186     'md-clear',
 187     'hv-tlbflush',
 188     'hv-evmcs',
 189     'aes'
 190 );
 191 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 192
 193 my $cpu_fmt = {
 194     cputype => {
 195         description => "Emulated CPU type.",
 196         type => 'string',
 197         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 198         default => 'kvm64',
 199         default_key => 1,
 200     },
 201     hidden => {
 202         description => "Do not identify as a KVM virtual machine.",
 203         type => 'boolean',
 204         optional => 1,
 205         default => 0
 206     },
 207     'hv-vendor-id' => {
 208         type => 'string',
 209         pattern => qr/[a-zA-Z0-9]{1,12}/,
 210         format_description => 'vendor-id',
 211         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 212         optional => 1,
 213     },
 214     flags => {
 215         description => "List of additional CPU flags separated by ';'."
 216                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 217                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 218         format_description => '+FLAG[;-FLAG...]',
 219         type => 'string',
 220         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 221         optional => 1,
 222     },
 223 };
 224
 225 my $watchdog_fmt = {
 226     model => {
 227         default_key => 1,
 228         type => 'string',
 229         enum => [qw(i6300esb ib700)],
 230         description => "Watchdog type to emulate.",
 231         default => 'i6300esb',
 232         optional => 1,
 233     },
 234     action => {
 235         type => 'string',
 236         enum => [qw(reset shutdown poweroff pause debug none)],
 237         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 238         optional => 1,
 239     },
 240 };
 241 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 242
 243 my $agent_fmt = {
 244     enabled => {
 245         description => "Enable/disable Qemu GuestAgent.",
 246         type => 'boolean',
 247         default => 0,
 248         default_key => 1,
 249     },
 250     fstrim_cloned_disks => {
 251         description => "Run fstrim after cloning/moving a disk.",
 252         type => 'boolean',
 253         optional => 1,
 254         default => 0
 255     },
 256     type => {
 257         description => "Select the agent type",
 258         type => 'string',
 259         default => 'virtio',
 260         optional => 1,
 261         enum => [qw(virtio isa)],
 262     },
 263 };
 264
 265 my $vga_fmt = {
 266     type => {
 267         description => "Select the VGA type.",
 268         type => 'string',
 269         default => 'std',
 270         optional => 1,
 271         default_key => 1,
 272         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 273     },
 274     memory => {
 275         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 276         type => 'integer',
 277         optional => 1,
 278         minimum => 4,
 279         maximum => 512,
 280     },
 281 };
 282
 283 my $ivshmem_fmt = {
 284     size => {
 285         type => 'integer',
 286         minimum => 1,
 287         description => "The size of the file in MB.",
 288     },
 289     name => {
 290         type => 'string',
 291         pattern => '[a-zA-Z0-9\-]+',
 292         optional => 1,
 293         format_description => 'string',
 294         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 295     },
 296 };
 297
 298 my $audio_fmt = {
 299     device => {
 300         type => 'string',
 301         enum => [qw(ich9-intel-hda intel-hda AC97)],
 302         description =>  "Configure an audio device."
 303     },
 304     driver =>  {
 305         type => 'string',
 306         enum => ['spice'],
 307         default => 'spice',
 308         optional => 1,
 309         description => "Driver backend for the audio device."
 310     },
 311 };
 312
 313 my $spice_enhancements_fmt = {
 314     foldersharing => {
 315         type => 'boolean',
 316         optional => 1,
 317         default => '0',
 318         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 319     },
 320     videostreaming =>  {
 321         type => 'string',
 322         enum => ['off', 'all', 'filter'],
 323         default => 'off',
 324         optional => 1,
 325         description => "Enable video streaming. Uses compression for detected video streams."
 326     },
 327 };
 328
 329 my $confdesc = {
 330     onboot => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Specifies whether a VM will be started during system bootup.",
 334         default => 0,
 335     },
 336     autostart => {
 337         optional => 1,
 338         type => 'boolean',
 339         description => "Automatic restart after crash (currently ignored).",
 340         default => 0,
 341     },
 342     hotplug => {
 343         optional => 1,
 344         type => 'string', format => 'pve-hotplug-features',
 345         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 346         default => 'network,disk,usb',
 347     },
 348     reboot => {
 349         optional => 1,
 350         type => 'boolean',
 351         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 352         default => 1,
 353     },
 354     lock => {
 355         optional => 1,
 356         type => 'string',
 357         description => "Lock/unlock the VM.",
 358         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 359     },
 360     cpulimit => {
 361         optional => 1,
 362         type => 'number',
 363         description => "Limit of CPU usage.",
 364         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 365         minimum => 0,
 366         maximum => 128,
 367         default => 0,
 368     },
 369     cpuunits => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "CPU weight for a VM.",
 373         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 374         minimum => 2,
 375         maximum => 262144,
 376         default => 1024,
 377     },
 378     memory => {
 379         optional => 1,
 380         type => 'integer',
 381         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 382         minimum => 16,
 383         default => 512,
 384     },
 385     balloon => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 389         minimum => 0,
 390     },
 391     shares => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 395         minimum => 0,
 396         maximum => 50000,
 397         default => 1000,
 398     },
 399     keyboard => {
 400         optional => 1,
 401         type => 'string',
 402         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 403                        "It should not be necessary to set it.",
 404         enum => PVE::Tools::kvmkeymaplist(),
 405         default => undef,
 406     },
 407     name => {
 408         optional => 1,
 409         type => 'string', format => 'dns-name',
 410         description => "Set a name for the VM. Only used on the configuration web interface.",
 411     },
 412     scsihw => {
 413         optional => 1,
 414         type => 'string',
 415         description => "SCSI controller model",
 416         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 417         default => 'lsi',
 418     },
 419     description => {
 420         optional => 1,
 421         type => 'string',
 422         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 423     },
 424     ostype => {
 425         optional => 1,
 426         type => 'string',
 427         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 428         description => "Specify guest operating system.",
 429         verbose_description => <<EODESC,
 430 Specify guest operating system. This is used to enable special
 431 optimization/features for specific operating systems:
 432
 433 [horizontal]
 434 other;; unspecified OS
 435 wxp;; Microsoft Windows XP
 436 w2k;; Microsoft Windows 2000
 437 w2k3;; Microsoft Windows 2003
 438 w2k8;; Microsoft Windows 2008
 439 wvista;; Microsoft Windows Vista
 440 win7;; Microsoft Windows 7
 441 win8;; Microsoft Windows 8/2012/2012r2
 442 win10;; Microsoft Windows 10/2016
 443 l24;; Linux 2.4 Kernel
 444 l26;; Linux 2.6/3.X Kernel
 445 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 446 EODESC
 447     },
 448     boot => {
 449         optional => 1,
 450         type => 'string',
 451         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 452         pattern => '[acdn]{1,4}',
 453         default => 'cdn',
 454     },
 455     bootdisk => {
 456         optional => 1,
 457         type => 'string', format => 'pve-qm-bootdisk',
 458         description => "Enable booting from specified disk.",
 459         pattern => '(ide|sata|scsi|virtio)\d+',
 460     },
 461     smp => {
 462         optional => 1,
 463         type => 'integer',
 464         description => "The number of CPUs. Please use option -sockets instead.",
 465         minimum => 1,
 466         default => 1,
 467     },
 468     sockets => {
 469         optional => 1,
 470         type => 'integer',
 471         description => "The number of CPU sockets.",
 472         minimum => 1,
 473         default => 1,
 474     },
 475     cores => {
 476         optional => 1,
 477         type => 'integer',
 478         description => "The number of cores per socket.",
 479         minimum => 1,
 480         default => 1,
 481     },
 482     numa => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Enable/disable NUMA.",
 486         default => 0,
 487     },
 488     hugepages => {
 489         optional => 1,
 490         type => 'string',
 491         description => "Enable/disable hugepages memory.",
 492         enum => [qw(any 2 1024)],
 493     },
 494     vcpus => {
 495         optional => 1,
 496         type => 'integer',
 497         description => "Number of hotplugged vcpus.",
 498         minimum => 1,
 499         default => 0,
 500     },
 501     acpi => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable ACPI.",
 505         default => 1,
 506     },
 507     agent => {
 508         optional => 1,
 509         description => "Enable/disable Qemu GuestAgent and its properties.",
 510         type => 'string',
 511         format => $agent_fmt,
 512     },
 513     kvm => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable KVM hardware virtualization.",
 517         default => 1,
 518     },
 519     tdf => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Enable/disable time drift fix.",
 523         default => 0,
 524     },
 525     localtime => {
 526         optional => 1,
 527         type => 'boolean',
 528         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 529     },
 530     freeze => {
 531         optional => 1,
 532         type => 'boolean',
 533         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 534     },
 535     vga => {
 536         optional => 1,
 537         type => 'string', format => $vga_fmt,
 538         description => "Configure the VGA hardware.",
 539         verbose_description => "Configure the VGA Hardware. If you want to use ".
 540             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 541             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 542             "is 'std' for all OS types besides some Windows versions (XP and " .
 543             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 544             "display server. For win* OS you can select how many independent " .
 545             "displays you want, Linux guests can add displays them self.\n".
 546             "You can also run without any graphic card, using a serial device as terminal.",
 547     },
 548     watchdog => {
 549         optional => 1,
 550         type => 'string', format => 'pve-qm-watchdog',
 551         description => "Create a virtual hardware watchdog device.",
 552         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 553             " (by a guest action), the watchdog must be periodically polled " .
 554             "by an agent inside the guest or else the watchdog will reset " .
 555             "the guest (or execute the respective action specified)",
 556     },
 557     startdate => {
 558         optional => 1,
 559         type => 'string',
 560         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 561         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 562         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 563         default => 'now',
 564     },
 565     startup =>  get_standard_option('pve-startup-order'),
 566     template => {
 567         optional => 1,
 568         type => 'boolean',
 569         description => "Enable/disable Template.",
 570         default => 0,
 571     },
 572     args => {
 573         optional => 1,
 574         type => 'string',
 575         description => "Arbitrary arguments passed to kvm.",
 576         verbose_description => <<EODESCR,
 577 Arbitrary arguments passed to kvm, for example:
 578
 579 args: -no-reboot -no-hpet
 580
 581 NOTE: this option is for experts only.
 582 EODESCR
 583     },
 584     tablet => {
 585         optional => 1,
 586         type => 'boolean',
 587         default => 1,
 588         description => "Enable/disable the USB tablet device.",
 589         verbose_description => "Enable/disable the USB tablet device. This device is " .
 590             "usually needed to allow absolute mouse positioning with VNC. " .
 591             "Else the mouse runs out of sync with normal VNC clients. " .
 592             "If you're running lots of console-only guests on one host, " .
 593             "you may consider disabling this to save some context switches. " .
 594             "This is turned off by default if you use spice (-vga=qxl).",
 595     },
 596     migrate_speed => {
 597         optional => 1,
 598         type => 'integer',
 599         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 600         minimum => 0,
 601         default => 0,
 602     },
 603     migrate_downtime => {
 604         optional => 1,
 605         type => 'number',
 606         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 607         minimum => 0,
 608         default => 0.1,
 609     },
 610     cdrom => {
 611         optional => 1,
 612         type => 'string', format => 'pve-qm-ide',
 613         typetext => '<volume>',
 614         description => "This is an alias for option -ide2",
 615     },
 616     cpu => {
 617         optional => 1,
 618         description => "Emulated CPU type.",
 619         type => 'string',
 620         format => $cpu_fmt,
 621     },
 622     parent => get_standard_option('pve-snapshot-name', {
 623         optional => 1,
 624         description => "Parent snapshot name. This is used internally, and should not be modified.",
 625     }),
 626     snaptime => {
 627         optional => 1,
 628         description => "Timestamp for snapshots.",
 629         type => 'integer',
 630         minimum => 0,
 631     },
 632     vmstate => {
 633         optional => 1,
 634         type => 'string', format => 'pve-volume-id',
 635         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 636     },
 637     vmstatestorage => get_standard_option('pve-storage-id', {
 638         description => "Default storage for VM state volumes/files.",
 639         optional => 1,
 640     }),
 641     runningmachine => get_standard_option('pve-qemu-machine', {
 642         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 643     }),
 644     machine => get_standard_option('pve-qemu-machine'),
 645     arch => {
 646         description => "Virtual processor architecture. Defaults to the host.",
 647         optional => 1,
 648         type => 'string',
 649         enum => [qw(x86_64 aarch64)],
 650     },
 651     smbios1 => {
 652         description => "Specify SMBIOS type 1 fields.",
 653         type => 'string', format => 'pve-qm-smbios1',
 654         maxLength => 512,
 655         optional => 1,
 656     },
 657     protection => {
 658         optional => 1,
 659         type => 'boolean',
 660         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 661         default => 0,
 662     },
 663     bios => {
 664         optional => 1,
 665         type => 'string',
 666         enum => [ qw(seabios ovmf) ],
 667         description => "Select BIOS implementation.",
 668         default => 'seabios',
 669     },
 670     vmgenid => {
 671         type => 'string',
 672         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 673         format_description => 'UUID',
 674         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 675         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 676             " 128-bit integer value identifier to the guest OS. This allows to".
 677             " notify the guest operating system when the virtual machine is".
 678             " executed with a different configuration (e.g. snapshot execution".
 679             " or creation from a template). The guest operating system notices".
 680             " the change, and is then able to react as appropriate by marking".
 681             " its copies of distributed databases as dirty, re-initializing its".
 682             " random number generator, etc.\n".
 683             "Note that auto-creation only works when done throug API/CLI create".
 684             " or update methods, but not when manually editing the config file.",
 685         default => "1 (autogenerated)",
 686         optional => 1,
 687     },
 688     hookscript => {
 689         type => 'string',
 690         format => 'pve-volume-id',
 691         optional => 1,
 692         description => "Script that will be executed during various steps in the vms lifetime.",
 693     },
 694     ivshmem => {
 695         type => 'string',
 696         format => $ivshmem_fmt,
 697         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 698         optional => 1,
 699     },
 700     audio0 => {
 701         type => 'string',
 702         format => $audio_fmt,
 703         description => "Configure a audio device, useful in combination with QXL/Spice.",
 704         optional => 1
 705     },
 706     spice_enhancements => {
 707         type => 'string',
 708         format => $spice_enhancements_fmt,
 709         description => "Configure additional enhancements for SPICE.",
 710         optional => 1
 711     },
 712 };
 713
 714 my $cicustom_fmt = {
 715     meta => {
 716         type => 'string',
 717         optional => 1,
 718         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 719         format => 'pve-volume-id',
 720         format_description => 'volume',
 721     },
 722     network => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     user => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 733         format => 'pve-volume-id',
 734         format_description => 'volume',
 735     },
 736 };
 737 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 738
 739 my $confdesc_cloudinit = {
 740     citype => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 744         enum => ['configdrive2', 'nocloud'],
 745     },
 746     ciuser => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 750     },
 751     cipassword => {
 752         optional => 1,
 753         type => 'string',
 754         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 755     },
 756     cicustom => {
 757         optional => 1,
 758         type => 'string',
 759         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 760         format => 'pve-qm-cicustom',
 761     },
 762     searchdomain => {
 763         optional => 1,
 764         type => 'string',
 765         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 766     },
 767     nameserver => {
 768         optional => 1,
 769         type => 'string', format => 'address-list',
 770         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 771     },
 772     sshkeys => {
 773         optional => 1,
 774         type => 'string',
 775         format => 'urlencoded',
 776         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 777     },
 778 };
 779
 780 # what about other qemu settings ?
 781 #cpu => 'string',
 782 #machine => 'string',
 783 #fda => 'file',
 784 #fdb => 'file',
 785 #mtdblock => 'file',
 786 #sd => 'file',
 787 #pflash => 'file',
 788 #snapshot => 'bool',
 789 #bootp => 'file',
 790 ##tftp => 'dir',
 791 ##smb => 'dir',
 792 #kernel => 'file',
 793 #append => 'string',
 794 #initrd => 'file',
 795 ##soundhw => 'string',
 796
 797 while (my ($k, $v) = each %$confdesc) {
 798     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 799 }
 800
 801 my $MAX_IDE_DISKS = 4;
 802 my $MAX_SCSI_DISKS = 14;
 803 my $MAX_VIRTIO_DISKS = 16;
 804 my $MAX_SATA_DISKS = 6;
 805 my $MAX_USB_DEVICES = 5;
 806 my $MAX_NETS = 32;
 807 my $MAX_UNUSED_DISKS = 256;
 808 my $MAX_HOSTPCI_DEVICES = 16;
 809 my $MAX_SERIAL_PORTS = 4;
 810 my $MAX_PARALLEL_PORTS = 3;
 811 my $MAX_NUMA = 8;
 812
 813 my $numa_fmt = {
 814     cpus => {
 815         type => "string",
 816         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 817         description => "CPUs accessing this NUMA node.",
 818         format_description => "id[-id];...",
 819     },
 820     memory => {
 821         type => "number",
 822         description => "Amount of memory this NUMA node provides.",
 823         optional => 1,
 824     },
 825     hostnodes => {
 826         type => "string",
 827         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 828         description => "Host NUMA nodes to use.",
 829         format_description => "id[-id];...",
 830         optional => 1,
 831     },
 832     policy => {
 833         type => 'string',
 834         enum => [qw(preferred bind interleave)],
 835         description => "NUMA allocation policy.",
 836         optional => 1,
 837     },
 838 };
 839 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 840 my $numadesc = {
 841     optional => 1,
 842     type => 'string', format => $numa_fmt,
 843     description => "NUMA topology.",
 844 };
 845 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 846
 847 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 848     $confdesc->{"numa$i"} = $numadesc;
 849 }
 850
 851 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 852                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 853                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 854 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 855
 856 my $net_fmt_bridge_descr = <<__EOD__;
 857 Bridge to attach the network device to. The Proxmox VE standard bridge
 858 is called 'vmbr0'.
 859
 860 If you do not specify a bridge, we create a kvm user (NATed) network
 861 device, which provides DHCP and DNS services. The following addresses
 862 are used:
 863
 864  10.0.2.2   Gateway
 865  10.0.2.3   DNS Server
 866  10.0.2.4   SMB Server
 867
 868 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 869 __EOD__
 870
 871 my $net_fmt = {
 872     macaddr  => get_standard_option('mac-addr', {
 873         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 874     }),
 875     model => {
 876         type => 'string',
 877         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 878         enum => $nic_model_list,
 879         default_key => 1,
 880     },
 881     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 882     bridge => {
 883         type => 'string',
 884         description => $net_fmt_bridge_descr,
 885         format_description => 'bridge',
 886         optional => 1,
 887     },
 888     queues => {
 889         type => 'integer',
 890         minimum => 0, maximum => 16,
 891         description => 'Number of packet queues to be used on the device.',
 892         optional => 1,
 893     },
 894     rate => {
 895         type => 'number',
 896         minimum => 0,
 897         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 898         optional => 1,
 899     },
 900     tag => {
 901         type => 'integer',
 902         minimum => 1, maximum => 4094,
 903         description => 'VLAN tag to apply to packets on this interface.',
 904         optional => 1,
 905     },
 906     trunks => {
 907         type => 'string',
 908         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 909         description => 'VLAN trunks to pass through this interface.',
 910         format_description => 'vlanid[;vlanid...]',
 911         optional => 1,
 912     },
 913     firewall => {
 914         type => 'boolean',
 915         description => 'Whether this interface should be protected by the firewall.',
 916         optional => 1,
 917     },
 918     link_down => {
 919         type => 'boolean',
 920         description => 'Whether this interface should be disconnected (like pulling the plug).',
 921         optional => 1,
 922     },
 923 };
 924
 925 my $netdesc = {
 926     optional => 1,
 927     type => 'string', format => $net_fmt,
 928     description => "Specify network devices.",
 929 };
 930
 931 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 932
 933 my $ipconfig_fmt = {
 934     ip => {
 935         type => 'string',
 936         format => 'pve-ipv4-config',
 937         format_description => 'IPv4Format/CIDR',
 938         description => 'IPv4 address in CIDR format.',
 939         optional => 1,
 940         default => 'dhcp',
 941     },
 942     gw => {
 943         type => 'string',
 944         format => 'ipv4',
 945         format_description => 'GatewayIPv4',
 946         description => 'Default gateway for IPv4 traffic.',
 947         optional => 1,
 948         requires => 'ip',
 949     },
 950     ip6 => {
 951         type => 'string',
 952         format => 'pve-ipv6-config',
 953         format_description => 'IPv6Format/CIDR',
 954         description => 'IPv6 address in CIDR format.',
 955         optional => 1,
 956         default => 'dhcp',
 957     },
 958     gw6 => {
 959         type => 'string',
 960         format => 'ipv6',
 961         format_description => 'GatewayIPv6',
 962         description => 'Default gateway for IPv6 traffic.',
 963         optional => 1,
 964         requires => 'ip6',
 965     },
 966 };
 967 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 968 my $ipconfigdesc = {
 969     optional => 1,
 970     type => 'string', format => 'pve-qm-ipconfig',
 971     description => <<'EODESCR',
 972 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 973
 974 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 975
 976 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 977 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 978
 979 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 980 EODESCR
 981 };
 982 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 983
 984 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 985     $confdesc->{"net$i"} = $netdesc;
 986     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 987 }
 988
 989 foreach my $key (keys %$confdesc_cloudinit) {
 990     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 991 }
 992
 993 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 994 sub verify_volume_id_or_qm_path {
 995     my ($volid, $noerr) = @_;
 996
 997     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 998         return $volid;
 999     }
1000
1001     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1002     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1003     if ($@) {
1004         return undef if $noerr;
1005         die $@;
1006     }
1007     return $volid;
1008 }
1009
1010 my $drivename_hash;
1011
1012 my %drivedesc_base = (
1013     volume => { alias => 'file' },
1014     file => {
1015         type => 'string',
1016         format => 'pve-volume-id-or-qm-path',
1017         default_key => 1,
1018         format_description => 'volume',
1019         description => "The drive's backing volume.",
1020     },
1021     media => {
1022         type => 'string',
1023         enum => [qw(cdrom disk)],
1024         description => "The drive's media type.",
1025         default => 'disk',
1026         optional => 1
1027     },
1028     cyls => {
1029         type => 'integer',
1030         description => "Force the drive's physical geometry to have a specific cylinder count.",
1031         optional => 1
1032     },
1033     heads => {
1034         type => 'integer',
1035         description => "Force the drive's physical geometry to have a specific head count.",
1036         optional => 1
1037     },
1038     secs => {
1039         type => 'integer',
1040         description => "Force the drive's physical geometry to have a specific sector count.",
1041         optional => 1
1042     },
1043     trans => {
1044         type => 'string',
1045         enum => [qw(none lba auto)],
1046         description => "Force disk geometry bios translation mode.",
1047         optional => 1,
1048     },
1049     snapshot => {
1050         type => 'boolean',
1051         description => "Controls qemu's snapshot mode feature."
1052             . " If activated, changes made to the disk are temporary and will"
1053             . " be discarded when the VM is shutdown.",
1054         optional => 1,
1055     },
1056     cache => {
1057         type => 'string',
1058         enum => [qw(none writethrough writeback unsafe directsync)],
1059         description => "The drive's cache mode",
1060         optional => 1,
1061     },
1062     format => get_standard_option('pve-qm-image-format'),
1063     size => {
1064         type => 'string',
1065         format => 'disk-size',
1066         format_description => 'DiskSize',
1067         description => "Disk size. This is purely informational and has no effect.",
1068         optional => 1,
1069     },
1070     backup => {
1071         type => 'boolean',
1072         description => "Whether the drive should be included when making backups.",
1073         optional => 1,
1074     },
1075     replicate => {
1076         type => 'boolean',
1077         description => 'Whether the drive should considered for replication jobs.',
1078         optional => 1,
1079         default => 1,
1080     },
1081     rerror => {
1082         type => 'string',
1083         enum => [qw(ignore report stop)],
1084         description => 'Read error action.',
1085         optional => 1,
1086     },
1087     werror => {
1088         type => 'string',
1089         enum => [qw(enospc ignore report stop)],
1090         description => 'Write error action.',
1091         optional => 1,
1092     },
1093     aio => {
1094         type => 'string',
1095         enum => [qw(native threads)],
1096         description => 'AIO type to use.',
1097         optional => 1,
1098     },
1099     discard => {
1100         type => 'string',
1101         enum => [qw(ignore on)],
1102         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1103         optional => 1,
1104     },
1105     detect_zeroes => {
1106         type => 'boolean',
1107         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1108         optional => 1,
1109     },
1110     serial => {
1111         type => 'string',
1112         format => 'urlencoded',
1113         format_description => 'serial',
1114         maxLength => 20*3, # *3 since it's %xx url enoded
1115         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1116         optional => 1,
1117     },
1118     shared => {
1119         type => 'boolean',
1120         description => 'Mark this locally-managed volume as available on all nodes',
1121         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1122         optional => 1,
1123         default => 0,
1124     }
1125 );
1126
1127 my %iothread_fmt = ( iothread => {
1128         type => 'boolean',
1129         description => "Whether to use iothreads for this drive",
1130         optional => 1,
1131 });
1132
1133 my %model_fmt = (
1134     model => {
1135         type => 'string',
1136         format => 'urlencoded',
1137         format_description => 'model',
1138         maxLength => 40*3, # *3 since it's %xx url enoded
1139         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1140         optional => 1,
1141     },
1142 );
1143
1144 my %queues_fmt = (
1145     queues => {
1146         type => 'integer',
1147         description => "Number of queues.",
1148         minimum => 2,
1149         optional => 1
1150     }
1151 );
1152
1153 my %scsiblock_fmt = (
1154     scsiblock => {
1155         type => 'boolean',
1156         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1157         optional => 1,
1158         default => 0,
1159     },
1160 );
1161
1162 my %ssd_fmt = (
1163     ssd => {
1164         type => 'boolean',
1165         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1166         optional => 1,
1167     },
1168 );
1169
1170 my %wwn_fmt = (
1171     wwn => {
1172         type => 'string',
1173         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1174         format_description => 'wwn',
1175         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1176         optional => 1,
1177     },
1178 );
1179
1180 my $add_throttle_desc = sub {
1181     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1182     my $d = {
1183         type => $type,
1184         format_description => $unit,
1185         description => "Maximum $what in $longunit.",
1186         optional => 1,
1187     };
1188     $d->{minimum} = $minimum if defined($minimum);
1189     $drivedesc_base{$key} = $d;
1190 };
1191 # throughput: (leaky bucket)
1192 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1193 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1194 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1195 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1197 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1198 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1199 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1200 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1201
1202 # pools: (pool of IO before throttling starts taking effect)
1203 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1204 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1205 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1206 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1207 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1208 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1209
1210 # burst lengths
1211 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1212 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1213 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1214 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1215 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1216 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1217
1218 # legacy support
1219 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1220 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1221 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1222 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1223
1224 my $ide_fmt = {
1225     %drivedesc_base,
1226     %model_fmt,
1227     %ssd_fmt,
1228     %wwn_fmt,
1229 };
1230 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1231
1232 my $idedesc = {
1233     optional => 1,
1234     type => 'string', format => $ide_fmt,
1235     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1236 };
1237 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1238
1239 my $scsi_fmt = {
1240     %drivedesc_base,
1241     %iothread_fmt,
1242     %queues_fmt,
1243     %scsiblock_fmt,
1244     %ssd_fmt,
1245     %wwn_fmt,
1246 };
1247 my $scsidesc = {
1248     optional => 1,
1249     type => 'string', format => $scsi_fmt,
1250     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1251 };
1252 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1253
1254 my $sata_fmt = {
1255     %drivedesc_base,
1256     %ssd_fmt,
1257     %wwn_fmt,
1258 };
1259 my $satadesc = {
1260     optional => 1,
1261     type => 'string', format => $sata_fmt,
1262     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1265
1266 my $virtio_fmt = {
1267     %drivedesc_base,
1268     %iothread_fmt,
1269 };
1270 my $virtiodesc = {
1271     optional => 1,
1272     type => 'string', format => $virtio_fmt,
1273     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1274 };
1275 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1276
1277 my $alldrive_fmt = {
1278     %drivedesc_base,
1279     %iothread_fmt,
1280     %model_fmt,
1281     %queues_fmt,
1282     %scsiblock_fmt,
1283     %ssd_fmt,
1284     %wwn_fmt,
1285 };
1286
1287 my $efidisk_fmt = {
1288     volume => { alias => 'file' },
1289     file => {
1290         type => 'string',
1291         format => 'pve-volume-id-or-qm-path',
1292         default_key => 1,
1293         format_description => 'volume',
1294         description => "The drive's backing volume.",
1295     },
1296     format => get_standard_option('pve-qm-image-format'),
1297     size => {
1298         type => 'string',
1299         format => 'disk-size',
1300         format_description => 'DiskSize',
1301         description => "Disk size. This is purely informational and has no effect.",
1302         optional => 1,
1303     },
1304 };
1305
1306 my $efidisk_desc = {
1307     optional => 1,
1308     type => 'string', format => $efidisk_fmt,
1309     description => "Configure a Disk for storing EFI vars",
1310 };
1311
1312 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1313
1314 my $usb_fmt = {
1315     host => {
1316         default_key => 1,
1317         type => 'string', format => 'pve-qm-usb-device',
1318         format_description => 'HOSTUSBDEVICE|spice',
1319         description => <<EODESCR,
1320 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1321
1322  'bus-port(.port)*' (decimal numbers) or
1323  'vendor_id:product_id' (hexadeciaml numbers) or
1324  'spice'
1325
1326 You can use the 'lsusb -t' command to list existing usb devices.
1327
1328 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1329
1330 The value 'spice' can be used to add a usb redirection devices for spice.
1331 EODESCR
1332     },
1333     usb3 => {
1334         optional => 1,
1335         type => 'boolean',
1336         description => "Specifies whether if given host option is a USB3 device or port.",
1337         default => 0,
1338     },
1339 };
1340
1341 my $usbdesc = {
1342     optional => 1,
1343     type => 'string', format => $usb_fmt,
1344     description => "Configure an USB device (n is 0 to 4).",
1345 };
1346 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1347
1348 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1349 my $hostpci_fmt = {
1350     host => {
1351         default_key => 1,
1352         type => 'string',
1353         pattern => qr/$PCIRE(;$PCIRE)*/,
1354         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1355         description => <<EODESCR,
1356 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1357 of PCI virtual functions of the host. HOSTPCIID syntax is:
1358
1359 'bus:dev.func' (hexadecimal numbers)
1360
1361 You can us the 'lspci' command to list existing PCI devices.
1362 EODESCR
1363     },
1364     rombar => {
1365         type => 'boolean',
1366         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1367         optional => 1,
1368         default => 1,
1369     },
1370     romfile => {
1371         type => 'string',
1372         pattern => '[^,;]+',
1373         format_description => 'string',
1374         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1375         optional => 1,
1376     },
1377     pcie => {
1378         type => 'boolean',
1379         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1380         optional => 1,
1381         default => 0,
1382     },
1383     'x-vga' => {
1384         type => 'boolean',
1385         description =>  "Enable vfio-vga device support.",
1386         optional => 1,
1387         default => 0,
1388     },
1389     'mdev' => {
1390         type => 'string',
1391         format_description => 'string',
1392         pattern => '[^/\.:]+',
1393         optional => 1,
1394         description => <<EODESCR
1395 The type of mediated device to use.
1396 An instance of this type will be created on startup of the VM and
1397 will be cleaned up when the VM stops.
1398 EODESCR
1399     }
1400 };
1401 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1402
1403 my $hostpcidesc = {
1404         optional => 1,
1405         type => 'string', format => 'pve-qm-hostpci',
1406         description => "Map host PCI devices into guest.",
1407         verbose_description =>  <<EODESCR,
1408 Map host PCI devices into guest.
1409
1410 NOTE: This option allows direct access to host hardware. So it is no longer
1411 possible to migrate such machines - use with special care.
1412
1413 CAUTION: Experimental! User reported problems with this option.
1414 EODESCR
1415 };
1416 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1417
1418 my $serialdesc = {
1419         optional => 1,
1420         type => 'string',
1421         pattern => '(/dev/.+|socket)',
1422         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1423         verbose_description =>  <<EODESCR,
1424 Create a serial device inside the VM (n is 0 to 3), and pass through a
1425 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1426 host side (use 'qm terminal' to open a terminal connection).
1427
1428 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1429
1430 CAUTION: Experimental! User reported problems with this option.
1431 EODESCR
1432 };
1433
1434 my $paralleldesc= {
1435         optional => 1,
1436         type => 'string',
1437         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1438         description =>  "Map host parallel devices (n is 0 to 2).",
1439         verbose_description =>  <<EODESCR,
1440 Map host parallel devices (n is 0 to 2).
1441
1442 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1443
1444 CAUTION: Experimental! User reported problems with this option.
1445 EODESCR
1446 };
1447
1448 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1449     $confdesc->{"parallel$i"} = $paralleldesc;
1450 }
1451
1452 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1453     $confdesc->{"serial$i"} = $serialdesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1457     $confdesc->{"hostpci$i"} = $hostpcidesc;
1458 }
1459
1460 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1461     $drivename_hash->{"ide$i"} = 1;
1462     $confdesc->{"ide$i"} = $idedesc;
1463 }
1464
1465 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1466     $drivename_hash->{"sata$i"} = 1;
1467     $confdesc->{"sata$i"} = $satadesc;
1468 }
1469
1470 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1471     $drivename_hash->{"scsi$i"} = 1;
1472     $confdesc->{"scsi$i"} = $scsidesc ;
1473 }
1474
1475 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1476     $drivename_hash->{"virtio$i"} = 1;
1477     $confdesc->{"virtio$i"} = $virtiodesc;
1478 }
1479
1480 $drivename_hash->{efidisk0} = 1;
1481 $confdesc->{efidisk0} = $efidisk_desc;
1482
1483 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1484     $confdesc->{"usb$i"} = $usbdesc;
1485 }
1486
1487 my $unuseddesc = {
1488     optional => 1,
1489     type => 'string', format => 'pve-volume-id',
1490     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1491 };
1492
1493 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1494     $confdesc->{"unused$i"} = $unuseddesc;
1495 }
1496
1497 my $kvm_api_version = 0;
1498
1499 sub kvm_version {
1500     return $kvm_api_version if $kvm_api_version;
1501
1502     open my $fh, '<', '/dev/kvm'
1503         or return undef;
1504
1505     # 0xae00 => KVM_GET_API_VERSION
1506     $kvm_api_version = ioctl($fh, 0xae00, 0);
1507
1508     return $kvm_api_version;
1509 }
1510
1511 my $kvm_user_version = {};
1512 my $kvm_mtime = {};
1513
1514 sub kvm_user_version {
1515     my ($binary) = @_;
1516
1517     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1518     my $st = stat($binary);
1519
1520     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1521     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1522         $cachedmtime == $st->mtime;
1523
1524     $kvm_user_version->{$binary} = 'unknown';
1525     $kvm_mtime->{$binary} = $st->mtime;
1526
1527     my $code = sub {
1528         my $line = shift;
1529         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1530             $kvm_user_version->{$binary} = $2;
1531         }
1532     };
1533
1534     eval { run_command([$binary, '--version'], outfunc => $code); };
1535     warn $@ if $@;
1536
1537     return $kvm_user_version->{$binary};
1538
1539 }
1540
1541 sub kernel_has_vhost_net {
1542     return -c '/dev/vhost-net';
1543 }
1544
1545 sub valid_drive_names {
1546     # order is important - used to autoselect boot disk
1547     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1548             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1549             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1550             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1551             'efidisk0');
1552 }
1553
1554 sub is_valid_drivename {
1555     my $dev = shift;
1556
1557     return defined($drivename_hash->{$dev});
1558 }
1559
1560 sub option_exists {
1561     my $key = shift;
1562     return defined($confdesc->{$key});
1563 }
1564
1565 sub nic_models {
1566     return $nic_model_list;
1567 }
1568
1569 sub os_list_description {
1570
1571     return {
1572         other => 'Other',
1573         wxp => 'Windows XP',
1574         w2k => 'Windows 2000',
1575         w2k3 =>, 'Windows 2003',
1576         w2k8 => 'Windows 2008',
1577         wvista => 'Windows Vista',
1578         win7 => 'Windows 7',
1579         win8 => 'Windows 8/2012',
1580         win10 => 'Windows 10/2016',
1581         l24 => 'Linux 2.4',
1582         l26 => 'Linux 2.6',
1583     };
1584 }
1585
1586 my $cdrom_path;
1587
1588 sub get_cdrom_path {
1589
1590     return  $cdrom_path if $cdrom_path;
1591
1592     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1593     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1594     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1595 }
1596
1597 sub get_iso_path {
1598     my ($storecfg, $vmid, $cdrom) = @_;
1599
1600     if ($cdrom eq 'cdrom') {
1601         return get_cdrom_path();
1602     } elsif ($cdrom eq 'none') {
1603         return '';
1604     } elsif ($cdrom =~ m|^/|) {
1605         return $cdrom;
1606     } else {
1607         return PVE::Storage::path($storecfg, $cdrom);
1608     }
1609 }
1610
1611 # try to convert old style file names to volume IDs
1612 sub filename_to_volume_id {
1613     my ($vmid, $file, $media) = @_;
1614
1615      if (!($file eq 'none' || $file eq 'cdrom' ||
1616           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1617
1618         return undef if $file =~ m|/|;
1619
1620         if ($media && $media eq 'cdrom') {
1621             $file = "local:iso/$file";
1622         } else {
1623             $file = "local:$vmid/$file";
1624         }
1625     }
1626
1627     return $file;
1628 }
1629
1630 sub verify_media_type {
1631     my ($opt, $vtype, $media) = @_;
1632
1633     return if !$media;
1634
1635     my $etype;
1636     if ($media eq 'disk') {
1637         $etype = 'images';
1638     } elsif ($media eq 'cdrom') {
1639         $etype = 'iso';
1640     } else {
1641         die "internal error";
1642     }
1643
1644     return if ($vtype eq $etype);
1645
1646     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1647 }
1648
1649 sub cleanup_drive_path {
1650     my ($opt, $storecfg, $drive) = @_;
1651
1652     # try to convert filesystem paths to volume IDs
1653
1654     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1655         ($drive->{file} !~ m|^/dev/.+|) &&
1656         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1657         ($drive->{file} !~ m/^\d+$/)) {
1658         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1659         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1660         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1661         verify_media_type($opt, $vtype, $drive->{media});
1662         $drive->{file} = $volid;
1663     }
1664
1665     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1666 }
1667
1668 sub parse_hotplug_features {
1669     my ($data) = @_;
1670
1671     my $res = {};
1672
1673     return $res if $data eq '0';
1674
1675     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1676
1677     foreach my $feature (PVE::Tools::split_list($data)) {
1678         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1679             $res->{$1} = 1;
1680         } else {
1681             die "invalid hotplug feature '$feature'\n";
1682         }
1683     }
1684     return $res;
1685 }
1686
1687 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1688 sub pve_verify_hotplug_features {
1689     my ($value, $noerr) = @_;
1690
1691     return $value if parse_hotplug_features($value);
1692
1693     return undef if $noerr;
1694
1695     die "unable to parse hotplug option\n";
1696 }
1697
1698 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1699 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1700 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1701 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1702 #        [,iothread=on][,serial=serial][,model=model]
1703
1704 sub parse_drive {
1705     my ($key, $data) = @_;
1706
1707     my ($interface, $index);
1708
1709     if ($key =~ m/^([^\d]+)(\d+)$/) {
1710         $interface = $1;
1711         $index = $2;
1712     } else {
1713         return undef;
1714     }
1715
1716     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1717                                      : $confdesc->{$key}->{format};
1718     if (!$desc) {
1719         warn "invalid drive key: $key\n";
1720         return undef;
1721     }
1722     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1723     return undef if !$res;
1724     $res->{interface} = $interface;
1725     $res->{index} = $index;
1726
1727     my $error = 0;
1728     foreach my $opt (qw(bps bps_rd bps_wr)) {
1729         if (my $bps = defined(delete $res->{$opt})) {
1730             if (defined($res->{"m$opt"})) {
1731                 warn "both $opt and m$opt specified\n";
1732                 ++$error;
1733                 next;
1734             }
1735             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1736         }
1737     }
1738
1739     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1740     for my $requirement (
1741         [mbps_max => 'mbps'],
1742         [mbps_rd_max => 'mbps_rd'],
1743         [mbps_wr_max => 'mbps_wr'],
1744         [miops_max => 'miops'],
1745         [miops_rd_max => 'miops_rd'],
1746         [miops_wr_max => 'miops_wr'],
1747         [bps_max_length => 'mbps_max'],
1748         [bps_rd_max_length => 'mbps_rd_max'],
1749         [bps_wr_max_length => 'mbps_wr_max'],
1750         [iops_max_length => 'iops_max'],
1751         [iops_rd_max_length => 'iops_rd_max'],
1752         [iops_wr_max_length => 'iops_wr_max']) {
1753         my ($option, $requires) = @$requirement;
1754         if ($res->{$option} && !$res->{$requires}) {
1755             warn "$option requires $requires\n";
1756             ++$error;
1757         }
1758     }
1759
1760     return undef if $error;
1761
1762     return undef if $res->{mbps_rd} && $res->{mbps};
1763     return undef if $res->{mbps_wr} && $res->{mbps};
1764     return undef if $res->{iops_rd} && $res->{iops};
1765     return undef if $res->{iops_wr} && $res->{iops};
1766
1767     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1768         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1769         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1770         return undef if $res->{interface} eq 'virtio';
1771     }
1772
1773     if (my $size = $res->{size}) {
1774         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1775     }
1776
1777     return $res;
1778 }
1779
1780 sub print_drive {
1781     my ($vmid, $drive) = @_;
1782     my $data = { %$drive };
1783     delete $data->{$_} for qw(index interface);
1784     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1785 }
1786
1787 sub scsi_inquiry {
1788     my($fh, $noerr) = @_;
1789
1790     my $SG_IO = 0x2285;
1791     my $SG_GET_VERSION_NUM = 0x2282;
1792
1793     my $versionbuf = "\x00" x 8;
1794     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1795     if (!$ret) {
1796         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1797         return undef;
1798     }
1799     my $version = unpack("I", $versionbuf);
1800     if ($version < 30000) {
1801         die "scsi generic interface too old\n"  if !$noerr;
1802         return undef;
1803     }
1804
1805     my $buf = "\x00" x 36;
1806     my $sensebuf = "\x00" x 8;
1807     my $cmd = pack("C x3 C x1", 0x12, 36);
1808
1809     # see /usr/include/scsi/sg.h
1810     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1811
1812     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1813                       length($sensebuf), 0, length($buf), $buf,
1814                       $cmd, $sensebuf, 6000);
1815
1816     $ret = ioctl($fh, $SG_IO, $packet);
1817     if (!$ret) {
1818         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1819         return undef;
1820     }
1821
1822     my @res = unpack($sg_io_hdr_t, $packet);
1823     if ($res[17] || $res[18]) {
1824         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1825         return undef;
1826     }
1827
1828     my $res = {};
1829     (my $byte0, my $byte1, $res->{vendor},
1830      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1831
1832     $res->{removable} = $byte1 & 128 ? 1 : 0;
1833     $res->{type} = $byte0 & 31;
1834
1835     return $res;
1836 }
1837
1838 sub path_is_scsi {
1839     my ($path) = @_;
1840
1841     my $fh = IO::File->new("+<$path") || return undef;
1842     my $res = scsi_inquiry($fh, 1);
1843     close($fh);
1844
1845     return $res;
1846 }
1847
1848 sub machine_type_is_q35 {
1849     my ($conf) = @_;
1850
1851     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1852 }
1853
1854 sub print_tabletdevice_full {
1855     my ($conf, $arch) = @_;
1856
1857     my $q35 = machine_type_is_q35($conf);
1858
1859     # we use uhci for old VMs because tablet driver was buggy in older qemu
1860     my $usbbus;
1861     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1862         $usbbus = 'ehci';
1863     } else {
1864         $usbbus = 'uhci';
1865     }
1866
1867     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1868 }
1869
1870 sub print_keyboarddevice_full {
1871     my ($conf, $arch, $machine) = @_;
1872
1873     return undef if $arch ne 'aarch64';
1874
1875     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1876 }
1877
1878 sub print_drivedevice_full {
1879     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1880
1881     my $device = '';
1882     my $maxdev = 0;
1883
1884     if ($drive->{interface} eq 'virtio') {
1885         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1886         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1887         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1888     } elsif ($drive->{interface} eq 'scsi') {
1889
1890         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1891         my $unit = $drive->{index} % $maxdev;
1892         my $devicetype = 'hd';
1893         my $path = '';
1894         if (drive_is_cdrom($drive)) {
1895             $devicetype = 'cd';
1896         } else {
1897             if ($drive->{file} =~ m|^/|) {
1898                 $path = $drive->{file};
1899                 if (my $info = path_is_scsi($path)) {
1900                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1901                         $devicetype = 'block';
1902                     } elsif ($info->{type} == 1) { # tape
1903                         $devicetype = 'generic';
1904                     }
1905                 }
1906             } else {
1907                  $path = PVE::Storage::path($storecfg, $drive->{file});
1908             }
1909
1910             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1911             if ($path =~ m/^iscsi\:\/\// &&
1912                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1913                 $devicetype = 'generic';
1914             }
1915         }
1916
1917         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1918             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1919         } else {
1920             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1921         }
1922
1923         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1924             $device .= ",rotation_rate=1";
1925         }
1926         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1927
1928     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1929         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1930         my $controller = int($drive->{index} / $maxdev);
1931         my $unit = $drive->{index} % $maxdev;
1932         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1933
1934         $device = "ide-$devicetype";
1935         if ($drive->{interface} eq 'ide') {
1936             $device .= ",bus=ide.$controller,unit=$unit";
1937         } else {
1938             $device .= ",bus=ahci$controller.$unit";
1939         }
1940         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1941
1942         if ($devicetype eq 'hd') {
1943             if (my $model = $drive->{model}) {
1944                 $model = URI::Escape::uri_unescape($model);
1945                 $device .= ",model=$model";
1946             }
1947             if ($drive->{ssd}) {
1948                 $device .= ",rotation_rate=1";
1949             }
1950         }
1951         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1952     } elsif ($drive->{interface} eq 'usb') {
1953         die "implement me";
1954         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1955     } else {
1956         die "unsupported interface type";
1957     }
1958
1959     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1960
1961     if (my $serial = $drive->{serial}) {
1962         $serial = URI::Escape::uri_unescape($serial);
1963         $device .= ",serial=$serial";
1964     }
1965
1966
1967     return $device;
1968 }
1969
1970 sub get_initiator_name {
1971     my $initiator;
1972
1973     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1974     while (defined(my $line = <$fh>)) {
1975         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1976         $initiator = $1;
1977         last;
1978     }
1979     $fh->close();
1980
1981     return $initiator;
1982 }
1983
1984 sub print_drive_full {
1985     my ($storecfg, $vmid, $drive) = @_;
1986
1987     my $path;
1988     my $volid = $drive->{file};
1989     my $format;
1990
1991     if (drive_is_cdrom($drive)) {
1992         $path = get_iso_path($storecfg, $vmid, $volid);
1993     } else {
1994         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1995         if ($storeid) {
1996             $path = PVE::Storage::path($storecfg, $volid);
1997             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1998             $format = qemu_img_format($scfg, $volname);
1999         } else {
2000             $path = $volid;
2001             $format = "raw";
2002         }
2003    }
2004
2005     my $opts = '';
2006     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
2007     foreach my $o (@qemu_drive_options) {
2008         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2009     }
2010
2011     # snapshot only accepts on|off
2012     if (defined($drive->{snapshot})) {
2013         my $v = $drive->{snapshot} ? 'on' : 'off';
2014         $opts .= ",snapshot=$v";
2015     }
2016
2017     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2018         my ($dir, $qmpname) = @$type;
2019         if (my $v = $drive->{"mbps$dir"}) {
2020             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2021         }
2022         if (my $v = $drive->{"mbps${dir}_max"}) {
2023             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2024         }
2025         if (my $v = $drive->{"bps${dir}_max_length"}) {
2026             $opts .= ",throttling.bps$qmpname-max-length=$v";
2027         }
2028         if (my $v = $drive->{"iops${dir}"}) {
2029             $opts .= ",throttling.iops$qmpname=$v";
2030         }
2031         if (my $v = $drive->{"iops${dir}_max"}) {
2032             $opts .= ",throttling.iops$qmpname-max=$v";
2033         }
2034         if (my $v = $drive->{"iops${dir}_max_length"}) {
2035             $opts .= ",throttling.iops$qmpname-max-length=$v";
2036         }
2037     }
2038
2039     $opts .= ",format=$format" if $format && !$drive->{format};
2040
2041     my $cache_direct = 0;
2042
2043     if (my $cache = $drive->{cache}) {
2044         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2045     } elsif (!drive_is_cdrom($drive)) {
2046         $opts .= ",cache=none";
2047         $cache_direct = 1;
2048     }
2049
2050     # aio native works only with O_DIRECT
2051     if (!$drive->{aio}) {
2052         if($cache_direct) {
2053             $opts .= ",aio=native";
2054         } else {
2055             $opts .= ",aio=threads";
2056         }
2057     }
2058
2059     if (!drive_is_cdrom($drive)) {
2060         my $detectzeroes;
2061         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2062             $detectzeroes = 'off';
2063         } elsif ($drive->{discard}) {
2064             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2065         } else {
2066             # This used to be our default with discard not being specified:
2067             $detectzeroes = 'on';
2068         }
2069         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2070     }
2071
2072     my $pathinfo = $path ? "file=$path," : '';
2073
2074     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2075 }
2076
2077 sub print_netdevice_full {
2078     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2079
2080     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2081
2082     my $device = $net->{model};
2083     if ($net->{model} eq 'virtio') {
2084          $device = 'virtio-net-pci';
2085      };
2086
2087     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2088     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2089     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2090         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2091         my $vectors = $net->{queues} * 2 + 2;
2092         $tmpstr .= ",vectors=$vectors,mq=on";
2093     }
2094     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2095
2096     if ($use_old_bios_files) {
2097         my $romfile;
2098         if ($device eq 'virtio-net-pci') {
2099             $romfile = 'pxe-virtio.rom';
2100         } elsif ($device eq 'e1000') {
2101             $romfile = 'pxe-e1000.rom';
2102         } elsif ($device eq 'ne2k') {
2103             $romfile = 'pxe-ne2k_pci.rom';
2104         } elsif ($device eq 'pcnet') {
2105             $romfile = 'pxe-pcnet.rom';
2106         } elsif ($device eq 'rtl8139') {
2107             $romfile = 'pxe-rtl8139.rom';
2108         }
2109         $tmpstr .= ",romfile=$romfile" if $romfile;
2110     }
2111
2112     return $tmpstr;
2113 }
2114
2115 sub print_netdev_full {
2116     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2117
2118     my $i = '';
2119     if ($netid =~ m/^net(\d+)$/) {
2120         $i = int($1);
2121     }
2122
2123     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2124
2125     my $ifname = "tap${vmid}i$i";
2126
2127     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2128     die "interface name '$ifname' is too long (max 15 character)\n"
2129         if length($ifname) >= 16;
2130
2131     my $vhostparam = '';
2132     if (is_native($arch)) {
2133         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2134     }
2135
2136     my $vmname = $conf->{name} || "vm$vmid";
2137
2138     my $netdev = "";
2139     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2140
2141     if ($net->{bridge}) {
2142         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2143     } else {
2144         $netdev = "type=user,id=$netid,hostname=$vmname";
2145     }
2146
2147     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2148
2149     return $netdev;
2150 }
2151
2152
2153 sub print_cpu_device {
2154     my ($conf, $id) = @_;
2155
2156     my $kvm = $conf->{kvm} // 1;
2157     my $cpu = $kvm ? "kvm64" : "qemu64";
2158     if (my $cputype = $conf->{cpu}) {
2159         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2160             or die "Cannot parse cpu description: $cputype\n";
2161         $cpu = $cpuconf->{cputype};
2162     }
2163
2164     my $cores = $conf->{cores} || 1;
2165
2166     my $current_core = ($id - 1) % $cores;
2167     my $current_socket = int(($id - 1 - $current_core)/$cores);
2168
2169     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2170 }
2171
2172 my $vga_map = {
2173     'cirrus' => 'cirrus-vga',
2174     'std' => 'VGA',
2175     'vmware' => 'vmware-svga',
2176     'virtio' => 'virtio-vga',
2177 };
2178
2179 sub print_vga_device {
2180     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2181
2182     my $type = $vga_map->{$vga->{type}};
2183     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2184         $type = 'virtio-gpu';
2185     }
2186     my $vgamem_mb = $vga->{memory};
2187     if ($qxlnum) {
2188         $type = $id ? 'qxl' : 'qxl-vga';
2189     }
2190     die "no devicetype for $vga->{type}\n" if !$type;
2191
2192     my $memory = "";
2193     if ($vgamem_mb) {
2194         if ($vga->{type} eq 'virtio') {
2195             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2196             $memory = ",max_hostmem=$bytes";
2197         } elsif ($qxlnum) {
2198             # from https://www.spice-space.org/multiple-monitors.html
2199             $memory = ",vgamem_mb=$vga->{memory}";
2200             my $ram = $vgamem_mb * 4;
2201             my $vram = $vgamem_mb * 2;
2202             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2203         } else {
2204             $memory = ",vgamem_mb=$vga->{memory}";
2205         }
2206     } elsif ($qxlnum && $id) {
2207         $memory = ",ram_size=67108864,vram_size=33554432";
2208     }
2209
2210     my $q35 = machine_type_is_q35($conf);
2211     my $vgaid = "vga" . ($id // '');
2212     my $pciaddr;
2213
2214     if ($q35 && $vgaid eq 'vga') {
2215         # the first display uses pcie.0 bus on q35 machines
2216         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2217     } else {
2218         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2219     }
2220
2221     return "$type,id=${vgaid}${memory}${pciaddr}";
2222 }
2223
2224 sub drive_is_cloudinit {
2225     my ($drive) = @_;
2226     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2227 }
2228
2229 sub drive_is_cdrom {
2230     my ($drive, $exclude_cloudinit) = @_;
2231
2232     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2233
2234     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2235
2236 }
2237
2238 sub parse_number_sets {
2239     my ($set) = @_;
2240     my $res = [];
2241     foreach my $part (split(/;/, $set)) {
2242         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2243             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2244             push @$res, [ $1, $2 ];
2245         } else {
2246             die "invalid range: $part\n";
2247         }
2248     }
2249     return $res;
2250 }
2251
2252 sub parse_numa {
2253     my ($data) = @_;
2254
2255     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2256     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2257     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2258     return $res;
2259 }
2260
2261 sub parse_hostpci {
2262     my ($value) = @_;
2263
2264     return undef if !$value;
2265
2266     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2267
2268     my @idlist = split(/;/, $res->{host});
2269     delete $res->{host};
2270     foreach my $id (@idlist) {
2271         if ($id =~ m/\./) { # full id 00:00.1
2272             push @{$res->{pciid}}, {
2273                 id => $id,
2274             };
2275         } else { # partial id 00:00
2276             $res->{pciid} = PVE::SysFSTools::lspci($id);
2277         }
2278     }
2279     return $res;
2280 }
2281
2282 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2283 sub parse_net {
2284     my ($data) = @_;
2285
2286     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2287     if ($@) {
2288         warn $@;
2289         return undef;
2290     }
2291     if (!defined($res->{macaddr})) {
2292         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2293         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2294     }
2295     return $res;
2296 }
2297
2298 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2299 sub parse_ipconfig {
2300     my ($data) = @_;
2301
2302     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2303     if ($@) {
2304         warn $@;
2305         return undef;
2306     }
2307
2308     if ($res->{gw} && !$res->{ip}) {
2309         warn 'gateway specified without specifying an IP address';
2310         return undef;
2311     }
2312     if ($res->{gw6} && !$res->{ip6}) {
2313         warn 'IPv6 gateway specified without specifying an IPv6 address';
2314         return undef;
2315     }
2316     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2317         warn 'gateway specified together with DHCP';
2318         return undef;
2319     }
2320     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2321         # gw6 + auto/dhcp
2322         warn "IPv6 gateway specified together with $res->{ip6} address";
2323         return undef;
2324     }
2325
2326     if (!$res->{ip} && !$res->{ip6}) {
2327         return { ip => 'dhcp', ip6 => 'dhcp' };
2328     }
2329
2330     return $res;
2331 }
2332
2333 sub print_net {
2334     my $net = shift;
2335
2336     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2337 }
2338
2339 sub add_random_macs {
2340     my ($settings) = @_;
2341
2342     foreach my $opt (keys %$settings) {
2343         next if $opt !~ m/^net(\d+)$/;
2344         my $net = parse_net($settings->{$opt});
2345         next if !$net;
2346         $settings->{$opt} = print_net($net);
2347     }
2348 }
2349
2350 sub vm_is_volid_owner {
2351     my ($storecfg, $vmid, $volid) = @_;
2352
2353     if ($volid !~  m|^/|) {
2354         my ($path, $owner);
2355         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2356         if ($owner && ($owner == $vmid)) {
2357             return 1;
2358         }
2359     }
2360
2361     return undef;
2362 }
2363
2364 sub vmconfig_register_unused_drive {
2365     my ($storecfg, $vmid, $conf, $drive) = @_;
2366
2367     if (drive_is_cloudinit($drive)) {
2368         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2369         warn $@ if $@;
2370     } elsif (!drive_is_cdrom($drive)) {
2371         my $volid = $drive->{file};
2372         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2373             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2374         }
2375     }
2376 }
2377
2378 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2379 my $smbios1_fmt = {
2380     uuid => {
2381         type => 'string',
2382         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2383         format_description => 'UUID',
2384         description => "Set SMBIOS1 UUID.",
2385         optional => 1,
2386     },
2387     version => {
2388         type => 'string',
2389         pattern => '[A-Za-z0-9+\/]+={0,2}',
2390         format_description => 'Base64 encoded string',
2391         description => "Set SMBIOS1 version.",
2392         optional => 1,
2393     },
2394     serial => {
2395         type => 'string',
2396         pattern => '[A-Za-z0-9+\/]+={0,2}',
2397         format_description => 'Base64 encoded string',
2398         description => "Set SMBIOS1 serial number.",
2399         optional => 1,
2400     },
2401     manufacturer => {
2402         type => 'string',
2403         pattern => '[A-Za-z0-9+\/]+={0,2}',
2404         format_description => 'Base64 encoded string',
2405         description => "Set SMBIOS1 manufacturer.",
2406         optional => 1,
2407     },
2408     product => {
2409         type => 'string',
2410         pattern => '[A-Za-z0-9+\/]+={0,2}',
2411         format_description => 'Base64 encoded string',
2412         description => "Set SMBIOS1 product ID.",
2413         optional => 1,
2414     },
2415     sku => {
2416         type => 'string',
2417         pattern => '[A-Za-z0-9+\/]+={0,2}',
2418         format_description => 'Base64 encoded string',
2419         description => "Set SMBIOS1 SKU string.",
2420         optional => 1,
2421     },
2422     family => {
2423         type => 'string',
2424         pattern => '[A-Za-z0-9+\/]+={0,2}',
2425         format_description => 'Base64 encoded string',
2426         description => "Set SMBIOS1 family string.",
2427         optional => 1,
2428     },
2429     base64 => {
2430         type => 'boolean',
2431         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2432         optional => 1,
2433     },
2434 };
2435
2436 sub parse_smbios1 {
2437     my ($data) = @_;
2438
2439     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2440     warn $@ if $@;
2441     return $res;
2442 }
2443
2444 sub print_smbios1 {
2445     my ($smbios1) = @_;
2446     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2447 }
2448
2449 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2450
2451 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2452 sub verify_bootdisk {
2453     my ($value, $noerr) = @_;
2454
2455     return $value if is_valid_drivename($value);
2456
2457     return undef if $noerr;
2458
2459     die "invalid boot disk '$value'\n";
2460 }
2461
2462 sub parse_watchdog {
2463     my ($value) = @_;
2464
2465     return undef if !$value;
2466
2467     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 sub parse_guest_agent {
2473     my ($value) = @_;
2474
2475     return {} if !defined($value->{agent});
2476
2477     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2478     warn $@ if $@;
2479
2480     # if the agent is disabled ignore the other potentially set properties
2481     return {} if !$res->{enabled};
2482     return $res;
2483 }
2484
2485 sub parse_vga {
2486     my ($value) = @_;
2487
2488     return {} if !$value;
2489     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2490     warn $@ if $@;
2491     return $res;
2492 }
2493
2494 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2495 sub verify_usb_device {
2496     my ($value, $noerr) = @_;
2497
2498     return $value if parse_usb_device($value);
2499
2500     return undef if $noerr;
2501
2502     die "unable to parse usb device\n";
2503 }
2504
2505 # add JSON properties for create and set function
2506 sub json_config_properties {
2507     my $prop = shift;
2508
2509     foreach my $opt (keys %$confdesc) {
2510         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2511         $prop->{$opt} = $confdesc->{$opt};
2512     }
2513
2514     return $prop;
2515 }
2516
2517 # return copy of $confdesc_cloudinit to generate documentation
2518 sub cloudinit_config_properties {
2519
2520     return dclone($confdesc_cloudinit);
2521 }
2522
2523 sub check_type {
2524     my ($key, $value) = @_;
2525
2526     die "unknown setting '$key'\n" if !$confdesc->{$key};
2527
2528     my $type = $confdesc->{$key}->{type};
2529
2530     if (!defined($value)) {
2531         die "got undefined value\n";
2532     }
2533
2534     if ($value =~ m/[\n\r]/) {
2535         die "property contains a line feed\n";
2536     }
2537
2538     if ($type eq 'boolean') {
2539         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2540         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2541         die "type check ('boolean') failed - got '$value'\n";
2542     } elsif ($type eq 'integer') {
2543         return int($1) if $value =~ m/^(\d+)$/;
2544         die "type check ('integer') failed - got '$value'\n";
2545     } elsif ($type eq 'number') {
2546         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2547         die "type check ('number') failed - got '$value'\n";
2548     } elsif ($type eq 'string') {
2549         if (my $fmt = $confdesc->{$key}->{format}) {
2550             PVE::JSONSchema::check_format($fmt, $value);
2551             return $value;
2552         }
2553         $value =~ s/^\"(.*)\"$/$1/;
2554         return $value;
2555     } else {
2556         die "internal error"
2557     }
2558 }
2559
2560 sub destroy_vm {
2561     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2562
2563     my $conf = PVE::QemuConfig->load_config($vmid);
2564
2565     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2566
2567     if ($conf->{template}) {
2568         # check if any base image is still used by a linked clone
2569         foreach_drive($conf, sub {
2570                 my ($ds, $drive) = @_;
2571                 return if drive_is_cdrom($drive);
2572
2573                 my $volid = $drive->{file};
2574                 return if !$volid || $volid =~ m|^/|;
2575
2576                 die "base volume '$volid' is still in use by linked cloned\n"
2577                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2578
2579         });
2580     }
2581
2582     # only remove disks owned by this VM
2583     foreach_drive($conf, sub {
2584         my ($ds, $drive) = @_;
2585         return if drive_is_cdrom($drive, 1);
2586
2587         my $volid = $drive->{file};
2588         return if !$volid || $volid =~ m|^/|;
2589
2590         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2591         return if !$path || !$owner || ($owner != $vmid);
2592
2593         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2594         warn "Could not remove disk '$volid', check manually: $@" if $@;
2595     });
2596
2597     # also remove unused disk
2598     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2599     PVE::Storage::foreach_volid($vmdisks, sub {
2600         my ($volid, $sid, $volname, $d) = @_;
2601         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2602         warn $@ if $@;
2603     });
2604
2605     if (defined $replacement_conf) {
2606         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2607     } else {
2608         PVE::QemuConfig->destroy_config($vmid);
2609     }
2610 }
2611
2612 sub parse_vm_config {
2613     my ($filename, $raw) = @_;
2614
2615     return undef if !defined($raw);
2616
2617     my $res = {
2618         digest => Digest::SHA::sha1_hex($raw),
2619         snapshots => {},
2620         pending => {},
2621     };
2622
2623     $filename =~ m|/qemu-server/(\d+)\.conf$|
2624         || die "got strange filename '$filename'";
2625
2626     my $vmid = $1;
2627
2628     my $conf = $res;
2629     my $descr;
2630     my $section = '';
2631
2632     my @lines = split(/\n/, $raw);
2633     foreach my $line (@lines) {
2634         next if $line =~ m/^\s*$/;
2635
2636         if ($line =~ m/^\[PENDING\]\s*$/i) {
2637             $section = 'pending';
2638             if (defined($descr)) {
2639                 $descr =~ s/\s+$//;
2640                 $conf->{description} = $descr;
2641             }
2642             $descr = undef;
2643             $conf = $res->{$section} = {};
2644             next;
2645
2646         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2647             $section = $1;
2648             if (defined($descr)) {
2649                 $descr =~ s/\s+$//;
2650                 $conf->{description} = $descr;
2651             }
2652             $descr = undef;
2653             $conf = $res->{snapshots}->{$section} = {};
2654             next;
2655         }
2656
2657         if ($line =~ m/^\#(.*)\s*$/) {
2658             $descr = '' if !defined($descr);
2659             $descr .= PVE::Tools::decode_text($1) . "\n";
2660             next;
2661         }
2662
2663         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2664             $descr = '' if !defined($descr);
2665             $descr .= PVE::Tools::decode_text($2);
2666         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2667             $conf->{snapstate} = $1;
2668         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2669             my $key = $1;
2670             my $value = $2;
2671             $conf->{$key} = $value;
2672         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2673             my $value = $1;
2674             if ($section eq 'pending') {
2675                 $conf->{delete} = $value; # we parse this later
2676             } else {
2677                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2678             }
2679         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2680             my $key = $1;
2681             my $value = $2;
2682             eval { $value = check_type($key, $value); };
2683             if ($@) {
2684                 warn "vm $vmid - unable to parse value of '$key' - $@";
2685             } else {
2686                 $key = 'ide2' if $key eq 'cdrom';
2687                 my $fmt = $confdesc->{$key}->{format};
2688                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2689                     my $v = parse_drive($key, $value);
2690                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2691                         $v->{file} = $volid;
2692                         $value = print_drive($vmid, $v);
2693                     } else {
2694                         warn "vm $vmid - unable to parse value of '$key'\n";
2695                         next;
2696                     }
2697                 }
2698
2699                 $conf->{$key} = $value;
2700             }
2701         }
2702     }
2703
2704     if (defined($descr)) {
2705         $descr =~ s/\s+$//;
2706         $conf->{description} = $descr;
2707     }
2708     delete $res->{snapstate}; # just to be sure
2709
2710     return $res;
2711 }
2712
2713 sub write_vm_config {
2714     my ($filename, $conf) = @_;
2715
2716     delete $conf->{snapstate}; # just to be sure
2717
2718     if ($conf->{cdrom}) {
2719         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2720         $conf->{ide2} = $conf->{cdrom};
2721         delete $conf->{cdrom};
2722     }
2723
2724     # we do not use 'smp' any longer
2725     if ($conf->{sockets}) {
2726         delete $conf->{smp};
2727     } elsif ($conf->{smp}) {
2728         $conf->{sockets} = $conf->{smp};
2729         delete $conf->{cores};
2730         delete $conf->{smp};
2731     }
2732
2733     my $used_volids = {};
2734
2735     my $cleanup_config = sub {
2736         my ($cref, $pending, $snapname) = @_;
2737
2738         foreach my $key (keys %$cref) {
2739             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2740                 $key eq 'snapstate' || $key eq 'pending';
2741             my $value = $cref->{$key};
2742             if ($key eq 'delete') {
2743                 die "propertry 'delete' is only allowed in [PENDING]\n"
2744                     if !$pending;
2745                 # fixme: check syntax?
2746                 next;
2747             }
2748             eval { $value = check_type($key, $value); };
2749             die "unable to parse value of '$key' - $@" if $@;
2750
2751             $cref->{$key} = $value;
2752
2753             if (!$snapname && is_valid_drivename($key)) {
2754                 my $drive = parse_drive($key, $value);
2755                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2756             }
2757         }
2758     };
2759
2760     &$cleanup_config($conf);
2761
2762     &$cleanup_config($conf->{pending}, 1);
2763
2764     foreach my $snapname (keys %{$conf->{snapshots}}) {
2765         die "internal error" if $snapname eq 'pending';
2766         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2767     }
2768
2769     # remove 'unusedX' settings if we re-add a volume
2770     foreach my $key (keys %$conf) {
2771         my $value = $conf->{$key};
2772         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2773             delete $conf->{$key};
2774         }
2775     }
2776
2777     my $generate_raw_config = sub {
2778         my ($conf, $pending) = @_;
2779
2780         my $raw = '';
2781
2782         # add description as comment to top of file
2783         if (defined(my $descr = $conf->{description})) {
2784             if ($descr) {
2785                 foreach my $cl (split(/\n/, $descr)) {
2786                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2787                 }
2788             } else {
2789                 $raw .= "#\n" if $pending;
2790             }
2791         }
2792
2793         foreach my $key (sort keys %$conf) {
2794             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2795             $raw .= "$key: $conf->{$key}\n";
2796         }
2797         return $raw;
2798     };
2799
2800     my $raw = &$generate_raw_config($conf);
2801
2802     if (scalar(keys %{$conf->{pending}})){
2803         $raw .= "\n[PENDING]\n";
2804         $raw .= &$generate_raw_config($conf->{pending}, 1);
2805     }
2806
2807     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2808         $raw .= "\n[$snapname]\n";
2809         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2810     }
2811
2812     return $raw;
2813 }
2814
2815 sub load_defaults {
2816
2817     my $res = {};
2818
2819     # we use static defaults from our JSON schema configuration
2820     foreach my $key (keys %$confdesc) {
2821         if (defined(my $default = $confdesc->{$key}->{default})) {
2822             $res->{$key} = $default;
2823         }
2824     }
2825
2826     return $res;
2827 }
2828
2829 sub config_list {
2830     my $vmlist = PVE::Cluster::get_vmlist();
2831     my $res = {};
2832     return $res if !$vmlist || !$vmlist->{ids};
2833     my $ids = $vmlist->{ids};
2834
2835     foreach my $vmid (keys %$ids) {
2836         my $d = $ids->{$vmid};
2837         next if !$d->{node} || $d->{node} ne $nodename;
2838         next if !$d->{type} || $d->{type} ne 'qemu';
2839         $res->{$vmid}->{exists} = 1;
2840     }
2841     return $res;
2842 }
2843
2844 # test if VM uses local resources (to prevent migration)
2845 sub check_local_resources {
2846     my ($conf, $noerr) = @_;
2847
2848     my @loc_res = ();
2849
2850     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2851     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2852
2853     push @loc_res, "ivshmem" if $conf->{ivshmem};
2854
2855     foreach my $k (keys %$conf) {
2856         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2857         # sockets are safe: they will recreated be on the target side post-migrate
2858         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2859         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2860     }
2861
2862     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2863
2864     return \@loc_res;
2865 }
2866
2867 # check if used storages are available on all nodes (use by migrate)
2868 sub check_storage_availability {
2869     my ($storecfg, $conf, $node) = @_;
2870
2871     foreach_drive($conf, sub {
2872         my ($ds, $drive) = @_;
2873
2874         my $volid = $drive->{file};
2875         return if !$volid;
2876
2877         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2878         return if !$sid;
2879
2880         # check if storage is available on both nodes
2881         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2882         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2883    });
2884 }
2885
2886 # list nodes where all VM images are available (used by has_feature API)
2887 sub shared_nodes {
2888     my ($conf, $storecfg) = @_;
2889
2890     my $nodelist = PVE::Cluster::get_nodelist();
2891     my $nodehash = { map { $_ => 1 } @$nodelist };
2892     my $nodename = PVE::INotify::nodename();
2893
2894     foreach_drive($conf, sub {
2895         my ($ds, $drive) = @_;
2896
2897         my $volid = $drive->{file};
2898         return if !$volid;
2899
2900         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2901         if ($storeid) {
2902             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2903             if ($scfg->{disable}) {
2904                 $nodehash = {};
2905             } elsif (my $avail = $scfg->{nodes}) {
2906                 foreach my $node (keys %$nodehash) {
2907                     delete $nodehash->{$node} if !$avail->{$node};
2908                 }
2909             } elsif (!$scfg->{shared}) {
2910                 foreach my $node (keys %$nodehash) {
2911                     delete $nodehash->{$node} if $node ne $nodename
2912                 }
2913             }
2914         }
2915     });
2916
2917     return $nodehash
2918 }
2919
2920 sub check_local_storage_availability {
2921     my ($conf, $storecfg) = @_;
2922
2923     my $nodelist = PVE::Cluster::get_nodelist();
2924     my $nodehash = { map { $_ => {} } @$nodelist };
2925
2926     foreach_drive($conf, sub {
2927         my ($ds, $drive) = @_;
2928
2929         my $volid = $drive->{file};
2930         return if !$volid;
2931
2932         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2933         if ($storeid) {
2934             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2935
2936             if ($scfg->{disable}) {
2937                 foreach my $node (keys %$nodehash) {
2938                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2939                 }
2940             } elsif (my $avail = $scfg->{nodes}) {
2941                 foreach my $node (keys %$nodehash) {
2942                     if (!$avail->{$node}) {
2943                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2944                     }
2945                 }
2946             }
2947         }
2948     });
2949
2950     foreach my $node (values %$nodehash) {
2951         if (my $unavail = $node->{unavailable_storages}) {
2952             $node->{unavailable_storages} = [ sort keys %$unavail ];
2953         }
2954     }
2955
2956     return $nodehash
2957 }
2958
2959 sub check_cmdline {
2960     my ($pidfile, $pid) = @_;
2961
2962     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2963     if (defined($fh)) {
2964         my $line = <$fh>;
2965         $fh->close;
2966         return undef if !$line;
2967         my @param = split(/\0/, $line);
2968
2969         my $cmd = $param[0];
2970         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2971
2972         for (my $i = 0; $i < scalar (@param); $i++) {
2973             my $p = $param[$i];
2974             next if !$p;
2975             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2976                 my $p = $param[$i+1];
2977                 return 1 if $p && ($p eq $pidfile);
2978                 return undef;
2979             }
2980         }
2981     }
2982     return undef;
2983 }
2984
2985 sub check_running {
2986     my ($vmid, $nocheck, $node) = @_;
2987
2988     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2989
2990     die "unable to find configuration file for VM $vmid - no such machine\n"
2991         if !$nocheck && ! -f $filename;
2992
2993     my $pidfile = pidfile_name($vmid);
2994
2995     if (my $fd = IO::File->new("<$pidfile")) {
2996         my $st = stat($fd);
2997         my $line = <$fd>;
2998         close($fd);
2999
3000         my $mtime = $st->mtime;
3001         if ($mtime > time()) {
3002             warn "file '$filename' modified in future\n";
3003         }
3004
3005         if ($line =~ m/^(\d+)$/) {
3006             my $pid = $1;
3007             if (check_cmdline($pidfile, $pid)) {
3008                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3009                     return $pid;
3010                 }
3011             }
3012         }
3013     }
3014
3015     return undef;
3016 }
3017
3018 sub vzlist {
3019
3020     my $vzlist = config_list();
3021
3022     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3023
3024     while (defined(my $de = $fd->read)) {
3025         next if $de !~ m/^(\d+)\.pid$/;
3026         my $vmid = $1;
3027         next if !defined($vzlist->{$vmid});
3028         if (my $pid = check_running($vmid)) {
3029             $vzlist->{$vmid}->{pid} = $pid;
3030         }
3031     }
3032
3033     return $vzlist;
3034 }
3035
3036 sub disksize {
3037     my ($storecfg, $conf) = @_;
3038
3039     my $bootdisk = $conf->{bootdisk};
3040     return undef if !$bootdisk;
3041     return undef if !is_valid_drivename($bootdisk);
3042
3043     return undef if !$conf->{$bootdisk};
3044
3045     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3046     return undef if !defined($drive);
3047
3048     return undef if drive_is_cdrom($drive);
3049
3050     my $volid = $drive->{file};
3051     return undef if !$volid;
3052
3053     return $drive->{size};
3054 }
3055
3056 our $vmstatus_return_properties = {
3057     vmid => get_standard_option('pve-vmid'),
3058     status => {
3059         description => "Qemu process status.",
3060         type => 'string',
3061         enum => ['stopped', 'running'],
3062     },
3063     maxmem => {
3064         description => "Maximum memory in bytes.",
3065         type => 'integer',
3066         optional => 1,
3067         renderer => 'bytes',
3068     },
3069     maxdisk => {
3070         description => "Root disk size in bytes.",
3071         type => 'integer',
3072         optional => 1,
3073         renderer => 'bytes',
3074     },
3075     name => {
3076         description => "VM name.",
3077         type => 'string',
3078         optional => 1,
3079     },
3080     qmpstatus => {
3081         description => "Qemu QMP agent status.",
3082         type => 'string',
3083         optional => 1,
3084     },
3085     pid => {
3086         description => "PID of running qemu process.",
3087         type => 'integer',
3088         optional => 1,
3089     },
3090     uptime => {
3091         description => "Uptime.",
3092         type => 'integer',
3093         optional => 1,
3094         renderer => 'duration',
3095     },
3096     cpus => {
3097         description => "Maximum usable CPUs.",
3098         type => 'number',
3099         optional => 1,
3100     },
3101     lock => {
3102         description => "The current config lock, if any.",
3103         type => 'string',
3104         optional => 1,
3105     }
3106 };
3107
3108 my $last_proc_pid_stat;
3109
3110 # get VM status information
3111 # This must be fast and should not block ($full == false)
3112 # We only query KVM using QMP if $full == true (this can be slow)
3113 sub vmstatus {
3114     my ($opt_vmid, $full) = @_;
3115
3116     my $res = {};
3117
3118     my $storecfg = PVE::Storage::config();
3119
3120     my $list = vzlist();
3121     my $defaults = load_defaults();
3122
3123     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3124
3125     my $cpucount = $cpuinfo->{cpus} || 1;
3126
3127     foreach my $vmid (keys %$list) {
3128         next if $opt_vmid && ($vmid ne $opt_vmid);
3129
3130         my $conf = PVE::QemuConfig->load_config($vmid);
3131
3132         my $d = { vmid => $vmid };
3133         $d->{pid} = $list->{$vmid}->{pid};
3134
3135         # fixme: better status?
3136         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3137
3138         my $size = disksize($storecfg, $conf);
3139         if (defined($size)) {
3140             $d->{disk} = 0; # no info available
3141             $d->{maxdisk} = $size;
3142         } else {
3143             $d->{disk} = 0;
3144             $d->{maxdisk} = 0;
3145         }
3146
3147         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3148             * ($conf->{cores} || $defaults->{cores});
3149         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3150         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3151
3152         $d->{name} = $conf->{name} || "VM $vmid";
3153         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3154             : $defaults->{memory}*(1024*1024);
3155
3156         if ($conf->{balloon}) {
3157             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3158             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3159                 : $defaults->{shares};
3160         }
3161
3162         $d->{uptime} = 0;
3163         $d->{cpu} = 0;
3164         $d->{mem} = 0;
3165
3166         $d->{netout} = 0;
3167         $d->{netin} = 0;
3168
3169         $d->{diskread} = 0;
3170         $d->{diskwrite} = 0;
3171
3172         $d->{template} = PVE::QemuConfig->is_template($conf);
3173
3174         $d->{serial} = 1 if conf_has_serial($conf);
3175         $d->{lock} = $conf->{lock} if $conf->{lock};
3176
3177         $res->{$vmid} = $d;
3178     }
3179
3180     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3181     foreach my $dev (keys %$netdev) {
3182         next if $dev !~ m/^tap([1-9]\d*)i/;
3183         my $vmid = $1;
3184         my $d = $res->{$vmid};
3185         next if !$d;
3186
3187         $d->{netout} += $netdev->{$dev}->{receive};
3188         $d->{netin} += $netdev->{$dev}->{transmit};
3189
3190         if ($full) {
3191             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3192             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3193         }
3194
3195     }
3196
3197     my $ctime = gettimeofday;
3198
3199     foreach my $vmid (keys %$list) {
3200
3201         my $d = $res->{$vmid};
3202         my $pid = $d->{pid};
3203         next if !$pid;
3204
3205         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3206         next if !$pstat; # not running
3207
3208         my $used = $pstat->{utime} + $pstat->{stime};
3209
3210         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3211
3212         if ($pstat->{vsize}) {
3213             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3214         }
3215
3216         my $old = $last_proc_pid_stat->{$pid};
3217         if (!$old) {
3218             $last_proc_pid_stat->{$pid} = {
3219                 time => $ctime,
3220                 used => $used,
3221                 cpu => 0,
3222             };
3223             next;
3224         }
3225
3226         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3227
3228         if ($dtime > 1000) {
3229             my $dutime = $used -  $old->{used};
3230
3231             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3232             $last_proc_pid_stat->{$pid} = {
3233                 time => $ctime,
3234                 used => $used,
3235                 cpu => $d->{cpu},
3236             };
3237         } else {
3238             $d->{cpu} = $old->{cpu};
3239         }
3240     }
3241
3242     return $res if !$full;
3243
3244     my $qmpclient = PVE::QMPClient->new();
3245
3246     my $ballooncb = sub {
3247         my ($vmid, $resp) = @_;
3248
3249         my $info = $resp->{'return'};
3250         return if !$info->{max_mem};
3251
3252         my $d = $res->{$vmid};
3253
3254         # use memory assigned to VM
3255         $d->{maxmem} = $info->{max_mem};
3256         $d->{balloon} = $info->{actual};
3257
3258         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3259             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3260             $d->{freemem} = $info->{free_mem};
3261         }
3262
3263         $d->{ballooninfo} = $info;
3264     };
3265
3266     my $blockstatscb = sub {
3267         my ($vmid, $resp) = @_;
3268         my $data = $resp->{'return'} || [];
3269         my $totalrdbytes = 0;
3270         my $totalwrbytes = 0;
3271
3272         for my $blockstat (@$data) {
3273             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3274             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3275
3276             $blockstat->{device} =~ s/drive-//;
3277             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3278         }
3279         $res->{$vmid}->{diskread} = $totalrdbytes;
3280         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3281     };
3282
3283     my $statuscb = sub {
3284         my ($vmid, $resp) = @_;
3285
3286         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3287         # this fails if ballon driver is not loaded, so this must be
3288         # the last commnand (following command are aborted if this fails).
3289         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3290
3291         my $status = 'unknown';
3292         if (!defined($status = $resp->{'return'}->{status})) {
3293             warn "unable to get VM status\n";
3294             return;
3295         }
3296
3297         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3298     };
3299
3300     foreach my $vmid (keys %$list) {
3301         next if $opt_vmid && ($vmid ne $opt_vmid);
3302         next if !$res->{$vmid}->{pid}; # not running
3303         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3304     }
3305
3306     $qmpclient->queue_execute(undef, 2);
3307
3308     foreach my $vmid (keys %$list) {
3309         next if $opt_vmid && ($vmid ne $opt_vmid);
3310         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3311     }
3312
3313     return $res;
3314 }
3315
3316 sub foreach_drive {
3317     my ($conf, $func, @param) = @_;
3318
3319     foreach my $ds (valid_drive_names()) {
3320         next if !defined($conf->{$ds});
3321
3322         my $drive = parse_drive($ds, $conf->{$ds});
3323         next if !$drive;
3324
3325         &$func($ds, $drive, @param);
3326     }
3327 }
3328
3329 sub foreach_volid {
3330     my ($conf, $func, @param) = @_;
3331
3332     my $volhash = {};
3333
3334     my $test_volid = sub {
3335         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3336
3337         return if !$volid;
3338
3339         $volhash->{$volid}->{cdrom} //= 1;
3340         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3341
3342         $volhash->{$volid}->{replicate} //= 0;
3343         $volhash->{$volid}->{replicate} = 1 if $replicate;
3344
3345         $volhash->{$volid}->{shared} //= 0;
3346         $volhash->{$volid}->{shared} = 1 if $shared;
3347
3348         $volhash->{$volid}->{referenced_in_config} //= 0;
3349         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3350
3351         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3352             if defined($snapname);
3353         $volhash->{$volid}->{size} = $size if $size;
3354     };
3355
3356     foreach_drive($conf, sub {
3357         my ($ds, $drive) = @_;
3358         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3359     });
3360
3361     foreach my $snapname (keys %{$conf->{snapshots}}) {
3362         my $snap = $conf->{snapshots}->{$snapname};
3363         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3364         foreach_drive($snap, sub {
3365             my ($ds, $drive) = @_;
3366             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3367         });
3368     }
3369
3370     foreach my $volid (keys %$volhash) {
3371         &$func($volid, $volhash->{$volid}, @param);
3372     }
3373 }
3374
3375 sub conf_has_serial {
3376     my ($conf) = @_;
3377
3378     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3379         if ($conf->{"serial$i"}) {
3380             return 1;
3381         }
3382     }
3383
3384     return 0;
3385 }
3386
3387 sub conf_has_audio {
3388     my ($conf, $id) = @_;
3389
3390     $id //= 0;
3391     my $audio = $conf->{"audio$id"};
3392     return undef if !defined($audio);
3393
3394     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3395     my $audiodriver = $audioproperties->{driver} // 'spice';
3396
3397     return {
3398         dev => $audioproperties->{device},
3399         dev_id => "audiodev$id",
3400         backend => $audiodriver,
3401         backend_id => "$audiodriver-backend${id}",
3402     };
3403 }
3404
3405 sub vga_conf_has_spice {
3406     my ($vga) = @_;
3407
3408     my $vgaconf = parse_vga($vga);
3409     my $vgatype = $vgaconf->{type};
3410     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3411
3412     return $1 || 1;
3413 }
3414
3415 sub is_native($) {
3416     my ($arch) = @_;
3417     return get_host_arch() eq $arch;
3418 }
3419
3420 my $default_machines = {
3421     x86_64 => 'pc',
3422     aarch64 => 'virt',
3423 };
3424
3425 sub get_basic_machine_info {
3426     my ($conf, $forcemachine) = @_;
3427
3428     my $arch = $conf->{arch} // get_host_arch();
3429     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3430     return ($arch, $machine);
3431 }
3432
3433 sub get_ovmf_files($) {
3434     my ($arch) = @_;
3435
3436     my $ovmf = $OVMF->{$arch}
3437         or die "no OVMF images known for architecture '$arch'\n";
3438
3439     return @$ovmf;
3440 }
3441
3442 my $Arch2Qemu = {
3443     aarch64 => '/usr/bin/qemu-system-aarch64',
3444     x86_64 => '/usr/bin/qemu-system-x86_64',
3445 };
3446 sub get_command_for_arch($) {
3447     my ($arch) = @_;
3448     return '/usr/bin/kvm' if is_native($arch);
3449
3450     my $cmd = $Arch2Qemu->{$arch}
3451         or die "don't know how to emulate architecture '$arch'\n";
3452     return $cmd;
3453 }
3454
3455 sub get_cpu_options {
3456     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3457
3458     my $cpuFlags = [];
3459     my $ostype = $conf->{ostype};
3460
3461     my $cpu = $kvm ? "kvm64" : "qemu64";
3462     if ($arch eq 'aarch64') {
3463         $cpu = 'cortex-a57';
3464     }
3465     my $hv_vendor_id;
3466     if (my $cputype = $conf->{cpu}) {
3467         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3468             or die "Cannot parse cpu description: $cputype\n";
3469         $cpu = $cpuconf->{cputype};
3470         $kvm_off = 1 if $cpuconf->{hidden};
3471         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3472
3473         if (defined(my $flags = $cpuconf->{flags})) {
3474             push @$cpuFlags, split(";", $flags);
3475         }
3476     }
3477
3478     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3479
3480     push @$cpuFlags , '-x2apic'
3481         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3482
3483     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3484
3485     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3486
3487     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3488
3489         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3490         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3491     }
3492
3493     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3494
3495     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3496
3497     push @$cpuFlags, 'kvm=off' if $kvm_off;
3498
3499     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3500         push @$cpuFlags, "vendor=${cpu_vendor}"
3501             if $cpu_vendor ne 'default';
3502     } elsif ($arch ne 'aarch64') {
3503         die "internal error"; # should not happen
3504     }
3505
3506     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3507
3508     return ('-cpu', $cpu);
3509 }
3510
3511 sub config_to_command {
3512     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3513
3514     my $cmd = [];
3515     my $globalFlags = [];
3516     my $machineFlags = [];
3517     my $rtcFlags = [];
3518     my $devices = [];
3519     my $pciaddr = '';
3520     my $bridges = {};
3521     my $vernum = 0; # unknown
3522     my $ostype = $conf->{ostype};
3523     my $winversion = windows_version($ostype);
3524     my $kvm = $conf->{kvm};
3525
3526     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3527     my $kvm_binary = get_command_for_arch($arch);
3528     my $kvmver = kvm_user_version($kvm_binary);
3529     $kvm //= 1 if is_native($arch);
3530
3531     if ($kvm) {
3532         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3533             if !defined kvm_version();
3534     }
3535
3536     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3537         $vernum = $1*1000000+$2*1000;
3538     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3539         $vernum = $1*1000000+$2*1000+$3;
3540     }
3541
3542     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3543
3544     my $q35 = machine_type_is_q35($conf);
3545     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3546     my $use_old_bios_files = undef;
3547     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3548
3549     my $cpuunits = defined($conf->{cpuunits}) ?
3550             $conf->{cpuunits} : $defaults->{cpuunits};
3551
3552     push @$cmd, $kvm_binary;
3553
3554     push @$cmd, '-id', $vmid;
3555
3556     my $vmname = $conf->{name} || "vm$vmid";
3557
3558     push @$cmd, '-name', $vmname;
3559
3560     my $use_virtio = 0;
3561
3562     my $qmpsocket = qmp_socket($vmid);
3563     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3564     push @$cmd, '-mon', "chardev=qmp,mode=control";
3565
3566     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3567         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3568         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3569     }
3570
3571     push @$cmd, '-pidfile' , pidfile_name($vmid);
3572
3573     push @$cmd, '-daemonize';
3574
3575     if ($conf->{smbios1}) {
3576         my $smbios_conf = parse_smbios1($conf->{smbios1});
3577         if ($smbios_conf->{base64}) {
3578             # Do not pass base64 flag to qemu
3579             delete $smbios_conf->{base64};
3580             my $smbios_string = "";
3581             foreach my $key (keys %$smbios_conf) {
3582                 my $value;
3583                 if ($key eq "uuid") {
3584                     $value = $smbios_conf->{uuid}
3585                 } else {
3586                     $value = decode_base64($smbios_conf->{$key});
3587                 }
3588                 # qemu accepts any binary data, only commas need escaping by double comma
3589                 $value =~ s/,/,,/g;
3590                 $smbios_string .= "," . $key . "=" . $value if $value;
3591             }
3592             push @$cmd, '-smbios', "type=1" . $smbios_string;
3593         } else {
3594             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3595         }
3596     }
3597
3598     if ($conf->{vmgenid}) {
3599         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3600     }
3601
3602     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3603     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3604         die "uefi base image not found\n" if ! -f $ovmf_code;
3605
3606         my $path;
3607         my $format;
3608         if (my $efidisk = $conf->{efidisk0}) {
3609             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3610             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3611             $format = $d->{format};
3612             if ($storeid) {
3613                 $path = PVE::Storage::path($storecfg, $d->{file});
3614                 if (!defined($format)) {
3615                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3616                     $format = qemu_img_format($scfg, $volname);
3617                 }
3618             } else {
3619                 $path = $d->{file};
3620                 die "efidisk format must be specified\n"
3621                     if !defined($format);
3622             }
3623         } else {
3624             warn "no efidisk configured! Using temporary efivars disk.\n";
3625             $path = "/tmp/$vmid-ovmf.fd";
3626             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3627             $format = 'raw';
3628         }
3629
3630         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3631         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3632     }
3633
3634     # load q35 config
3635     if ($q35) {
3636         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3637         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3638             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3639         } else {
3640             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3641         }
3642     }
3643
3644     # add usb controllers
3645     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3646     push @$devices, @usbcontrollers if @usbcontrollers;
3647     my $vga = parse_vga($conf->{vga});
3648
3649     my $qxlnum = vga_conf_has_spice($conf->{vga});
3650     $vga->{type} = 'qxl' if $qxlnum;
3651
3652     if (!$vga->{type}) {
3653         if ($arch eq 'aarch64') {
3654             $vga->{type} = 'virtio';
3655         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3656             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657         } else {
3658             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659         }
3660     }
3661
3662     # enable absolute mouse coordinates (needed by vnc)
3663     my $tablet;
3664     if (defined($conf->{tablet})) {
3665         $tablet = $conf->{tablet};
3666     } else {
3667         $tablet = $defaults->{tablet};
3668         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3669         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3670     }
3671
3672     if ($tablet) {
3673         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3674         my $kbd = print_keyboarddevice_full($conf, $arch);
3675         push @$devices, '-device', $kbd if defined($kbd);
3676     }
3677
3678     my $kvm_off = 0;
3679     my $gpu_passthrough;
3680
3681     # host pci devices
3682     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3683         my $id = "hostpci$i";
3684         my $d = parse_hostpci($conf->{$id});
3685         next if !$d;
3686
3687         if (my $pcie = $d->{pcie}) {
3688             die "q35 machine model is not enabled" if !$q35;
3689             # win7 wants to have the pcie devices directly on the pcie bus
3690             # instead of in the root port
3691             if ($winversion == 7) {
3692                 $pciaddr = print_pcie_addr("${id}bus0");
3693             } else {
3694                 # add more root ports if needed, 4 are present by default
3695                 # by pve-q35 cfgs, rest added here on demand.
3696                 if ($i > 3) {
3697                     push @$devices, '-device', print_pcie_root_port($i);
3698                 }
3699                 $pciaddr = print_pcie_addr($id);
3700             }
3701         } else {
3702             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3703         }
3704
3705         my $xvga = '';
3706         if ($d->{'x-vga'}) {
3707             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3708             $kvm_off = 1;
3709             $vga->{type} = 'none' if !defined($conf->{vga});
3710             $gpu_passthrough = 1;
3711         }
3712
3713         my $pcidevices = $d->{pciid};
3714         my $multifunction = 1 if @$pcidevices > 1;
3715
3716         my $sysfspath;
3717         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3718             my $pci_id = $pcidevices->[0]->{id};
3719             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3720             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3721         } elsif ($d->{mdev}) {
3722             warn "ignoring mediated device '$id' with multifunction device\n";
3723         }
3724
3725         my $j=0;
3726         foreach my $pcidevice (@$pcidevices) {
3727             my $devicestr = "vfio-pci";
3728
3729             if ($sysfspath) {
3730                 $devicestr .= ",sysfsdev=$sysfspath";
3731             } else {
3732                 $devicestr .= ",host=$pcidevice->{id}";
3733             }
3734
3735             my $mf_addr = $multifunction ? ".$j" : '';
3736             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3737
3738             if ($j == 0) {
3739                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3740                 $devicestr .= "$xvga";
3741                 $devicestr .= ",multifunction=on" if $multifunction;
3742                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3743             }
3744
3745             push @$devices, '-device', $devicestr;
3746             $j++;
3747         }
3748     }
3749
3750     # usb devices
3751     my $usb_dev_features = {};
3752     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3753
3754     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3755     push @$devices, @usbdevices if @usbdevices;
3756     # serial devices
3757     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3758         if (my $path = $conf->{"serial$i"}) {
3759             if ($path eq 'socket') {
3760                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3761                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3762                 # On aarch64, serial0 is the UART device. Qemu only allows
3763                 # connecting UART devices via the '-serial' command line, as
3764                 # the device has a fixed slot on the hardware...
3765                 if ($arch eq 'aarch64' && $i == 0) {
3766                     push @$devices, '-serial', "chardev:serial$i";
3767                 } else {
3768                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3769                 }
3770             } else {
3771                 die "no such serial device\n" if ! -c $path;
3772                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3773                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3774             }
3775         }
3776     }
3777
3778     # parallel devices
3779     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3780         if (my $path = $conf->{"parallel$i"}) {
3781             die "no such parallel device\n" if ! -c $path;
3782             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3783             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3784             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3785         }
3786     }
3787
3788     if (my $audio = conf_has_audio($conf)) {
3789
3790         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3791
3792         my $id = $audio->{dev_id};
3793         if ($audio->{dev} eq 'AC97') {
3794             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3795         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3796             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3797             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3798             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3799         } else {
3800             die "unkown audio device '$audio->{dev}', implement me!";
3801         }
3802
3803         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3804     }
3805
3806     my $sockets = 1;
3807     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3808     $sockets = $conf->{sockets} if  $conf->{sockets};
3809
3810     my $cores = $conf->{cores} || 1;
3811
3812     my $maxcpus = $sockets * $cores;
3813
3814     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3815
3816     my $allowed_vcpus = $cpuinfo->{cpus};
3817
3818     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3819         if ($allowed_vcpus < $maxcpus);
3820
3821     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3822
3823         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3824         for (my $i = 2; $i <= $vcpus; $i++)  {
3825             my $cpustr = print_cpu_device($conf,$i);
3826             push @$cmd, '-device', $cpustr;
3827         }
3828
3829     } else {
3830
3831         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3832     }
3833     push @$cmd, '-nodefaults';
3834
3835     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3836
3837     my $bootindex_hash = {};
3838     my $i = 1;
3839     foreach my $o (split(//, $bootorder)) {
3840         $bootindex_hash->{$o} = $i*100;
3841         $i++;
3842     }
3843
3844     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3845
3846     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3847
3848     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3849
3850     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3851         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3852         my $socket = vnc_socket($vmid);
3853         push @$cmd,  '-vnc', "unix:$socket,password";
3854     } else {
3855         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3856         push @$cmd, '-nographic';
3857     }
3858
3859     # time drift fix
3860     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3861
3862     my $useLocaltime = $conf->{localtime};
3863
3864     if ($winversion >= 5) { # windows
3865         $useLocaltime = 1 if !defined($conf->{localtime});
3866
3867         # use time drift fix when acpi is enabled
3868         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3869             $tdf = 1 if !defined($conf->{tdf});
3870         }
3871     }
3872
3873     if ($winversion >= 6) {
3874         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3875         push @$cmd, '-no-hpet';
3876     }
3877
3878     push @$rtcFlags, 'driftfix=slew' if $tdf;
3879
3880     if (!$kvm) {
3881         push @$machineFlags, 'accel=tcg';
3882     }
3883
3884     if ($machine_type) {
3885         push @$machineFlags, "type=${machine_type}";
3886     }
3887
3888     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3889         push @$rtcFlags, "base=$conf->{startdate}";
3890     } elsif ($useLocaltime) {
3891         push @$rtcFlags, 'base=localtime';
3892     }
3893
3894     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3895
3896     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3897
3898     push @$cmd, '-S' if $conf->{freeze};
3899
3900     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3901
3902     my $guest_agent = parse_guest_agent($conf);
3903
3904     if ($guest_agent->{enabled}) {
3905         my $qgasocket = qmp_socket($vmid, 1);
3906         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3907
3908         if ($guest_agent->{type} eq 'virtio') {
3909             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3910             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3911             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3912         } elsif ($guest_agent->{type} eq 'isa') {
3913             push @$devices, '-device', "isa-serial,chardev=qga0";
3914         }
3915     }
3916
3917     my $spice_port;
3918
3919     if ($qxlnum) {
3920         if ($qxlnum > 1) {
3921             if ($winversion){
3922                 for(my $i = 1; $i < $qxlnum; $i++){
3923                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3924                 }
3925             } else {
3926                 # assume other OS works like Linux
3927                 my ($ram, $vram) = ("134217728", "67108864");
3928                 if ($vga->{memory}) {
3929                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3930                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3931                 }
3932                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3933                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3934             }
3935         }
3936
3937         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3938
3939         my $nodename = PVE::INotify::nodename();
3940         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3941         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3942         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3943
3944         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3945         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3946         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3947
3948         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3949         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3950
3951         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3952         if ($spice_enhancement->{foldersharing}) {
3953             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3954             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3955         }
3956
3957         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3958         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3959         push @$devices, '-spice', "$spice_opts";
3960     }
3961
3962     # enable balloon by default, unless explicitly disabled
3963     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3964         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3965         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3966     }
3967
3968     if ($conf->{watchdog}) {
3969         my $wdopts = parse_watchdog($conf->{watchdog});
3970         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3971         my $watchdog = $wdopts->{model} || 'i6300esb';
3972         push @$devices, '-device', "$watchdog$pciaddr";
3973         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3974     }
3975
3976     my $vollist = [];
3977     my $scsicontroller = {};
3978     my $ahcicontroller = {};
3979     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3980
3981     # Add iscsi initiator name if available
3982     if (my $initiator = get_initiator_name()) {
3983         push @$devices, '-iscsi', "initiator-name=$initiator";
3984     }
3985
3986     foreach_drive($conf, sub {
3987         my ($ds, $drive) = @_;
3988
3989         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3990             push @$vollist, $drive->{file};
3991         }
3992
3993         # ignore efidisk here, already added in bios/fw handling code above
3994         return if $drive->{interface} eq 'efidisk';
3995
3996         $use_virtio = 1 if $ds =~ m/^virtio/;
3997
3998         if (drive_is_cdrom ($drive)) {
3999             if ($bootindex_hash->{d}) {
4000                 $drive->{bootindex} = $bootindex_hash->{d};
4001                 $bootindex_hash->{d} += 1;
4002             }
4003         } else {
4004             if ($bootindex_hash->{c}) {
4005                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4006                 $bootindex_hash->{c} += 1;
4007             }
4008         }
4009
4010         if($drive->{interface} eq 'virtio'){
4011            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4012         }
4013
4014         if ($drive->{interface} eq 'scsi') {
4015
4016             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4017
4018             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4019             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4020
4021             my $iothread = '';
4022             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4023                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4024                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4025             } elsif ($drive->{iothread}) {
4026                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4027             }
4028
4029             my $queues = '';
4030             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4031                 $queues = ",num_queues=$drive->{queues}";
4032             }
4033
4034             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4035             $scsicontroller->{$controller}=1;
4036         }
4037
4038         if ($drive->{interface} eq 'sata') {
4039            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4040            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4041            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4042            $ahcicontroller->{$controller}=1;
4043         }
4044
4045         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4046         push @$devices, '-drive',$drive_cmd;
4047         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4048     });
4049
4050     for (my $i = 0; $i < $MAX_NETS; $i++) {
4051          next if !$conf->{"net$i"};
4052          my $d = parse_net($conf->{"net$i"});
4053          next if !$d;
4054
4055          $use_virtio = 1 if $d->{model} eq 'virtio';
4056
4057          if ($bootindex_hash->{n}) {
4058             $d->{bootindex} = $bootindex_hash->{n};
4059             $bootindex_hash->{n} += 1;
4060          }
4061
4062          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4063          push @$devices, '-netdev', $netdevfull;
4064
4065          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4066          push @$devices, '-device', $netdevicefull;
4067     }
4068
4069     if ($conf->{ivshmem}) {
4070         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4071
4072         my $bus;
4073         if ($q35) {
4074             $bus = print_pcie_addr("ivshmem");
4075         } else {
4076             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4077         }
4078
4079         my $ivshmem_name = $ivshmem->{name} // $vmid;
4080         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4081
4082         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4083         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4084     }
4085
4086     if (!$q35) {
4087         # add pci bridges
4088         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4089            $bridges->{1} = 1;
4090            $bridges->{2} = 1;
4091         }
4092
4093         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4094
4095         for my $k (sort {$b cmp $a} keys %$bridges) {
4096             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4097             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4098         }
4099     }
4100
4101     push @$cmd, @$devices;
4102     push @$cmd, '-rtc', join(',', @$rtcFlags)
4103         if scalar(@$rtcFlags);
4104     push @$cmd, '-machine', join(',', @$machineFlags)
4105         if scalar(@$machineFlags);
4106     push @$cmd, '-global', join(',', @$globalFlags)
4107         if scalar(@$globalFlags);
4108
4109     if (my $vmstate = $conf->{vmstate}) {
4110         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4111         push @$vollist, $vmstate;
4112         push @$cmd, '-loadstate', $statepath;
4113     }
4114
4115     # add custom args
4116     if ($conf->{args}) {
4117         my $aa = PVE::Tools::split_args($conf->{args});
4118         push @$cmd, @$aa;
4119     }
4120
4121     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4122 }
4123
4124 sub vnc_socket {
4125     my ($vmid) = @_;
4126     return "${var_run_tmpdir}/$vmid.vnc";
4127 }
4128
4129 sub spice_port {
4130     my ($vmid) = @_;
4131
4132     my $res = vm_mon_cmd($vmid, 'query-spice');
4133
4134     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4135 }
4136
4137 sub qmp_socket {
4138     my ($vmid, $qga) = @_;
4139     my $sockettype = $qga ? 'qga' : 'qmp';
4140     return "${var_run_tmpdir}/$vmid.$sockettype";
4141 }
4142
4143 sub pidfile_name {
4144     my ($vmid) = @_;
4145     return "${var_run_tmpdir}/$vmid.pid";
4146 }
4147
4148 sub vm_devices_list {
4149     my ($vmid) = @_;
4150
4151     my $res = vm_mon_cmd($vmid, 'query-pci');
4152     my $devices_to_check = [];
4153     my $devices = {};
4154     foreach my $pcibus (@$res) {
4155         push @$devices_to_check, @{$pcibus->{devices}},
4156     }
4157
4158     while (@$devices_to_check) {
4159         my $to_check = [];
4160         for my $d (@$devices_to_check) {
4161             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4162             next if !$d->{'pci_bridge'};
4163
4164             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4165             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4166         }
4167         $devices_to_check = $to_check;
4168     }
4169
4170     my $resblock = vm_mon_cmd($vmid, 'query-block');
4171     foreach my $block (@$resblock) {
4172         if($block->{device} =~ m/^drive-(\S+)/){
4173                 $devices->{$1} = 1;
4174         }
4175     }
4176
4177     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4178     foreach my $mice (@$resmice) {
4179         if ($mice->{name} eq 'QEMU HID Tablet') {
4180             $devices->{tablet} = 1;
4181             last;
4182         }
4183     }
4184
4185     # for usb devices there is no query-usb
4186     # but we can iterate over the entries in
4187     # qom-list path=/machine/peripheral
4188     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4189     foreach my $per (@$resperipheral) {
4190         if ($per->{name} =~ m/^usb\d+$/) {
4191             $devices->{$per->{name}} = 1;
4192         }
4193     }
4194
4195     return $devices;
4196 }
4197
4198 sub vm_deviceplug {
4199     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4200
4201     my $q35 = machine_type_is_q35($conf);
4202
4203     my $devices_list = vm_devices_list($vmid);
4204     return 1 if defined($devices_list->{$deviceid});
4205
4206     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4207
4208     if ($deviceid eq 'tablet') {
4209
4210         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4211
4212     } elsif ($deviceid eq 'keyboard') {
4213
4214         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4215
4216     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4217
4218         die "usb hotplug currently not reliable\n";
4219         # since we can't reliably hot unplug all added usb devices
4220         # and usb passthrough disables live migration
4221         # we disable usb hotplugging for now
4222         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4223
4224     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4225
4226         qemu_iothread_add($vmid, $deviceid, $device);
4227
4228         qemu_driveadd($storecfg, $vmid, $device);
4229         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4230
4231         qemu_deviceadd($vmid, $devicefull);
4232         eval { qemu_deviceaddverify($vmid, $deviceid); };
4233         if (my $err = $@) {
4234             eval { qemu_drivedel($vmid, $deviceid); };
4235             warn $@ if $@;
4236             die $err;
4237         }
4238
4239     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4240
4241
4242         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4243         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4244         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4245
4246         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4247
4248         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4249             qemu_iothread_add($vmid, $deviceid, $device);
4250             $devicefull .= ",iothread=iothread-$deviceid";
4251         }
4252
4253         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4254             $devicefull .= ",num_queues=$device->{queues}";
4255         }
4256
4257         qemu_deviceadd($vmid, $devicefull);
4258         qemu_deviceaddverify($vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4261
4262         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4263         qemu_driveadd($storecfg, $vmid, $device);
4264
4265         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4266         eval { qemu_deviceadd($vmid, $devicefull); };
4267         if (my $err = $@) {
4268             eval { qemu_drivedel($vmid, $deviceid); };
4269             warn $@ if $@;
4270             die $err;
4271         }
4272
4273     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4274
4275         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4276
4277         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4278         my $use_old_bios_files = undef;
4279         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4280
4281         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4282         qemu_deviceadd($vmid, $netdevicefull);
4283         eval {
4284             qemu_deviceaddverify($vmid, $deviceid);
4285             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4286         };
4287         if (my $err = $@) {
4288             eval { qemu_netdevdel($vmid, $deviceid); };
4289             warn $@ if $@;
4290             die $err;
4291         }
4292
4293     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4294
4295         my $bridgeid = $2;
4296         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4297         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4298
4299         qemu_deviceadd($vmid, $devicefull);
4300         qemu_deviceaddverify($vmid, $deviceid);
4301
4302     } else {
4303         die "can't hotplug device '$deviceid'\n";
4304     }
4305
4306     return 1;
4307 }
4308
4309 # fixme: this should raise exceptions on error!
4310 sub vm_deviceunplug {
4311     my ($vmid, $conf, $deviceid) = @_;
4312
4313     my $devices_list = vm_devices_list($vmid);
4314     return 1 if !defined($devices_list->{$deviceid});
4315
4316     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4317
4318     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4319
4320         qemu_devicedel($vmid, $deviceid);
4321
4322     } elsif ($deviceid =~ m/^usb\d+$/) {
4323
4324         die "usb hotplug currently not reliable\n";
4325         # when unplugging usb devices this way,
4326         # there may be remaining usb controllers/hubs
4327         # so we disable it for now
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_devicedelverify($vmid, $deviceid);
4330
4331     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4332
4333         qemu_devicedel($vmid, $deviceid);
4334         qemu_devicedelverify($vmid, $deviceid);
4335         qemu_drivedel($vmid, $deviceid);
4336         qemu_iothread_del($conf, $vmid, $deviceid);
4337
4338     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4339
4340         qemu_devicedel($vmid, $deviceid);
4341         qemu_devicedelverify($vmid, $deviceid);
4342         qemu_iothread_del($conf, $vmid, $deviceid);
4343
4344     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4345
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_drivedel($vmid, $deviceid);
4348         qemu_deletescsihw($conf, $vmid, $deviceid);
4349
4350     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4351
4352         qemu_devicedel($vmid, $deviceid);
4353         qemu_devicedelverify($vmid, $deviceid);
4354         qemu_netdevdel($vmid, $deviceid);
4355
4356     } else {
4357         die "can't unplug device '$deviceid'\n";
4358     }
4359
4360     return 1;
4361 }
4362
4363 sub qemu_deviceadd {
4364     my ($vmid, $devicefull) = @_;
4365
4366     $devicefull = "driver=".$devicefull;
4367     my %options =  split(/[=,]/, $devicefull);
4368
4369     vm_mon_cmd($vmid, "device_add" , %options);
4370 }
4371
4372 sub qemu_devicedel {
4373     my ($vmid, $deviceid) = @_;
4374
4375     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4376 }
4377
4378 sub qemu_iothread_add {
4379     my($vmid, $deviceid, $device) = @_;
4380
4381     if ($device->{iothread}) {
4382         my $iothreads = vm_iothreads_list($vmid);
4383         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4384     }
4385 }
4386
4387 sub qemu_iothread_del {
4388     my($conf, $vmid, $deviceid) = @_;
4389
4390     my $confid = $deviceid;
4391     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4392         $confid = 'scsi' . $1;
4393     }
4394     my $device = parse_drive($confid, $conf->{$confid});
4395     if ($device->{iothread}) {
4396         my $iothreads = vm_iothreads_list($vmid);
4397         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4398     }
4399 }
4400
4401 sub qemu_objectadd {
4402     my($vmid, $objectid, $qomtype) = @_;
4403
4404     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4405
4406     return 1;
4407 }
4408
4409 sub qemu_objectdel {
4410     my($vmid, $objectid) = @_;
4411
4412     vm_mon_cmd($vmid, "object-del", id => $objectid);
4413
4414     return 1;
4415 }
4416
4417 sub qemu_driveadd {
4418     my ($storecfg, $vmid, $device) = @_;
4419
4420     my $drive = print_drive_full($storecfg, $vmid, $device);
4421     $drive =~ s/\\/\\\\/g;
4422     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4423
4424     # If the command succeeds qemu prints: "OK"
4425     return 1 if $ret =~ m/OK/s;
4426
4427     die "adding drive failed: $ret\n";
4428 }
4429
4430 sub qemu_drivedel {
4431     my($vmid, $deviceid) = @_;
4432
4433     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4434     $ret =~ s/^\s+//;
4435
4436     return 1 if $ret eq "";
4437
4438     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4439     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4440
4441     die "deleting drive $deviceid failed : $ret\n";
4442 }
4443
4444 sub qemu_deviceaddverify {
4445     my ($vmid, $deviceid) = @_;
4446
4447     for (my $i = 0; $i <= 5; $i++) {
4448          my $devices_list = vm_devices_list($vmid);
4449          return 1 if defined($devices_list->{$deviceid});
4450          sleep 1;
4451     }
4452
4453     die "error on hotplug device '$deviceid'\n";
4454 }
4455
4456
4457 sub qemu_devicedelverify {
4458     my ($vmid, $deviceid) = @_;
4459
4460     # need to verify that the device is correctly removed as device_del
4461     # is async and empty return is not reliable
4462
4463     for (my $i = 0; $i <= 5; $i++) {
4464          my $devices_list = vm_devices_list($vmid);
4465          return 1 if !defined($devices_list->{$deviceid});
4466          sleep 1;
4467     }
4468
4469     die "error on hot-unplugging device '$deviceid'\n";
4470 }
4471
4472 sub qemu_findorcreatescsihw {
4473     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4474
4475     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4476
4477     my $scsihwid="$controller_prefix$controller";
4478     my $devices_list = vm_devices_list($vmid);
4479
4480     if(!defined($devices_list->{$scsihwid})) {
4481         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4482     }
4483
4484     return 1;
4485 }
4486
4487 sub qemu_deletescsihw {
4488     my ($conf, $vmid, $opt) = @_;
4489
4490     my $device = parse_drive($opt, $conf->{$opt});
4491
4492     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4493         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4494         return 1;
4495     }
4496
4497     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4498
4499     my $devices_list = vm_devices_list($vmid);
4500     foreach my $opt (keys %{$devices_list}) {
4501         if (PVE::QemuServer::is_valid_drivename($opt)) {
4502             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4503             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4504                 return 1;
4505             }
4506         }
4507     }
4508
4509     my $scsihwid="scsihw$controller";
4510
4511     vm_deviceunplug($vmid, $conf, $scsihwid);
4512
4513     return 1;
4514 }
4515
4516 sub qemu_add_pci_bridge {
4517     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4518
4519     my $bridges = {};
4520
4521     my $bridgeid;
4522
4523     print_pci_addr($device, $bridges, $arch, $machine_type);
4524
4525     while (my ($k, $v) = each %$bridges) {
4526         $bridgeid = $k;
4527     }
4528     return 1 if !defined($bridgeid) || $bridgeid < 1;
4529
4530     my $bridge = "pci.$bridgeid";
4531     my $devices_list = vm_devices_list($vmid);
4532
4533     if (!defined($devices_list->{$bridge})) {
4534         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4535     }
4536
4537     return 1;
4538 }
4539
4540 sub qemu_set_link_status {
4541     my ($vmid, $device, $up) = @_;
4542
4543     vm_mon_cmd($vmid, "set_link", name => $device,
4544                up => $up ? JSON::true : JSON::false);
4545 }
4546
4547 sub qemu_netdevadd {
4548     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4549
4550     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4551     my %options =  split(/[=,]/, $netdev);
4552
4553     vm_mon_cmd($vmid, "netdev_add",  %options);
4554     return 1;
4555 }
4556
4557 sub qemu_netdevdel {
4558     my ($vmid, $deviceid) = @_;
4559
4560     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4561 }
4562
4563 sub qemu_usb_hotplug {
4564     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4565
4566     return if !$device;
4567
4568     # remove the old one first
4569     vm_deviceunplug($vmid, $conf, $deviceid);
4570
4571     # check if xhci controller is necessary and available
4572     if ($device->{usb3}) {
4573
4574         my $devicelist = vm_devices_list($vmid);
4575
4576         if (!$devicelist->{xhci}) {
4577             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4578             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4579         }
4580     }
4581     my $d = parse_usb_device($device->{host});
4582     $d->{usb3} = $device->{usb3};
4583
4584     # add the new one
4585     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4586 }
4587
4588 sub qemu_cpu_hotplug {
4589     my ($vmid, $conf, $vcpus) = @_;
4590
4591     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4592
4593     my $sockets = 1;
4594     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4595     $sockets = $conf->{sockets} if  $conf->{sockets};
4596     my $cores = $conf->{cores} || 1;
4597     my $maxcpus = $sockets * $cores;
4598
4599     $vcpus = $maxcpus if !$vcpus;
4600
4601     die "you can't add more vcpus than maxcpus\n"
4602         if $vcpus > $maxcpus;
4603
4604     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4605
4606     if ($vcpus < $currentvcpus) {
4607
4608         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4609
4610             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4611                 qemu_devicedel($vmid, "cpu$i");
4612                 my $retry = 0;
4613                 my $currentrunningvcpus = undef;
4614                 while (1) {
4615                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4616                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4617                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4618                     $retry++;
4619                     sleep 1;
4620                 }
4621                 #update conf after each succesfull cpu unplug
4622                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4623                 PVE::QemuConfig->write_config($vmid, $conf);
4624             }
4625         } else {
4626             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4627         }
4628
4629         return;
4630     }
4631
4632     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4633     die "vcpus in running vm does not match its configuration\n"
4634         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4635
4636     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4637
4638         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4639             my $cpustr = print_cpu_device($conf, $i);
4640             qemu_deviceadd($vmid, $cpustr);
4641
4642             my $retry = 0;
4643             my $currentrunningvcpus = undef;
4644             while (1) {
4645                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4646                 last if scalar(@{$currentrunningvcpus}) == $i;
4647                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4648                 sleep 1;
4649                 $retry++;
4650             }
4651             #update conf after each succesfull cpu hotplug
4652             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4653             PVE::QemuConfig->write_config($vmid, $conf);
4654         }
4655     } else {
4656
4657         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4658             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4659         }
4660     }
4661 }
4662
4663 sub qemu_block_set_io_throttle {
4664     my ($vmid, $deviceid,
4665         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4666         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4667         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4668         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4669
4670     return if !check_running($vmid) ;
4671
4672     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4673         bps => int($bps),
4674         bps_rd => int($bps_rd),
4675         bps_wr => int($bps_wr),
4676         iops => int($iops),
4677         iops_rd => int($iops_rd),
4678         iops_wr => int($iops_wr),
4679         bps_max => int($bps_max),
4680         bps_rd_max => int($bps_rd_max),
4681         bps_wr_max => int($bps_wr_max),
4682         iops_max => int($iops_max),
4683         iops_rd_max => int($iops_rd_max),
4684         iops_wr_max => int($iops_wr_max),
4685         bps_max_length => int($bps_max_length),
4686         bps_rd_max_length => int($bps_rd_max_length),
4687         bps_wr_max_length => int($bps_wr_max_length),
4688         iops_max_length => int($iops_max_length),
4689         iops_rd_max_length => int($iops_rd_max_length),
4690         iops_wr_max_length => int($iops_wr_max_length),
4691     );
4692
4693 }
4694
4695 # old code, only used to shutdown old VM after update
4696 sub __read_avail {
4697     my ($fh, $timeout) = @_;
4698
4699     my $sel = new IO::Select;
4700     $sel->add($fh);
4701
4702     my $res = '';
4703     my $buf;
4704
4705     my @ready;
4706     while (scalar (@ready = $sel->can_read($timeout))) {
4707         my $count;
4708         if ($count = $fh->sysread($buf, 8192)) {
4709             if ($buf =~ /^(.*)\(qemu\) $/s) {
4710                 $res .= $1;
4711                 last;
4712             } else {
4713                 $res .= $buf;
4714             }
4715         } else {
4716             if (!defined($count)) {
4717                 die "$!\n";
4718             }
4719             last;
4720         }
4721     }
4722
4723     die "monitor read timeout\n" if !scalar(@ready);
4724
4725     return $res;
4726 }
4727
4728 sub qemu_block_resize {
4729     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4730
4731     my $running = check_running($vmid);
4732
4733     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4734
4735     return if !$running;
4736
4737     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4738
4739 }
4740
4741 sub qemu_volume_snapshot {
4742     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4743
4744     my $running = check_running($vmid);
4745
4746     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4747         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4748     } else {
4749         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4750     }
4751 }
4752
4753 sub qemu_volume_snapshot_delete {
4754     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4755
4756     my $running = check_running($vmid);
4757
4758     if($running) {
4759
4760         $running = undef;
4761         my $conf = PVE::QemuConfig->load_config($vmid);
4762         foreach_drive($conf, sub {
4763             my ($ds, $drive) = @_;
4764             $running = 1 if $drive->{file} eq $volid;
4765         });
4766     }
4767
4768     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4769         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4770     } else {
4771         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4772     }
4773 }
4774
4775 sub set_migration_caps {
4776     my ($vmid) = @_;
4777
4778     my $cap_ref = [];
4779
4780     my $enabled_cap = {
4781         "auto-converge" => 1,
4782         "xbzrle" => 1,
4783         "x-rdma-pin-all" => 0,
4784         "zero-blocks" => 0,
4785         "compress" => 0
4786     };
4787
4788     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4789
4790     for my $supported_capability (@$supported_capabilities) {
4791         push @$cap_ref, {
4792             capability => $supported_capability->{capability},
4793             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4794         };
4795     }
4796
4797     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4798 }
4799
4800 my $fast_plug_option = {
4801     'lock' => 1,
4802     'name' => 1,
4803     'onboot' => 1,
4804     'shares' => 1,
4805     'startup' => 1,
4806     'description' => 1,
4807     'protection' => 1,
4808     'vmstatestorage' => 1,
4809     'hookscript' => 1,
4810 };
4811
4812 # hotplug changes in [PENDING]
4813 # $selection hash can be used to only apply specified options, for
4814 # example: { cores => 1 } (only apply changed 'cores')
4815 # $errors ref is used to return error messages
4816 sub vmconfig_hotplug_pending {
4817     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4818
4819     my $defaults = load_defaults();
4820     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4821
4822     # commit values which do not have any impact on running VM first
4823     # Note: those option cannot raise errors, we we do not care about
4824     # $selection and always apply them.
4825
4826     my $add_error = sub {
4827         my ($opt, $msg) = @_;
4828         $errors->{$opt} = "hotplug problem - $msg";
4829     };
4830
4831     my $changes = 0;
4832     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4833         if ($fast_plug_option->{$opt}) {
4834             $conf->{$opt} = $conf->{pending}->{$opt};
4835             delete $conf->{pending}->{$opt};
4836             $changes = 1;
4837         }
4838     }
4839
4840     if ($changes) {
4841         PVE::QemuConfig->write_config($vmid, $conf);
4842         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4843     }
4844
4845     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4846
4847     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4848     foreach my $opt (sort keys %$pending_delete_hash) {
4849         next if $selection && !$selection->{$opt};
4850         my $force = $pending_delete_hash->{$opt}->{force};
4851         eval {
4852             if ($opt eq 'hotplug') {
4853                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4854             } elsif ($opt eq 'tablet') {
4855                 die "skip\n" if !$hotplug_features->{usb};
4856                 if ($defaults->{tablet}) {
4857                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4858                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4859                         if $arch eq 'aarch64';
4860                 } else {
4861                     vm_deviceunplug($vmid, $conf, 'tablet');
4862                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4863                 }
4864             } elsif ($opt =~ m/^usb\d+/) {
4865                 die "skip\n";
4866                 # since we cannot reliably hot unplug usb devices
4867                 # we are disabling it
4868                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4869                 vm_deviceunplug($vmid, $conf, $opt);
4870             } elsif ($opt eq 'vcpus') {
4871                 die "skip\n" if !$hotplug_features->{cpu};
4872                 qemu_cpu_hotplug($vmid, $conf, undef);
4873             } elsif ($opt eq 'balloon') {
4874                 # enable balloon device is not hotpluggable
4875                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4876                 # here we reset the ballooning value to memory
4877                 my $balloon = $conf->{memory} || $defaults->{memory};
4878                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4879             } elsif ($fast_plug_option->{$opt}) {
4880                 # do nothing
4881             } elsif ($opt =~ m/^net(\d+)$/) {
4882                 die "skip\n" if !$hotplug_features->{network};
4883                 vm_deviceunplug($vmid, $conf, $opt);
4884             } elsif (is_valid_drivename($opt)) {
4885                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4886                 vm_deviceunplug($vmid, $conf, $opt);
4887                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4888             } elsif ($opt =~ m/^memory$/) {
4889                 die "skip\n" if !$hotplug_features->{memory};
4890                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4891             } elsif ($opt eq 'cpuunits') {
4892                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4893             } elsif ($opt eq 'cpulimit') {
4894                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4895             } else {
4896                 die "skip\n";
4897             }
4898         };
4899         if (my $err = $@) {
4900             &$add_error($opt, $err) if $err ne "skip\n";
4901         } else {
4902             # save new config if hotplug was successful
4903             delete $conf->{$opt};
4904             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4905             PVE::QemuConfig->write_config($vmid, $conf);
4906             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4907         }
4908     }
4909
4910     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4911     $apply_pending_cloudinit = sub {
4912         return if $apply_pending_cloudinit_done; # once is enough
4913         $apply_pending_cloudinit_done = 1; # once is enough
4914
4915         my ($key, $value) = @_;
4916
4917         my @cloudinit_opts = keys %$confdesc_cloudinit;
4918         foreach my $opt (keys %{$conf->{pending}}) {
4919             next if !grep { $_ eq $opt } @cloudinit_opts;
4920             $conf->{$opt} = delete $conf->{pending}->{$opt};
4921         }
4922
4923         my $new_conf = { %$conf };
4924         $new_conf->{$key} = $value;
4925         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4926     };
4927
4928     foreach my $opt (keys %{$conf->{pending}}) {
4929         next if $selection && !$selection->{$opt};
4930         my $value = $conf->{pending}->{$opt};
4931         eval {
4932             if ($opt eq 'hotplug') {
4933                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4934             } elsif ($opt eq 'tablet') {
4935                 die "skip\n" if !$hotplug_features->{usb};
4936                 if ($value == 1) {
4937                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4938                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4939                         if $arch eq 'aarch64';
4940                 } elsif ($value == 0) {
4941                     vm_deviceunplug($vmid, $conf, 'tablet');
4942                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4943                 }
4944             } elsif ($opt =~ m/^usb\d+$/) {
4945                 die "skip\n";
4946                 # since we cannot reliably hot unplug usb devices
4947                 # we are disabling it
4948                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4949                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4950                 die "skip\n" if !$d;
4951                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4952             } elsif ($opt eq 'vcpus') {
4953                 die "skip\n" if !$hotplug_features->{cpu};
4954                 qemu_cpu_hotplug($vmid, $conf, $value);
4955             } elsif ($opt eq 'balloon') {
4956                 # enable/disable balloning device is not hotpluggable
4957                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4958                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4959                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4960
4961                 # allow manual ballooning if shares is set to zero
4962                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4963                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4964                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4965                 }
4966             } elsif ($opt =~ m/^net(\d+)$/) {
4967                 # some changes can be done without hotplug
4968                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4969                                     $vmid, $opt, $value, $arch, $machine_type);
4970             } elsif (is_valid_drivename($opt)) {
4971                 # some changes can be done without hotplug
4972                 my $drive = parse_drive($opt, $value);
4973                 if (drive_is_cloudinit($drive)) {
4974                     &$apply_pending_cloudinit($opt, $value);
4975                 }
4976                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4977                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4978             } elsif ($opt =~ m/^memory$/) { #dimms
4979                 die "skip\n" if !$hotplug_features->{memory};
4980                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4981             } elsif ($opt eq 'cpuunits') {
4982                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4983             } elsif ($opt eq 'cpulimit') {
4984                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4985                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4986             } else {
4987                 die "skip\n";  # skip non-hot-pluggable options
4988             }
4989         };
4990         if (my $err = $@) {
4991             &$add_error($opt, $err) if $err ne "skip\n";
4992         } else {
4993             # save new config if hotplug was successful
4994             $conf->{$opt} = $value;
4995             delete $conf->{pending}->{$opt};
4996             PVE::QemuConfig->write_config($vmid, $conf);
4997             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4998         }
4999     }
5000 }
5001
5002 sub try_deallocate_drive {
5003     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5004
5005     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5006         my $volid = $drive->{file};
5007         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5008             my $sid = PVE::Storage::parse_volume_id($volid);
5009             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5010
5011             # check if the disk is really unused
5012             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5013                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5014             PVE::Storage::vdisk_free($storecfg, $volid);
5015             return 1;
5016         } else {
5017             # If vm is not owner of this disk remove from config
5018             return 1;
5019         }
5020     }
5021
5022     return undef;
5023 }
5024
5025 sub vmconfig_delete_or_detach_drive {
5026     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5027
5028     my $drive = parse_drive($opt, $conf->{$opt});
5029
5030     my $rpcenv = PVE::RPCEnvironment::get();
5031     my $authuser = $rpcenv->get_user();
5032
5033     if ($force) {
5034         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5035         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5036     } else {
5037         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5038     }
5039 }
5040
5041
5042
5043 sub vmconfig_apply_pending {
5044     my ($vmid, $conf, $storecfg) = @_;
5045
5046     # cold plug
5047
5048     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5049     foreach my $opt (sort keys %$pending_delete_hash) {
5050         die "internal error" if $opt =~ m/^unused/;
5051         my $force = $pending_delete_hash->{$opt}->{force};
5052         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5053         if (!defined($conf->{$opt})) {
5054             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5055             PVE::QemuConfig->write_config($vmid, $conf);
5056         } elsif (is_valid_drivename($opt)) {
5057             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5058             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5059             delete $conf->{$opt};
5060             PVE::QemuConfig->write_config($vmid, $conf);
5061         } else {
5062             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5063             delete $conf->{$opt};
5064             PVE::QemuConfig->write_config($vmid, $conf);
5065         }
5066     }
5067
5068     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5069
5070     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5071         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5072
5073         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5074             # skip if nothing changed
5075         } elsif (is_valid_drivename($opt)) {
5076             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5077                 if defined($conf->{$opt});
5078             $conf->{$opt} = $conf->{pending}->{$opt};
5079         } else {
5080             $conf->{$opt} = $conf->{pending}->{$opt};
5081         }
5082
5083         delete $conf->{pending}->{$opt};
5084         PVE::QemuConfig->write_config($vmid, $conf);
5085     }
5086 }
5087
5088 my $safe_num_ne = sub {
5089     my ($a, $b) = @_;
5090
5091     return 0 if !defined($a) && !defined($b);
5092     return 1 if !defined($a);
5093     return 1 if !defined($b);
5094
5095     return $a != $b;
5096 };
5097
5098 my $safe_string_ne = sub {
5099     my ($a, $b) = @_;
5100
5101     return 0 if !defined($a) && !defined($b);
5102     return 1 if !defined($a);
5103     return 1 if !defined($b);
5104
5105     return $a ne $b;
5106 };
5107
5108 sub vmconfig_update_net {
5109     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5110
5111     my $newnet = parse_net($value);
5112
5113     if ($conf->{$opt}) {
5114         my $oldnet = parse_net($conf->{$opt});
5115
5116         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5117             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5118             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5119             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5120
5121             # for non online change, we try to hot-unplug
5122             die "skip\n" if !$hotplug;
5123             vm_deviceunplug($vmid, $conf, $opt);
5124         } else {
5125
5126             die "internal error" if $opt !~ m/net(\d+)/;
5127             my $iface = "tap${vmid}i$1";
5128
5129             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5130                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5131                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5132                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5133                 PVE::Network::tap_unplug($iface);
5134                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5135             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5136                 # Rate can be applied on its own but any change above needs to
5137                 # include the rate in tap_plug since OVS resets everything.
5138                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5139             }
5140
5141             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5142                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5143             }
5144
5145             return 1;
5146         }
5147     }
5148
5149     if ($hotplug) {
5150         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5151     } else {
5152         die "skip\n";
5153     }
5154 }
5155
5156 sub vmconfig_update_disk {
5157     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5158
5159     # fixme: do we need force?
5160
5161     my $drive = parse_drive($opt, $value);
5162
5163     if ($conf->{$opt}) {
5164
5165         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5166
5167             my $media = $drive->{media} || 'disk';
5168             my $oldmedia = $old_drive->{media} || 'disk';
5169             die "unable to change media type\n" if $media ne $oldmedia;
5170
5171             if (!drive_is_cdrom($old_drive)) {
5172
5173                 if ($drive->{file} ne $old_drive->{file}) {
5174
5175                     die "skip\n" if !$hotplug;
5176
5177                     # unplug and register as unused
5178                     vm_deviceunplug($vmid, $conf, $opt);
5179                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5180
5181                 } else {
5182                     # update existing disk
5183
5184                     # skip non hotpluggable value
5185                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5186                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5187                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5188                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5189                         die "skip\n";
5190                     }
5191
5192                     # apply throttle
5193                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5194                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5195                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5196                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5197                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5198                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5199                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5200                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5201                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5202                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5203                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5204                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5205                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5206                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5207                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5208                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5209                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5210                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5211
5212                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5213                                                    ($drive->{mbps} || 0)*1024*1024,
5214                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5215                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5216                                                    $drive->{iops} || 0,
5217                                                    $drive->{iops_rd} || 0,
5218                                                    $drive->{iops_wr} || 0,
5219                                                    ($drive->{mbps_max} || 0)*1024*1024,
5220                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5221                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5222                                                    $drive->{iops_max} || 0,
5223                                                    $drive->{iops_rd_max} || 0,
5224                                                    $drive->{iops_wr_max} || 0,
5225                                                    $drive->{bps_max_length} || 1,
5226                                                    $drive->{bps_rd_max_length} || 1,
5227                                                    $drive->{bps_wr_max_length} || 1,
5228                                                    $drive->{iops_max_length} || 1,
5229                                                    $drive->{iops_rd_max_length} || 1,
5230                                                    $drive->{iops_wr_max_length} || 1);
5231
5232                     }
5233
5234                     return 1;
5235                 }
5236
5237             } else { # cdrom
5238
5239                 if ($drive->{file} eq 'none') {
5240                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5241                     if (drive_is_cloudinit($old_drive)) {
5242                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5243                     }
5244                 } else {
5245                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5246                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5247                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5248                 }
5249
5250                 return 1;
5251             }
5252         }
5253     }
5254
5255     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5256     # hotplug new disks
5257     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5258     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5259 }
5260
5261 sub vm_start {
5262     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5263         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5264
5265     PVE::QemuConfig->lock_config($vmid, sub {
5266         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5267
5268         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5269
5270         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5271
5272         PVE::QemuConfig->check_lock($conf)
5273             if !($skiplock || $is_suspended);
5274
5275         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5276
5277         # clean up leftover reboot request files
5278         eval { clear_reboot_request($vmid); };
5279         warn $@ if $@;
5280
5281         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5282             vmconfig_apply_pending($vmid, $conf, $storecfg);
5283             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5284         }
5285
5286         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5287
5288         my $defaults = load_defaults();
5289
5290         # set environment variable useful inside network script
5291         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5292
5293         my $local_volumes = {};
5294
5295         if ($targetstorage) {
5296             foreach_drive($conf, sub {
5297                 my ($ds, $drive) = @_;
5298
5299                 return if drive_is_cdrom($drive);
5300
5301                 my $volid = $drive->{file};
5302
5303                 return if !$volid;
5304
5305                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5306
5307                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5308                 return if $scfg->{shared};
5309                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5310             });
5311
5312             my $format = undef;
5313
5314             foreach my $opt (sort keys %$local_volumes) {
5315
5316                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5317                 my $drive = parse_drive($opt, $conf->{$opt});
5318
5319                 #if remote storage is specified, use default format
5320                 if ($targetstorage && $targetstorage ne "1") {
5321                     $storeid = $targetstorage;
5322                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5323                     $format = $defFormat;
5324                 } else {
5325                     #else we use same format than original
5326                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5327                     $format = qemu_img_format($scfg, $volid);
5328                 }
5329
5330                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5331                 my $newdrive = $drive;
5332                 $newdrive->{format} = $format;
5333                 $newdrive->{file} = $newvolid;
5334                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5335                 $local_volumes->{$opt} = $drivestr;
5336                 #pass drive to conf for command line
5337                 $conf->{$opt} = $drivestr;
5338             }
5339         }
5340
5341         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5342
5343         if ($is_suspended) {
5344             # enforce machine type on suspended vm to ensure HW compatibility
5345             $forcemachine = $conf->{runningmachine};
5346             print "Resuming suspended VM\n";
5347         }
5348
5349         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5350
5351         my $migration_ip;
5352         my $get_migration_ip = sub {
5353             my ($cidr, $nodename) = @_;
5354
5355             return $migration_ip if defined($migration_ip);
5356
5357             if (!defined($cidr)) {
5358                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5359                 $cidr = $dc_conf->{migration}->{network};
5360             }
5361
5362             if (defined($cidr)) {
5363                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5364
5365                 die "could not get IP: no address configured on local " .
5366                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5367
5368                 die "could not get IP: multiple addresses configured on local " .
5369                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5370
5371                 $migration_ip = @$ips[0];
5372             }
5373
5374             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5375                 if !defined($migration_ip);
5376
5377             return $migration_ip;
5378         };
5379
5380         my $migrate_uri;
5381         if ($statefile) {
5382             if ($statefile eq 'tcp') {
5383                 my $localip = "localhost";
5384                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5385                 my $nodename = PVE::INotify::nodename();
5386
5387                 if (!defined($migration_type)) {
5388                     if (defined($datacenterconf->{migration}->{type})) {
5389                         $migration_type = $datacenterconf->{migration}->{type};
5390                     } else {
5391                         $migration_type = 'secure';
5392                     }
5393                 }
5394
5395                 if ($migration_type eq 'insecure') {
5396                     $localip = $get_migration_ip->($migration_network, $nodename);
5397                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5398                 }
5399
5400                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5401                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5402                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5403                 push @$cmd, '-incoming', $migrate_uri;
5404                 push @$cmd, '-S';
5405
5406             } elsif ($statefile eq 'unix') {
5407                 # should be default for secure migrations as a ssh TCP forward
5408                 # tunnel is not deterministic reliable ready and fails regurarly
5409                 # to set up in time, so use UNIX socket forwards
5410                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5411                 unlink $socket_addr;
5412
5413                 $migrate_uri = "unix:$socket_addr";
5414
5415                 push @$cmd, '-incoming', $migrate_uri;
5416                 push @$cmd, '-S';
5417
5418             } elsif (-e $statefile) {
5419                 push @$cmd, '-loadstate', $statefile;
5420             } else {
5421                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5422                 push @$vollist, $statefile;
5423                 push @$cmd, '-loadstate', $statepath;
5424             }
5425         } elsif ($paused) {
5426             push @$cmd, '-S';
5427         }
5428
5429         # host pci devices
5430         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5431           my $d = parse_hostpci($conf->{"hostpci$i"});
5432           next if !$d;
5433           my $pcidevices = $d->{pciid};
5434           foreach my $pcidevice (@$pcidevices) {
5435                 my $pciid = $pcidevice->{id};
5436
5437                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5438                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5439                 die "no pci device info for device '$pciid'\n" if !$info;
5440
5441                 if ($d->{mdev}) {
5442                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5443                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5444                 } else {
5445                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5446                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5447                     die "can't reset pci device '$pciid'\n"
5448                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5449                 }
5450           }
5451         }
5452
5453         PVE::Storage::activate_volumes($storecfg, $vollist);
5454
5455         eval {
5456             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5457                 outfunc => sub {}, errfunc => sub {});
5458         };
5459         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5460         # timeout should be more than enough here...
5461         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5462
5463         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5464                                                   : $defaults->{cpuunits};
5465
5466         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5467         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5468
5469         my %properties = (
5470             Slice => 'qemu.slice',
5471             KillMode => 'none',
5472             CPUShares => $cpuunits
5473         );
5474
5475         if (my $cpulimit = $conf->{cpulimit}) {
5476             $properties{CPUQuota} = int($cpulimit * 100);
5477         }
5478         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5479
5480         my $run_qemu = sub {
5481             PVE::Tools::run_fork sub {
5482                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5483                 run_command($cmd, %run_params);
5484             };
5485         };
5486
5487         if ($conf->{hugepages}) {
5488
5489             my $code = sub {
5490                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5491                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5492
5493                 PVE::QemuServer::Memory::hugepages_mount();
5494                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5495
5496                 eval { $run_qemu->() };
5497                 if (my $err = $@) {
5498                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5499                     die $err;
5500                 }
5501
5502                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5503             };
5504             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5505
5506         } else {
5507             eval { $run_qemu->() };
5508         }
5509
5510         if (my $err = $@) {
5511             # deactivate volumes if start fails
5512             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5513             die "start failed: $err";
5514         }
5515
5516         print "migration listens on $migrate_uri\n" if $migrate_uri;
5517
5518         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5519             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5520             warn $@ if $@;
5521         }
5522
5523         #start nbd server for storage migration
5524         if ($targetstorage) {
5525             my $nodename = PVE::INotify::nodename();
5526             my $localip = $get_migration_ip->($migration_network, $nodename);
5527             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5528             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5529
5530             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5531
5532             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5533
5534             foreach my $opt (sort keys %$local_volumes) {
5535                 my $volid = $local_volumes->{$opt};
5536                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5537                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5538                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5539             }
5540         }
5541
5542         if ($migratedfrom) {
5543             eval {
5544                 set_migration_caps($vmid);
5545             };
5546             warn $@ if $@;
5547
5548             if ($spice_port) {
5549                 print "spice listens on port $spice_port\n";
5550                 if ($spice_ticket) {
5551                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5552                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5553                 }
5554             }
5555
5556         } else {
5557             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5558                 if !$statefile && $conf->{balloon};
5559
5560             foreach my $opt (keys %$conf) {
5561                 next if $opt !~  m/^net\d+$/;
5562                 my $nicconf = parse_net($conf->{$opt});
5563                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5564             }
5565         }
5566
5567         vm_mon_cmd_nocheck($vmid, 'qom-set',
5568                     path => "machine/peripheral/balloon0",
5569                     property => "guest-stats-polling-interval",
5570                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5571
5572         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5573             print "Resumed VM, removing state\n";
5574             delete $conf->@{qw(lock vmstate runningmachine)};
5575             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5576             PVE::Storage::vdisk_free($storecfg, $vmstate);
5577             PVE::QemuConfig->write_config($vmid, $conf);
5578         }
5579
5580         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5581     });
5582 }
5583
5584 sub vm_mon_cmd {
5585     my ($vmid, $execute, %params) = @_;
5586
5587     my $cmd = { execute => $execute, arguments => \%params };
5588     vm_qmp_command($vmid, $cmd);
5589 }
5590
5591 sub vm_mon_cmd_nocheck {
5592     my ($vmid, $execute, %params) = @_;
5593
5594     my $cmd = { execute => $execute, arguments => \%params };
5595     vm_qmp_command($vmid, $cmd, 1);
5596 }
5597
5598 sub vm_qmp_command {
5599     my ($vmid, $cmd, $nocheck) = @_;
5600
5601     my $res;
5602
5603     my $timeout;
5604     if ($cmd->{arguments}) {
5605         $timeout = delete $cmd->{arguments}->{timeout};
5606     }
5607
5608     eval {
5609         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5610         my $sname = qmp_socket($vmid);
5611         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5612             my $qmpclient = PVE::QMPClient->new();
5613
5614             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5615         } else {
5616             die "unable to open monitor socket\n";
5617         }
5618     };
5619     if (my $err = $@) {
5620         syslog("err", "VM $vmid qmp command failed - $err");
5621         die $err;
5622     }
5623
5624     return $res;
5625 }
5626
5627 sub vm_human_monitor_command {
5628     my ($vmid, $cmdline) = @_;
5629
5630     my $cmd = {
5631         execute => 'human-monitor-command',
5632         arguments => { 'command-line' => $cmdline},
5633     };
5634
5635     return vm_qmp_command($vmid, $cmd);
5636 }
5637
5638 sub vm_commandline {
5639     my ($storecfg, $vmid, $snapname) = @_;
5640
5641     my $conf = PVE::QemuConfig->load_config($vmid);
5642
5643     if ($snapname) {
5644         my $snapshot = $conf->{snapshots}->{$snapname};
5645         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5646
5647         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5648
5649         $conf = $snapshot;
5650     }
5651
5652     my $defaults = load_defaults();
5653
5654     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5655
5656     return PVE::Tools::cmd2string($cmd);
5657 }
5658
5659 sub vm_reset {
5660     my ($vmid, $skiplock) = @_;
5661
5662     PVE::QemuConfig->lock_config($vmid, sub {
5663
5664         my $conf = PVE::QemuConfig->load_config($vmid);
5665
5666         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5667
5668         vm_mon_cmd($vmid, "system_reset");
5669     });
5670 }
5671
5672 sub get_vm_volumes {
5673     my ($conf) = @_;
5674
5675     my $vollist = [];
5676     foreach_volid($conf, sub {
5677         my ($volid, $attr) = @_;
5678
5679         return if $volid =~ m|^/|;
5680
5681         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5682         return if !$sid;
5683
5684         push @$vollist, $volid;
5685     });
5686
5687     return $vollist;
5688 }
5689
5690 sub vm_stop_cleanup {
5691     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5692
5693     eval {
5694
5695         if (!$keepActive) {
5696             my $vollist = get_vm_volumes($conf);
5697             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5698         }
5699
5700         foreach my $ext (qw(mon qmp pid vnc qga)) {
5701             unlink "/var/run/qemu-server/${vmid}.$ext";
5702         }
5703
5704         if ($conf->{ivshmem}) {
5705             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5706             # just delete it for now, VMs which have this already open do not
5707             # are affected, but new VMs will get a separated one. If this
5708             # becomes an issue we either add some sort of ref-counting or just
5709             # add a "don't delete on stop" flag to the ivshmem format.
5710             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5711         }
5712
5713         foreach my $key (keys %$conf) {
5714             next if $key !~ m/^hostpci(\d+)$/;
5715             my $hostpciindex = $1;
5716             my $d = parse_hostpci($conf->{$key});
5717             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5718
5719             foreach my $pci (@{$d->{pciid}}) {
5720                 my $pciid = $pci->{id};
5721                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5722             }
5723         }
5724
5725         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5726     };
5727     warn $@ if $@; # avoid errors - just warn
5728 }
5729
5730 # call only in locked context
5731 sub _do_vm_stop {
5732     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5733
5734     my $pid = check_running($vmid, $nocheck);
5735     return if !$pid;
5736
5737     my $conf;
5738     if (!$nocheck) {
5739         $conf = PVE::QemuConfig->load_config($vmid);
5740         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5741         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5742             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5743             $timeout = $opts->{down} if $opts->{down};
5744         }
5745         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5746     }
5747
5748     eval {
5749         if ($shutdown) {
5750             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5751                 vm_qmp_command($vmid, {
5752                         execute => "guest-shutdown",
5753                         arguments => { timeout => $timeout }
5754                     }, $nocheck);
5755             } else {
5756                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5757             }
5758         } else {
5759             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5760         }
5761     };
5762     my $err = $@;
5763
5764     if (!$err) {
5765         $timeout = 60 if !defined($timeout);
5766
5767         my $count = 0;
5768         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5769             $count++;
5770             sleep 1;
5771         }
5772
5773         if ($count >= $timeout) {
5774             if ($force) {
5775                 warn "VM still running - terminating now with SIGTERM\n";
5776                 kill 15, $pid;
5777             } else {
5778                 die "VM quit/powerdown failed - got timeout\n";
5779             }
5780         } else {
5781             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5782             return;
5783         }
5784     } else {
5785         if ($force) {
5786             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5787             kill 15, $pid;
5788         } else {
5789             die "VM quit/powerdown failed\n";
5790         }
5791     }
5792
5793     # wait again
5794     $timeout = 10;
5795
5796     my $count = 0;
5797     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5798         $count++;
5799         sleep 1;
5800     }
5801
5802     if ($count >= $timeout) {
5803         warn "VM still running - terminating now with SIGKILL\n";
5804         kill 9, $pid;
5805         sleep 1;
5806     }
5807
5808     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5809 }
5810
5811 # Note: use $nocheck to skip tests if VM configuration file exists.
5812 # We need that when migration VMs to other nodes (files already moved)
5813 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5814 sub vm_stop {
5815     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5816
5817     $force = 1 if !defined($force) && !$shutdown;
5818
5819     if ($migratedfrom){
5820         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5821         kill 15, $pid if $pid;
5822         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5823         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5824         return;
5825     }
5826
5827     PVE::QemuConfig->lock_config($vmid, sub {
5828         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5829    });
5830 }
5831
5832 sub vm_reboot {
5833     my ($vmid, $timeout) = @_;
5834
5835     PVE::QemuConfig->lock_config($vmid, sub {
5836         eval {
5837
5838             # only reboot if running, as qmeventd starts it again on a stop event
5839             return if !check_running($vmid);
5840
5841             create_reboot_request($vmid);
5842
5843             my $storecfg = PVE::Storage::config();
5844             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5845
5846         };
5847         if (my $err = $@) {
5848             # avoid that the next normal shutdown will be confused for a reboot
5849             clear_reboot_request($vmid);
5850             die $err;
5851         }
5852    });
5853 }
5854
5855 sub vm_suspend {
5856     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5857
5858     my $conf;
5859     my $path;
5860     my $storecfg;
5861     my $vmstate;
5862
5863     PVE::QemuConfig->lock_config($vmid, sub {
5864
5865         $conf = PVE::QemuConfig->load_config($vmid);
5866
5867         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5868         PVE::QemuConfig->check_lock($conf)
5869             if !($skiplock || $is_backing_up);
5870
5871         die "cannot suspend to disk during backup\n"
5872             if $is_backing_up && $includestate;
5873
5874         if ($includestate) {
5875             $conf->{lock} = 'suspending';
5876             my $date = strftime("%Y-%m-%d", localtime(time()));
5877             $storecfg = PVE::Storage::config();
5878             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5879             $path = PVE::Storage::path($storecfg, $vmstate);
5880             PVE::QemuConfig->write_config($vmid, $conf);
5881         } else {
5882             vm_mon_cmd($vmid, "stop");
5883         }
5884     });
5885
5886     if ($includestate) {
5887         # save vm state
5888         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5889
5890         eval {
5891             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5892             for(;;) {
5893                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5894                 if (!$state->{status}) {
5895                     die "savevm not active\n";
5896                 } elsif ($state->{status} eq 'active') {
5897                     sleep(1);
5898                     next;
5899                 } elsif ($state->{status} eq 'completed') {
5900                     print "State saved, quitting\n";
5901                     last;
5902                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5903                     die "query-savevm failed with error '$state->{error}'\n"
5904                 } else {
5905                     die "query-savevm returned status '$state->{status}'\n";
5906                 }
5907             }
5908         };
5909         my $err = $@;
5910
5911         PVE::QemuConfig->lock_config($vmid, sub {
5912             $conf = PVE::QemuConfig->load_config($vmid);
5913             if ($err) {
5914                 # cleanup, but leave suspending lock, to indicate something went wrong
5915                 eval {
5916                     vm_mon_cmd($vmid, "savevm-end");
5917                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5918                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5919                     delete $conf->@{qw(vmstate runningmachine)};
5920                     PVE::QemuConfig->write_config($vmid, $conf);
5921                 };
5922                 warn $@ if $@;
5923                 die $err;
5924             }
5925
5926             die "lock changed unexpectedly\n"
5927                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5928
5929             vm_qmp_command($vmid, { execute => "quit" });
5930             $conf->{lock} = 'suspended';
5931             PVE::QemuConfig->write_config($vmid, $conf);
5932         });
5933     }
5934 }
5935
5936 sub vm_resume {
5937     my ($vmid, $skiplock, $nocheck) = @_;
5938
5939     PVE::QemuConfig->lock_config($vmid, sub {
5940         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5941         my $res = $vm_mon_cmd->($vmid, 'query-status');
5942         my $resume_cmd = 'cont';
5943
5944         if ($res->{status} && $res->{status} eq 'suspended') {
5945             $resume_cmd = 'system_wakeup';
5946         }
5947
5948         if (!$nocheck) {
5949
5950             my $conf = PVE::QemuConfig->load_config($vmid);
5951
5952             PVE::QemuConfig->check_lock($conf)
5953                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5954         }
5955
5956         $vm_mon_cmd->($vmid, $resume_cmd);
5957     });
5958 }
5959
5960 sub vm_sendkey {
5961     my ($vmid, $skiplock, $key) = @_;
5962
5963     PVE::QemuConfig->lock_config($vmid, sub {
5964
5965         my $conf = PVE::QemuConfig->load_config($vmid);
5966
5967         # there is no qmp command, so we use the human monitor command
5968         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5969         die $res if $res ne '';
5970     });
5971 }
5972
5973 # vzdump restore implementaion
5974
5975 sub tar_archive_read_firstfile {
5976     my $archive = shift;
5977
5978     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5979
5980     # try to detect archive type first
5981     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5982         die "unable to open file '$archive'\n";
5983     my $firstfile = <$fh>;
5984     kill 15, $pid;
5985     close $fh;
5986
5987     die "ERROR: archive contaions no data\n" if !$firstfile;
5988     chomp $firstfile;
5989
5990     return $firstfile;
5991 }
5992
5993 sub tar_restore_cleanup {
5994     my ($storecfg, $statfile) = @_;
5995
5996     print STDERR "starting cleanup\n";
5997
5998     if (my $fd = IO::File->new($statfile, "r")) {
5999         while (defined(my $line = <$fd>)) {
6000             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6001                 my $volid = $2;
6002                 eval {
6003                     if ($volid =~ m|^/|) {
6004                         unlink $volid || die 'unlink failed\n';
6005                     } else {
6006                         PVE::Storage::vdisk_free($storecfg, $volid);
6007                     }
6008                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6009                 };
6010                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6011             } else {
6012                 print STDERR "unable to parse line in statfile - $line";
6013             }
6014         }
6015         $fd->close();
6016     }
6017 }
6018
6019 sub restore_archive {
6020     my ($archive, $vmid, $user, $opts) = @_;
6021
6022     my $format = $opts->{format};
6023     my $comp;
6024
6025     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6026         $format = 'tar' if !$format;
6027         $comp = 'gzip';
6028     } elsif ($archive =~ m/\.tar$/) {
6029         $format = 'tar' if !$format;
6030     } elsif ($archive =~ m/.tar.lzo$/) {
6031         $format = 'tar' if !$format;
6032         $comp = 'lzop';
6033     } elsif ($archive =~ m/\.vma$/) {
6034         $format = 'vma' if !$format;
6035     } elsif ($archive =~ m/\.vma\.gz$/) {
6036         $format = 'vma' if !$format;
6037         $comp = 'gzip';
6038     } elsif ($archive =~ m/\.vma\.lzo$/) {
6039         $format = 'vma' if !$format;
6040         $comp = 'lzop';
6041     } else {
6042         $format = 'vma' if !$format; # default
6043     }
6044
6045     # try to detect archive format
6046     if ($format eq 'tar') {
6047         return restore_tar_archive($archive, $vmid, $user, $opts);
6048     } else {
6049         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6050     }
6051 }
6052
6053 sub restore_update_config_line {
6054     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6055
6056     return if $line =~ m/^\#qmdump\#/;
6057     return if $line =~ m/^\#vzdump\#/;
6058     return if $line =~ m/^lock:/;
6059     return if $line =~ m/^unused\d+:/;
6060     return if $line =~ m/^parent:/;
6061
6062     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6063     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6064         # try to convert old 1.X settings
6065         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6066         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6067             my ($model, $macaddr) = split(/\=/, $devconfig);
6068             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6069             my $net = {
6070                 model => $model,
6071                 bridge => "vmbr$ind",
6072                 macaddr => $macaddr,
6073             };
6074             my $netstr = print_net($net);
6075
6076             print $outfd "net$cookie->{netcount}: $netstr\n";
6077             $cookie->{netcount}++;
6078         }
6079     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6080         my ($id, $netstr) = ($1, $2);
6081         my $net = parse_net($netstr);
6082         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6083         $netstr = print_net($net);
6084         print $outfd "$id: $netstr\n";
6085     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6086         my $virtdev = $1;
6087         my $value = $3;
6088         my $di = parse_drive($virtdev, $value);
6089         if (defined($di->{backup}) && !$di->{backup}) {
6090             print $outfd "#$line";
6091         } elsif ($map->{$virtdev}) {
6092             delete $di->{format}; # format can change on restore
6093             $di->{file} = $map->{$virtdev};
6094             $value = print_drive($vmid, $di);
6095             print $outfd "$virtdev: $value\n";
6096         } else {
6097             print $outfd $line;
6098         }
6099     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6100         my $vmgenid = $1;
6101         if ($vmgenid ne '0') {
6102             # always generate a new vmgenid if there was a valid one setup
6103             $vmgenid = generate_uuid();
6104         }
6105         print $outfd "vmgenid: $vmgenid\n";
6106     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6107         my ($uuid, $uuid_str);
6108         UUID::generate($uuid);
6109         UUID::unparse($uuid, $uuid_str);
6110         my $smbios1 = parse_smbios1($2);
6111         $smbios1->{uuid} = $uuid_str;
6112         print $outfd $1.print_smbios1($smbios1)."\n";
6113     } else {
6114         print $outfd $line;
6115     }
6116 }
6117
6118 sub scan_volids {
6119     my ($cfg, $vmid) = @_;
6120
6121     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6122
6123     my $volid_hash = {};
6124     foreach my $storeid (keys %$info) {
6125         foreach my $item (@{$info->{$storeid}}) {
6126             next if !($item->{volid} && $item->{size});
6127             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6128             $volid_hash->{$item->{volid}} = $item;
6129         }
6130     }
6131
6132     return $volid_hash;
6133 }
6134
6135 sub is_volume_in_use {
6136     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6137
6138     my $path = PVE::Storage::path($storecfg, $volid);
6139
6140     my $scan_config = sub {
6141         my ($cref, $snapname) = @_;
6142
6143         foreach my $key (keys %$cref) {
6144             my $value = $cref->{$key};
6145             if (is_valid_drivename($key)) {
6146                 next if $skip_drive && $key eq $skip_drive;
6147                 my $drive = parse_drive($key, $value);
6148                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6149                 return 1 if $volid eq $drive->{file};
6150                 if ($drive->{file} =~ m!^/!) {
6151                     return 1 if $drive->{file} eq $path;
6152                 } else {
6153                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6154                     next if !$storeid;
6155                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6156                     next if !$scfg;
6157                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6158                 }
6159             }
6160         }
6161
6162         return 0;
6163     };
6164
6165     return 1 if &$scan_config($conf);
6166
6167     undef $skip_drive;
6168
6169     foreach my $snapname (keys %{$conf->{snapshots}}) {
6170         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6171     }
6172
6173     return 0;
6174 }
6175
6176 sub update_disksize {
6177     my ($vmid, $conf, $volid_hash) = @_;
6178
6179     my $changes;
6180     my $prefix = "VM $vmid:";
6181
6182     # used and unused disks
6183     my $referenced = {};
6184
6185     # Note: it is allowed to define multiple storages with same path (alias), so
6186     # we need to check both 'volid' and real 'path' (two different volid can point
6187     # to the same path).
6188
6189     my $referencedpath = {};
6190
6191     # update size info
6192     foreach my $opt (keys %$conf) {
6193         if (is_valid_drivename($opt)) {
6194             my $drive = parse_drive($opt, $conf->{$opt});
6195             my $volid = $drive->{file};
6196             next if !$volid;
6197
6198             $referenced->{$volid} = 1;
6199             if ($volid_hash->{$volid} &&
6200                 (my $path = $volid_hash->{$volid}->{path})) {
6201                 $referencedpath->{$path} = 1;
6202             }
6203
6204             next if drive_is_cdrom($drive);
6205             next if !$volid_hash->{$volid};
6206
6207             $drive->{size} = $volid_hash->{$volid}->{size};
6208             my $new = print_drive($vmid, $drive);
6209             if ($new ne $conf->{$opt}) {
6210                 $changes = 1;
6211                 $conf->{$opt} = $new;
6212                 print "$prefix update disk '$opt' information.\n";
6213             }
6214         }
6215     }
6216
6217     # remove 'unusedX' entry if volume is used
6218     foreach my $opt (keys %$conf) {
6219         next if $opt !~ m/^unused\d+$/;
6220         my $volid = $conf->{$opt};
6221         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6222         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6223             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6224             $changes = 1;
6225             delete $conf->{$opt};
6226         }
6227
6228         $referenced->{$volid} = 1;
6229         $referencedpath->{$path} = 1 if $path;
6230     }
6231
6232     foreach my $volid (sort keys %$volid_hash) {
6233         next if $volid =~ m/vm-$vmid-state-/;
6234         next if $referenced->{$volid};
6235         my $path = $volid_hash->{$volid}->{path};
6236         next if !$path; # just to be sure
6237         next if $referencedpath->{$path};
6238         $changes = 1;
6239         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6240         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6241         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6242     }
6243
6244     return $changes;
6245 }
6246
6247 sub rescan {
6248     my ($vmid, $nolock, $dryrun) = @_;
6249
6250     my $cfg = PVE::Storage::config();
6251
6252     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6253     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6254     foreach my $stor (keys %{$cfg->{ids}}) {
6255         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6256     }
6257
6258     print "rescan volumes...\n";
6259     my $volid_hash = scan_volids($cfg, $vmid);
6260
6261     my $updatefn =  sub {
6262         my ($vmid) = @_;
6263
6264         my $conf = PVE::QemuConfig->load_config($vmid);
6265
6266         PVE::QemuConfig->check_lock($conf);
6267
6268         my $vm_volids = {};
6269         foreach my $volid (keys %$volid_hash) {
6270             my $info = $volid_hash->{$volid};
6271             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6272         }
6273
6274         my $changes = update_disksize($vmid, $conf, $vm_volids);
6275
6276         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6277     };
6278
6279     if (defined($vmid)) {
6280         if ($nolock) {
6281             &$updatefn($vmid);
6282         } else {
6283             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6284         }
6285     } else {
6286         my $vmlist = config_list();
6287         foreach my $vmid (keys %$vmlist) {
6288             if ($nolock) {
6289                 &$updatefn($vmid);
6290             } else {
6291                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6292             }
6293         }
6294     }
6295 }
6296
6297 sub restore_vma_archive {
6298     my ($archive, $vmid, $user, $opts, $comp) = @_;
6299
6300     my $readfrom = $archive;
6301
6302     my $cfg = PVE::Storage::config();
6303     my $commands = [];
6304     my $bwlimit = $opts->{bwlimit};
6305
6306     my $dbg_cmdstring = '';
6307     my $add_pipe = sub {
6308         my ($cmd) = @_;
6309         push @$commands, $cmd;
6310         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6311         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6312         $readfrom = '-';
6313     };
6314
6315     my $input = undef;
6316     if ($archive eq '-') {
6317         $input = '<&STDIN';
6318     } else {
6319         # If we use a backup from a PVE defined storage we also consider that
6320         # storage's rate limit:
6321         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6322         if (defined($volid)) {
6323             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6324             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6325             if ($readlimit) {
6326                 print STDERR "applying read rate limit: $readlimit\n";
6327                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6328                 $add_pipe->($cstream);
6329             }
6330         }
6331     }
6332
6333     if ($comp) {
6334         my $cmd;
6335         if ($comp eq 'gzip') {
6336             $cmd = ['zcat', $readfrom];
6337         } elsif ($comp eq 'lzop') {
6338             $cmd = ['lzop', '-d', '-c', $readfrom];
6339         } else {
6340             die "unknown compression method '$comp'\n";
6341         }
6342         $add_pipe->($cmd);
6343     }
6344
6345     my $tmpdir = "/var/tmp/vzdumptmp$$";
6346     rmtree $tmpdir;
6347
6348     # disable interrupts (always do cleanups)
6349     local $SIG{INT} =
6350         local $SIG{TERM} =
6351         local $SIG{QUIT} =
6352         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6353
6354     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6355     POSIX::mkfifo($mapfifo, 0600);
6356     my $fifofh;
6357
6358     my $openfifo = sub {
6359         open($fifofh, '>', $mapfifo) || die $!;
6360     };
6361
6362     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6363
6364     my $oldtimeout;
6365     my $timeout = 5;
6366
6367     my $devinfo = {};
6368
6369     my $rpcenv = PVE::RPCEnvironment::get();
6370
6371     my $conffile = PVE::QemuConfig->config_file($vmid);
6372     my $tmpfn = "$conffile.$$.tmp";
6373
6374     # Note: $oldconf is undef if VM does not exists
6375     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6376     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6377
6378     my %storage_limits;
6379
6380     my $print_devmap = sub {
6381         my $virtdev_hash = {};
6382
6383         my $cfgfn = "$tmpdir/qemu-server.conf";
6384
6385         # we can read the config - that is already extracted
6386         my $fh = IO::File->new($cfgfn, "r") ||
6387             "unable to read qemu-server.conf - $!\n";
6388
6389         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6390         if (-f $fwcfgfn) {
6391             my $pve_firewall_dir = '/etc/pve/firewall';
6392             mkdir $pve_firewall_dir; # make sure the dir exists
6393             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6394         }
6395
6396         while (defined(my $line = <$fh>)) {
6397             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6398                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6399                 die "archive does not contain data for drive '$virtdev'\n"
6400                     if !$devinfo->{$devname};
6401                 if (defined($opts->{storage})) {
6402                     $storeid = $opts->{storage} || 'local';
6403                 } elsif (!$storeid) {
6404                     $storeid = 'local';
6405                 }
6406                 $format = 'raw' if !$format;
6407                 $devinfo->{$devname}->{devname} = $devname;
6408                 $devinfo->{$devname}->{virtdev} = $virtdev;
6409                 $devinfo->{$devname}->{format} = $format;
6410                 $devinfo->{$devname}->{storeid} = $storeid;
6411
6412                 # check permission on storage
6413                 my $pool = $opts->{pool}; # todo: do we need that?
6414                 if ($user ne 'root@pam') {
6415                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6416                 }
6417
6418                 $storage_limits{$storeid} = $bwlimit;
6419
6420                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6421             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6422                 my $virtdev = $1;
6423                 my $drive = parse_drive($virtdev, $2);
6424                 if (drive_is_cloudinit($drive)) {
6425                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6426                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6427                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6428
6429                     my $d = {
6430                         format => $format,
6431                         storeid => $opts->{storage} // $storeid,
6432                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6433                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6434                         name => "vm-$vmid-cloudinit",
6435                         is_cloudinit => 1,
6436                     };
6437                     $virtdev_hash->{$virtdev} = $d;
6438                 }
6439             }
6440         }
6441
6442         foreach my $key (keys %storage_limits) {
6443             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6444             next if !$limit;
6445             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6446             $storage_limits{$key} = $limit * 1024;
6447         }
6448
6449         foreach my $devname (keys %$devinfo) {
6450             die "found no device mapping information for device '$devname'\n"
6451                 if !$devinfo->{$devname}->{virtdev};
6452         }
6453
6454         # create empty/temp config
6455         if ($oldconf) {
6456             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6457             foreach_drive($oldconf, sub {
6458                 my ($ds, $drive) = @_;
6459
6460                 return if drive_is_cdrom($drive, 1);
6461
6462                 my $volid = $drive->{file};
6463                 return if !$volid || $volid =~ m|^/|;
6464
6465                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6466                 return if !$path || !$owner || ($owner != $vmid);
6467
6468                 # Note: only delete disk we want to restore
6469                 # other volumes will become unused
6470                 if ($virtdev_hash->{$ds}) {
6471                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6472                     if (my $err = $@) {
6473                         warn $err;
6474                     }
6475                 }
6476             });
6477
6478             # delete vmstate files, after the restore we have no snapshots anymore
6479             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6480                 my $snap = $oldconf->{snapshots}->{$snapname};
6481                 if ($snap->{vmstate}) {
6482                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6483                     if (my $err = $@) {
6484                         warn $err;
6485                     }
6486                 }
6487             }
6488         }
6489
6490         my $map = {};
6491         foreach my $virtdev (sort keys %$virtdev_hash) {
6492             my $d = $virtdev_hash->{$virtdev};
6493             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6494             my $storeid = $d->{storeid};
6495             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6496
6497             my $map_opts = '';
6498             if (my $limit = $storage_limits{$storeid}) {
6499                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6500             }
6501
6502             # test if requested format is supported
6503             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6504             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6505             $d->{format} = $defFormat if !$supported;
6506
6507             my $name;
6508             if ($d->{is_cloudinit}) {
6509                 $name = $d->{name};
6510                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6511             }
6512
6513             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6514             print STDERR "new volume ID is '$volid'\n";
6515             $d->{volid} = $volid;
6516
6517             PVE::Storage::activate_volumes($cfg, [$volid]);
6518
6519             my $write_zeros = 1;
6520             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6521                 $write_zeros = 0;
6522             }
6523
6524             if (!$d->{is_cloudinit}) {
6525                 my $path = PVE::Storage::path($cfg, $volid);
6526
6527                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6528
6529                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6530             }
6531             $map->{$virtdev} = $volid;
6532         }
6533
6534         $fh->seek(0, 0) || die "seek failed - $!\n";
6535
6536         my $outfd = new IO::File ($tmpfn, "w") ||
6537             die "unable to write config for VM $vmid\n";
6538
6539         my $cookie = { netcount => 0 };
6540         while (defined(my $line = <$fh>)) {
6541             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6542         }
6543
6544         $fh->close();
6545         $outfd->close();
6546     };
6547
6548     eval {
6549         # enable interrupts
6550         local $SIG{INT} =
6551             local $SIG{TERM} =
6552             local $SIG{QUIT} =
6553             local $SIG{HUP} =
6554             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6555         local $SIG{ALRM} = sub { die "got timeout\n"; };
6556
6557         $oldtimeout = alarm($timeout);
6558
6559         my $parser = sub {
6560             my $line = shift;
6561
6562             print "$line\n";
6563
6564             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6565                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6566                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6567             } elsif ($line =~ m/^CTIME: /) {
6568                 # we correctly received the vma config, so we can disable
6569                 # the timeout now for disk allocation (set to 10 minutes, so
6570                 # that we always timeout if something goes wrong)
6571                 alarm(600);
6572                 &$print_devmap();
6573                 print $fifofh "done\n";
6574                 my $tmp = $oldtimeout || 0;
6575                 $oldtimeout = undef;
6576                 alarm($tmp);
6577                 close($fifofh);
6578             }
6579         };
6580
6581         print "restore vma archive: $dbg_cmdstring\n";
6582         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6583     };
6584     my $err = $@;
6585
6586     alarm($oldtimeout) if $oldtimeout;
6587
6588     my $vollist = [];
6589     foreach my $devname (keys %$devinfo) {
6590         my $volid = $devinfo->{$devname}->{volid};
6591         push @$vollist, $volid if $volid;
6592     }
6593
6594     PVE::Storage::deactivate_volumes($cfg, $vollist);
6595
6596     unlink $mapfifo;
6597
6598     if ($err) {
6599         rmtree $tmpdir;
6600         unlink $tmpfn;
6601
6602         foreach my $devname (keys %$devinfo) {
6603             my $volid = $devinfo->{$devname}->{volid};
6604             next if !$volid;
6605             eval {
6606                 if ($volid =~ m|^/|) {
6607                     unlink $volid || die 'unlink failed\n';
6608                 } else {
6609                     PVE::Storage::vdisk_free($cfg, $volid);
6610                 }
6611                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6612             };
6613             print STDERR "unable to cleanup '$volid' - $@" if $@;
6614         }
6615         die $err;
6616     }
6617
6618     rmtree $tmpdir;
6619
6620     rename($tmpfn, $conffile) ||
6621         die "unable to commit configuration file '$conffile'\n";
6622
6623     PVE::Cluster::cfs_update(); # make sure we read new file
6624
6625     eval { rescan($vmid, 1); };
6626     warn $@ if $@;
6627 }
6628
6629 sub restore_tar_archive {
6630     my ($archive, $vmid, $user, $opts) = @_;
6631
6632     if ($archive ne '-') {
6633         my $firstfile = tar_archive_read_firstfile($archive);
6634         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6635             if $firstfile ne 'qemu-server.conf';
6636     }
6637
6638     my $storecfg = PVE::Storage::config();
6639
6640     # avoid zombie disks when restoring over an existing VM -> cleanup first
6641     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6642     # skiplock=1 because qmrestore has set the 'create' lock itself already
6643     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6644     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6645
6646     my $tocmd = "/usr/lib/qemu-server/qmextract";
6647
6648     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6649     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6650     $tocmd .= ' --prealloc' if $opts->{prealloc};
6651     $tocmd .= ' --info' if $opts->{info};
6652
6653     # tar option "xf" does not autodetect compression when read from STDIN,
6654     # so we pipe to zcat
6655     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6656         PVE::Tools::shellquote("--to-command=$tocmd");
6657
6658     my $tmpdir = "/var/tmp/vzdumptmp$$";
6659     mkpath $tmpdir;
6660
6661     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6662     local $ENV{VZDUMP_VMID} = $vmid;
6663     local $ENV{VZDUMP_USER} = $user;
6664
6665     my $conffile = PVE::QemuConfig->config_file($vmid);
6666     my $tmpfn = "$conffile.$$.tmp";
6667
6668     # disable interrupts (always do cleanups)
6669     local $SIG{INT} =
6670         local $SIG{TERM} =
6671         local $SIG{QUIT} =
6672         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6673
6674     eval {
6675         # enable interrupts
6676         local $SIG{INT} =
6677             local $SIG{TERM} =
6678             local $SIG{QUIT} =
6679             local $SIG{HUP} =
6680             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6681
6682         if ($archive eq '-') {
6683             print "extracting archive from STDIN\n";
6684             run_command($cmd, input => "<&STDIN");
6685         } else {
6686             print "extracting archive '$archive'\n";
6687             run_command($cmd);
6688         }
6689
6690         return if $opts->{info};
6691
6692         # read new mapping
6693         my $map = {};
6694         my $statfile = "$tmpdir/qmrestore.stat";
6695         if (my $fd = IO::File->new($statfile, "r")) {
6696             while (defined (my $line = <$fd>)) {
6697                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6698                     $map->{$1} = $2 if $1;
6699                 } else {
6700                     print STDERR "unable to parse line in statfile - $line\n";
6701                 }
6702             }
6703             $fd->close();
6704         }
6705
6706         my $confsrc = "$tmpdir/qemu-server.conf";
6707
6708         my $srcfd = new IO::File($confsrc, "r") ||
6709             die "unable to open file '$confsrc'\n";
6710
6711         my $outfd = new IO::File ($tmpfn, "w") ||
6712             die "unable to write config for VM $vmid\n";
6713
6714         my $cookie = { netcount => 0 };
6715         while (defined (my $line = <$srcfd>)) {
6716             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6717         }
6718
6719         $srcfd->close();
6720         $outfd->close();
6721     };
6722     if (my $err = $@) {
6723         unlink $tmpfn;
6724         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6725         die $err;
6726     }
6727
6728     rmtree $tmpdir;
6729
6730     rename $tmpfn, $conffile ||
6731         die "unable to commit configuration file '$conffile'\n";
6732
6733     PVE::Cluster::cfs_update(); # make sure we read new file
6734
6735     eval { rescan($vmid, 1); };
6736     warn $@ if $@;
6737 };
6738
6739 sub foreach_storage_used_by_vm {
6740     my ($conf, $func) = @_;
6741
6742     my $sidhash = {};
6743
6744     foreach_drive($conf, sub {
6745         my ($ds, $drive) = @_;
6746         return if drive_is_cdrom($drive);
6747
6748         my $volid = $drive->{file};
6749
6750         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6751         $sidhash->{$sid} = $sid if $sid;
6752     });
6753
6754     foreach my $sid (sort keys %$sidhash) {
6755         &$func($sid);
6756     }
6757 }
6758
6759 sub do_snapshots_with_qemu {
6760     my ($storecfg, $volid) = @_;
6761
6762     my $storage_name = PVE::Storage::parse_volume_id($volid);
6763     my $scfg = $storecfg->{ids}->{$storage_name};
6764
6765     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6766         return 1;
6767     }
6768
6769     if ($volid =~ m/\.(qcow2|qed)$/){
6770         return 1;
6771     }
6772
6773     return undef;
6774 }
6775
6776 sub qga_check_running {
6777     my ($vmid, $nowarn) = @_;
6778
6779     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6780     if ($@) {
6781         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6782         return 0;
6783     }
6784     return 1;
6785 }
6786
6787 sub template_create {
6788     my ($vmid, $conf, $disk) = @_;
6789
6790     my $storecfg = PVE::Storage::config();
6791
6792     foreach_drive($conf, sub {
6793         my ($ds, $drive) = @_;
6794
6795         return if drive_is_cdrom($drive);
6796         return if $disk && $ds ne $disk;
6797
6798         my $volid = $drive->{file};
6799         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6800
6801         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6802         $drive->{file} = $voliddst;
6803         $conf->{$ds} = print_drive($vmid, $drive);
6804         PVE::QemuConfig->write_config($vmid, $conf);
6805     });
6806 }
6807
6808 sub convert_iscsi_path {
6809     my ($path) = @_;
6810
6811     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6812         my $portal = $1;
6813         my $target = $2;
6814         my $lun = $3;
6815
6816         my $initiator_name = get_initiator_name();
6817
6818         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6819                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6820     }
6821
6822     die "cannot convert iscsi path '$path', unkown format\n";
6823 }
6824
6825 sub qemu_img_convert {
6826     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6827
6828     my $storecfg = PVE::Storage::config();
6829     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6830     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6831
6832     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6833
6834     my $cachemode;
6835     my $src_path;
6836     my $src_is_iscsi = 0;
6837     my $src_format = 'raw';
6838
6839     if ($src_storeid) {
6840         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6841         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6842         $src_format = qemu_img_format($src_scfg, $src_volname);
6843         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6844         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6845         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6846     } elsif (-f $src_volid) {
6847         $src_path = $src_volid;
6848         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6849             $src_format = $1;
6850         }
6851     }
6852
6853     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6854
6855     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6856     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6857     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6858     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6859
6860     my $cmd = [];
6861     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6862     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6863     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6864     push @$cmd, '-T', $cachemode if defined($cachemode);
6865
6866     if ($src_is_iscsi) {
6867         push @$cmd, '--image-opts';
6868         $src_path = convert_iscsi_path($src_path);
6869     } else {
6870         push @$cmd, '-f', $src_format;
6871     }
6872
6873     if ($dst_is_iscsi) {
6874         push @$cmd, '--target-image-opts';
6875         $dst_path = convert_iscsi_path($dst_path);
6876     } else {
6877         push @$cmd, '-O', $dst_format;
6878     }
6879
6880     push @$cmd, $src_path;
6881
6882     if (!$dst_is_iscsi && $is_zero_initialized) {
6883         push @$cmd, "zeroinit:$dst_path";
6884     } else {
6885         push @$cmd, $dst_path;
6886     }
6887
6888     my $parser = sub {
6889         my $line = shift;
6890         if($line =~ m/\((\S+)\/100\%\)/){
6891             my $percent = $1;
6892             my $transferred = int($size * $percent / 100);
6893             my $remaining = $size - $transferred;
6894
6895             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6896         }
6897
6898     };
6899
6900     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6901     my $err = $@;
6902     die "copy failed: $err" if $err;
6903 }
6904
6905 sub qemu_img_format {
6906     my ($scfg, $volname) = @_;
6907
6908     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6909         return $1;
6910     } else {
6911         return "raw";
6912     }
6913 }
6914
6915 sub qemu_drive_mirror {
6916     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6917
6918     $jobs = {} if !$jobs;
6919
6920     my $qemu_target;
6921     my $format;
6922     $jobs->{"drive-$drive"} = {};
6923
6924     if ($dst_volid =~ /^nbd:/) {
6925         $qemu_target = $dst_volid;
6926         $format = "nbd";
6927     } else {
6928         my $storecfg = PVE::Storage::config();
6929         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6930
6931         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6932
6933         $format = qemu_img_format($dst_scfg, $dst_volname);
6934
6935         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6936
6937         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6938     }
6939
6940     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6941     $opts->{format} = $format if $format;
6942
6943     if (defined($bwlimit)) {
6944         $opts->{speed} = $bwlimit * 1024;
6945         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6946     } else {
6947         print "drive mirror is starting for drive-$drive\n";
6948     }
6949
6950     # if a job already runs for this device we get an error, catch it for cleanup
6951     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6952     if (my $err = $@) {
6953         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6954         warn "$@\n" if $@;
6955         die "mirroring error: $err\n";
6956     }
6957
6958     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6959 }
6960
6961 sub qemu_drive_mirror_monitor {
6962     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6963
6964     eval {
6965         my $err_complete = 0;
6966
6967         while (1) {
6968             die "storage migration timed out\n" if $err_complete > 300;
6969
6970             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6971
6972             my $running_mirror_jobs = {};
6973             foreach my $stat (@$stats) {
6974                 next if $stat->{type} ne 'mirror';
6975                 $running_mirror_jobs->{$stat->{device}} = $stat;
6976             }
6977
6978             my $readycounter = 0;
6979
6980             foreach my $job (keys %$jobs) {
6981
6982                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6983                     print "$job : finished\n";
6984                     delete $jobs->{$job};
6985                     next;
6986                 }
6987
6988                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6989
6990                 my $busy = $running_mirror_jobs->{$job}->{busy};
6991                 my $ready = $running_mirror_jobs->{$job}->{ready};
6992                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6993                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6994                     my $remaining = $total - $transferred;
6995                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6996
6997                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6998                 }
6999
7000                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7001             }
7002
7003             last if scalar(keys %$jobs) == 0;
7004
7005             if ($readycounter == scalar(keys %$jobs)) {
7006                 print "all mirroring jobs are ready \n";
7007                 last if $skipcomplete; #do the complete later
7008
7009                 if ($vmiddst && $vmiddst != $vmid) {
7010                     my $agent_running = $qga && qga_check_running($vmid);
7011                     if ($agent_running) {
7012                         print "freeze filesystem\n";
7013                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7014                     } else {
7015                         print "suspend vm\n";
7016                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7017                     }
7018
7019                     # if we clone a disk for a new target vm, we don't switch the disk
7020                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7021
7022                     if ($agent_running) {
7023                         print "unfreeze filesystem\n";
7024                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7025                     } else {
7026                         print "resume vm\n";
7027                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7028                     }
7029
7030                     last;
7031                 } else {
7032
7033                     foreach my $job (keys %$jobs) {
7034                         # try to switch the disk if source and destination are on the same guest
7035                         print "$job: Completing block job...\n";
7036
7037                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7038                         if ($@ =~ m/cannot be completed/) {
7039                             print "$job: Block job cannot be completed, try again.\n";
7040                             $err_complete++;
7041                         }else {
7042                             print "$job: Completed successfully.\n";
7043                             $jobs->{$job}->{complete} = 1;
7044                         }
7045                     }
7046                 }
7047             }
7048             sleep 1;
7049         }
7050     };
7051     my $err = $@;
7052
7053     if ($err) {
7054         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7055         die "mirroring error: $err";
7056     }
7057
7058 }
7059
7060 sub qemu_blockjobs_cancel {
7061     my ($vmid, $jobs) = @_;
7062
7063     foreach my $job (keys %$jobs) {
7064         print "$job: Cancelling block job\n";
7065         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7066         $jobs->{$job}->{cancel} = 1;
7067     }
7068
7069     while (1) {
7070         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7071
7072         my $running_jobs = {};
7073         foreach my $stat (@$stats) {
7074             $running_jobs->{$stat->{device}} = $stat;
7075         }
7076
7077         foreach my $job (keys %$jobs) {
7078
7079             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7080                 print "$job: Done.\n";
7081                 delete $jobs->{$job};
7082             }
7083         }
7084
7085         last if scalar(keys %$jobs) == 0;
7086
7087         sleep 1;
7088     }
7089 }
7090
7091 sub clone_disk {
7092     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7093         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7094
7095     my $newvolid;
7096
7097     if (!$full) {
7098         print "create linked clone of drive $drivename ($drive->{file})\n";
7099         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7100         push @$newvollist, $newvolid;
7101     } else {
7102
7103         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7104         $storeid = $storage if $storage;
7105
7106         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7107         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7108
7109         print "create full clone of drive $drivename ($drive->{file})\n";
7110         my $name = undef;
7111         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7112         push @$newvollist, $newvolid;
7113
7114         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7115
7116         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7117         if (!$running || $snapname) {
7118             # TODO: handle bwlimits
7119             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7120         } else {
7121
7122             my $kvmver = get_running_qemu_version ($vmid);
7123             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7124                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7125                     if $drive->{iothread};
7126             }
7127
7128             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7129         }
7130     }
7131
7132     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7133
7134     my $disk = $drive;
7135     $disk->{format} = undef;
7136     $disk->{file} = $newvolid;
7137     $disk->{size} = $size;
7138
7139     return $disk;
7140 }
7141
7142 # this only works if VM is running
7143 sub get_current_qemu_machine {
7144     my ($vmid) = @_;
7145
7146     my $cmd = { execute => 'query-machines', arguments => {} };
7147     my $res = vm_qmp_command($vmid, $cmd);
7148
7149     my ($current, $default);
7150     foreach my $e (@$res) {
7151         $default = $e->{name} if $e->{'is-default'};
7152         $current = $e->{name} if $e->{'is-current'};
7153     }
7154
7155     # fallback to the default machine if current is not supported by qemu
7156     return $current || $default || 'pc';
7157 }
7158
7159 sub get_running_qemu_version {
7160     my ($vmid) = @_;
7161     my $cmd = { execute => 'query-version', arguments => {} };
7162     my $res = vm_qmp_command($vmid, $cmd);
7163     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7164 }
7165
7166 sub qemu_machine_feature_enabled {
7167     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7168
7169     my $current_major;
7170     my $current_minor;
7171
7172     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7173
7174         $current_major = $3;
7175         $current_minor = $4;
7176
7177     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7178
7179         $current_major = $1;
7180         $current_minor = $2;
7181     }
7182
7183     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7184 }
7185
7186 # gets in pairs the versions you want to compares, i.e.:
7187 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7188 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7189 sub version_cmp {
7190     my @versions = @_;
7191
7192     my $size = scalar(@versions);
7193
7194     return 0 if $size == 0;
7195     die "cannot compare odd count of versions" if $size & 1;
7196
7197     for (my $i = 0; $i < $size; $i += 2) {
7198         my ($a, $b) = splice(@versions, 0, 2);
7199         $a //= 0;
7200         $b //= 0;
7201
7202         return 1 if $a > $b;
7203         return -1 if $a < $b;
7204     }
7205     return 0;
7206 }
7207
7208 # dies if a) VM not running or not exisiting b) Version query failed
7209 # So, any defined return value is valid, any invalid state can be caught by eval
7210 sub runs_at_least_qemu_version {
7211     my ($vmid, $major, $minor, $extra) = @_;
7212
7213     my $v = vm_qmp_command($vmid, { execute => 'query-version' });
7214     die "could not query currently running version for VM $vmid\n" if !defined($v);
7215     $v = $v->{qemu};
7216
7217     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7218 }
7219
7220 sub qemu_machine_pxe {
7221     my ($vmid, $conf) = @_;
7222
7223     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7224
7225     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7226         $machine .= '.pxe';
7227     }
7228
7229     return $machine;
7230 }
7231
7232 sub qemu_use_old_bios_files {
7233     my ($machine_type) = @_;
7234
7235     return if !$machine_type;
7236
7237     my $use_old_bios_files = undef;
7238
7239     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7240         $machine_type = $1;
7241         $use_old_bios_files = 1;
7242     } else {
7243         my $kvmver = kvm_user_version();
7244         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7245         # load new efi bios files on migration. So this hack is required to allow
7246         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7247         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7248         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7249     }
7250
7251     return ($use_old_bios_files, $machine_type);
7252 }
7253
7254 sub create_efidisk($$$$$) {
7255     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7256
7257     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7258     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7259
7260     my $vars_size_b = -s $ovmf_vars;
7261     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7262     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7263     PVE::Storage::activate_volumes($storecfg, [$volid]);
7264
7265     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7266
7267     return ($volid, $vars_size);
7268 }
7269
7270 sub vm_iothreads_list {
7271     my ($vmid) = @_;
7272
7273     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7274
7275     my $iothreads = {};
7276     foreach my $iothread (@$res) {
7277         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7278     }
7279
7280     return $iothreads;
7281 }
7282
7283 sub scsihw_infos {
7284     my ($conf, $drive) = @_;
7285
7286     my $maxdev = 0;
7287
7288     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7289         $maxdev = 7;
7290     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7291         $maxdev = 1;
7292     } else {
7293         $maxdev = 256;
7294     }
7295
7296     my $controller = int($drive->{index} / $maxdev);
7297     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7298
7299     return ($maxdev, $controller, $controller_prefix);
7300 }
7301
7302 sub add_hyperv_enlightenments {
7303     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7304
7305     return if $winversion < 6;
7306     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7307
7308     if ($gpu_passthrough || defined($hv_vendor_id)) {
7309         $hv_vendor_id //= 'proxmox';
7310         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7311     }
7312
7313     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7314         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7315         push @$cpuFlags , 'hv_vapic';
7316         push @$cpuFlags , 'hv_time';
7317     } else {
7318         push @$cpuFlags , 'hv_spinlocks=0xffff';
7319     }
7320
7321     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7322         push @$cpuFlags , 'hv_reset';
7323         push @$cpuFlags , 'hv_vpindex';
7324         push @$cpuFlags , 'hv_runtime';
7325     }
7326
7327     if ($winversion >= 7) {
7328         push @$cpuFlags , 'hv_relaxed';
7329
7330         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7331             push @$cpuFlags , 'hv_synic';
7332             push @$cpuFlags , 'hv_stimer';
7333         }
7334
7335         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7336             push @$cpuFlags , 'hv_ipi';
7337         }
7338     }
7339 }
7340
7341 sub windows_version {
7342     my ($ostype) = @_;
7343
7344     return 0 if !$ostype;
7345
7346     my $winversion = 0;
7347
7348     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7349         $winversion = 5;
7350     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7351         $winversion = 6;
7352     } elsif ($ostype =~ m/^win(\d+)$/) {
7353         $winversion = $1;
7354     }
7355
7356     return $winversion;
7357 }
7358
7359 sub resolve_dst_disk_format {
7360         my ($storecfg, $storeid, $src_volname, $format) = @_;
7361         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7362
7363         if (!$format) {
7364             # if no target format is specified, use the source disk format as hint
7365             if ($src_volname) {
7366                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7367                 $format = qemu_img_format($scfg, $src_volname);
7368             } else {
7369                 return $defFormat;
7370             }
7371         }
7372
7373         # test if requested format is supported - else use default
7374         my $supported = grep { $_ eq $format } @$validFormats;
7375         $format = $defFormat if !$supported;
7376         return $format;
7377 }
7378
7379 sub resolve_first_disk {
7380     my $conf = shift;
7381     my @disks = PVE::QemuServer::valid_drive_names();
7382     my $firstdisk;
7383     foreach my $ds (reverse @disks) {
7384         next if !$conf->{$ds};
7385         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7386         next if PVE::QemuServer::drive_is_cdrom($disk);
7387         $firstdisk = $ds;
7388     }
7389     return $firstdisk;
7390 }
7391
7392 sub generate_uuid {
7393     my ($uuid, $uuid_str);
7394     UUID::generate($uuid);
7395     UUID::unparse($uuid, $uuid_str);
7396     return $uuid_str;
7397 }
7398
7399 sub generate_smbios1_uuid {
7400     return "uuid=".generate_uuid();
7401 }
7402
7403 sub nbd_stop {
7404     my ($vmid) = @_;
7405
7406     vm_mon_cmd($vmid, 'nbd-server-stop');
7407 }
7408
7409 sub create_reboot_request {
7410     my ($vmid) = @_;
7411     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7412         or die "failed to create reboot trigger file: $!\n";
7413     close($fh);
7414 }
7415
7416 sub clear_reboot_request {
7417     my ($vmid) = @_;
7418     my $path = "/run/qemu-server/$vmid.reboot";
7419     my $res = 0;
7420
7421     $res = unlink($path);
7422     die "could not remove reboot request for $vmid: $!"
7423         if !$res && $! != POSIX::ENOENT;
7424
7425     return $res;
7426 }
7427
7428 # bash completion helper
7429
7430 sub complete_backup_archives {
7431     my ($cmdname, $pname, $cvalue) = @_;
7432
7433     my $cfg = PVE::Storage::config();
7434
7435     my $storeid;
7436
7437     if ($cvalue =~ m/^([^:]+):/) {
7438         $storeid = $1;
7439     }
7440
7441     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7442
7443     my $res = [];
7444     foreach my $id (keys %$data) {
7445         foreach my $item (@{$data->{$id}}) {
7446             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7447             push @$res, $item->{volid} if defined($item->{volid});
7448         }
7449     }
7450
7451     return $res;
7452 }
7453
7454 my $complete_vmid_full = sub {
7455     my ($running) = @_;
7456
7457     my $idlist = vmstatus();
7458
7459     my $res = [];
7460
7461     foreach my $id (keys %$idlist) {
7462         my $d = $idlist->{$id};
7463         if (defined($running)) {
7464             next if $d->{template};
7465             next if $running && $d->{status} ne 'running';
7466             next if !$running && $d->{status} eq 'running';
7467         }
7468         push @$res, $id;
7469
7470     }
7471     return $res;
7472 };
7473
7474 sub complete_vmid {
7475     return &$complete_vmid_full();
7476 }
7477
7478 sub complete_vmid_stopped {
7479     return &$complete_vmid_full(0);
7480 }
7481
7482 sub complete_vmid_running {
7483     return &$complete_vmid_full(1);
7484 }
7485
7486 sub complete_storage {
7487
7488     my $cfg = PVE::Storage::config();
7489     my $ids = $cfg->{ids};
7490
7491     my $res = [];
7492     foreach my $sid (keys %$ids) {
7493         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7494         next if !$ids->{$sid}->{content}->{images};
7495         push @$res, $sid;
7496     }
7497
7498     return $res;
7499 }
7500
7501 sub complete_migration_storage {
7502     my ($cmd, $param, $current_value, $all_args) = @_;
7503
7504     my $targetnode = @$all_args[1];
7505
7506     my $cfg = PVE::Storage::config();
7507     my $ids = $cfg->{ids};
7508
7509     my $res = [];
7510     foreach my $sid (keys %$ids) {
7511         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7512         next if !$ids->{$sid}->{content}->{images};
7513         push @$res, $sid;
7514     }
7515
7516     return $res;
7517 }
7518
7519 1;