PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     },
 641     audio0 => {
 642         type => 'string',
 643         enum => [qw(ich9-intel-hda intel-hda AC97)],
 644         description => "Configure a audio device.",
 645         optional => 1
 646     },
 647 };
 648
 649 my $cicustom_fmt = {
 650     meta => {
 651         type => 'string',
 652         optional => 1,
 653         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 654         format => 'pve-volume-id',
 655         format_description => 'volume',
 656     },
 657     network => {
 658         type => 'string',
 659         optional => 1,
 660         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 661         format => 'pve-volume-id',
 662         format_description => 'volume',
 663     },
 664     user => {
 665         type => 'string',
 666         optional => 1,
 667         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 668         format => 'pve-volume-id',
 669         format_description => 'volume',
 670     },
 671 };
 672 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 673
 674 my $confdesc_cloudinit = {
 675     citype => {
 676         optional => 1,
 677         type => 'string',
 678         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 679         enum => ['configdrive2', 'nocloud'],
 680     },
 681     ciuser => {
 682         optional => 1,
 683         type => 'string',
 684         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 685     },
 686     cipassword => {
 687         optional => 1,
 688         type => 'string',
 689         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 690     },
 691     cicustom => {
 692         optional => 1,
 693         type => 'string',
 694         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 695         format => 'pve-qm-cicustom',
 696     },
 697     searchdomain => {
 698         optional => 1,
 699         type => 'string',
 700         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 701     },
 702     nameserver => {
 703         optional => 1,
 704         type => 'string', format => 'address-list',
 705         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 706     },
 707     sshkeys => {
 708         optional => 1,
 709         type => 'string',
 710         format => 'urlencoded',
 711         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 712     },
 713 };
 714
 715 # what about other qemu settings ?
 716 #cpu => 'string',
 717 #machine => 'string',
 718 #fda => 'file',
 719 #fdb => 'file',
 720 #mtdblock => 'file',
 721 #sd => 'file',
 722 #pflash => 'file',
 723 #snapshot => 'bool',
 724 #bootp => 'file',
 725 ##tftp => 'dir',
 726 ##smb => 'dir',
 727 #kernel => 'file',
 728 #append => 'string',
 729 #initrd => 'file',
 730 ##soundhw => 'string',
 731
 732 while (my ($k, $v) = each %$confdesc) {
 733     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 734 }
 735
 736 my $MAX_IDE_DISKS = 4;
 737 my $MAX_SCSI_DISKS = 14;
 738 my $MAX_VIRTIO_DISKS = 16;
 739 my $MAX_SATA_DISKS = 6;
 740 my $MAX_USB_DEVICES = 5;
 741 my $MAX_NETS = 32;
 742 my $MAX_UNUSED_DISKS = 256;
 743 my $MAX_HOSTPCI_DEVICES = 4;
 744 my $MAX_SERIAL_PORTS = 4;
 745 my $MAX_PARALLEL_PORTS = 3;
 746 my $MAX_NUMA = 8;
 747
 748 my $numa_fmt = {
 749     cpus => {
 750         type => "string",
 751         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 752         description => "CPUs accessing this NUMA node.",
 753         format_description => "id[-id];...",
 754     },
 755     memory => {
 756         type => "number",
 757         description => "Amount of memory this NUMA node provides.",
 758         optional => 1,
 759     },
 760     hostnodes => {
 761         type => "string",
 762         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 763         description => "Host NUMA nodes to use.",
 764         format_description => "id[-id];...",
 765         optional => 1,
 766     },
 767     policy => {
 768         type => 'string',
 769         enum => [qw(preferred bind interleave)],
 770         description => "NUMA allocation policy.",
 771         optional => 1,
 772     },
 773 };
 774 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 775 my $numadesc = {
 776     optional => 1,
 777     type => 'string', format => $numa_fmt,
 778     description => "NUMA topology.",
 779 };
 780 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 781
 782 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 783     $confdesc->{"numa$i"} = $numadesc;
 784 }
 785
 786 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 787                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 788                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 789 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 790
 791 my $net_fmt_bridge_descr = <<__EOD__;
 792 Bridge to attach the network device to. The Proxmox VE standard bridge
 793 is called 'vmbr0'.
 794
 795 If you do not specify a bridge, we create a kvm user (NATed) network
 796 device, which provides DHCP and DNS services. The following addresses
 797 are used:
 798
 799  10.0.2.2   Gateway
 800  10.0.2.3   DNS Server
 801  10.0.2.4   SMB Server
 802
 803 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 804 __EOD__
 805
 806 my $net_fmt = {
 807     macaddr  => get_standard_option('mac-addr', {
 808         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 809     }),
 810     model => {
 811         type => 'string',
 812         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 813         enum => $nic_model_list,
 814         default_key => 1,
 815     },
 816     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 817     bridge => {
 818         type => 'string',
 819         description => $net_fmt_bridge_descr,
 820         format_description => 'bridge',
 821         optional => 1,
 822     },
 823     queues => {
 824         type => 'integer',
 825         minimum => 0, maximum => 16,
 826         description => 'Number of packet queues to be used on the device.',
 827         optional => 1,
 828     },
 829     rate => {
 830         type => 'number',
 831         minimum => 0,
 832         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 833         optional => 1,
 834     },
 835     tag => {
 836         type => 'integer',
 837         minimum => 1, maximum => 4094,
 838         description => 'VLAN tag to apply to packets on this interface.',
 839         optional => 1,
 840     },
 841     trunks => {
 842         type => 'string',
 843         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 844         description => 'VLAN trunks to pass through this interface.',
 845         format_description => 'vlanid[;vlanid...]',
 846         optional => 1,
 847     },
 848     firewall => {
 849         type => 'boolean',
 850         description => 'Whether this interface should be protected by the firewall.',
 851         optional => 1,
 852     },
 853     link_down => {
 854         type => 'boolean',
 855         description => 'Whether this interface should be disconnected (like pulling the plug).',
 856         optional => 1,
 857     },
 858 };
 859
 860 my $netdesc = {
 861     optional => 1,
 862     type => 'string', format => $net_fmt,
 863     description => "Specify network devices.",
 864 };
 865
 866 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 867
 868 my $ipconfig_fmt = {
 869     ip => {
 870         type => 'string',
 871         format => 'pve-ipv4-config',
 872         format_description => 'IPv4Format/CIDR',
 873         description => 'IPv4 address in CIDR format.',
 874         optional => 1,
 875         default => 'dhcp',
 876     },
 877     gw => {
 878         type => 'string',
 879         format => 'ipv4',
 880         format_description => 'GatewayIPv4',
 881         description => 'Default gateway for IPv4 traffic.',
 882         optional => 1,
 883         requires => 'ip',
 884     },
 885     ip6 => {
 886         type => 'string',
 887         format => 'pve-ipv6-config',
 888         format_description => 'IPv6Format/CIDR',
 889         description => 'IPv6 address in CIDR format.',
 890         optional => 1,
 891         default => 'dhcp',
 892     },
 893     gw6 => {
 894         type => 'string',
 895         format => 'ipv6',
 896         format_description => 'GatewayIPv6',
 897         description => 'Default gateway for IPv6 traffic.',
 898         optional => 1,
 899         requires => 'ip6',
 900     },
 901 };
 902 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 903 my $ipconfigdesc = {
 904     optional => 1,
 905     type => 'string', format => 'pve-qm-ipconfig',
 906     description => <<'EODESCR',
 907 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 908
 909 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 910
 911 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 912 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 913
 914 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 915 EODESCR
 916 };
 917 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 918
 919 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 920     $confdesc->{"net$i"} = $netdesc;
 921     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 922 }
 923
 924 foreach my $key (keys %$confdesc_cloudinit) {
 925     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 926 }
 927
 928 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 929 sub verify_volume_id_or_qm_path {
 930     my ($volid, $noerr) = @_;
 931
 932     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 933         return $volid;
 934     }
 935
 936     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 937     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 938     if ($@) {
 939         return undef if $noerr;
 940         die $@;
 941     }
 942     return $volid;
 943 }
 944
 945 my $drivename_hash;
 946
 947 my %drivedesc_base = (
 948     volume => { alias => 'file' },
 949     file => {
 950         type => 'string',
 951         format => 'pve-volume-id-or-qm-path',
 952         default_key => 1,
 953         format_description => 'volume',
 954         description => "The drive's backing volume.",
 955     },
 956     media => {
 957         type => 'string',
 958         enum => [qw(cdrom disk)],
 959         description => "The drive's media type.",
 960         default => 'disk',
 961         optional => 1
 962     },
 963     cyls => {
 964         type => 'integer',
 965         description => "Force the drive's physical geometry to have a specific cylinder count.",
 966         optional => 1
 967     },
 968     heads => {
 969         type => 'integer',
 970         description => "Force the drive's physical geometry to have a specific head count.",
 971         optional => 1
 972     },
 973     secs => {
 974         type => 'integer',
 975         description => "Force the drive's physical geometry to have a specific sector count.",
 976         optional => 1
 977     },
 978     trans => {
 979         type => 'string',
 980         enum => [qw(none lba auto)],
 981         description => "Force disk geometry bios translation mode.",
 982         optional => 1,
 983     },
 984     snapshot => {
 985         type => 'boolean',
 986         description => "Controls qemu's snapshot mode feature."
 987             . " If activated, changes made to the disk are temporary and will"
 988             . " be discarded when the VM is shutdown.",
 989         optional => 1,
 990     },
 991     cache => {
 992         type => 'string',
 993         enum => [qw(none writethrough writeback unsafe directsync)],
 994         description => "The drive's cache mode",
 995         optional => 1,
 996     },
 997     format => get_standard_option('pve-qm-image-format'),
 998     size => {
 999         type => 'string',
1000         format => 'disk-size',
1001         format_description => 'DiskSize',
1002         description => "Disk size. This is purely informational and has no effect.",
1003         optional => 1,
1004     },
1005     backup => {
1006         type => 'boolean',
1007         description => "Whether the drive should be included when making backups.",
1008         optional => 1,
1009     },
1010     replicate => {
1011         type => 'boolean',
1012         description => 'Whether the drive should considered for replication jobs.',
1013         optional => 1,
1014         default => 1,
1015     },
1016     rerror => {
1017         type => 'string',
1018         enum => [qw(ignore report stop)],
1019         description => 'Read error action.',
1020         optional => 1,
1021     },
1022     werror => {
1023         type => 'string',
1024         enum => [qw(enospc ignore report stop)],
1025         description => 'Write error action.',
1026         optional => 1,
1027     },
1028     aio => {
1029         type => 'string',
1030         enum => [qw(native threads)],
1031         description => 'AIO type to use.',
1032         optional => 1,
1033     },
1034     discard => {
1035         type => 'string',
1036         enum => [qw(ignore on)],
1037         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1038         optional => 1,
1039     },
1040     detect_zeroes => {
1041         type => 'boolean',
1042         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1043         optional => 1,
1044     },
1045     serial => {
1046         type => 'string',
1047         format => 'urlencoded',
1048         format_description => 'serial',
1049         maxLength => 20*3, # *3 since it's %xx url enoded
1050         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1051         optional => 1,
1052     },
1053     shared => {
1054         type => 'boolean',
1055         description => 'Mark this locally-managed volume as available on all nodes',
1056         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1057         optional => 1,
1058         default => 0,
1059     }
1060 );
1061
1062 my %iothread_fmt = ( iothread => {
1063         type => 'boolean',
1064         description => "Whether to use iothreads for this drive",
1065         optional => 1,
1066 });
1067
1068 my %model_fmt = (
1069     model => {
1070         type => 'string',
1071         format => 'urlencoded',
1072         format_description => 'model',
1073         maxLength => 40*3, # *3 since it's %xx url enoded
1074         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1075         optional => 1,
1076     },
1077 );
1078
1079 my %queues_fmt = (
1080     queues => {
1081         type => 'integer',
1082         description => "Number of queues.",
1083         minimum => 2,
1084         optional => 1
1085     }
1086 );
1087
1088 my %scsiblock_fmt = (
1089     scsiblock => {
1090         type => 'boolean',
1091         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1092         optional => 1,
1093         default => 0,
1094     },
1095 );
1096
1097 my %ssd_fmt = (
1098     ssd => {
1099         type => 'boolean',
1100         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1101         optional => 1,
1102     },
1103 );
1104
1105 my %wwn_fmt = (
1106     wwn => {
1107         type => 'string',
1108         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1109         format_description => 'wwn',
1110         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1111         optional => 1,
1112     },
1113 );
1114
1115 my $add_throttle_desc = sub {
1116     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1117     my $d = {
1118         type => $type,
1119         format_description => $unit,
1120         description => "Maximum $what in $longunit.",
1121         optional => 1,
1122     };
1123     $d->{minimum} = $minimum if defined($minimum);
1124     $drivedesc_base{$key} = $d;
1125 };
1126 # throughput: (leaky bucket)
1127 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1128 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1129 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1130 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1131 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1132 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1133 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1134 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1135 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1136
1137 # pools: (pool of IO before throttling starts taking effect)
1138 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1139 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1140 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1141 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1142 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1143 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1144
1145 # burst lengths
1146 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1147 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1148 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1150 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1151 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1152
1153 # legacy support
1154 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1155 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1156 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1157 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1158
1159 my $ide_fmt = {
1160     %drivedesc_base,
1161     %model_fmt,
1162     %ssd_fmt,
1163     %wwn_fmt,
1164 };
1165 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1166
1167 my $idedesc = {
1168     optional => 1,
1169     type => 'string', format => $ide_fmt,
1170     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1171 };
1172 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1173
1174 my $scsi_fmt = {
1175     %drivedesc_base,
1176     %iothread_fmt,
1177     %queues_fmt,
1178     %scsiblock_fmt,
1179     %ssd_fmt,
1180     %wwn_fmt,
1181 };
1182 my $scsidesc = {
1183     optional => 1,
1184     type => 'string', format => $scsi_fmt,
1185     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1186 };
1187 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1188
1189 my $sata_fmt = {
1190     %drivedesc_base,
1191     %ssd_fmt,
1192     %wwn_fmt,
1193 };
1194 my $satadesc = {
1195     optional => 1,
1196     type => 'string', format => $sata_fmt,
1197     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1198 };
1199 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1200
1201 my $virtio_fmt = {
1202     %drivedesc_base,
1203     %iothread_fmt,
1204 };
1205 my $virtiodesc = {
1206     optional => 1,
1207     type => 'string', format => $virtio_fmt,
1208     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1209 };
1210 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1211
1212 my $alldrive_fmt = {
1213     %drivedesc_base,
1214     %iothread_fmt,
1215     %model_fmt,
1216     %queues_fmt,
1217     %scsiblock_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221
1222 my $efidisk_fmt = {
1223     volume => { alias => 'file' },
1224     file => {
1225         type => 'string',
1226         format => 'pve-volume-id-or-qm-path',
1227         default_key => 1,
1228         format_description => 'volume',
1229         description => "The drive's backing volume.",
1230     },
1231     format => get_standard_option('pve-qm-image-format'),
1232     size => {
1233         type => 'string',
1234         format => 'disk-size',
1235         format_description => 'DiskSize',
1236         description => "Disk size. This is purely informational and has no effect.",
1237         optional => 1,
1238     },
1239 };
1240
1241 my $efidisk_desc = {
1242     optional => 1,
1243     type => 'string', format => $efidisk_fmt,
1244     description => "Configure a Disk for storing EFI vars",
1245 };
1246
1247 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1248
1249 my $usb_fmt = {
1250     host => {
1251         default_key => 1,
1252         type => 'string', format => 'pve-qm-usb-device',
1253         format_description => 'HOSTUSBDEVICE|spice',
1254         description => <<EODESCR,
1255 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1256
1257  'bus-port(.port)*' (decimal numbers) or
1258  'vendor_id:product_id' (hexadeciaml numbers) or
1259  'spice'
1260
1261 You can use the 'lsusb -t' command to list existing usb devices.
1262
1263 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1264
1265 The value 'spice' can be used to add a usb redirection devices for spice.
1266 EODESCR
1267     },
1268     usb3 => {
1269         optional => 1,
1270         type => 'boolean',
1271         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1272         default => 0,
1273     },
1274 };
1275
1276 my $usbdesc = {
1277     optional => 1,
1278     type => 'string', format => $usb_fmt,
1279     description => "Configure an USB device (n is 0 to 4).",
1280 };
1281 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1282
1283 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1284 my $hostpci_fmt = {
1285     host => {
1286         default_key => 1,
1287         type => 'string',
1288         pattern => qr/$PCIRE(;$PCIRE)*/,
1289         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1290         description => <<EODESCR,
1291 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1292 of PCI virtual functions of the host. HOSTPCIID syntax is:
1293
1294 'bus:dev.func' (hexadecimal numbers)
1295
1296 You can us the 'lspci' command to list existing PCI devices.
1297 EODESCR
1298     },
1299     rombar => {
1300         type => 'boolean',
1301         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1302         optional => 1,
1303         default => 1,
1304     },
1305     romfile => {
1306         type => 'string',
1307         pattern => '[^,;]+',
1308         format_description => 'string',
1309         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1310         optional => 1,
1311     },
1312     pcie => {
1313         type => 'boolean',
1314         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'x-vga' => {
1319         type => 'boolean',
1320         description =>  "Enable vfio-vga device support.",
1321         optional => 1,
1322         default => 0,
1323     },
1324     'mdev' => {
1325         type => 'string',
1326         format_description => 'string',
1327         pattern => '[^/\.:]+',
1328         optional => 1,
1329         description => <<EODESCR
1330 The type of mediated device to use.
1331 An instance of this type will be created on startup of the VM and
1332 will be cleaned up when the VM stops.
1333 EODESCR
1334     }
1335 };
1336 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1337
1338 my $hostpcidesc = {
1339         optional => 1,
1340         type => 'string', format => 'pve-qm-hostpci',
1341         description => "Map host PCI devices into guest.",
1342         verbose_description =>  <<EODESCR,
1343 Map host PCI devices into guest.
1344
1345 NOTE: This option allows direct access to host hardware. So it is no longer
1346 possible to migrate such machines - use with special care.
1347
1348 CAUTION: Experimental! User reported problems with this option.
1349 EODESCR
1350 };
1351 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1352
1353 my $serialdesc = {
1354         optional => 1,
1355         type => 'string',
1356         pattern => '(/dev/.+|socket)',
1357         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1358         verbose_description =>  <<EODESCR,
1359 Create a serial device inside the VM (n is 0 to 3), and pass through a
1360 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1361 host side (use 'qm terminal' to open a terminal connection).
1362
1363 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1364
1365 CAUTION: Experimental! User reported problems with this option.
1366 EODESCR
1367 };
1368
1369 my $paralleldesc= {
1370         optional => 1,
1371         type => 'string',
1372         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1373         description =>  "Map host parallel devices (n is 0 to 2).",
1374         verbose_description =>  <<EODESCR,
1375 Map host parallel devices (n is 0 to 2).
1376
1377 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1378
1379 CAUTION: Experimental! User reported problems with this option.
1380 EODESCR
1381 };
1382
1383 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1384     $confdesc->{"parallel$i"} = $paralleldesc;
1385 }
1386
1387 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1388     $confdesc->{"serial$i"} = $serialdesc;
1389 }
1390
1391 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1392     $confdesc->{"hostpci$i"} = $hostpcidesc;
1393 }
1394
1395 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1396     $drivename_hash->{"ide$i"} = 1;
1397     $confdesc->{"ide$i"} = $idedesc;
1398 }
1399
1400 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1401     $drivename_hash->{"sata$i"} = 1;
1402     $confdesc->{"sata$i"} = $satadesc;
1403 }
1404
1405 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1406     $drivename_hash->{"scsi$i"} = 1;
1407     $confdesc->{"scsi$i"} = $scsidesc ;
1408 }
1409
1410 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1411     $drivename_hash->{"virtio$i"} = 1;
1412     $confdesc->{"virtio$i"} = $virtiodesc;
1413 }
1414
1415 $drivename_hash->{efidisk0} = 1;
1416 $confdesc->{efidisk0} = $efidisk_desc;
1417
1418 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1419     $confdesc->{"usb$i"} = $usbdesc;
1420 }
1421
1422 my $unuseddesc = {
1423     optional => 1,
1424     type => 'string', format => 'pve-volume-id',
1425     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1426 };
1427
1428 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1429     $confdesc->{"unused$i"} = $unuseddesc;
1430 }
1431
1432 my $kvm_api_version = 0;
1433
1434 sub kvm_version {
1435     return $kvm_api_version if $kvm_api_version;
1436
1437     open my $fh, '<', '/dev/kvm'
1438         or return undef;
1439
1440     # 0xae00 => KVM_GET_API_VERSION
1441     $kvm_api_version = ioctl($fh, 0xae00, 0);
1442
1443     return $kvm_api_version;
1444 }
1445
1446 my $kvm_user_version;
1447
1448 sub kvm_user_version {
1449
1450     return $kvm_user_version if $kvm_user_version;
1451
1452     $kvm_user_version = 'unknown';
1453
1454     my $code = sub {
1455         my $line = shift;
1456         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1457             $kvm_user_version = $2;
1458         }
1459     };
1460
1461     eval { run_command("kvm -version", outfunc => $code); };
1462     warn $@ if $@;
1463
1464     return $kvm_user_version;
1465
1466 }
1467
1468 sub kernel_has_vhost_net {
1469     return -c '/dev/vhost-net';
1470 }
1471
1472 sub valid_drive_names {
1473     # order is important - used to autoselect boot disk
1474     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1475             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1476             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1477             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1478             'efidisk0');
1479 }
1480
1481 sub is_valid_drivename {
1482     my $dev = shift;
1483
1484     return defined($drivename_hash->{$dev});
1485 }
1486
1487 sub option_exists {
1488     my $key = shift;
1489     return defined($confdesc->{$key});
1490 }
1491
1492 sub nic_models {
1493     return $nic_model_list;
1494 }
1495
1496 sub os_list_description {
1497
1498     return {
1499         other => 'Other',
1500         wxp => 'Windows XP',
1501         w2k => 'Windows 2000',
1502         w2k3 =>, 'Windows 2003',
1503         w2k8 => 'Windows 2008',
1504         wvista => 'Windows Vista',
1505         win7 => 'Windows 7',
1506         win8 => 'Windows 8/2012',
1507         win10 => 'Windows 10/2016',
1508         l24 => 'Linux 2.4',
1509         l26 => 'Linux 2.6',
1510     };
1511 }
1512
1513 my $cdrom_path;
1514
1515 sub get_cdrom_path {
1516
1517     return  $cdrom_path if $cdrom_path;
1518
1519     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1520     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1521     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1522 }
1523
1524 sub get_iso_path {
1525     my ($storecfg, $vmid, $cdrom) = @_;
1526
1527     if ($cdrom eq 'cdrom') {
1528         return get_cdrom_path();
1529     } elsif ($cdrom eq 'none') {
1530         return '';
1531     } elsif ($cdrom =~ m|^/|) {
1532         return $cdrom;
1533     } else {
1534         return PVE::Storage::path($storecfg, $cdrom);
1535     }
1536 }
1537
1538 # try to convert old style file names to volume IDs
1539 sub filename_to_volume_id {
1540     my ($vmid, $file, $media) = @_;
1541
1542      if (!($file eq 'none' || $file eq 'cdrom' ||
1543           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1544
1545         return undef if $file =~ m|/|;
1546
1547         if ($media && $media eq 'cdrom') {
1548             $file = "local:iso/$file";
1549         } else {
1550             $file = "local:$vmid/$file";
1551         }
1552     }
1553
1554     return $file;
1555 }
1556
1557 sub verify_media_type {
1558     my ($opt, $vtype, $media) = @_;
1559
1560     return if !$media;
1561
1562     my $etype;
1563     if ($media eq 'disk') {
1564         $etype = 'images';
1565     } elsif ($media eq 'cdrom') {
1566         $etype = 'iso';
1567     } else {
1568         die "internal error";
1569     }
1570
1571     return if ($vtype eq $etype);
1572
1573     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1574 }
1575
1576 sub cleanup_drive_path {
1577     my ($opt, $storecfg, $drive) = @_;
1578
1579     # try to convert filesystem paths to volume IDs
1580
1581     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1582         ($drive->{file} !~ m|^/dev/.+|) &&
1583         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1584         ($drive->{file} !~ m/^\d+$/)) {
1585         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1586         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1587         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1588         verify_media_type($opt, $vtype, $drive->{media});
1589         $drive->{file} = $volid;
1590     }
1591
1592     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1593 }
1594
1595 sub parse_hotplug_features {
1596     my ($data) = @_;
1597
1598     my $res = {};
1599
1600     return $res if $data eq '0';
1601
1602     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1603
1604     foreach my $feature (PVE::Tools::split_list($data)) {
1605         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1606             $res->{$1} = 1;
1607         } else {
1608             die "invalid hotplug feature '$feature'\n";
1609         }
1610     }
1611     return $res;
1612 }
1613
1614 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1615 sub pve_verify_hotplug_features {
1616     my ($value, $noerr) = @_;
1617
1618     return $value if parse_hotplug_features($value);
1619
1620     return undef if $noerr;
1621
1622     die "unable to parse hotplug option\n";
1623 }
1624
1625 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1626 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1627 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1628 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1629 #        [,iothread=on][,serial=serial][,model=model]
1630
1631 sub parse_drive {
1632     my ($key, $data) = @_;
1633
1634     my ($interface, $index);
1635
1636     if ($key =~ m/^([^\d]+)(\d+)$/) {
1637         $interface = $1;
1638         $index = $2;
1639     } else {
1640         return undef;
1641     }
1642
1643     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1644                                      : $confdesc->{$key}->{format};
1645     if (!$desc) {
1646         warn "invalid drive key: $key\n";
1647         return undef;
1648     }
1649     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1650     return undef if !$res;
1651     $res->{interface} = $interface;
1652     $res->{index} = $index;
1653
1654     my $error = 0;
1655     foreach my $opt (qw(bps bps_rd bps_wr)) {
1656         if (my $bps = defined(delete $res->{$opt})) {
1657             if (defined($res->{"m$opt"})) {
1658                 warn "both $opt and m$opt specified\n";
1659                 ++$error;
1660                 next;
1661             }
1662             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1663         }
1664     }
1665
1666     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1667     for my $requirement (
1668         [mbps_max => 'mbps'],
1669         [mbps_rd_max => 'mbps_rd'],
1670         [mbps_wr_max => 'mbps_wr'],
1671         [miops_max => 'miops'],
1672         [miops_rd_max => 'miops_rd'],
1673         [miops_wr_max => 'miops_wr'],
1674         [bps_max_length => 'mbps_max'],
1675         [bps_rd_max_length => 'mbps_rd_max'],
1676         [bps_wr_max_length => 'mbps_wr_max'],
1677         [iops_max_length => 'iops_max'],
1678         [iops_rd_max_length => 'iops_rd_max'],
1679         [iops_wr_max_length => 'iops_wr_max']) {
1680         my ($option, $requires) = @$requirement;
1681         if ($res->{$option} && !$res->{$requires}) {
1682             warn "$option requires $requires\n";
1683             ++$error;
1684         }
1685     }
1686
1687     return undef if $error;
1688
1689     return undef if $res->{mbps_rd} && $res->{mbps};
1690     return undef if $res->{mbps_wr} && $res->{mbps};
1691     return undef if $res->{iops_rd} && $res->{iops};
1692     return undef if $res->{iops_wr} && $res->{iops};
1693
1694     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1695         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1696         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1697         return undef if $res->{interface} eq 'virtio';
1698     }
1699
1700     if (my $size = $res->{size}) {
1701         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1702     }
1703
1704     return $res;
1705 }
1706
1707 sub print_drive {
1708     my ($vmid, $drive) = @_;
1709     my $data = { %$drive };
1710     delete $data->{$_} for qw(index interface);
1711     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1712 }
1713
1714 sub scsi_inquiry {
1715     my($fh, $noerr) = @_;
1716
1717     my $SG_IO = 0x2285;
1718     my $SG_GET_VERSION_NUM = 0x2282;
1719
1720     my $versionbuf = "\x00" x 8;
1721     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1722     if (!$ret) {
1723         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1724         return undef;
1725     }
1726     my $version = unpack("I", $versionbuf);
1727     if ($version < 30000) {
1728         die "scsi generic interface too old\n"  if !$noerr;
1729         return undef;
1730     }
1731
1732     my $buf = "\x00" x 36;
1733     my $sensebuf = "\x00" x 8;
1734     my $cmd = pack("C x3 C x1", 0x12, 36);
1735
1736     # see /usr/include/scsi/sg.h
1737     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1738
1739     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1740                       length($sensebuf), 0, length($buf), $buf,
1741                       $cmd, $sensebuf, 6000);
1742
1743     $ret = ioctl($fh, $SG_IO, $packet);
1744     if (!$ret) {
1745         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my @res = unpack($sg_io_hdr_t, $packet);
1750     if ($res[17] || $res[18]) {
1751         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1752         return undef;
1753     }
1754
1755     my $res = {};
1756     (my $byte0, my $byte1, $res->{vendor},
1757      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1758
1759     $res->{removable} = $byte1 & 128 ? 1 : 0;
1760     $res->{type} = $byte0 & 31;
1761
1762     return $res;
1763 }
1764
1765 sub path_is_scsi {
1766     my ($path) = @_;
1767
1768     my $fh = IO::File->new("+<$path") || return undef;
1769     my $res = scsi_inquiry($fh, 1);
1770     close($fh);
1771
1772     return $res;
1773 }
1774
1775 sub machine_type_is_q35 {
1776     my ($conf) = @_;
1777
1778     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1779 }
1780
1781 sub print_tabletdevice_full {
1782     my ($conf, $arch) = @_;
1783
1784     my $q35 = machine_type_is_q35($conf);
1785
1786     # we use uhci for old VMs because tablet driver was buggy in older qemu
1787     my $usbbus;
1788     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1789         $usbbus = 'ehci';
1790     } else {
1791         $usbbus = 'uhci';
1792     }
1793
1794     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1795 }
1796
1797 sub print_keyboarddevice_full {
1798     my ($conf, $arch, $machine) = @_;
1799
1800     return undef if $arch ne 'aarch64';
1801
1802     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1803 }
1804
1805 sub print_drivedevice_full {
1806     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1807
1808     my $device = '';
1809     my $maxdev = 0;
1810
1811     if ($drive->{interface} eq 'virtio') {
1812         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1813         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1814         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1815     } elsif ($drive->{interface} eq 'scsi') {
1816
1817         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1818         my $unit = $drive->{index} % $maxdev;
1819         my $devicetype = 'hd';
1820         my $path = '';
1821         if (drive_is_cdrom($drive)) {
1822             $devicetype = 'cd';
1823         } else {
1824             if ($drive->{file} =~ m|^/|) {
1825                 $path = $drive->{file};
1826                 if (my $info = path_is_scsi($path)) {
1827                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1828                         $devicetype = 'block';
1829                     } elsif ($info->{type} == 1) { # tape
1830                         $devicetype = 'generic';
1831                     }
1832                 }
1833             } else {
1834                  $path = PVE::Storage::path($storecfg, $drive->{file});
1835             }
1836
1837             if($path =~ m/^iscsi\:\/\//){
1838                 $devicetype = 'generic';
1839             }
1840         }
1841
1842         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         } else {
1845             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1846         }
1847
1848         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1849             $device .= ",rotation_rate=1";
1850         }
1851         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1852
1853     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1854         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1855         my $controller = int($drive->{index} / $maxdev);
1856         my $unit = $drive->{index} % $maxdev;
1857         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1858
1859         $device = "ide-$devicetype";
1860         if ($drive->{interface} eq 'ide') {
1861             $device .= ",bus=ide.$controller,unit=$unit";
1862         } else {
1863             $device .= ",bus=ahci$controller.$unit";
1864         }
1865         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1866
1867         if ($devicetype eq 'hd') {
1868             if (my $model = $drive->{model}) {
1869                 $model = URI::Escape::uri_unescape($model);
1870                 $device .= ",model=$model";
1871             }
1872             if ($drive->{ssd}) {
1873                 $device .= ",rotation_rate=1";
1874             }
1875         }
1876         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1877     } elsif ($drive->{interface} eq 'usb') {
1878         die "implement me";
1879         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1880     } else {
1881         die "unsupported interface type";
1882     }
1883
1884     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1885
1886     if (my $serial = $drive->{serial}) {
1887         $serial = URI::Escape::uri_unescape($serial);
1888         $device .= ",serial=$serial";
1889     }
1890
1891
1892     return $device;
1893 }
1894
1895 sub get_initiator_name {
1896     my $initiator;
1897
1898     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1899     while (defined(my $line = <$fh>)) {
1900         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1901         $initiator = $1;
1902         last;
1903     }
1904     $fh->close();
1905
1906     return $initiator;
1907 }
1908
1909 sub print_drive_full {
1910     my ($storecfg, $vmid, $drive) = @_;
1911
1912     my $path;
1913     my $volid = $drive->{file};
1914     my $format;
1915
1916     if (drive_is_cdrom($drive)) {
1917         $path = get_iso_path($storecfg, $vmid, $volid);
1918     } else {
1919         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1920         if ($storeid) {
1921             $path = PVE::Storage::path($storecfg, $volid);
1922             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1923             $format = qemu_img_format($scfg, $volname);
1924         } else {
1925             $path = $volid;
1926             $format = "raw";
1927         }
1928    }
1929
1930     my $opts = '';
1931     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1932     foreach my $o (@qemu_drive_options) {
1933         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1934     }
1935
1936     # snapshot only accepts on|off
1937     if (defined($drive->{snapshot})) {
1938         my $v = $drive->{snapshot} ? 'on' : 'off';
1939         $opts .= ",snapshot=$v";
1940     }
1941
1942     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1943         my ($dir, $qmpname) = @$type;
1944         if (my $v = $drive->{"mbps$dir"}) {
1945             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1946         }
1947         if (my $v = $drive->{"mbps${dir}_max"}) {
1948             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1949         }
1950         if (my $v = $drive->{"bps${dir}_max_length"}) {
1951             $opts .= ",throttling.bps$qmpname-max-length=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}"}) {
1954             $opts .= ",throttling.iops$qmpname=$v";
1955         }
1956         if (my $v = $drive->{"iops${dir}_max"}) {
1957             $opts .= ",throttling.iops$qmpname-max=$v";
1958         }
1959         if (my $v = $drive->{"iops${dir}_max_length"}) {
1960             $opts .= ",throttling.iops$qmpname-max-length=$v";
1961         }
1962     }
1963
1964     $opts .= ",format=$format" if $format && !$drive->{format};
1965
1966     my $cache_direct = 0;
1967
1968     if (my $cache = $drive->{cache}) {
1969         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1970     } elsif (!drive_is_cdrom($drive)) {
1971         $opts .= ",cache=none";
1972         $cache_direct = 1;
1973     }
1974
1975     # aio native works only with O_DIRECT
1976     if (!$drive->{aio}) {
1977         if($cache_direct) {
1978             $opts .= ",aio=native";
1979         } else {
1980             $opts .= ",aio=threads";
1981         }
1982     }
1983
1984     if (!drive_is_cdrom($drive)) {
1985         my $detectzeroes;
1986         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1987             $detectzeroes = 'off';
1988         } elsif ($drive->{discard}) {
1989             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1990         } else {
1991             # This used to be our default with discard not being specified:
1992             $detectzeroes = 'on';
1993         }
1994         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1995     }
1996
1997     my $pathinfo = $path ? "file=$path," : '';
1998
1999     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2000 }
2001
2002 sub print_netdevice_full {
2003     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2004
2005     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2006
2007     my $device = $net->{model};
2008     if ($net->{model} eq 'virtio') {
2009          $device = 'virtio-net-pci';
2010      };
2011
2012     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2013     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2014     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2015         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2016         my $vectors = $net->{queues} * 2 + 2;
2017         $tmpstr .= ",vectors=$vectors,mq=on";
2018     }
2019     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2020
2021     if ($use_old_bios_files) {
2022         my $romfile;
2023         if ($device eq 'virtio-net-pci') {
2024             $romfile = 'pxe-virtio.rom';
2025         } elsif ($device eq 'e1000') {
2026             $romfile = 'pxe-e1000.rom';
2027         } elsif ($device eq 'ne2k') {
2028             $romfile = 'pxe-ne2k_pci.rom';
2029         } elsif ($device eq 'pcnet') {
2030             $romfile = 'pxe-pcnet.rom';
2031         } elsif ($device eq 'rtl8139') {
2032             $romfile = 'pxe-rtl8139.rom';
2033         }
2034         $tmpstr .= ",romfile=$romfile" if $romfile;
2035     }
2036
2037     return $tmpstr;
2038 }
2039
2040 sub print_netdev_full {
2041     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2042
2043     my $i = '';
2044     if ($netid =~ m/^net(\d+)$/) {
2045         $i = int($1);
2046     }
2047
2048     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2049
2050     my $ifname = "tap${vmid}i$i";
2051
2052     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2053     die "interface name '$ifname' is too long (max 15 character)\n"
2054         if length($ifname) >= 16;
2055
2056     my $vhostparam = '';
2057     if (is_native($arch)) {
2058         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2059     }
2060
2061     my $vmname = $conf->{name} || "vm$vmid";
2062
2063     my $netdev = "";
2064     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2065
2066     if ($net->{bridge}) {
2067         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2068     } else {
2069         $netdev = "type=user,id=$netid,hostname=$vmname";
2070     }
2071
2072     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2073
2074     return $netdev;
2075 }
2076
2077
2078 sub print_cpu_device {
2079     my ($conf, $id) = @_;
2080
2081     my $kvm = $conf->{kvm} // 1;
2082     my $cpu = $kvm ? "kvm64" : "qemu64";
2083     if (my $cputype = $conf->{cpu}) {
2084         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2085             or die "Cannot parse cpu description: $cputype\n";
2086         $cpu = $cpuconf->{cputype};
2087     }
2088
2089     my $cores = $conf->{cores} || 1;
2090
2091     my $current_core = ($id - 1) % $cores;
2092     my $current_socket = int(($id - 1 - $current_core)/$cores);
2093
2094     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2095 }
2096
2097 my $vga_map = {
2098     'cirrus' => 'cirrus-vga',
2099     'std' => 'VGA',
2100     'vmware' => 'vmware-svga',
2101     'virtio' => 'virtio-vga',
2102 };
2103
2104 sub print_vga_device {
2105     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2106
2107     my $type = $vga_map->{$vga->{type}};
2108     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2109         $type = 'virtio-gpu';
2110     }
2111     my $vgamem_mb = $vga->{memory};
2112     if ($qxlnum) {
2113         $type = $id ? 'qxl' : 'qxl-vga';
2114     }
2115     die "no devicetype for $vga->{type}\n" if !$type;
2116
2117     my $memory = "";
2118     if ($vgamem_mb) {
2119         if ($vga->{type} eq 'virtio') {
2120             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2121             $memory = ",max_hostmem=$bytes";
2122         } elsif ($qxlnum) {
2123             # from https://www.spice-space.org/multiple-monitors.html
2124             $memory = ",vgamem_mb=$vga->{memory}";
2125             my $ram = $vgamem_mb * 4;
2126             my $vram = $vgamem_mb * 2;
2127             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2128         } else {
2129             $memory = ",vgamem_mb=$vga->{memory}";
2130         }
2131     } elsif ($qxlnum && $id) {
2132         $memory = ",ram_size=67108864,vram_size=33554432";
2133     }
2134
2135     my $q35 = machine_type_is_q35($conf);
2136     my $vgaid = "vga" . ($id // '');
2137     my $pciaddr;
2138
2139     if ($q35 && $vgaid eq 'vga') {
2140         # the first display uses pcie.0 bus on q35 machines
2141         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2142     } else {
2143         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2144     }
2145
2146     return "$type,id=${vgaid}${memory}${pciaddr}";
2147 }
2148
2149 sub drive_is_cloudinit {
2150     my ($drive) = @_;
2151     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2152 }
2153
2154 sub drive_is_cdrom {
2155     my ($drive, $exclude_cloudinit) = @_;
2156
2157     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2158
2159     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2160
2161 }
2162
2163 sub parse_number_sets {
2164     my ($set) = @_;
2165     my $res = [];
2166     foreach my $part (split(/;/, $set)) {
2167         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2168             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2169             push @$res, [ $1, $2 ];
2170         } else {
2171             die "invalid range: $part\n";
2172         }
2173     }
2174     return $res;
2175 }
2176
2177 sub parse_numa {
2178     my ($data) = @_;
2179
2180     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2181     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2182     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2183     return $res;
2184 }
2185
2186 sub parse_hostpci {
2187     my ($value) = @_;
2188
2189     return undef if !$value;
2190
2191     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2192
2193     my @idlist = split(/;/, $res->{host});
2194     delete $res->{host};
2195     foreach my $id (@idlist) {
2196         if ($id =~ m/\./) { # full id 00:00.1
2197             push @{$res->{pciid}}, {
2198                 id => $id,
2199             };
2200         } else { # partial id 00:00
2201             $res->{pciid} = PVE::SysFSTools::lspci($id);
2202         }
2203     }
2204     return $res;
2205 }
2206
2207 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2208 sub parse_net {
2209     my ($data) = @_;
2210
2211     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2212     if ($@) {
2213         warn $@;
2214         return undef;
2215     }
2216     if (!defined($res->{macaddr})) {
2217         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2218         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2219     }
2220     return $res;
2221 }
2222
2223 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2224 sub parse_ipconfig {
2225     my ($data) = @_;
2226
2227     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2228     if ($@) {
2229         warn $@;
2230         return undef;
2231     }
2232
2233     if ($res->{gw} && !$res->{ip}) {
2234         warn 'gateway specified without specifying an IP address';
2235         return undef;
2236     }
2237     if ($res->{gw6} && !$res->{ip6}) {
2238         warn 'IPv6 gateway specified without specifying an IPv6 address';
2239         return undef;
2240     }
2241     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2242         warn 'gateway specified together with DHCP';
2243         return undef;
2244     }
2245     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2246         # gw6 + auto/dhcp
2247         warn "IPv6 gateway specified together with $res->{ip6} address";
2248         return undef;
2249     }
2250
2251     if (!$res->{ip} && !$res->{ip6}) {
2252         return { ip => 'dhcp', ip6 => 'dhcp' };
2253     }
2254
2255     return $res;
2256 }
2257
2258 sub print_net {
2259     my $net = shift;
2260
2261     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2262 }
2263
2264 sub add_random_macs {
2265     my ($settings) = @_;
2266
2267     foreach my $opt (keys %$settings) {
2268         next if $opt !~ m/^net(\d+)$/;
2269         my $net = parse_net($settings->{$opt});
2270         next if !$net;
2271         $settings->{$opt} = print_net($net);
2272     }
2273 }
2274
2275 sub vm_is_volid_owner {
2276     my ($storecfg, $vmid, $volid) = @_;
2277
2278     if ($volid !~  m|^/|) {
2279         my ($path, $owner);
2280         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2281         if ($owner && ($owner == $vmid)) {
2282             return 1;
2283         }
2284     }
2285
2286     return undef;
2287 }
2288
2289 sub split_flagged_list {
2290     my $text = shift || '';
2291     $text =~ s/[,;]/ /g;
2292     $text =~ s/^\s+//;
2293     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2294 }
2295
2296 sub join_flagged_list {
2297     my ($how, $lst) = @_;
2298     join $how, map { $lst->{$_} . $_ } keys %$lst;
2299 }
2300
2301 sub vmconfig_delete_pending_option {
2302     my ($conf, $key, $force) = @_;
2303
2304     delete $conf->{pending}->{$key};
2305     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2306     $pending_delete_hash->{$key} = $force ? '!' : '';
2307     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2308 }
2309
2310 sub vmconfig_undelete_pending_option {
2311     my ($conf, $key) = @_;
2312
2313     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2314     delete $pending_delete_hash->{$key};
2315
2316     if (%$pending_delete_hash) {
2317         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2318     } else {
2319         delete $conf->{pending}->{delete};
2320     }
2321 }
2322
2323 sub vmconfig_register_unused_drive {
2324     my ($storecfg, $vmid, $conf, $drive) = @_;
2325
2326     if (drive_is_cloudinit($drive)) {
2327         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2328         warn $@ if $@;
2329     } elsif (!drive_is_cdrom($drive)) {
2330         my $volid = $drive->{file};
2331         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2332             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2333         }
2334     }
2335 }
2336
2337 sub vmconfig_cleanup_pending {
2338     my ($conf) = @_;
2339
2340     # remove pending changes when nothing changed
2341     my $changes;
2342     foreach my $opt (keys %{$conf->{pending}}) {
2343         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2344             $changes = 1;
2345             delete $conf->{pending}->{$opt};
2346         }
2347     }
2348
2349     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2350     my $pending_delete_hash = {};
2351     while (my ($opt, $force) = each %$current_delete_hash) {
2352         if (defined($conf->{$opt})) {
2353             $pending_delete_hash->{$opt} = $force;
2354         } else {
2355             $changes = 1;
2356         }
2357     }
2358
2359     if (%$pending_delete_hash) {
2360         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2361     } else {
2362         delete $conf->{pending}->{delete};
2363     }
2364
2365     return $changes;
2366 }
2367
2368 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2369 my $smbios1_fmt = {
2370     uuid => {
2371         type => 'string',
2372         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2373         format_description => 'UUID',
2374         description => "Set SMBIOS1 UUID.",
2375         optional => 1,
2376     },
2377     version => {
2378         type => 'string',
2379         pattern => '[A-Za-z0-9+\/]+={0,2}',
2380         format_description => 'Base64 encoded string',
2381         description => "Set SMBIOS1 version.",
2382         optional => 1,
2383     },
2384     serial => {
2385         type => 'string',
2386         pattern => '[A-Za-z0-9+\/]+={0,2}',
2387         format_description => 'Base64 encoded string',
2388         description => "Set SMBIOS1 serial number.",
2389         optional => 1,
2390     },
2391     manufacturer => {
2392         type => 'string',
2393         pattern => '[A-Za-z0-9+\/]+={0,2}',
2394         format_description => 'Base64 encoded string',
2395         description => "Set SMBIOS1 manufacturer.",
2396         optional => 1,
2397     },
2398     product => {
2399         type => 'string',
2400         pattern => '[A-Za-z0-9+\/]+={0,2}',
2401         format_description => 'Base64 encoded string',
2402         description => "Set SMBIOS1 product ID.",
2403         optional => 1,
2404     },
2405     sku => {
2406         type => 'string',
2407         pattern => '[A-Za-z0-9+\/]+={0,2}',
2408         format_description => 'Base64 encoded string',
2409         description => "Set SMBIOS1 SKU string.",
2410         optional => 1,
2411     },
2412     family => {
2413         type => 'string',
2414         pattern => '[A-Za-z0-9+\/]+={0,2}',
2415         format_description => 'Base64 encoded string',
2416         description => "Set SMBIOS1 family string.",
2417         optional => 1,
2418     },
2419     base64 => {
2420         type => 'boolean',
2421         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2422         optional => 1,
2423     },
2424 };
2425
2426 sub parse_smbios1 {
2427     my ($data) = @_;
2428
2429     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2430     warn $@ if $@;
2431     return $res;
2432 }
2433
2434 sub print_smbios1 {
2435     my ($smbios1) = @_;
2436     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2437 }
2438
2439 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2440
2441 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2442 sub verify_bootdisk {
2443     my ($value, $noerr) = @_;
2444
2445     return $value if is_valid_drivename($value);
2446
2447     return undef if $noerr;
2448
2449     die "invalid boot disk '$value'\n";
2450 }
2451
2452 sub parse_watchdog {
2453     my ($value) = @_;
2454
2455     return undef if !$value;
2456
2457     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2458     warn $@ if $@;
2459     return $res;
2460 }
2461
2462 sub parse_guest_agent {
2463     my ($value) = @_;
2464
2465     return {} if !defined($value->{agent});
2466
2467     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2468     warn $@ if $@;
2469
2470     # if the agent is disabled ignore the other potentially set properties
2471     return {} if !$res->{enabled};
2472     return $res;
2473 }
2474
2475 sub parse_vga {
2476     my ($value) = @_;
2477
2478     return {} if !$value;
2479     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2480     warn $@ if $@;
2481     return $res;
2482 }
2483
2484 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2485 sub verify_usb_device {
2486     my ($value, $noerr) = @_;
2487
2488     return $value if parse_usb_device($value);
2489
2490     return undef if $noerr;
2491
2492     die "unable to parse usb device\n";
2493 }
2494
2495 # add JSON properties for create and set function
2496 sub json_config_properties {
2497     my $prop = shift;
2498
2499     foreach my $opt (keys %$confdesc) {
2500         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2501         $prop->{$opt} = $confdesc->{$opt};
2502     }
2503
2504     return $prop;
2505 }
2506
2507 # return copy of $confdesc_cloudinit to generate documentation
2508 sub cloudinit_config_properties {
2509
2510     return dclone($confdesc_cloudinit);
2511 }
2512
2513 sub check_type {
2514     my ($key, $value) = @_;
2515
2516     die "unknown setting '$key'\n" if !$confdesc->{$key};
2517
2518     my $type = $confdesc->{$key}->{type};
2519
2520     if (!defined($value)) {
2521         die "got undefined value\n";
2522     }
2523
2524     if ($value =~ m/[\n\r]/) {
2525         die "property contains a line feed\n";
2526     }
2527
2528     if ($type eq 'boolean') {
2529         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2530         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2531         die "type check ('boolean') failed - got '$value'\n";
2532     } elsif ($type eq 'integer') {
2533         return int($1) if $value =~ m/^(\d+)$/;
2534         die "type check ('integer') failed - got '$value'\n";
2535     } elsif ($type eq 'number') {
2536         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2537         die "type check ('number') failed - got '$value'\n";
2538     } elsif ($type eq 'string') {
2539         if (my $fmt = $confdesc->{$key}->{format}) {
2540             PVE::JSONSchema::check_format($fmt, $value);
2541             return $value;
2542         }
2543         $value =~ s/^\"(.*)\"$/$1/;
2544         return $value;
2545     } else {
2546         die "internal error"
2547     }
2548 }
2549
2550 sub touch_config {
2551     my ($vmid) = @_;
2552
2553     my $conf = PVE::QemuConfig->config_file($vmid);
2554     utime undef, undef, $conf;
2555 }
2556
2557 sub destroy_vm {
2558     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2559
2560     my $conffile = PVE::QemuConfig->config_file($vmid);
2561
2562     my $conf = PVE::QemuConfig->load_config($vmid);
2563
2564     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2565
2566     if ($conf->{template}) {
2567         # check if any base image is still used by a linked clone
2568         foreach_drive($conf, sub {
2569                 my ($ds, $drive) = @_;
2570
2571                 return if drive_is_cdrom($drive);
2572
2573                 my $volid = $drive->{file};
2574
2575                 return if !$volid || $volid =~ m|^/|;
2576
2577                 die "base volume '$volid' is still in use by linked cloned\n"
2578                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2579
2580         });
2581     }
2582
2583     # only remove disks owned by this VM
2584     foreach_drive($conf, sub {
2585         my ($ds, $drive) = @_;
2586
2587         return if drive_is_cdrom($drive, 1);
2588
2589         my $volid = $drive->{file};
2590
2591         return if !$volid || $volid =~ m|^/|;
2592
2593         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2594         return if !$path || !$owner || ($owner != $vmid);
2595
2596         eval {
2597             PVE::Storage::vdisk_free($storecfg, $volid);
2598         };
2599         warn "Could not remove disk '$volid', check manually: $@" if $@;
2600
2601     });
2602
2603     if ($keep_empty_config) {
2604         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2605     } else {
2606         unlink $conffile;
2607     }
2608
2609     # also remove unused disk
2610     eval {
2611         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2612
2613         eval {
2614             PVE::Storage::foreach_volid($dl, sub {
2615                 my ($volid, $sid, $volname, $d) = @_;
2616                 PVE::Storage::vdisk_free($storecfg, $volid);
2617             });
2618         };
2619         warn $@ if $@;
2620
2621     };
2622     warn $@ if $@;
2623 }
2624
2625 sub parse_vm_config {
2626     my ($filename, $raw) = @_;
2627
2628     return undef if !defined($raw);
2629
2630     my $res = {
2631         digest => Digest::SHA::sha1_hex($raw),
2632         snapshots => {},
2633         pending => {},
2634     };
2635
2636     $filename =~ m|/qemu-server/(\d+)\.conf$|
2637         || die "got strange filename '$filename'";
2638
2639     my $vmid = $1;
2640
2641     my $conf = $res;
2642     my $descr;
2643     my $section = '';
2644
2645     my @lines = split(/\n/, $raw);
2646     foreach my $line (@lines) {
2647         next if $line =~ m/^\s*$/;
2648
2649         if ($line =~ m/^\[PENDING\]\s*$/i) {
2650             $section = 'pending';
2651             if (defined($descr)) {
2652                 $descr =~ s/\s+$//;
2653                 $conf->{description} = $descr;
2654             }
2655             $descr = undef;
2656             $conf = $res->{$section} = {};
2657             next;
2658
2659         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2660             $section = $1;
2661             if (defined($descr)) {
2662                 $descr =~ s/\s+$//;
2663                 $conf->{description} = $descr;
2664             }
2665             $descr = undef;
2666             $conf = $res->{snapshots}->{$section} = {};
2667             next;
2668         }
2669
2670         if ($line =~ m/^\#(.*)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($1) . "\n";
2673             next;
2674         }
2675
2676         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2677             $descr = '' if !defined($descr);
2678             $descr .= PVE::Tools::decode_text($2);
2679         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2680             $conf->{snapstate} = $1;
2681         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2682             my $key = $1;
2683             my $value = $2;
2684             $conf->{$key} = $value;
2685         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2686             my $value = $1;
2687             if ($section eq 'pending') {
2688                 $conf->{delete} = $value; # we parse this later
2689             } else {
2690                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2691             }
2692         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2693             my $key = $1;
2694             my $value = $2;
2695             eval { $value = check_type($key, $value); };
2696             if ($@) {
2697                 warn "vm $vmid - unable to parse value of '$key' - $@";
2698             } else {
2699                 $key = 'ide2' if $key eq 'cdrom';
2700                 my $fmt = $confdesc->{$key}->{format};
2701                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2702                     my $v = parse_drive($key, $value);
2703                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2704                         $v->{file} = $volid;
2705                         $value = print_drive($vmid, $v);
2706                     } else {
2707                         warn "vm $vmid - unable to parse value of '$key'\n";
2708                         next;
2709                     }
2710                 }
2711
2712                 $conf->{$key} = $value;
2713             }
2714         }
2715     }
2716
2717     if (defined($descr)) {
2718         $descr =~ s/\s+$//;
2719         $conf->{description} = $descr;
2720     }
2721     delete $res->{snapstate}; # just to be sure
2722
2723     return $res;
2724 }
2725
2726 sub write_vm_config {
2727     my ($filename, $conf) = @_;
2728
2729     delete $conf->{snapstate}; # just to be sure
2730
2731     if ($conf->{cdrom}) {
2732         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2733         $conf->{ide2} = $conf->{cdrom};
2734         delete $conf->{cdrom};
2735     }
2736
2737     # we do not use 'smp' any longer
2738     if ($conf->{sockets}) {
2739         delete $conf->{smp};
2740     } elsif ($conf->{smp}) {
2741         $conf->{sockets} = $conf->{smp};
2742         delete $conf->{cores};
2743         delete $conf->{smp};
2744     }
2745
2746     my $used_volids = {};
2747
2748     my $cleanup_config = sub {
2749         my ($cref, $pending, $snapname) = @_;
2750
2751         foreach my $key (keys %$cref) {
2752             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2753                 $key eq 'snapstate' || $key eq 'pending';
2754             my $value = $cref->{$key};
2755             if ($key eq 'delete') {
2756                 die "propertry 'delete' is only allowed in [PENDING]\n"
2757                     if !$pending;
2758                 # fixme: check syntax?
2759                 next;
2760             }
2761             eval { $value = check_type($key, $value); };
2762             die "unable to parse value of '$key' - $@" if $@;
2763
2764             $cref->{$key} = $value;
2765
2766             if (!$snapname && is_valid_drivename($key)) {
2767                 my $drive = parse_drive($key, $value);
2768                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2769             }
2770         }
2771     };
2772
2773     &$cleanup_config($conf);
2774
2775     &$cleanup_config($conf->{pending}, 1);
2776
2777     foreach my $snapname (keys %{$conf->{snapshots}}) {
2778         die "internal error" if $snapname eq 'pending';
2779         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2780     }
2781
2782     # remove 'unusedX' settings if we re-add a volume
2783     foreach my $key (keys %$conf) {
2784         my $value = $conf->{$key};
2785         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2786             delete $conf->{$key};
2787         }
2788     }
2789
2790     my $generate_raw_config = sub {
2791         my ($conf, $pending) = @_;
2792
2793         my $raw = '';
2794
2795         # add description as comment to top of file
2796         if (defined(my $descr = $conf->{description})) {
2797             if ($descr) {
2798                 foreach my $cl (split(/\n/, $descr)) {
2799                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2800                 }
2801             } else {
2802                 $raw .= "#\n" if $pending;
2803             }
2804         }
2805
2806         foreach my $key (sort keys %$conf) {
2807             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2808             $raw .= "$key: $conf->{$key}\n";
2809         }
2810         return $raw;
2811     };
2812
2813     my $raw = &$generate_raw_config($conf);
2814
2815     if (scalar(keys %{$conf->{pending}})){
2816         $raw .= "\n[PENDING]\n";
2817         $raw .= &$generate_raw_config($conf->{pending}, 1);
2818     }
2819
2820     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2821         $raw .= "\n[$snapname]\n";
2822         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2823     }
2824
2825     return $raw;
2826 }
2827
2828 sub load_defaults {
2829
2830     my $res = {};
2831
2832     # we use static defaults from our JSON schema configuration
2833     foreach my $key (keys %$confdesc) {
2834         if (defined(my $default = $confdesc->{$key}->{default})) {
2835             $res->{$key} = $default;
2836         }
2837     }
2838
2839     return $res;
2840 }
2841
2842 sub config_list {
2843     my $vmlist = PVE::Cluster::get_vmlist();
2844     my $res = {};
2845     return $res if !$vmlist || !$vmlist->{ids};
2846     my $ids = $vmlist->{ids};
2847
2848     foreach my $vmid (keys %$ids) {
2849         my $d = $ids->{$vmid};
2850         next if !$d->{node} || $d->{node} ne $nodename;
2851         next if !$d->{type} || $d->{type} ne 'qemu';
2852         $res->{$vmid}->{exists} = 1;
2853     }
2854     return $res;
2855 }
2856
2857 # test if VM uses local resources (to prevent migration)
2858 sub check_local_resources {
2859     my ($conf, $noerr) = @_;
2860
2861     my @loc_res = ();
2862
2863     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2864     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2865
2866     push @loc_res, "ivshmem" if $conf->{ivshmem};
2867
2868     foreach my $k (keys %$conf) {
2869         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2870         # sockets are safe: they will recreated be on the target side post-migrate
2871         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2872         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2873     }
2874
2875     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2876
2877     return \@loc_res;
2878 }
2879
2880 # check if used storages are available on all nodes (use by migrate)
2881 sub check_storage_availability {
2882     my ($storecfg, $conf, $node) = @_;
2883
2884     foreach_drive($conf, sub {
2885         my ($ds, $drive) = @_;
2886
2887         my $volid = $drive->{file};
2888         return if !$volid;
2889
2890         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2891         return if !$sid;
2892
2893         # check if storage is available on both nodes
2894         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2895         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2896    });
2897 }
2898
2899 # list nodes where all VM images are available (used by has_feature API)
2900 sub shared_nodes {
2901     my ($conf, $storecfg) = @_;
2902
2903     my $nodelist = PVE::Cluster::get_nodelist();
2904     my $nodehash = { map { $_ => 1 } @$nodelist };
2905     my $nodename = PVE::INotify::nodename();
2906
2907     foreach_drive($conf, sub {
2908         my ($ds, $drive) = @_;
2909
2910         my $volid = $drive->{file};
2911         return if !$volid;
2912
2913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2914         if ($storeid) {
2915             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2916             if ($scfg->{disable}) {
2917                 $nodehash = {};
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     delete $nodehash->{$node} if !$avail->{$node};
2921                 }
2922             } elsif (!$scfg->{shared}) {
2923                 foreach my $node (keys %$nodehash) {
2924                     delete $nodehash->{$node} if $node ne $nodename
2925                 }
2926             }
2927         }
2928     });
2929
2930     return $nodehash
2931 }
2932
2933 sub check_local_storage_availability {
2934     my ($conf, $storecfg) = @_;
2935
2936     my $nodelist = PVE::Cluster::get_nodelist();
2937     my $nodehash = { map { $_ => {} } @$nodelist };
2938
2939     foreach_drive($conf, sub {
2940         my ($ds, $drive) = @_;
2941
2942         my $volid = $drive->{file};
2943         return if !$volid;
2944
2945         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2946         if ($storeid) {
2947             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2948
2949             if ($scfg->{disable}) {
2950                 foreach my $node (keys %$nodehash) {
2951                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2952                 }
2953             } elsif (my $avail = $scfg->{nodes}) {
2954                 foreach my $node (keys %$nodehash) {
2955                     if (!$avail->{$node}) {
2956                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2957                     }
2958                 }
2959             }
2960         }
2961     });
2962
2963     foreach my $node (values %$nodehash) {
2964         if (my $unavail = $node->{unavailable_storages}) {
2965             $node->{unavailable_storages} = [ sort keys %$unavail ];
2966         }
2967     }
2968
2969     return $nodehash
2970 }
2971
2972 sub check_cmdline {
2973     my ($pidfile, $pid) = @_;
2974
2975     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2976     if (defined($fh)) {
2977         my $line = <$fh>;
2978         $fh->close;
2979         return undef if !$line;
2980         my @param = split(/\0/, $line);
2981
2982         my $cmd = $param[0];
2983         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2984
2985         for (my $i = 0; $i < scalar (@param); $i++) {
2986             my $p = $param[$i];
2987             next if !$p;
2988             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2989                 my $p = $param[$i+1];
2990                 return 1 if $p && ($p eq $pidfile);
2991                 return undef;
2992             }
2993         }
2994     }
2995     return undef;
2996 }
2997
2998 sub check_running {
2999     my ($vmid, $nocheck, $node) = @_;
3000
3001     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3002
3003     die "unable to find configuration file for VM $vmid - no such machine\n"
3004         if !$nocheck && ! -f $filename;
3005
3006     my $pidfile = pidfile_name($vmid);
3007
3008     if (my $fd = IO::File->new("<$pidfile")) {
3009         my $st = stat($fd);
3010         my $line = <$fd>;
3011         close($fd);
3012
3013         my $mtime = $st->mtime;
3014         if ($mtime > time()) {
3015             warn "file '$filename' modified in future\n";
3016         }
3017
3018         if ($line =~ m/^(\d+)$/) {
3019             my $pid = $1;
3020             if (check_cmdline($pidfile, $pid)) {
3021                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3022                     return $pid;
3023                 }
3024             }
3025         }
3026     }
3027
3028     return undef;
3029 }
3030
3031 sub vzlist {
3032
3033     my $vzlist = config_list();
3034
3035     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3036
3037     while (defined(my $de = $fd->read)) {
3038         next if $de !~ m/^(\d+)\.pid$/;
3039         my $vmid = $1;
3040         next if !defined($vzlist->{$vmid});
3041         if (my $pid = check_running($vmid)) {
3042             $vzlist->{$vmid}->{pid} = $pid;
3043         }
3044     }
3045
3046     return $vzlist;
3047 }
3048
3049 sub disksize {
3050     my ($storecfg, $conf) = @_;
3051
3052     my $bootdisk = $conf->{bootdisk};
3053     return undef if !$bootdisk;
3054     return undef if !is_valid_drivename($bootdisk);
3055
3056     return undef if !$conf->{$bootdisk};
3057
3058     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3059     return undef if !defined($drive);
3060
3061     return undef if drive_is_cdrom($drive);
3062
3063     my $volid = $drive->{file};
3064     return undef if !$volid;
3065
3066     return $drive->{size};
3067 }
3068
3069 our $vmstatus_return_properties = {
3070     vmid => get_standard_option('pve-vmid'),
3071     status => {
3072         description => "Qemu process status.",
3073         type => 'string',
3074         enum => ['stopped', 'running'],
3075     },
3076     maxmem => {
3077         description => "Maximum memory in bytes.",
3078         type => 'integer',
3079         optional => 1,
3080         renderer => 'bytes',
3081     },
3082     maxdisk => {
3083         description => "Root disk size in bytes.",
3084         type => 'integer',
3085         optional => 1,
3086         renderer => 'bytes',
3087     },
3088     name => {
3089         description => "VM name.",
3090         type => 'string',
3091         optional => 1,
3092     },
3093     qmpstatus => {
3094         description => "Qemu QMP agent status.",
3095         type => 'string',
3096         optional => 1,
3097     },
3098     pid => {
3099         description => "PID of running qemu process.",
3100         type => 'integer',
3101         optional => 1,
3102     },
3103     uptime => {
3104         description => "Uptime.",
3105         type => 'integer',
3106         optional => 1,
3107         renderer => 'duration',
3108     },
3109     cpus => {
3110         description => "Maximum usable CPUs.",
3111         type => 'number',
3112         optional => 1,
3113     },
3114     lock => {
3115         description => "The current config lock, if any.",
3116         type => 'string',
3117         optional => 1,
3118     }
3119 };
3120
3121 my $last_proc_pid_stat;
3122
3123 # get VM status information
3124 # This must be fast and should not block ($full == false)
3125 # We only query KVM using QMP if $full == true (this can be slow)
3126 sub vmstatus {
3127     my ($opt_vmid, $full) = @_;
3128
3129     my $res = {};
3130
3131     my $storecfg = PVE::Storage::config();
3132
3133     my $list = vzlist();
3134     my $defaults = load_defaults();
3135
3136     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3137
3138     my $cpucount = $cpuinfo->{cpus} || 1;
3139
3140     foreach my $vmid (keys %$list) {
3141         next if $opt_vmid && ($vmid ne $opt_vmid);
3142
3143         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3144         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3145
3146         my $d = { vmid => $vmid };
3147         $d->{pid} = $list->{$vmid}->{pid};
3148
3149         # fixme: better status?
3150         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3151
3152         my $size = disksize($storecfg, $conf);
3153         if (defined($size)) {
3154             $d->{disk} = 0; # no info available
3155             $d->{maxdisk} = $size;
3156         } else {
3157             $d->{disk} = 0;
3158             $d->{maxdisk} = 0;
3159         }
3160
3161         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3162             * ($conf->{cores} || $defaults->{cores});
3163         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3164         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3165
3166         $d->{name} = $conf->{name} || "VM $vmid";
3167         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3168             : $defaults->{memory}*(1024*1024);
3169
3170         if ($conf->{balloon}) {
3171             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3172             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3173                 : $defaults->{shares};
3174         }
3175
3176         $d->{uptime} = 0;
3177         $d->{cpu} = 0;
3178         $d->{mem} = 0;
3179
3180         $d->{netout} = 0;
3181         $d->{netin} = 0;
3182
3183         $d->{diskread} = 0;
3184         $d->{diskwrite} = 0;
3185
3186         $d->{template} = PVE::QemuConfig->is_template($conf);
3187
3188         $d->{serial} = 1 if conf_has_serial($conf);
3189         $d->{lock} = $conf->{lock} if $conf->{lock};
3190
3191         $res->{$vmid} = $d;
3192     }
3193
3194     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3195     foreach my $dev (keys %$netdev) {
3196         next if $dev !~ m/^tap([1-9]\d*)i/;
3197         my $vmid = $1;
3198         my $d = $res->{$vmid};
3199         next if !$d;
3200
3201         $d->{netout} += $netdev->{$dev}->{receive};
3202         $d->{netin} += $netdev->{$dev}->{transmit};
3203
3204         if ($full) {
3205             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3206             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3207         }
3208
3209     }
3210
3211     my $ctime = gettimeofday;
3212
3213     foreach my $vmid (keys %$list) {
3214
3215         my $d = $res->{$vmid};
3216         my $pid = $d->{pid};
3217         next if !$pid;
3218
3219         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3220         next if !$pstat; # not running
3221
3222         my $used = $pstat->{utime} + $pstat->{stime};
3223
3224         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3225
3226         if ($pstat->{vsize}) {
3227             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3228         }
3229
3230         my $old = $last_proc_pid_stat->{$pid};
3231         if (!$old) {
3232             $last_proc_pid_stat->{$pid} = {
3233                 time => $ctime,
3234                 used => $used,
3235                 cpu => 0,
3236             };
3237             next;
3238         }
3239
3240         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3241
3242         if ($dtime > 1000) {
3243             my $dutime = $used -  $old->{used};
3244
3245             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3246             $last_proc_pid_stat->{$pid} = {
3247                 time => $ctime,
3248                 used => $used,
3249                 cpu => $d->{cpu},
3250             };
3251         } else {
3252             $d->{cpu} = $old->{cpu};
3253         }
3254     }
3255
3256     return $res if !$full;
3257
3258     my $qmpclient = PVE::QMPClient->new();
3259
3260     my $ballooncb = sub {
3261         my ($vmid, $resp) = @_;
3262
3263         my $info = $resp->{'return'};
3264         return if !$info->{max_mem};
3265
3266         my $d = $res->{$vmid};
3267
3268         # use memory assigned to VM
3269         $d->{maxmem} = $info->{max_mem};
3270         $d->{balloon} = $info->{actual};
3271
3272         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3273             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3274             $d->{freemem} = $info->{free_mem};
3275         }
3276
3277         $d->{ballooninfo} = $info;
3278     };
3279
3280     my $blockstatscb = sub {
3281         my ($vmid, $resp) = @_;
3282         my $data = $resp->{'return'} || [];
3283         my $totalrdbytes = 0;
3284         my $totalwrbytes = 0;
3285
3286         for my $blockstat (@$data) {
3287             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3288             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3289
3290             $blockstat->{device} =~ s/drive-//;
3291             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3292         }
3293         $res->{$vmid}->{diskread} = $totalrdbytes;
3294         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3295     };
3296
3297     my $statuscb = sub {
3298         my ($vmid, $resp) = @_;
3299
3300         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3301         # this fails if ballon driver is not loaded, so this must be
3302         # the last commnand (following command are aborted if this fails).
3303         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3304
3305         my $status = 'unknown';
3306         if (!defined($status = $resp->{'return'}->{status})) {
3307             warn "unable to get VM status\n";
3308             return;
3309         }
3310
3311         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3312     };
3313
3314     foreach my $vmid (keys %$list) {
3315         next if $opt_vmid && ($vmid ne $opt_vmid);
3316         next if !$res->{$vmid}->{pid}; # not running
3317         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3318     }
3319
3320     $qmpclient->queue_execute(undef, 2);
3321
3322     foreach my $vmid (keys %$list) {
3323         next if $opt_vmid && ($vmid ne $opt_vmid);
3324         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3325     }
3326
3327     return $res;
3328 }
3329
3330 sub foreach_drive {
3331     my ($conf, $func, @param) = @_;
3332
3333     foreach my $ds (valid_drive_names()) {
3334         next if !defined($conf->{$ds});
3335
3336         my $drive = parse_drive($ds, $conf->{$ds});
3337         next if !$drive;
3338
3339         &$func($ds, $drive, @param);
3340     }
3341 }
3342
3343 sub foreach_volid {
3344     my ($conf, $func, @param) = @_;
3345
3346     my $volhash = {};
3347
3348     my $test_volid = sub {
3349         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3350
3351         return if !$volid;
3352
3353         $volhash->{$volid}->{cdrom} //= 1;
3354         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3355
3356         $volhash->{$volid}->{replicate} //= 0;
3357         $volhash->{$volid}->{replicate} = 1 if $replicate;
3358
3359         $volhash->{$volid}->{shared} //= 0;
3360         $volhash->{$volid}->{shared} = 1 if $shared;
3361
3362         $volhash->{$volid}->{referenced_in_config} //= 0;
3363         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3364
3365         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3366             if defined($snapname);
3367         $volhash->{$volid}->{size} = $size if $size;
3368     };
3369
3370     foreach_drive($conf, sub {
3371         my ($ds, $drive) = @_;
3372         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3373     });
3374
3375     foreach my $snapname (keys %{$conf->{snapshots}}) {
3376         my $snap = $conf->{snapshots}->{$snapname};
3377         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3378         foreach_drive($snap, sub {
3379             my ($ds, $drive) = @_;
3380             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3381         });
3382     }
3383
3384     foreach my $volid (keys %$volhash) {
3385         &$func($volid, $volhash->{$volid}, @param);
3386     }
3387 }
3388
3389 sub conf_has_serial {
3390     my ($conf) = @_;
3391
3392     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3393         if ($conf->{"serial$i"}) {
3394             return 1;
3395         }
3396     }
3397
3398     return 0;
3399 }
3400
3401 sub vga_conf_has_spice {
3402     my ($vga) = @_;
3403
3404     my $vgaconf = parse_vga($vga);
3405     my $vgatype = $vgaconf->{type};
3406     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3407
3408     return $1 || 1;
3409 }
3410
3411 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3412 sub get_host_arch() {
3413     $host_arch = (POSIX::uname())[4] if !$host_arch;
3414     return $host_arch;
3415 }
3416
3417 sub is_native($) {
3418     my ($arch) = @_;
3419     return get_host_arch() eq $arch;
3420 }
3421
3422 my $default_machines = {
3423     x86_64 => 'pc',
3424     aarch64 => 'virt',
3425 };
3426
3427 sub get_basic_machine_info {
3428     my ($conf, $forcemachine) = @_;
3429
3430     my $arch = $conf->{arch} // get_host_arch();
3431     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3432     return ($arch, $machine);
3433 }
3434
3435 sub get_ovmf_files($) {
3436     my ($arch) = @_;
3437
3438     my $ovmf = $OVMF->{$arch}
3439         or die "no OVMF images known for architecture '$arch'\n";
3440
3441     return @$ovmf;
3442 }
3443
3444 my $Arch2Qemu = {
3445     aarch64 => '/usr/bin/qemu-system-aarch64',
3446     x86_64 => '/usr/bin/qemu-system-x86_64',
3447 };
3448 sub get_command_for_arch($) {
3449     my ($arch) = @_;
3450     return '/usr/bin/kvm' if is_native($arch);
3451
3452     my $cmd = $Arch2Qemu->{$arch}
3453         or die "don't know how to emulate architecture '$arch'\n";
3454     return $cmd;
3455 }
3456
3457 sub get_cpu_options {
3458     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3459
3460     my $cpuFlags = [];
3461     my $ostype = $conf->{ostype};
3462
3463     my $cpu = $kvm ? "kvm64" : "qemu64";
3464     if ($arch eq 'aarch64') {
3465         $cpu = 'cortex-a57';
3466     }
3467     my $hv_vendor_id;
3468     if (my $cputype = $conf->{cpu}) {
3469         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3470             or die "Cannot parse cpu description: $cputype\n";
3471         $cpu = $cpuconf->{cputype};
3472         $kvm_off = 1 if $cpuconf->{hidden};
3473         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3474
3475         if (defined(my $flags = $cpuconf->{flags})) {
3476             push @$cpuFlags, split(";", $flags);
3477         }
3478     }
3479
3480     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3481
3482     push @$cpuFlags , '-x2apic'
3483         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3484
3485     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3486
3487     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3488
3489     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3490
3491         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3492         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3493     }
3494
3495     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3496
3497     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3498
3499     push @$cpuFlags, 'kvm=off' if $kvm_off;
3500
3501     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3502         push @$cpuFlags, "vendor=${cpu_vendor}"
3503             if $cpu_vendor ne 'default';
3504     } elsif ($arch ne 'aarch64') {
3505         die "internal error"; # should not happen
3506     }
3507
3508     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3509
3510     return ('-cpu', $cpu);
3511 }
3512
3513 sub config_to_command {
3514     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3515
3516     my $cmd = [];
3517     my $globalFlags = [];
3518     my $machineFlags = [];
3519     my $rtcFlags = [];
3520     my $devices = [];
3521     my $pciaddr = '';
3522     my $bridges = {};
3523     my $kvmver = kvm_user_version();
3524     my $vernum = 0; # unknown
3525     my $ostype = $conf->{ostype};
3526     my $winversion = windows_version($ostype);
3527     my $kvm = $conf->{kvm};
3528
3529     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3530     $kvm //= 1 if is_native($arch);
3531
3532     if ($kvm) {
3533         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3534             if !defined kvm_version();
3535     }
3536
3537     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3538         $vernum = $1*1000000+$2*1000;
3539     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3540         $vernum = $1*1000000+$2*1000+$3;
3541     }
3542
3543     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3544
3545     my $have_ovz = -f '/proc/vz/vestat';
3546
3547     my $q35 = machine_type_is_q35($conf);
3548     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3549     my $use_old_bios_files = undef;
3550     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3551
3552     my $cpuunits = defined($conf->{cpuunits}) ?
3553             $conf->{cpuunits} : $defaults->{cpuunits};
3554
3555     push @$cmd, get_command_for_arch($arch);
3556
3557     push @$cmd, '-id', $vmid;
3558
3559     my $vmname = $conf->{name} || "vm$vmid";
3560
3561     push @$cmd, '-name', $vmname;
3562
3563     my $use_virtio = 0;
3564
3565     my $qmpsocket = qmp_socket($vmid);
3566     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3567     push @$cmd, '-mon', "chardev=qmp,mode=control";
3568
3569     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3570         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3571         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3572     }
3573
3574     push @$cmd, '-pidfile' , pidfile_name($vmid);
3575
3576     push @$cmd, '-daemonize';
3577
3578     if ($conf->{smbios1}) {
3579         my $smbios_conf = parse_smbios1($conf->{smbios1});
3580         if ($smbios_conf->{base64}) {
3581             # Do not pass base64 flag to qemu
3582             delete $smbios_conf->{base64};
3583             my $smbios_string = "";
3584             foreach my $key (keys %$smbios_conf) {
3585                 my $value;
3586                 if ($key eq "uuid") {
3587                     $value = $smbios_conf->{uuid}
3588                 } else {
3589                     $value = decode_base64($smbios_conf->{$key});
3590                 }
3591                 # qemu accepts any binary data, only commas need escaping by double comma
3592                 $value =~ s/,/,,/g;
3593                 $smbios_string .= "," . $key . "=" . $value if $value;
3594             }
3595             push @$cmd, '-smbios', "type=1" . $smbios_string;
3596         } else {
3597             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3598         }
3599     }
3600
3601     if ($conf->{vmgenid}) {
3602         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3603     }
3604
3605     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3606     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3607         die "uefi base image not found\n" if ! -f $ovmf_code;
3608
3609         my $path;
3610         my $format;
3611         if (my $efidisk = $conf->{efidisk0}) {
3612             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3613             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3614             $format = $d->{format};
3615             if ($storeid) {
3616                 $path = PVE::Storage::path($storecfg, $d->{file});
3617                 if (!defined($format)) {
3618                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3619                     $format = qemu_img_format($scfg, $volname);
3620                 }
3621             } else {
3622                 $path = $d->{file};
3623                 die "efidisk format must be specified\n"
3624                     if !defined($format);
3625             }
3626         } else {
3627             warn "no efidisk configured! Using temporary efivars disk.\n";
3628             $path = "/tmp/$vmid-ovmf.fd";
3629             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3630             $format = 'raw';
3631         }
3632
3633         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3634         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3635     }
3636
3637     # load q35 config
3638     if ($q35) {
3639         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3640         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3641             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3642         } else {
3643             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3644         }
3645     }
3646
3647     # add usb controllers
3648     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3649     push @$devices, @usbcontrollers if @usbcontrollers;
3650     my $vga = parse_vga($conf->{vga});
3651
3652     my $qxlnum = vga_conf_has_spice($conf->{vga});
3653     $vga->{type} = 'qxl' if $qxlnum;
3654
3655     if (!$vga->{type}) {
3656         if ($arch eq 'aarch64') {
3657             $vga->{type} = 'virtio';
3658         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3659             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3660         } else {
3661             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3662         }
3663     }
3664
3665     # enable absolute mouse coordinates (needed by vnc)
3666     my $tablet;
3667     if (defined($conf->{tablet})) {
3668         $tablet = $conf->{tablet};
3669     } else {
3670         $tablet = $defaults->{tablet};
3671         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3672         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3673     }
3674
3675     if ($tablet) {
3676         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3677         my $kbd = print_keyboarddevice_full($conf, $arch);
3678         push @$devices, '-device', $kbd if defined($kbd);
3679     }
3680
3681     my $kvm_off = 0;
3682     my $gpu_passthrough;
3683
3684     # host pci devices
3685     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3686         my $d = parse_hostpci($conf->{"hostpci$i"});
3687         next if !$d;
3688
3689         my $pcie = $d->{pcie};
3690         if ($pcie) {
3691             die "q35 machine model is not enabled" if !$q35;
3692             # win7 wants to have the pcie devices directly on the pcie bus
3693             # instead of in the root port
3694             if ($winversion == 7) {
3695                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3696             } else {
3697                 $pciaddr = print_pcie_addr("hostpci$i");
3698             }
3699         } else {
3700             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3701         }
3702
3703         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3704         my $romfile = $d->{romfile};
3705
3706         my $xvga = '';
3707         if ($d->{'x-vga'}) {
3708             $xvga = ',x-vga=on';
3709             $kvm_off = 1;
3710             $vga->{type} = 'none' if !defined($conf->{vga});
3711             $gpu_passthrough = 1;
3712
3713             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3714                 $xvga = "";
3715             }
3716         }
3717         my $pcidevices = $d->{pciid};
3718         my $multifunction = 1 if @$pcidevices > 1;
3719         my $sysfspath;
3720         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3721             my $id = $pcidevices->[0]->{id};
3722             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3723             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3724         } elsif ($d->{mdev}) {
3725             warn "ignoring mediated device with multifunction device\n";
3726         }
3727
3728         my $j=0;
3729         foreach my $pcidevice (@$pcidevices) {
3730
3731             my $id = "hostpci$i";
3732             $id .= ".$j" if $multifunction;
3733             my $addr = $pciaddr;
3734             $addr .= ".$j" if $multifunction;
3735             my $devicestr = "vfio-pci";
3736             if ($sysfspath) {
3737                 $devicestr .= ",sysfsdev=$sysfspath";
3738             } else {
3739                 $devicestr .= ",host=$pcidevice->{id}";
3740             }
3741             $devicestr .= ",id=$id$addr";
3742
3743             if($j == 0){
3744                 $devicestr .= "$rombar$xvga";
3745                 $devicestr .= ",multifunction=on" if $multifunction;
3746                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3747             }
3748
3749             push @$devices, '-device', $devicestr;
3750             $j++;
3751         }
3752     }
3753
3754     # usb devices
3755     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3756     push @$devices, @usbdevices if @usbdevices;
3757     # serial devices
3758     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3759         if (my $path = $conf->{"serial$i"}) {
3760             if ($path eq 'socket') {
3761                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3762                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3763                 # On aarch64, serial0 is the UART device. Qemu only allows
3764                 # connecting UART devices via the '-serial' command line, as
3765                 # the device has a fixed slot on the hardware...
3766                 if ($arch eq 'aarch64' && $i == 0) {
3767                     push @$devices, '-serial', "chardev:serial$i";
3768                 } else {
3769                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3770                 }
3771             } else {
3772                 die "no such serial device\n" if ! -c $path;
3773                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3775             }
3776         }
3777     }
3778
3779     # parallel devices
3780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3781         if (my $path = $conf->{"parallel$i"}) {
3782             die "no such parallel device\n" if ! -c $path;
3783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3786         }
3787     }
3788
3789     if ($conf->{"audio0"}) {
3790         my $audiodevice = $conf->{audio0};
3791         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3792
3793         if ($audiodevice eq 'AC97') {
3794             push @$devices, '-device', "AC97,id=sound0${audiopciaddr}";
3795         } else {
3796             push @$devices, '-device', "${audiodevice},id=sound5${audiopciaddr}";
3797             push @$devices, '-device', "hda-micro,id=sound5-codec0,bus=sound5.0,cad=0";
3798             push @$devices, '-device', "hda-duplex,id=sound5-codec1,bus=sound5.0,cad=1";
3799         }
3800     }
3801
3802     my $sockets = 1;
3803     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3804     $sockets = $conf->{sockets} if  $conf->{sockets};
3805
3806     my $cores = $conf->{cores} || 1;
3807
3808     my $maxcpus = $sockets * $cores;
3809
3810     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3811
3812     my $allowed_vcpus = $cpuinfo->{cpus};
3813
3814     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3815         if ($allowed_vcpus < $maxcpus);
3816
3817     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3818
3819         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3820         for (my $i = 2; $i <= $vcpus; $i++)  {
3821             my $cpustr = print_cpu_device($conf,$i);
3822             push @$cmd, '-device', $cpustr;
3823         }
3824
3825     } else {
3826
3827         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3828     }
3829     push @$cmd, '-nodefaults';
3830
3831     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3832
3833     my $bootindex_hash = {};
3834     my $i = 1;
3835     foreach my $o (split(//, $bootorder)) {
3836         $bootindex_hash->{$o} = $i*100;
3837         $i++;
3838     }
3839
3840     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3841
3842     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3843
3844     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3845
3846     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3847         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3848         my $socket = vnc_socket($vmid);
3849         push @$cmd,  '-vnc', "unix:$socket,password";
3850     } else {
3851         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3852         push @$cmd, '-nographic';
3853     }
3854
3855     # time drift fix
3856     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3857
3858     my $useLocaltime = $conf->{localtime};
3859
3860     if ($winversion >= 5) { # windows
3861         $useLocaltime = 1 if !defined($conf->{localtime});
3862
3863         # use time drift fix when acpi is enabled
3864         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3865             $tdf = 1 if !defined($conf->{tdf});
3866         }
3867     }
3868
3869     if ($winversion >= 6) {
3870         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3871         push @$cmd, '-no-hpet';
3872     }
3873
3874     push @$rtcFlags, 'driftfix=slew' if $tdf;
3875
3876     if (!$kvm) {
3877         push @$machineFlags, 'accel=tcg';
3878     }
3879
3880     if ($machine_type) {
3881         push @$machineFlags, "type=${machine_type}";
3882     }
3883
3884     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3885         push @$rtcFlags, "base=$conf->{startdate}";
3886     } elsif ($useLocaltime) {
3887         push @$rtcFlags, 'base=localtime';
3888     }
3889
3890     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3891
3892     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3893
3894     push @$cmd, '-S' if $conf->{freeze};
3895
3896     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3897
3898     if (parse_guest_agent($conf)->{enabled}) {
3899         my $qgasocket = qmp_socket($vmid, 1);
3900         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3901         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3902         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3903         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3904     }
3905
3906     my $spice_port;
3907
3908     if ($qxlnum) {
3909         if ($qxlnum > 1) {
3910             if ($winversion){
3911                 for(my $i = 1; $i < $qxlnum; $i++){
3912                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3913                 }
3914             } else {
3915                 # assume other OS works like Linux
3916                 my ($ram, $vram) = ("134217728", "67108864");
3917                 if ($vga->{memory}) {
3918                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3919                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3920                 }
3921                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3922                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3923             }
3924         }
3925
3926         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3927
3928         my $nodename = PVE::INotify::nodename();
3929         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3930         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3931         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3932         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3933         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3934
3935         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3936
3937         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3938         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3939         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3940     }
3941
3942     # enable balloon by default, unless explicitly disabled
3943     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3944         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3945         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3946     }
3947
3948     if ($conf->{watchdog}) {
3949         my $wdopts = parse_watchdog($conf->{watchdog});
3950         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3951         my $watchdog = $wdopts->{model} || 'i6300esb';
3952         push @$devices, '-device', "$watchdog$pciaddr";
3953         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3954     }
3955
3956     my $vollist = [];
3957     my $scsicontroller = {};
3958     my $ahcicontroller = {};
3959     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3960
3961     # Add iscsi initiator name if available
3962     if (my $initiator = get_initiator_name()) {
3963         push @$devices, '-iscsi', "initiator-name=$initiator";
3964     }
3965
3966     foreach_drive($conf, sub {
3967         my ($ds, $drive) = @_;
3968
3969         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3970             push @$vollist, $drive->{file};
3971         }
3972
3973         # ignore efidisk here, already added in bios/fw handling code above
3974         return if $drive->{interface} eq 'efidisk';
3975
3976         $use_virtio = 1 if $ds =~ m/^virtio/;
3977
3978         if (drive_is_cdrom ($drive)) {
3979             if ($bootindex_hash->{d}) {
3980                 $drive->{bootindex} = $bootindex_hash->{d};
3981                 $bootindex_hash->{d} += 1;
3982             }
3983         } else {
3984             if ($bootindex_hash->{c}) {
3985                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3986                 $bootindex_hash->{c} += 1;
3987             }
3988         }
3989
3990         if($drive->{interface} eq 'virtio'){
3991            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3992         }
3993
3994         if ($drive->{interface} eq 'scsi') {
3995
3996             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3997
3998             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3999             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4000
4001             my $iothread = '';
4002             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4003                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4004                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4005             } elsif ($drive->{iothread}) {
4006                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4007             }
4008
4009             my $queues = '';
4010             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4011                 $queues = ",num_queues=$drive->{queues}";
4012             }
4013
4014             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4015             $scsicontroller->{$controller}=1;
4016         }
4017
4018         if ($drive->{interface} eq 'sata') {
4019            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4020            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4021            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4022            $ahcicontroller->{$controller}=1;
4023         }
4024
4025         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4026         push @$devices, '-drive',$drive_cmd;
4027         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4028     });
4029
4030     for (my $i = 0; $i < $MAX_NETS; $i++) {
4031          next if !$conf->{"net$i"};
4032          my $d = parse_net($conf->{"net$i"});
4033          next if !$d;
4034
4035          $use_virtio = 1 if $d->{model} eq 'virtio';
4036
4037          if ($bootindex_hash->{n}) {
4038             $d->{bootindex} = $bootindex_hash->{n};
4039             $bootindex_hash->{n} += 1;
4040          }
4041
4042          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4043          push @$devices, '-netdev', $netdevfull;
4044
4045          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4046          push @$devices, '-device', $netdevicefull;
4047     }
4048
4049     if ($conf->{ivshmem}) {
4050         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4051
4052         my $bus;
4053         if ($q35) {
4054             $bus = print_pcie_addr("ivshmem");
4055         } else {
4056             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4057         }
4058
4059         my $ivshmem_name = $ivshmem->{name} // $vmid;
4060         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4061
4062         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4063         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4064     }
4065
4066     if (!$q35) {
4067         # add pci bridges
4068         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4069            $bridges->{1} = 1;
4070            $bridges->{2} = 1;
4071         }
4072
4073         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4074
4075         while (my ($k, $v) = each %$bridges) {
4076             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4077             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4078         }
4079     }
4080
4081     push @$cmd, @$devices;
4082     push @$cmd, '-rtc', join(',', @$rtcFlags)
4083         if scalar(@$rtcFlags);
4084     push @$cmd, '-machine', join(',', @$machineFlags)
4085         if scalar(@$machineFlags);
4086     push @$cmd, '-global', join(',', @$globalFlags)
4087         if scalar(@$globalFlags);
4088
4089     if (my $vmstate = $conf->{vmstate}) {
4090         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4091         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4092         push @$cmd, '-loadstate', $statepath;
4093     }
4094
4095     # add custom args
4096     if ($conf->{args}) {
4097         my $aa = PVE::Tools::split_args($conf->{args});
4098         push @$cmd, @$aa;
4099     }
4100
4101     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4102 }
4103
4104 sub vnc_socket {
4105     my ($vmid) = @_;
4106     return "${var_run_tmpdir}/$vmid.vnc";
4107 }
4108
4109 sub spice_port {
4110     my ($vmid) = @_;
4111
4112     my $res = vm_mon_cmd($vmid, 'query-spice');
4113
4114     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4115 }
4116
4117 sub qmp_socket {
4118     my ($vmid, $qga, $name) = @_;
4119     my $sockettype = $qga ? 'qga' : 'qmp';
4120     my $ext = $name ? '-'.$name : '';
4121     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4122 }
4123
4124 sub pidfile_name {
4125     my ($vmid) = @_;
4126     return "${var_run_tmpdir}/$vmid.pid";
4127 }
4128
4129 sub vm_devices_list {
4130     my ($vmid) = @_;
4131
4132     my $res = vm_mon_cmd($vmid, 'query-pci');
4133     my $devices_to_check = [];
4134     my $devices = {};
4135     foreach my $pcibus (@$res) {
4136         push @$devices_to_check, @{$pcibus->{devices}},
4137     }
4138
4139     while (@$devices_to_check) {
4140         my $to_check = [];
4141         for my $d (@$devices_to_check) {
4142             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4143             next if !$d->{'pci_bridge'};
4144
4145             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4146             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4147         }
4148         $devices_to_check = $to_check;
4149     }
4150
4151     my $resblock = vm_mon_cmd($vmid, 'query-block');
4152     foreach my $block (@$resblock) {
4153         if($block->{device} =~ m/^drive-(\S+)/){
4154                 $devices->{$1} = 1;
4155         }
4156     }
4157
4158     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4159     foreach my $mice (@$resmice) {
4160         if ($mice->{name} eq 'QEMU HID Tablet') {
4161             $devices->{tablet} = 1;
4162             last;
4163         }
4164     }
4165
4166     # for usb devices there is no query-usb
4167     # but we can iterate over the entries in
4168     # qom-list path=/machine/peripheral
4169     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4170     foreach my $per (@$resperipheral) {
4171         if ($per->{name} =~ m/^usb\d+$/) {
4172             $devices->{$per->{name}} = 1;
4173         }
4174     }
4175
4176     return $devices;
4177 }
4178
4179 sub vm_deviceplug {
4180     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4181
4182     my $q35 = machine_type_is_q35($conf);
4183
4184     my $devices_list = vm_devices_list($vmid);
4185     return 1 if defined($devices_list->{$deviceid});
4186
4187     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4188
4189     if ($deviceid eq 'tablet') {
4190
4191         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4192
4193     } elsif ($deviceid eq 'keyboard') {
4194
4195         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4196
4197     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4198
4199         die "usb hotplug currently not reliable\n";
4200         # since we can't reliably hot unplug all added usb devices
4201         # and usb passthrough disables live migration
4202         # we disable usb hotplugging for now
4203         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4204
4205     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4206
4207         qemu_iothread_add($vmid, $deviceid, $device);
4208
4209         qemu_driveadd($storecfg, $vmid, $device);
4210         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4211
4212         qemu_deviceadd($vmid, $devicefull);
4213         eval { qemu_deviceaddverify($vmid, $deviceid); };
4214         if (my $err = $@) {
4215             eval { qemu_drivedel($vmid, $deviceid); };
4216             warn $@ if $@;
4217             die $err;
4218         }
4219
4220     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4221
4222
4223         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4224         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4225         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4226
4227         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4228
4229         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4230             qemu_iothread_add($vmid, $deviceid, $device);
4231             $devicefull .= ",iothread=iothread-$deviceid";
4232         }
4233
4234         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4235             $devicefull .= ",num_queues=$device->{queues}";
4236         }
4237
4238         qemu_deviceadd($vmid, $devicefull);
4239         qemu_deviceaddverify($vmid, $deviceid);
4240
4241     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4242
4243         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4244         qemu_driveadd($storecfg, $vmid, $device);
4245
4246         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4247         eval { qemu_deviceadd($vmid, $devicefull); };
4248         if (my $err = $@) {
4249             eval { qemu_drivedel($vmid, $deviceid); };
4250             warn $@ if $@;
4251             die $err;
4252         }
4253
4254     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4255
4256         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4257
4258         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4259         my $use_old_bios_files = undef;
4260         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4261
4262         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4263         qemu_deviceadd($vmid, $netdevicefull);
4264         eval {
4265             qemu_deviceaddverify($vmid, $deviceid);
4266             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4267         };
4268         if (my $err = $@) {
4269             eval { qemu_netdevdel($vmid, $deviceid); };
4270             warn $@ if $@;
4271             die $err;
4272         }
4273
4274     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4275
4276         my $bridgeid = $2;
4277         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4278         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4279
4280         qemu_deviceadd($vmid, $devicefull);
4281         qemu_deviceaddverify($vmid, $deviceid);
4282
4283     } else {
4284         die "can't hotplug device '$deviceid'\n";
4285     }
4286
4287     return 1;
4288 }
4289
4290 # fixme: this should raise exceptions on error!
4291 sub vm_deviceunplug {
4292     my ($vmid, $conf, $deviceid) = @_;
4293
4294     my $devices_list = vm_devices_list($vmid);
4295     return 1 if !defined($devices_list->{$deviceid});
4296
4297     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4298
4299     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4300
4301         qemu_devicedel($vmid, $deviceid);
4302
4303     } elsif ($deviceid =~ m/^usb\d+$/) {
4304
4305         die "usb hotplug currently not reliable\n";
4306         # when unplugging usb devices this way,
4307         # there may be remaining usb controllers/hubs
4308         # so we disable it for now
4309         qemu_devicedel($vmid, $deviceid);
4310         qemu_devicedelverify($vmid, $deviceid);
4311
4312     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4313
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_devicedelverify($vmid, $deviceid);
4316         qemu_drivedel($vmid, $deviceid);
4317         qemu_iothread_del($conf, $vmid, $deviceid);
4318
4319     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4320
4321         qemu_devicedel($vmid, $deviceid);
4322         qemu_devicedelverify($vmid, $deviceid);
4323         qemu_iothread_del($conf, $vmid, $deviceid);
4324
4325     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4326
4327         qemu_devicedel($vmid, $deviceid);
4328         qemu_drivedel($vmid, $deviceid);
4329         qemu_deletescsihw($conf, $vmid, $deviceid);
4330
4331     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4332
4333         qemu_devicedel($vmid, $deviceid);
4334         qemu_devicedelverify($vmid, $deviceid);
4335         qemu_netdevdel($vmid, $deviceid);
4336
4337     } else {
4338         die "can't unplug device '$deviceid'\n";
4339     }
4340
4341     return 1;
4342 }
4343
4344 sub qemu_deviceadd {
4345     my ($vmid, $devicefull) = @_;
4346
4347     $devicefull = "driver=".$devicefull;
4348     my %options =  split(/[=,]/, $devicefull);
4349
4350     vm_mon_cmd($vmid, "device_add" , %options);
4351 }
4352
4353 sub qemu_devicedel {
4354     my ($vmid, $deviceid) = @_;
4355
4356     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4357 }
4358
4359 sub qemu_iothread_add {
4360     my($vmid, $deviceid, $device) = @_;
4361
4362     if ($device->{iothread}) {
4363         my $iothreads = vm_iothreads_list($vmid);
4364         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4365     }
4366 }
4367
4368 sub qemu_iothread_del {
4369     my($conf, $vmid, $deviceid) = @_;
4370
4371     my $confid = $deviceid;
4372     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4373         $confid = 'scsi' . $1;
4374     }
4375     my $device = parse_drive($confid, $conf->{$confid});
4376     if ($device->{iothread}) {
4377         my $iothreads = vm_iothreads_list($vmid);
4378         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4379     }
4380 }
4381
4382 sub qemu_objectadd {
4383     my($vmid, $objectid, $qomtype) = @_;
4384
4385     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4386
4387     return 1;
4388 }
4389
4390 sub qemu_objectdel {
4391     my($vmid, $objectid) = @_;
4392
4393     vm_mon_cmd($vmid, "object-del", id => $objectid);
4394
4395     return 1;
4396 }
4397
4398 sub qemu_driveadd {
4399     my ($storecfg, $vmid, $device) = @_;
4400
4401     my $drive = print_drive_full($storecfg, $vmid, $device);
4402     $drive =~ s/\\/\\\\/g;
4403     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4404
4405     # If the command succeeds qemu prints: "OK"
4406     return 1 if $ret =~ m/OK/s;
4407
4408     die "adding drive failed: $ret\n";
4409 }
4410
4411 sub qemu_drivedel {
4412     my($vmid, $deviceid) = @_;
4413
4414     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4415     $ret =~ s/^\s+//;
4416
4417     return 1 if $ret eq "";
4418
4419     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4420     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4421
4422     die "deleting drive $deviceid failed : $ret\n";
4423 }
4424
4425 sub qemu_deviceaddverify {
4426     my ($vmid, $deviceid) = @_;
4427
4428     for (my $i = 0; $i <= 5; $i++) {
4429          my $devices_list = vm_devices_list($vmid);
4430          return 1 if defined($devices_list->{$deviceid});
4431          sleep 1;
4432     }
4433
4434     die "error on hotplug device '$deviceid'\n";
4435 }
4436
4437
4438 sub qemu_devicedelverify {
4439     my ($vmid, $deviceid) = @_;
4440
4441     # need to verify that the device is correctly removed as device_del
4442     # is async and empty return is not reliable
4443
4444     for (my $i = 0; $i <= 5; $i++) {
4445          my $devices_list = vm_devices_list($vmid);
4446          return 1 if !defined($devices_list->{$deviceid});
4447          sleep 1;
4448     }
4449
4450     die "error on hot-unplugging device '$deviceid'\n";
4451 }
4452
4453 sub qemu_findorcreatescsihw {
4454     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $scsihwid="$controller_prefix$controller";
4459     my $devices_list = vm_devices_list($vmid);
4460
4461     if(!defined($devices_list->{$scsihwid})) {
4462         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4463     }
4464
4465     return 1;
4466 }
4467
4468 sub qemu_deletescsihw {
4469     my ($conf, $vmid, $opt) = @_;
4470
4471     my $device = parse_drive($opt, $conf->{$opt});
4472
4473     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4474         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4475         return 1;
4476     }
4477
4478     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4479
4480     my $devices_list = vm_devices_list($vmid);
4481     foreach my $opt (keys %{$devices_list}) {
4482         if (PVE::QemuServer::is_valid_drivename($opt)) {
4483             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4484             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4485                 return 1;
4486             }
4487         }
4488     }
4489
4490     my $scsihwid="scsihw$controller";
4491
4492     vm_deviceunplug($vmid, $conf, $scsihwid);
4493
4494     return 1;
4495 }
4496
4497 sub qemu_add_pci_bridge {
4498     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4499
4500     my $bridges = {};
4501
4502     my $bridgeid;
4503
4504     print_pci_addr($device, $bridges, $arch, $machine_type);
4505
4506     while (my ($k, $v) = each %$bridges) {
4507         $bridgeid = $k;
4508     }
4509     return 1 if !defined($bridgeid) || $bridgeid < 1;
4510
4511     my $bridge = "pci.$bridgeid";
4512     my $devices_list = vm_devices_list($vmid);
4513
4514     if (!defined($devices_list->{$bridge})) {
4515         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4516     }
4517
4518     return 1;
4519 }
4520
4521 sub qemu_set_link_status {
4522     my ($vmid, $device, $up) = @_;
4523
4524     vm_mon_cmd($vmid, "set_link", name => $device,
4525                up => $up ? JSON::true : JSON::false);
4526 }
4527
4528 sub qemu_netdevadd {
4529     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4530
4531     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4532     my %options =  split(/[=,]/, $netdev);
4533
4534     vm_mon_cmd($vmid, "netdev_add",  %options);
4535     return 1;
4536 }
4537
4538 sub qemu_netdevdel {
4539     my ($vmid, $deviceid) = @_;
4540
4541     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4542 }
4543
4544 sub qemu_usb_hotplug {
4545     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4546
4547     return if !$device;
4548
4549     # remove the old one first
4550     vm_deviceunplug($vmid, $conf, $deviceid);
4551
4552     # check if xhci controller is necessary and available
4553     if ($device->{usb3}) {
4554
4555         my $devicelist = vm_devices_list($vmid);
4556
4557         if (!$devicelist->{xhci}) {
4558             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4559             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4560         }
4561     }
4562     my $d = parse_usb_device($device->{host});
4563     $d->{usb3} = $device->{usb3};
4564
4565     # add the new one
4566     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4567 }
4568
4569 sub qemu_cpu_hotplug {
4570     my ($vmid, $conf, $vcpus) = @_;
4571
4572     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4573
4574     my $sockets = 1;
4575     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4576     $sockets = $conf->{sockets} if  $conf->{sockets};
4577     my $cores = $conf->{cores} || 1;
4578     my $maxcpus = $sockets * $cores;
4579
4580     $vcpus = $maxcpus if !$vcpus;
4581
4582     die "you can't add more vcpus than maxcpus\n"
4583         if $vcpus > $maxcpus;
4584
4585     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4586
4587     if ($vcpus < $currentvcpus) {
4588
4589         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4590
4591             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4592                 qemu_devicedel($vmid, "cpu$i");
4593                 my $retry = 0;
4594                 my $currentrunningvcpus = undef;
4595                 while (1) {
4596                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4597                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4598                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4599                     $retry++;
4600                     sleep 1;
4601                 }
4602                 #update conf after each succesfull cpu unplug
4603                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4604                 PVE::QemuConfig->write_config($vmid, $conf);
4605             }
4606         } else {
4607             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4608         }
4609
4610         return;
4611     }
4612
4613     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4614     die "vcpus in running vm does not match its configuration\n"
4615         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4616
4617     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4618
4619         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4620             my $cpustr = print_cpu_device($conf, $i);
4621             qemu_deviceadd($vmid, $cpustr);
4622
4623             my $retry = 0;
4624             my $currentrunningvcpus = undef;
4625             while (1) {
4626                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4627                 last if scalar(@{$currentrunningvcpus}) == $i;
4628                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4629                 sleep 1;
4630                 $retry++;
4631             }
4632             #update conf after each succesfull cpu hotplug
4633             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4634             PVE::QemuConfig->write_config($vmid, $conf);
4635         }
4636     } else {
4637
4638         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4639             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4640         }
4641     }
4642 }
4643
4644 sub qemu_block_set_io_throttle {
4645     my ($vmid, $deviceid,
4646         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4647         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4648         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4649         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4650
4651     return if !check_running($vmid) ;
4652
4653     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4654         bps => int($bps),
4655         bps_rd => int($bps_rd),
4656         bps_wr => int($bps_wr),
4657         iops => int($iops),
4658         iops_rd => int($iops_rd),
4659         iops_wr => int($iops_wr),
4660         bps_max => int($bps_max),
4661         bps_rd_max => int($bps_rd_max),
4662         bps_wr_max => int($bps_wr_max),
4663         iops_max => int($iops_max),
4664         iops_rd_max => int($iops_rd_max),
4665         iops_wr_max => int($iops_wr_max),
4666         bps_max_length => int($bps_max_length),
4667         bps_rd_max_length => int($bps_rd_max_length),
4668         bps_wr_max_length => int($bps_wr_max_length),
4669         iops_max_length => int($iops_max_length),
4670         iops_rd_max_length => int($iops_rd_max_length),
4671         iops_wr_max_length => int($iops_wr_max_length),
4672     );
4673
4674 }
4675
4676 # old code, only used to shutdown old VM after update
4677 sub __read_avail {
4678     my ($fh, $timeout) = @_;
4679
4680     my $sel = new IO::Select;
4681     $sel->add($fh);
4682
4683     my $res = '';
4684     my $buf;
4685
4686     my @ready;
4687     while (scalar (@ready = $sel->can_read($timeout))) {
4688         my $count;
4689         if ($count = $fh->sysread($buf, 8192)) {
4690             if ($buf =~ /^(.*)\(qemu\) $/s) {
4691                 $res .= $1;
4692                 last;
4693             } else {
4694                 $res .= $buf;
4695             }
4696         } else {
4697             if (!defined($count)) {
4698                 die "$!\n";
4699             }
4700             last;
4701         }
4702     }
4703
4704     die "monitor read timeout\n" if !scalar(@ready);
4705
4706     return $res;
4707 }
4708
4709 sub qemu_block_resize {
4710     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4711
4712     my $running = check_running($vmid);
4713
4714     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4715
4716     return if !$running;
4717
4718     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4719
4720 }
4721
4722 sub qemu_volume_snapshot {
4723     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4724
4725     my $running = check_running($vmid);
4726
4727     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4728         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4729     } else {
4730         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4731     }
4732 }
4733
4734 sub qemu_volume_snapshot_delete {
4735     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4736
4737     my $running = check_running($vmid);
4738
4739     if($running) {
4740
4741         $running = undef;
4742         my $conf = PVE::QemuConfig->load_config($vmid);
4743         foreach_drive($conf, sub {
4744             my ($ds, $drive) = @_;
4745             $running = 1 if $drive->{file} eq $volid;
4746         });
4747     }
4748
4749     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4750         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4751     } else {
4752         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4753     }
4754 }
4755
4756 sub set_migration_caps {
4757     my ($vmid) = @_;
4758
4759     my $cap_ref = [];
4760
4761     my $enabled_cap = {
4762         "auto-converge" => 1,
4763         "xbzrle" => 1,
4764         "x-rdma-pin-all" => 0,
4765         "zero-blocks" => 0,
4766         "compress" => 0
4767     };
4768
4769     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4770
4771     for my $supported_capability (@$supported_capabilities) {
4772         push @$cap_ref, {
4773             capability => $supported_capability->{capability},
4774             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4775         };
4776     }
4777
4778     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4779 }
4780
4781 my $fast_plug_option = {
4782     'lock' => 1,
4783     'name' => 1,
4784     'onboot' => 1,
4785     'shares' => 1,
4786     'startup' => 1,
4787     'description' => 1,
4788     'protection' => 1,
4789     'vmstatestorage' => 1,
4790     'hookscript' => 1,
4791 };
4792
4793 # hotplug changes in [PENDING]
4794 # $selection hash can be used to only apply specified options, for
4795 # example: { cores => 1 } (only apply changed 'cores')
4796 # $errors ref is used to return error messages
4797 sub vmconfig_hotplug_pending {
4798     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4799
4800     my $defaults = load_defaults();
4801     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4802
4803     # commit values which do not have any impact on running VM first
4804     # Note: those option cannot raise errors, we we do not care about
4805     # $selection and always apply them.
4806
4807     my $add_error = sub {
4808         my ($opt, $msg) = @_;
4809         $errors->{$opt} = "hotplug problem - $msg";
4810     };
4811
4812     my $changes = 0;
4813     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4814         if ($fast_plug_option->{$opt}) {
4815             $conf->{$opt} = $conf->{pending}->{$opt};
4816             delete $conf->{pending}->{$opt};
4817             $changes = 1;
4818         }
4819     }
4820
4821     if ($changes) {
4822         PVE::QemuConfig->write_config($vmid, $conf);
4823         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4824     }
4825
4826     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4827
4828     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4829     while (my ($opt, $force) = each %$pending_delete_hash) {
4830         next if $selection && !$selection->{$opt};
4831         eval {
4832             if ($opt eq 'hotplug') {
4833                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4834             } elsif ($opt eq 'tablet') {
4835                 die "skip\n" if !$hotplug_features->{usb};
4836                 if ($defaults->{tablet}) {
4837                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4838                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4839                         if $arch eq 'aarch64';
4840                 } else {
4841                     vm_deviceunplug($vmid, $conf, 'tablet');
4842                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4843                 }
4844             } elsif ($opt =~ m/^usb\d+/) {
4845                 die "skip\n";
4846                 # since we cannot reliably hot unplug usb devices
4847                 # we are disabling it
4848                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4849                 vm_deviceunplug($vmid, $conf, $opt);
4850             } elsif ($opt eq 'vcpus') {
4851                 die "skip\n" if !$hotplug_features->{cpu};
4852                 qemu_cpu_hotplug($vmid, $conf, undef);
4853             } elsif ($opt eq 'balloon') {
4854                 # enable balloon device is not hotpluggable
4855                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4856                 # here we reset the ballooning value to memory
4857                 my $balloon = $conf->{memory} || $defaults->{memory};
4858                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4859             } elsif ($fast_plug_option->{$opt}) {
4860                 # do nothing
4861             } elsif ($opt =~ m/^net(\d+)$/) {
4862                 die "skip\n" if !$hotplug_features->{network};
4863                 vm_deviceunplug($vmid, $conf, $opt);
4864             } elsif (is_valid_drivename($opt)) {
4865                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4866                 vm_deviceunplug($vmid, $conf, $opt);
4867                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4868             } elsif ($opt =~ m/^memory$/) {
4869                 die "skip\n" if !$hotplug_features->{memory};
4870                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4871             } elsif ($opt eq 'cpuunits') {
4872                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4873             } elsif ($opt eq 'cpulimit') {
4874                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4875             } else {
4876                 die "skip\n";
4877             }
4878         };
4879         if (my $err = $@) {
4880             &$add_error($opt, $err) if $err ne "skip\n";
4881         } else {
4882             # save new config if hotplug was successful
4883             delete $conf->{$opt};
4884             vmconfig_undelete_pending_option($conf, $opt);
4885             PVE::QemuConfig->write_config($vmid, $conf);
4886             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4887         }
4888     }
4889
4890     my $apply_pending_cloudinit;
4891     $apply_pending_cloudinit = sub {
4892         my ($key, $value) = @_;
4893         $apply_pending_cloudinit = sub {}; # once is enough
4894
4895         my @cloudinit_opts = keys %$confdesc_cloudinit;
4896         foreach my $opt (keys %{$conf->{pending}}) {
4897             next if !grep { $_ eq $opt } @cloudinit_opts;
4898             $conf->{$opt} = delete $conf->{pending}->{$opt};
4899         }
4900
4901         my $new_conf = { %$conf };
4902         $new_conf->{$key} = $value;
4903         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4904     };
4905
4906     foreach my $opt (keys %{$conf->{pending}}) {
4907         next if $selection && !$selection->{$opt};
4908         my $value = $conf->{pending}->{$opt};
4909         eval {
4910             if ($opt eq 'hotplug') {
4911                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4912             } elsif ($opt eq 'tablet') {
4913                 die "skip\n" if !$hotplug_features->{usb};
4914                 if ($value == 1) {
4915                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4916                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4917                         if $arch eq 'aarch64';
4918                 } elsif ($value == 0) {
4919                     vm_deviceunplug($vmid, $conf, 'tablet');
4920                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4921                 }
4922             } elsif ($opt =~ m/^usb\d+$/) {
4923                 die "skip\n";
4924                 # since we cannot reliably hot unplug usb devices
4925                 # we are disabling it
4926                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4927                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4928                 die "skip\n" if !$d;
4929                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4930             } elsif ($opt eq 'vcpus') {
4931                 die "skip\n" if !$hotplug_features->{cpu};
4932                 qemu_cpu_hotplug($vmid, $conf, $value);
4933             } elsif ($opt eq 'balloon') {
4934                 # enable/disable balloning device is not hotpluggable
4935                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4936                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4937                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4938
4939                 # allow manual ballooning if shares is set to zero
4940                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4941                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4942                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4943                 }
4944             } elsif ($opt =~ m/^net(\d+)$/) {
4945                 # some changes can be done without hotplug
4946                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4947                                     $vmid, $opt, $value, $arch, $machine_type);
4948             } elsif (is_valid_drivename($opt)) {
4949                 # some changes can be done without hotplug
4950                 my $drive = parse_drive($opt, $value);
4951                 if (drive_is_cloudinit($drive)) {
4952                     &$apply_pending_cloudinit($opt, $value);
4953                 }
4954                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4955                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4956             } elsif ($opt =~ m/^memory$/) { #dimms
4957                 die "skip\n" if !$hotplug_features->{memory};
4958                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4959             } elsif ($opt eq 'cpuunits') {
4960                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4961             } elsif ($opt eq 'cpulimit') {
4962                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4963                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4964             } else {
4965                 die "skip\n";  # skip non-hot-pluggable options
4966             }
4967         };
4968         if (my $err = $@) {
4969             &$add_error($opt, $err) if $err ne "skip\n";
4970         } else {
4971             # save new config if hotplug was successful
4972             $conf->{$opt} = $value;
4973             delete $conf->{pending}->{$opt};
4974             PVE::QemuConfig->write_config($vmid, $conf);
4975             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4976         }
4977     }
4978 }
4979
4980 sub try_deallocate_drive {
4981     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4982
4983     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4984         my $volid = $drive->{file};
4985         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4986             my $sid = PVE::Storage::parse_volume_id($volid);
4987             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4988
4989             # check if the disk is really unused
4990             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4991                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4992             PVE::Storage::vdisk_free($storecfg, $volid);
4993             return 1;
4994         } else {
4995             # If vm is not owner of this disk remove from config
4996             return 1;
4997         }
4998     }
4999
5000     return undef;
5001 }
5002
5003 sub vmconfig_delete_or_detach_drive {
5004     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5005
5006     my $drive = parse_drive($opt, $conf->{$opt});
5007
5008     my $rpcenv = PVE::RPCEnvironment::get();
5009     my $authuser = $rpcenv->get_user();
5010
5011     if ($force) {
5012         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5013         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5014     } else {
5015         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5016     }
5017 }
5018
5019 sub vmconfig_apply_pending {
5020     my ($vmid, $conf, $storecfg) = @_;
5021
5022     # cold plug
5023
5024     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5025     while (my ($opt, $force) = each %$pending_delete_hash) {
5026         die "internal error" if $opt =~ m/^unused/;
5027         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5028         if (!defined($conf->{$opt})) {
5029             vmconfig_undelete_pending_option($conf, $opt);
5030             PVE::QemuConfig->write_config($vmid, $conf);
5031         } elsif (is_valid_drivename($opt)) {
5032             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5033             vmconfig_undelete_pending_option($conf, $opt);
5034             delete $conf->{$opt};
5035             PVE::QemuConfig->write_config($vmid, $conf);
5036         } else {
5037             vmconfig_undelete_pending_option($conf, $opt);
5038             delete $conf->{$opt};
5039             PVE::QemuConfig->write_config($vmid, $conf);
5040         }
5041     }
5042
5043     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5044
5045     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5046         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5047
5048         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5049             # skip if nothing changed
5050         } elsif (is_valid_drivename($opt)) {
5051             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5052                 if defined($conf->{$opt});
5053             $conf->{$opt} = $conf->{pending}->{$opt};
5054         } else {
5055             $conf->{$opt} = $conf->{pending}->{$opt};
5056         }
5057
5058         delete $conf->{pending}->{$opt};
5059         PVE::QemuConfig->write_config($vmid, $conf);
5060     }
5061 }
5062
5063 my $safe_num_ne = sub {
5064     my ($a, $b) = @_;
5065
5066     return 0 if !defined($a) && !defined($b);
5067     return 1 if !defined($a);
5068     return 1 if !defined($b);
5069
5070     return $a != $b;
5071 };
5072
5073 my $safe_string_ne = sub {
5074     my ($a, $b) = @_;
5075
5076     return 0 if !defined($a) && !defined($b);
5077     return 1 if !defined($a);
5078     return 1 if !defined($b);
5079
5080     return $a ne $b;
5081 };
5082
5083 sub vmconfig_update_net {
5084     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5085
5086     my $newnet = parse_net($value);
5087
5088     if ($conf->{$opt}) {
5089         my $oldnet = parse_net($conf->{$opt});
5090
5091         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5092             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5093             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5094             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5095
5096             # for non online change, we try to hot-unplug
5097             die "skip\n" if !$hotplug;
5098             vm_deviceunplug($vmid, $conf, $opt);
5099         } else {
5100
5101             die "internal error" if $opt !~ m/net(\d+)/;
5102             my $iface = "tap${vmid}i$1";
5103
5104             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5105                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5106                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5107                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5108                 PVE::Network::tap_unplug($iface);
5109                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5110             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5111                 # Rate can be applied on its own but any change above needs to
5112                 # include the rate in tap_plug since OVS resets everything.
5113                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5114             }
5115
5116             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5117                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5118             }
5119
5120             return 1;
5121         }
5122     }
5123
5124     if ($hotplug) {
5125         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5126     } else {
5127         die "skip\n";
5128     }
5129 }
5130
5131 sub vmconfig_update_disk {
5132     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5133
5134     # fixme: do we need force?
5135
5136     my $drive = parse_drive($opt, $value);
5137
5138     if ($conf->{$opt}) {
5139
5140         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5141
5142             my $media = $drive->{media} || 'disk';
5143             my $oldmedia = $old_drive->{media} || 'disk';
5144             die "unable to change media type\n" if $media ne $oldmedia;
5145
5146             if (!drive_is_cdrom($old_drive)) {
5147
5148                 if ($drive->{file} ne $old_drive->{file}) {
5149
5150                     die "skip\n" if !$hotplug;
5151
5152                     # unplug and register as unused
5153                     vm_deviceunplug($vmid, $conf, $opt);
5154                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5155
5156                 } else {
5157                     # update existing disk
5158
5159                     # skip non hotpluggable value
5160                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5161                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5162                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5163                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5164                         die "skip\n";
5165                     }
5166
5167                     # apply throttle
5168                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5169                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5170                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5171                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5172                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5173                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5174                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5175                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5176                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5177                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5178                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5179                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5180                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5181                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5182                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5183                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5184                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5185                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5186
5187                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5188                                                    ($drive->{mbps} || 0)*1024*1024,
5189                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5190                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5191                                                    $drive->{iops} || 0,
5192                                                    $drive->{iops_rd} || 0,
5193                                                    $drive->{iops_wr} || 0,
5194                                                    ($drive->{mbps_max} || 0)*1024*1024,
5195                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5196                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5197                                                    $drive->{iops_max} || 0,
5198                                                    $drive->{iops_rd_max} || 0,
5199                                                    $drive->{iops_wr_max} || 0,
5200                                                    $drive->{bps_max_length} || 1,
5201                                                    $drive->{bps_rd_max_length} || 1,
5202                                                    $drive->{bps_wr_max_length} || 1,
5203                                                    $drive->{iops_max_length} || 1,
5204                                                    $drive->{iops_rd_max_length} || 1,
5205                                                    $drive->{iops_wr_max_length} || 1);
5206
5207                     }
5208
5209                     return 1;
5210                 }
5211
5212             } else { # cdrom
5213
5214                 if ($drive->{file} eq 'none') {
5215                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5216                     if (drive_is_cloudinit($old_drive)) {
5217                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5218                     }
5219                 } else {
5220                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5221                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5222                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5223                 }
5224
5225                 return 1;
5226             }
5227         }
5228     }
5229
5230     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5231     # hotplug new disks
5232     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5233     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5234 }
5235
5236 sub vm_start {
5237     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5238         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5239
5240     PVE::QemuConfig->lock_config($vmid, sub {
5241         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5242
5243         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5244
5245         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5246
5247         PVE::QemuConfig->check_lock($conf)
5248             if !($skiplock || $is_suspended);
5249
5250         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5251
5252         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5253             vmconfig_apply_pending($vmid, $conf, $storecfg);
5254             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5255         }
5256
5257         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5258
5259         my $defaults = load_defaults();
5260
5261         # set environment variable useful inside network script
5262         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5263
5264         my $local_volumes = {};
5265
5266         if ($targetstorage) {
5267             foreach_drive($conf, sub {
5268                 my ($ds, $drive) = @_;
5269
5270                 return if drive_is_cdrom($drive);
5271
5272                 my $volid = $drive->{file};
5273
5274                 return if !$volid;
5275
5276                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5277
5278                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5279                 return if $scfg->{shared};
5280                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5281             });
5282
5283             my $format = undef;
5284
5285             foreach my $opt (sort keys %$local_volumes) {
5286
5287                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5288                 my $drive = parse_drive($opt, $conf->{$opt});
5289
5290                 #if remote storage is specified, use default format
5291                 if ($targetstorage && $targetstorage ne "1") {
5292                     $storeid = $targetstorage;
5293                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5294                     $format = $defFormat;
5295                 } else {
5296                     #else we use same format than original
5297                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5298                     $format = qemu_img_format($scfg, $volid);
5299                 }
5300
5301                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5302                 my $newdrive = $drive;
5303                 $newdrive->{format} = $format;
5304                 $newdrive->{file} = $newvolid;
5305                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5306                 $local_volumes->{$opt} = $drivestr;
5307                 #pass drive to conf for command line
5308                 $conf->{$opt} = $drivestr;
5309             }
5310         }
5311
5312         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5313
5314         if ($is_suspended) {
5315             # enforce machine type on suspended vm to ensure HW compatibility
5316             $forcemachine = $conf->{runningmachine};
5317             print "Resuming suspended VM\n";
5318         }
5319
5320         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5321
5322         my $migrate_port = 0;
5323         my $migrate_uri;
5324         if ($statefile) {
5325             if ($statefile eq 'tcp') {
5326                 my $localip = "localhost";
5327                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5328                 my $nodename = PVE::INotify::nodename();
5329
5330                 if (!defined($migration_type)) {
5331                     if (defined($datacenterconf->{migration}->{type})) {
5332                         $migration_type = $datacenterconf->{migration}->{type};
5333                     } else {
5334                         $migration_type = 'secure';
5335                     }
5336                 }
5337
5338                 if ($migration_type eq 'insecure') {
5339                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5340                     if ($migrate_network_addr) {
5341                         $localip = $migrate_network_addr;
5342                     } else {
5343                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5344                     }
5345
5346                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5347                 }
5348
5349                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5350                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5351                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5352                 push @$cmd, '-incoming', $migrate_uri;
5353                 push @$cmd, '-S';
5354
5355             } elsif ($statefile eq 'unix') {
5356                 # should be default for secure migrations as a ssh TCP forward
5357                 # tunnel is not deterministic reliable ready and fails regurarly
5358                 # to set up in time, so use UNIX socket forwards
5359                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5360                 unlink $socket_addr;
5361
5362                 $migrate_uri = "unix:$socket_addr";
5363
5364                 push @$cmd, '-incoming', $migrate_uri;
5365                 push @$cmd, '-S';
5366
5367             } else {
5368                 push @$cmd, '-loadstate', $statefile;
5369             }
5370         } elsif ($paused) {
5371             push @$cmd, '-S';
5372         }
5373
5374         # host pci devices
5375         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5376           my $d = parse_hostpci($conf->{"hostpci$i"});
5377           next if !$d;
5378           my $pcidevices = $d->{pciid};
5379           foreach my $pcidevice (@$pcidevices) {
5380                 my $pciid = $pcidevice->{id};
5381
5382                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5383                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5384                 die "no pci device info for device '$pciid'\n" if !$info;
5385
5386                 if ($d->{mdev}) {
5387                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5388                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5389                 } else {
5390                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5391                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5392                     die "can't reset pci device '$pciid'\n"
5393                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5394                 }
5395           }
5396         }
5397
5398         PVE::Storage::activate_volumes($storecfg, $vollist);
5399
5400         eval {
5401             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5402                 outfunc => sub {}, errfunc => sub {});
5403         };
5404         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5405         # timeout should be more than enough here...
5406         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5407
5408         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5409                                                   : $defaults->{cpuunits};
5410
5411         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5412         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5413
5414         my %properties = (
5415             Slice => 'qemu.slice',
5416             KillMode => 'none',
5417             CPUShares => $cpuunits
5418         );
5419
5420         if (my $cpulimit = $conf->{cpulimit}) {
5421             $properties{CPUQuota} = int($cpulimit * 100);
5422         }
5423         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5424
5425         my $run_qemu = sub {
5426             PVE::Tools::run_fork sub {
5427                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5428                 run_command($cmd, %run_params);
5429             };
5430         };
5431
5432         if ($conf->{hugepages}) {
5433
5434             my $code = sub {
5435                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5436                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5437
5438                 PVE::QemuServer::Memory::hugepages_mount();
5439                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5440
5441                 eval { $run_qemu->() };
5442                 if (my $err = $@) {
5443                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5444                     die $err;
5445                 }
5446
5447                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5448             };
5449             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5450
5451         } else {
5452             eval { $run_qemu->() };
5453         }
5454
5455         if (my $err = $@) {
5456             # deactivate volumes if start fails
5457             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5458             die "start failed: $err";
5459         }
5460
5461         print "migration listens on $migrate_uri\n" if $migrate_uri;
5462
5463         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5464             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5465             warn $@ if $@;
5466         }
5467
5468         #start nbd server for storage migration
5469         if ($targetstorage) {
5470             my $nodename = PVE::INotify::nodename();
5471             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5472             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5473             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5474             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5475
5476             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5477
5478             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5479
5480             foreach my $opt (sort keys %$local_volumes) {
5481                 my $volid = $local_volumes->{$opt};
5482                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5483                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5484                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5485             }
5486         }
5487
5488         if ($migratedfrom) {
5489             eval {
5490                 set_migration_caps($vmid);
5491             };
5492             warn $@ if $@;
5493
5494             if ($spice_port) {
5495                 print "spice listens on port $spice_port\n";
5496                 if ($spice_ticket) {
5497                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5498                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5499                 }
5500             }
5501
5502         } else {
5503             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5504                 if !$statefile && $conf->{balloon};
5505
5506             foreach my $opt (keys %$conf) {
5507                 next if $opt !~  m/^net\d+$/;
5508                 my $nicconf = parse_net($conf->{$opt});
5509                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5510             }
5511         }
5512
5513         vm_mon_cmd_nocheck($vmid, 'qom-set',
5514                     path => "machine/peripheral/balloon0",
5515                     property => "guest-stats-polling-interval",
5516                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5517
5518         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5519             print "Resumed VM, removing state\n";
5520             delete $conf->@{qw(lock vmstate runningmachine)};
5521             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5522             PVE::Storage::vdisk_free($storecfg, $vmstate);
5523             PVE::QemuConfig->write_config($vmid, $conf);
5524         }
5525
5526         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5527     });
5528 }
5529
5530 sub vm_mon_cmd {
5531     my ($vmid, $execute, %params) = @_;
5532
5533     my $cmd = { execute => $execute, arguments => \%params };
5534     vm_qmp_command($vmid, $cmd);
5535 }
5536
5537 sub vm_mon_cmd_nocheck {
5538     my ($vmid, $execute, %params) = @_;
5539
5540     my $cmd = { execute => $execute, arguments => \%params };
5541     vm_qmp_command($vmid, $cmd, 1);
5542 }
5543
5544 sub vm_qmp_command {
5545     my ($vmid, $cmd, $nocheck) = @_;
5546
5547     my $res;
5548
5549     my $timeout;
5550     if ($cmd->{arguments}) {
5551         $timeout = delete $cmd->{arguments}->{timeout};
5552     }
5553
5554     eval {
5555         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5556         my $sname = qmp_socket($vmid);
5557         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5558             my $qmpclient = PVE::QMPClient->new();
5559
5560             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5561         } else {
5562             die "unable to open monitor socket\n";
5563         }
5564     };
5565     if (my $err = $@) {
5566         syslog("err", "VM $vmid qmp command failed - $err");
5567         die $err;
5568     }
5569
5570     return $res;
5571 }
5572
5573 sub vm_human_monitor_command {
5574     my ($vmid, $cmdline) = @_;
5575
5576     my $res;
5577
5578     my $cmd = {
5579         execute => 'human-monitor-command',
5580         arguments => { 'command-line' => $cmdline},
5581     };
5582
5583     return vm_qmp_command($vmid, $cmd);
5584 }
5585
5586 sub vm_commandline {
5587     my ($storecfg, $vmid, $snapname) = @_;
5588
5589     my $conf = PVE::QemuConfig->load_config($vmid);
5590
5591     if ($snapname) {
5592         my $snapshot = $conf->{snapshots}->{$snapname};
5593         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5594
5595         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5596
5597         $conf = $snapshot;
5598     }
5599
5600     my $defaults = load_defaults();
5601
5602     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5603
5604     return PVE::Tools::cmd2string($cmd);
5605 }
5606
5607 sub vm_reset {
5608     my ($vmid, $skiplock) = @_;
5609
5610     PVE::QemuConfig->lock_config($vmid, sub {
5611
5612         my $conf = PVE::QemuConfig->load_config($vmid);
5613
5614         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5615
5616         vm_mon_cmd($vmid, "system_reset");
5617     });
5618 }
5619
5620 sub get_vm_volumes {
5621     my ($conf) = @_;
5622
5623     my $vollist = [];
5624     foreach_volid($conf, sub {
5625         my ($volid, $attr) = @_;
5626
5627         return if $volid =~ m|^/|;
5628
5629         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5630         return if !$sid;
5631
5632         push @$vollist, $volid;
5633     });
5634
5635     return $vollist;
5636 }
5637
5638 sub vm_stop_cleanup {
5639     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5640
5641     eval {
5642
5643         if (!$keepActive) {
5644             my $vollist = get_vm_volumes($conf);
5645             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5646         }
5647
5648         foreach my $ext (qw(mon qmp pid vnc qga)) {
5649             unlink "/var/run/qemu-server/${vmid}.$ext";
5650         }
5651
5652         if ($conf->{ivshmem}) {
5653             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5654             # just delete it for now, VMs which have this already open do not
5655             # are affected, but new VMs will get a separated one. If this
5656             # becomes an issue we either add some sort of ref-counting or just
5657             # add a "don't delete on stop" flag to the ivshmem format.
5658             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5659         }
5660
5661         foreach my $key (keys %$conf) {
5662             next if $key !~ m/^hostpci(\d+)$/;
5663             my $hostpciindex = $1;
5664             my $d = parse_hostpci($conf->{$key});
5665             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5666
5667             foreach my $pci (@{$d->{pciid}}) {
5668                 my $pciid = $pci->{id};
5669                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5670             }
5671         }
5672
5673         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5674     };
5675     warn $@ if $@; # avoid errors - just warn
5676 }
5677
5678 # Note: use $nockeck to skip tests if VM configuration file exists.
5679 # We need that when migration VMs to other nodes (files already moved)
5680 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5681 sub vm_stop {
5682     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5683
5684     $force = 1 if !defined($force) && !$shutdown;
5685
5686     if ($migratedfrom){
5687         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5688         kill 15, $pid if $pid;
5689         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5690         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5691         return;
5692     }
5693
5694     PVE::QemuConfig->lock_config($vmid, sub {
5695
5696         my $pid = check_running($vmid, $nocheck);
5697         return if !$pid;
5698
5699         my $conf;
5700         if (!$nocheck) {
5701             $conf = PVE::QemuConfig->load_config($vmid);
5702             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5703             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5704                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5705                 $timeout = $opts->{down} if $opts->{down};
5706             }
5707             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5708         }
5709
5710         eval {
5711             if ($shutdown) {
5712                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5713                     vm_qmp_command($vmid, {
5714                         execute => "guest-shutdown",
5715                         arguments => { timeout => $timeout }
5716                     }, $nocheck);
5717                 } else {
5718                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5719                 }
5720             } else {
5721                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5722             }
5723         };
5724         my $err = $@;
5725
5726         if (!$err) {
5727             $timeout = 60 if !defined($timeout);
5728
5729             my $count = 0;
5730             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5731                 $count++;
5732                 sleep 1;
5733             }
5734
5735             if ($count >= $timeout) {
5736                 if ($force) {
5737                     warn "VM still running - terminating now with SIGTERM\n";
5738                     kill 15, $pid;
5739                 } else {
5740                     die "VM quit/powerdown failed - got timeout\n";
5741                 }
5742             } else {
5743                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5744                 return;
5745             }
5746         } else {
5747             if ($force) {
5748                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5749                 kill 15, $pid;
5750             } else {
5751                 die "VM quit/powerdown failed\n";
5752             }
5753         }
5754
5755         # wait again
5756         $timeout = 10;
5757
5758         my $count = 0;
5759         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5760             $count++;
5761             sleep 1;
5762         }
5763
5764         if ($count >= $timeout) {
5765             warn "VM still running - terminating now with SIGKILL\n";
5766             kill 9, $pid;
5767             sleep 1;
5768         }
5769
5770         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5771    });
5772 }
5773
5774 sub vm_suspend {
5775     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5776
5777     my $conf;
5778     my $path;
5779     my $storecfg;
5780     my $vmstate;
5781
5782     PVE::QemuConfig->lock_config($vmid, sub {
5783
5784         $conf = PVE::QemuConfig->load_config($vmid);
5785
5786         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5787         PVE::QemuConfig->check_lock($conf)
5788             if !($skiplock || $is_backing_up);
5789
5790         die "cannot suspend to disk during backup\n"
5791             if $is_backing_up && $includestate;
5792
5793         if ($includestate) {
5794             $conf->{lock} = 'suspending';
5795             my $date = strftime("%Y-%m-%d", localtime(time()));
5796             $storecfg = PVE::Storage::config();
5797             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5798             $path = PVE::Storage::path($storecfg, $vmstate);
5799             PVE::QemuConfig->write_config($vmid, $conf);
5800         } else {
5801             vm_mon_cmd($vmid, "stop");
5802         }
5803     });
5804
5805     if ($includestate) {
5806         # save vm state
5807         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5808
5809         eval {
5810             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5811             for(;;) {
5812                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5813                 if (!$state->{status}) {
5814                     die "savevm not active\n";
5815                 } elsif ($state->{status} eq 'active') {
5816                     sleep(1);
5817                     next;
5818                 } elsif ($state->{status} eq 'completed') {
5819                     print "State saved, quitting\n";
5820                     last;
5821                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5822                     die "query-savevm failed with error '$state->{error}'\n"
5823                 } else {
5824                     die "query-savevm returned status '$state->{status}'\n";
5825                 }
5826             }
5827         };
5828         my $err = $@;
5829
5830         PVE::QemuConfig->lock_config($vmid, sub {
5831             $conf = PVE::QemuConfig->load_config($vmid);
5832             if ($err) {
5833                 # cleanup, but leave suspending lock, to indicate something went wrong
5834                 eval {
5835                     vm_mon_cmd($vmid, "savevm-end");
5836                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5837                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5838                     delete $conf->@{qw(vmstate runningmachine)};
5839                     PVE::QemuConfig->write_config($vmid, $conf);
5840                 };
5841                 warn $@ if $@;
5842                 die $err;
5843             }
5844
5845             die "lock changed unexpectedly\n"
5846                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5847
5848             vm_qmp_command($vmid, { execute => "quit" });
5849             $conf->{lock} = 'suspended';
5850             PVE::QemuConfig->write_config($vmid, $conf);
5851         });
5852     }
5853 }
5854
5855 sub vm_resume {
5856     my ($vmid, $skiplock, $nocheck) = @_;
5857
5858     PVE::QemuConfig->lock_config($vmid, sub {
5859         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5860         my $res = $vm_mon_cmd->($vmid, 'query-status');
5861         my $resume_cmd = 'cont';
5862
5863         if ($res->{status} && $res->{status} eq 'suspended') {
5864             $resume_cmd = 'system_wakeup';
5865         }
5866
5867         if (!$nocheck) {
5868
5869             my $conf = PVE::QemuConfig->load_config($vmid);
5870
5871             PVE::QemuConfig->check_lock($conf)
5872                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5873         }
5874
5875         $vm_mon_cmd->($vmid, $resume_cmd);
5876     });
5877 }
5878
5879 sub vm_sendkey {
5880     my ($vmid, $skiplock, $key) = @_;
5881
5882     PVE::QemuConfig->lock_config($vmid, sub {
5883
5884         my $conf = PVE::QemuConfig->load_config($vmid);
5885
5886         # there is no qmp command, so we use the human monitor command
5887         vm_human_monitor_command($vmid, "sendkey $key");
5888     });
5889 }
5890
5891 sub vm_destroy {
5892     my ($storecfg, $vmid, $skiplock) = @_;
5893
5894     PVE::QemuConfig->lock_config($vmid, sub {
5895
5896         my $conf = PVE::QemuConfig->load_config($vmid);
5897
5898         if (!check_running($vmid)) {
5899             destroy_vm($storecfg, $vmid, undef, $skiplock);
5900         } else {
5901             die "VM $vmid is running - destroy failed\n";
5902         }
5903     });
5904 }
5905
5906 # vzdump restore implementaion
5907
5908 sub tar_archive_read_firstfile {
5909     my $archive = shift;
5910
5911     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5912
5913     # try to detect archive type first
5914     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5915         die "unable to open file '$archive'\n";
5916     my $firstfile = <$fh>;
5917     kill 15, $pid;
5918     close $fh;
5919
5920     die "ERROR: archive contaions no data\n" if !$firstfile;
5921     chomp $firstfile;
5922
5923     return $firstfile;
5924 }
5925
5926 sub tar_restore_cleanup {
5927     my ($storecfg, $statfile) = @_;
5928
5929     print STDERR "starting cleanup\n";
5930
5931     if (my $fd = IO::File->new($statfile, "r")) {
5932         while (defined(my $line = <$fd>)) {
5933             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5934                 my $volid = $2;
5935                 eval {
5936                     if ($volid =~ m|^/|) {
5937                         unlink $volid || die 'unlink failed\n';
5938                     } else {
5939                         PVE::Storage::vdisk_free($storecfg, $volid);
5940                     }
5941                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5942                 };
5943                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5944             } else {
5945                 print STDERR "unable to parse line in statfile - $line";
5946             }
5947         }
5948         $fd->close();
5949     }
5950 }
5951
5952 sub restore_archive {
5953     my ($archive, $vmid, $user, $opts) = @_;
5954
5955     my $format = $opts->{format};
5956     my $comp;
5957
5958     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5959         $format = 'tar' if !$format;
5960         $comp = 'gzip';
5961     } elsif ($archive =~ m/\.tar$/) {
5962         $format = 'tar' if !$format;
5963     } elsif ($archive =~ m/.tar.lzo$/) {
5964         $format = 'tar' if !$format;
5965         $comp = 'lzop';
5966     } elsif ($archive =~ m/\.vma$/) {
5967         $format = 'vma' if !$format;
5968     } elsif ($archive =~ m/\.vma\.gz$/) {
5969         $format = 'vma' if !$format;
5970         $comp = 'gzip';
5971     } elsif ($archive =~ m/\.vma\.lzo$/) {
5972         $format = 'vma' if !$format;
5973         $comp = 'lzop';
5974     } else {
5975         $format = 'vma' if !$format; # default
5976     }
5977
5978     # try to detect archive format
5979     if ($format eq 'tar') {
5980         return restore_tar_archive($archive, $vmid, $user, $opts);
5981     } else {
5982         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5983     }
5984 }
5985
5986 sub restore_update_config_line {
5987     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5988
5989     return if $line =~ m/^\#qmdump\#/;
5990     return if $line =~ m/^\#vzdump\#/;
5991     return if $line =~ m/^lock:/;
5992     return if $line =~ m/^unused\d+:/;
5993     return if $line =~ m/^parent:/;
5994
5995     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5996     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5997         # try to convert old 1.X settings
5998         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5999         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6000             my ($model, $macaddr) = split(/\=/, $devconfig);
6001             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6002             my $net = {
6003                 model => $model,
6004                 bridge => "vmbr$ind",
6005                 macaddr => $macaddr,
6006             };
6007             my $netstr = print_net($net);
6008
6009             print $outfd "net$cookie->{netcount}: $netstr\n";
6010             $cookie->{netcount}++;
6011         }
6012     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6013         my ($id, $netstr) = ($1, $2);
6014         my $net = parse_net($netstr);
6015         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6016         $netstr = print_net($net);
6017         print $outfd "$id: $netstr\n";
6018     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6019         my $virtdev = $1;
6020         my $value = $3;
6021         my $di = parse_drive($virtdev, $value);
6022         if (defined($di->{backup}) && !$di->{backup}) {
6023             print $outfd "#$line";
6024         } elsif ($map->{$virtdev}) {
6025             delete $di->{format}; # format can change on restore
6026             $di->{file} = $map->{$virtdev};
6027             $value = print_drive($vmid, $di);
6028             print $outfd "$virtdev: $value\n";
6029         } else {
6030             print $outfd $line;
6031         }
6032     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6033         my $vmgenid = $1;
6034         if ($vmgenid ne '0') {
6035             # always generate a new vmgenid if there was a valid one setup
6036             $vmgenid = generate_uuid();
6037         }
6038         print $outfd "vmgenid: $vmgenid\n";
6039     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6040         my ($uuid, $uuid_str);
6041         UUID::generate($uuid);
6042         UUID::unparse($uuid, $uuid_str);
6043         my $smbios1 = parse_smbios1($2);
6044         $smbios1->{uuid} = $uuid_str;
6045         print $outfd $1.print_smbios1($smbios1)."\n";
6046     } else {
6047         print $outfd $line;
6048     }
6049 }
6050
6051 sub scan_volids {
6052     my ($cfg, $vmid) = @_;
6053
6054     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6055
6056     my $volid_hash = {};
6057     foreach my $storeid (keys %$info) {
6058         foreach my $item (@{$info->{$storeid}}) {
6059             next if !($item->{volid} && $item->{size});
6060             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6061             $volid_hash->{$item->{volid}} = $item;
6062         }
6063     }
6064
6065     return $volid_hash;
6066 }
6067
6068 sub is_volume_in_use {
6069     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6070
6071     my $path = PVE::Storage::path($storecfg, $volid);
6072
6073     my $scan_config = sub {
6074         my ($cref, $snapname) = @_;
6075
6076         foreach my $key (keys %$cref) {
6077             my $value = $cref->{$key};
6078             if (is_valid_drivename($key)) {
6079                 next if $skip_drive && $key eq $skip_drive;
6080                 my $drive = parse_drive($key, $value);
6081                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6082                 return 1 if $volid eq $drive->{file};
6083                 if ($drive->{file} =~ m!^/!) {
6084                     return 1 if $drive->{file} eq $path;
6085                 } else {
6086                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6087                     next if !$storeid;
6088                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6089                     next if !$scfg;
6090                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6091                 }
6092             }
6093         }
6094
6095         return 0;
6096     };
6097
6098     return 1 if &$scan_config($conf);
6099
6100     undef $skip_drive;
6101
6102     foreach my $snapname (keys %{$conf->{snapshots}}) {
6103         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6104     }
6105
6106     return 0;
6107 }
6108
6109 sub update_disksize {
6110     my ($vmid, $conf, $volid_hash) = @_;
6111
6112     my $changes;
6113     my $prefix = "VM $vmid:";
6114
6115     # used and unused disks
6116     my $referenced = {};
6117
6118     # Note: it is allowed to define multiple storages with same path (alias), so
6119     # we need to check both 'volid' and real 'path' (two different volid can point
6120     # to the same path).
6121
6122     my $referencedpath = {};
6123
6124     # update size info
6125     foreach my $opt (keys %$conf) {
6126         if (is_valid_drivename($opt)) {
6127             my $drive = parse_drive($opt, $conf->{$opt});
6128             my $volid = $drive->{file};
6129             next if !$volid;
6130
6131             $referenced->{$volid} = 1;
6132             if ($volid_hash->{$volid} &&
6133                 (my $path = $volid_hash->{$volid}->{path})) {
6134                 $referencedpath->{$path} = 1;
6135             }
6136
6137             next if drive_is_cdrom($drive);
6138             next if !$volid_hash->{$volid};
6139
6140             $drive->{size} = $volid_hash->{$volid}->{size};
6141             my $new = print_drive($vmid, $drive);
6142             if ($new ne $conf->{$opt}) {
6143                 $changes = 1;
6144                 $conf->{$opt} = $new;
6145                 print "$prefix update disk '$opt' information.\n";
6146             }
6147         }
6148     }
6149
6150     # remove 'unusedX' entry if volume is used
6151     foreach my $opt (keys %$conf) {
6152         next if $opt !~ m/^unused\d+$/;
6153         my $volid = $conf->{$opt};
6154         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6155         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6156             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6157             $changes = 1;
6158             delete $conf->{$opt};
6159         }
6160
6161         $referenced->{$volid} = 1;
6162         $referencedpath->{$path} = 1 if $path;
6163     }
6164
6165     foreach my $volid (sort keys %$volid_hash) {
6166         next if $volid =~ m/vm-$vmid-state-/;
6167         next if $referenced->{$volid};
6168         my $path = $volid_hash->{$volid}->{path};
6169         next if !$path; # just to be sure
6170         next if $referencedpath->{$path};
6171         $changes = 1;
6172         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6173         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6174         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6175     }
6176
6177     return $changes;
6178 }
6179
6180 sub rescan {
6181     my ($vmid, $nolock, $dryrun) = @_;
6182
6183     my $cfg = PVE::Storage::config();
6184
6185     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6186     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6187     foreach my $stor (keys %{$cfg->{ids}}) {
6188         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6189     }
6190
6191     print "rescan volumes...\n";
6192     my $volid_hash = scan_volids($cfg, $vmid);
6193
6194     my $updatefn =  sub {
6195         my ($vmid) = @_;
6196
6197         my $conf = PVE::QemuConfig->load_config($vmid);
6198
6199         PVE::QemuConfig->check_lock($conf);
6200
6201         my $vm_volids = {};
6202         foreach my $volid (keys %$volid_hash) {
6203             my $info = $volid_hash->{$volid};
6204             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6205         }
6206
6207         my $changes = update_disksize($vmid, $conf, $vm_volids);
6208
6209         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6210     };
6211
6212     if (defined($vmid)) {
6213         if ($nolock) {
6214             &$updatefn($vmid);
6215         } else {
6216             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6217         }
6218     } else {
6219         my $vmlist = config_list();
6220         foreach my $vmid (keys %$vmlist) {
6221             if ($nolock) {
6222                 &$updatefn($vmid);
6223             } else {
6224                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6225             }
6226         }
6227     }
6228 }
6229
6230 sub restore_vma_archive {
6231     my ($archive, $vmid, $user, $opts, $comp) = @_;
6232
6233     my $readfrom = $archive;
6234
6235     my $cfg = PVE::Storage::config();
6236     my $commands = [];
6237     my $bwlimit = $opts->{bwlimit};
6238
6239     my $dbg_cmdstring = '';
6240     my $add_pipe = sub {
6241         my ($cmd) = @_;
6242         push @$commands, $cmd;
6243         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6244         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6245         $readfrom = '-';
6246     };
6247
6248     my $input = undef;
6249     if ($archive eq '-') {
6250         $input = '<&STDIN';
6251     } else {
6252         # If we use a backup from a PVE defined storage we also consider that
6253         # storage's rate limit:
6254         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6255         if (defined($volid)) {
6256             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6257             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6258             if ($readlimit) {
6259                 print STDERR "applying read rate limit: $readlimit\n";
6260                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6261                 $add_pipe->($cstream);
6262             }
6263         }
6264     }
6265
6266     if ($comp) {
6267         my $cmd;
6268         if ($comp eq 'gzip') {
6269             $cmd = ['zcat', $readfrom];
6270         } elsif ($comp eq 'lzop') {
6271             $cmd = ['lzop', '-d', '-c', $readfrom];
6272         } else {
6273             die "unknown compression method '$comp'\n";
6274         }
6275         $add_pipe->($cmd);
6276     }
6277
6278     my $tmpdir = "/var/tmp/vzdumptmp$$";
6279     rmtree $tmpdir;
6280
6281     # disable interrupts (always do cleanups)
6282     local $SIG{INT} =
6283         local $SIG{TERM} =
6284         local $SIG{QUIT} =
6285         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6286
6287     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6288     POSIX::mkfifo($mapfifo, 0600);
6289     my $fifofh;
6290
6291     my $openfifo = sub {
6292         open($fifofh, '>', $mapfifo) || die $!;
6293     };
6294
6295     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6296
6297     my $oldtimeout;
6298     my $timeout = 5;
6299
6300     my $devinfo = {};
6301
6302     my $rpcenv = PVE::RPCEnvironment::get();
6303
6304     my $conffile = PVE::QemuConfig->config_file($vmid);
6305     my $tmpfn = "$conffile.$$.tmp";
6306
6307     # Note: $oldconf is undef if VM does not exists
6308     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6309     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6310
6311     my %storage_limits;
6312
6313     my $print_devmap = sub {
6314         my $virtdev_hash = {};
6315
6316         my $cfgfn = "$tmpdir/qemu-server.conf";
6317
6318         # we can read the config - that is already extracted
6319         my $fh = IO::File->new($cfgfn, "r") ||
6320             "unable to read qemu-server.conf - $!\n";
6321
6322         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6323         if (-f $fwcfgfn) {
6324             my $pve_firewall_dir = '/etc/pve/firewall';
6325             mkdir $pve_firewall_dir; # make sure the dir exists
6326             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6327         }
6328
6329         while (defined(my $line = <$fh>)) {
6330             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6331                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6332                 die "archive does not contain data for drive '$virtdev'\n"
6333                     if !$devinfo->{$devname};
6334                 if (defined($opts->{storage})) {
6335                     $storeid = $opts->{storage} || 'local';
6336                 } elsif (!$storeid) {
6337                     $storeid = 'local';
6338                 }
6339                 $format = 'raw' if !$format;
6340                 $devinfo->{$devname}->{devname} = $devname;
6341                 $devinfo->{$devname}->{virtdev} = $virtdev;
6342                 $devinfo->{$devname}->{format} = $format;
6343                 $devinfo->{$devname}->{storeid} = $storeid;
6344
6345                 # check permission on storage
6346                 my $pool = $opts->{pool}; # todo: do we need that?
6347                 if ($user ne 'root@pam') {
6348                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6349                 }
6350
6351                 $storage_limits{$storeid} = $bwlimit;
6352
6353                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6354             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6355                 my $virtdev = $1;
6356                 my $drive = parse_drive($virtdev, $2);
6357                 if (drive_is_cloudinit($drive)) {
6358                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6359                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6360                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6361
6362                     my $d = {
6363                         format => $format,
6364                         storeid => $opts->{storage} // $storeid,
6365                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6366                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6367                         name => "vm-$vmid-cloudinit",
6368                         is_cloudinit => 1,
6369                     };
6370                     $virtdev_hash->{$virtdev} = $d;
6371                 }
6372             }
6373         }
6374
6375         foreach my $key (keys %storage_limits) {
6376             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6377             next if !$limit;
6378             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6379             $storage_limits{$key} = $limit * 1024;
6380         }
6381
6382         foreach my $devname (keys %$devinfo) {
6383             die "found no device mapping information for device '$devname'\n"
6384                 if !$devinfo->{$devname}->{virtdev};
6385         }
6386
6387         # create empty/temp config
6388         if ($oldconf) {
6389             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6390             foreach_drive($oldconf, sub {
6391                 my ($ds, $drive) = @_;
6392
6393                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6394
6395                 my $volid = $drive->{file};
6396                 return if !$volid || $volid =~ m|^/|;
6397
6398                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6399                 return if !$path || !$owner || ($owner != $vmid);
6400
6401                 # Note: only delete disk we want to restore
6402                 # other volumes will become unused
6403                 if ($virtdev_hash->{$ds}) {
6404                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6405                     if (my $err = $@) {
6406                         warn $err;
6407                     }
6408                 }
6409             });
6410
6411             # delete vmstate files, after the restore we have no snapshots anymore
6412             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6413                 my $snap = $oldconf->{snapshots}->{$snapname};
6414                 if ($snap->{vmstate}) {
6415                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6416                     if (my $err = $@) {
6417                         warn $err;
6418                     }
6419                 }
6420             }
6421         }
6422
6423         my $map = {};
6424         foreach my $virtdev (sort keys %$virtdev_hash) {
6425             my $d = $virtdev_hash->{$virtdev};
6426             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6427             my $storeid = $d->{storeid};
6428             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6429
6430             my $map_opts = '';
6431             if (my $limit = $storage_limits{$storeid}) {
6432                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6433             }
6434
6435             # test if requested format is supported
6436             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6437             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6438             $d->{format} = $defFormat if !$supported;
6439
6440             my $name;
6441             if ($d->{is_cloudinit}) {
6442                 $name = $d->{name};
6443                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6444             }
6445
6446             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6447             print STDERR "new volume ID is '$volid'\n";
6448             $d->{volid} = $volid;
6449
6450             PVE::Storage::activate_volumes($cfg, [$volid]);
6451
6452             my $write_zeros = 1;
6453             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6454                 $write_zeros = 0;
6455             }
6456
6457             if (!$d->{is_cloudinit}) {
6458                 my $path = PVE::Storage::path($cfg, $volid);
6459
6460                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6461
6462                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6463             }
6464             $map->{$virtdev} = $volid;
6465         }
6466
6467         $fh->seek(0, 0) || die "seek failed - $!\n";
6468
6469         my $outfd = new IO::File ($tmpfn, "w") ||
6470             die "unable to write config for VM $vmid\n";
6471
6472         my $cookie = { netcount => 0 };
6473         while (defined(my $line = <$fh>)) {
6474             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6475         }
6476
6477         $fh->close();
6478         $outfd->close();
6479     };
6480
6481     eval {
6482         # enable interrupts
6483         local $SIG{INT} =
6484             local $SIG{TERM} =
6485             local $SIG{QUIT} =
6486             local $SIG{HUP} =
6487             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6488         local $SIG{ALRM} = sub { die "got timeout\n"; };
6489
6490         $oldtimeout = alarm($timeout);
6491
6492         my $parser = sub {
6493             my $line = shift;
6494
6495             print "$line\n";
6496
6497             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6498                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6499                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6500             } elsif ($line =~ m/^CTIME: /) {
6501                 # we correctly received the vma config, so we can disable
6502                 # the timeout now for disk allocation (set to 10 minutes, so
6503                 # that we always timeout if something goes wrong)
6504                 alarm(600);
6505                 &$print_devmap();
6506                 print $fifofh "done\n";
6507                 my $tmp = $oldtimeout || 0;
6508                 $oldtimeout = undef;
6509                 alarm($tmp);
6510                 close($fifofh);
6511             }
6512         };
6513
6514         print "restore vma archive: $dbg_cmdstring\n";
6515         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6516     };
6517     my $err = $@;
6518
6519     alarm($oldtimeout) if $oldtimeout;
6520
6521     my $vollist = [];
6522     foreach my $devname (keys %$devinfo) {
6523         my $volid = $devinfo->{$devname}->{volid};
6524         push @$vollist, $volid if $volid;
6525     }
6526
6527     PVE::Storage::deactivate_volumes($cfg, $vollist);
6528
6529     unlink $mapfifo;
6530
6531     if ($err) {
6532         rmtree $tmpdir;
6533         unlink $tmpfn;
6534
6535         foreach my $devname (keys %$devinfo) {
6536             my $volid = $devinfo->{$devname}->{volid};
6537             next if !$volid;
6538             eval {
6539                 if ($volid =~ m|^/|) {
6540                     unlink $volid || die 'unlink failed\n';
6541                 } else {
6542                     PVE::Storage::vdisk_free($cfg, $volid);
6543                 }
6544                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6545             };
6546             print STDERR "unable to cleanup '$volid' - $@" if $@;
6547         }
6548         die $err;
6549     }
6550
6551     rmtree $tmpdir;
6552
6553     rename($tmpfn, $conffile) ||
6554         die "unable to commit configuration file '$conffile'\n";
6555
6556     PVE::Cluster::cfs_update(); # make sure we read new file
6557
6558     eval { rescan($vmid, 1); };
6559     warn $@ if $@;
6560 }
6561
6562 sub restore_tar_archive {
6563     my ($archive, $vmid, $user, $opts) = @_;
6564
6565     if ($archive ne '-') {
6566         my $firstfile = tar_archive_read_firstfile($archive);
6567         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6568             if $firstfile ne 'qemu-server.conf';
6569     }
6570
6571     my $storecfg = PVE::Storage::config();
6572
6573     # destroy existing data - keep empty config
6574     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6575     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6576
6577     my $tocmd = "/usr/lib/qemu-server/qmextract";
6578
6579     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6580     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6581     $tocmd .= ' --prealloc' if $opts->{prealloc};
6582     $tocmd .= ' --info' if $opts->{info};
6583
6584     # tar option "xf" does not autodetect compression when read from STDIN,
6585     # so we pipe to zcat
6586     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6587         PVE::Tools::shellquote("--to-command=$tocmd");
6588
6589     my $tmpdir = "/var/tmp/vzdumptmp$$";
6590     mkpath $tmpdir;
6591
6592     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6593     local $ENV{VZDUMP_VMID} = $vmid;
6594     local $ENV{VZDUMP_USER} = $user;
6595
6596     my $conffile = PVE::QemuConfig->config_file($vmid);
6597     my $tmpfn = "$conffile.$$.tmp";
6598
6599     # disable interrupts (always do cleanups)
6600     local $SIG{INT} =
6601         local $SIG{TERM} =
6602         local $SIG{QUIT} =
6603         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6604
6605     eval {
6606         # enable interrupts
6607         local $SIG{INT} =
6608             local $SIG{TERM} =
6609             local $SIG{QUIT} =
6610             local $SIG{HUP} =
6611             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6612
6613         if ($archive eq '-') {
6614             print "extracting archive from STDIN\n";
6615             run_command($cmd, input => "<&STDIN");
6616         } else {
6617             print "extracting archive '$archive'\n";
6618             run_command($cmd);
6619         }
6620
6621         return if $opts->{info};
6622
6623         # read new mapping
6624         my $map = {};
6625         my $statfile = "$tmpdir/qmrestore.stat";
6626         if (my $fd = IO::File->new($statfile, "r")) {
6627             while (defined (my $line = <$fd>)) {
6628                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6629                     $map->{$1} = $2 if $1;
6630                 } else {
6631                     print STDERR "unable to parse line in statfile - $line\n";
6632                 }
6633             }
6634             $fd->close();
6635         }
6636
6637         my $confsrc = "$tmpdir/qemu-server.conf";
6638
6639         my $srcfd = new IO::File($confsrc, "r") ||
6640             die "unable to open file '$confsrc'\n";
6641
6642         my $outfd = new IO::File ($tmpfn, "w") ||
6643             die "unable to write config for VM $vmid\n";
6644
6645         my $cookie = { netcount => 0 };
6646         while (defined (my $line = <$srcfd>)) {
6647             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6648         }
6649
6650         $srcfd->close();
6651         $outfd->close();
6652     };
6653     my $err = $@;
6654
6655     if ($err) {
6656
6657         unlink $tmpfn;
6658
6659         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6660
6661         die $err;
6662     }
6663
6664     rmtree $tmpdir;
6665
6666     rename $tmpfn, $conffile ||
6667         die "unable to commit configuration file '$conffile'\n";
6668
6669     PVE::Cluster::cfs_update(); # make sure we read new file
6670
6671     eval { rescan($vmid, 1); };
6672     warn $@ if $@;
6673 };
6674
6675 sub foreach_storage_used_by_vm {
6676     my ($conf, $func) = @_;
6677
6678     my $sidhash = {};
6679
6680     foreach_drive($conf, sub {
6681         my ($ds, $drive) = @_;
6682         return if drive_is_cdrom($drive);
6683
6684         my $volid = $drive->{file};
6685
6686         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6687         $sidhash->{$sid} = $sid if $sid;
6688     });
6689
6690     foreach my $sid (sort keys %$sidhash) {
6691         &$func($sid);
6692     }
6693 }
6694
6695 sub do_snapshots_with_qemu {
6696     my ($storecfg, $volid) = @_;
6697
6698     my $storage_name = PVE::Storage::parse_volume_id($volid);
6699     my $scfg = $storecfg->{ids}->{$storage_name};
6700
6701     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6702         return 1;
6703     }
6704
6705     if ($volid =~ m/\.(qcow2|qed)$/){
6706         return 1;
6707     }
6708
6709     return undef;
6710 }
6711
6712 sub qga_check_running {
6713     my ($vmid, $nowarn) = @_;
6714
6715     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6716     if ($@) {
6717         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6718         return 0;
6719     }
6720     return 1;
6721 }
6722
6723 sub template_create {
6724     my ($vmid, $conf, $disk) = @_;
6725
6726     my $storecfg = PVE::Storage::config();
6727
6728     foreach_drive($conf, sub {
6729         my ($ds, $drive) = @_;
6730
6731         return if drive_is_cdrom($drive);
6732         return if $disk && $ds ne $disk;
6733
6734         my $volid = $drive->{file};
6735         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6736
6737         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6738         $drive->{file} = $voliddst;
6739         $conf->{$ds} = print_drive($vmid, $drive);
6740         PVE::QemuConfig->write_config($vmid, $conf);
6741     });
6742 }
6743
6744 sub convert_iscsi_path {
6745     my ($path) = @_;
6746
6747     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6748         my $portal = $1;
6749         my $target = $2;
6750         my $lun = $3;
6751
6752         my $initiator_name = get_initiator_name();
6753
6754         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6755                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6756     }
6757
6758     die "cannot convert iscsi path '$path', unkown format\n";
6759 }
6760
6761 sub qemu_img_convert {
6762     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6763
6764     my $storecfg = PVE::Storage::config();
6765     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6766     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6767
6768     if ($src_storeid && $dst_storeid) {
6769
6770         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6771
6772         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6773         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6774
6775         my $src_format = qemu_img_format($src_scfg, $src_volname);
6776         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6777
6778         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6779         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6780
6781         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6782         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6783
6784         my $cmd = [];
6785         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6786         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6787         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6788         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6789
6790         if ($src_is_iscsi) {
6791             push @$cmd, '--image-opts';
6792             $src_path = convert_iscsi_path($src_path);
6793         } else {
6794             push @$cmd, '-f', $src_format;
6795         }
6796
6797         if ($dst_is_iscsi) {
6798             push @$cmd, '--target-image-opts';
6799             $dst_path = convert_iscsi_path($dst_path);
6800         } else {
6801             push @$cmd, '-O', $dst_format;
6802         }
6803
6804         push @$cmd, $src_path;
6805
6806         if (!$dst_is_iscsi && $is_zero_initialized) {
6807             push @$cmd, "zeroinit:$dst_path";
6808         } else {
6809             push @$cmd, $dst_path;
6810         }
6811
6812         my $parser = sub {
6813             my $line = shift;
6814             if($line =~ m/\((\S+)\/100\%\)/){
6815                 my $percent = $1;
6816                 my $transferred = int($size * $percent / 100);
6817                 my $remaining = $size - $transferred;
6818
6819                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6820             }
6821
6822         };
6823
6824         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6825         my $err = $@;
6826         die "copy failed: $err" if $err;
6827     }
6828 }
6829
6830 sub qemu_img_format {
6831     my ($scfg, $volname) = @_;
6832
6833     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6834         return $1;
6835     } else {
6836         return "raw";
6837     }
6838 }
6839
6840 sub qemu_drive_mirror {
6841     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6842
6843     $jobs = {} if !$jobs;
6844
6845     my $qemu_target;
6846     my $format;
6847     $jobs->{"drive-$drive"} = {};
6848
6849     if ($dst_volid =~ /^nbd:/) {
6850         $qemu_target = $dst_volid;
6851         $format = "nbd";
6852     } else {
6853         my $storecfg = PVE::Storage::config();
6854         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6855
6856         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6857
6858         $format = qemu_img_format($dst_scfg, $dst_volname);
6859
6860         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6861
6862         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6863     }
6864
6865     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6866     $opts->{format} = $format if $format;
6867
6868     if (defined($bwlimit)) {
6869         $opts->{speed} = $bwlimit * 1024;
6870         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6871     } else {
6872         print "drive mirror is starting for drive-$drive\n";
6873     }
6874
6875     # if a job already runs for this device we get an error, catch it for cleanup
6876     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6877     if (my $err = $@) {
6878         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6879         warn "$@\n" if $@;
6880         die "mirroring error: $err\n";
6881     }
6882
6883     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6884 }
6885
6886 sub qemu_drive_mirror_monitor {
6887     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6888
6889     eval {
6890         my $err_complete = 0;
6891
6892         while (1) {
6893             die "storage migration timed out\n" if $err_complete > 300;
6894
6895             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6896
6897             my $running_mirror_jobs = {};
6898             foreach my $stat (@$stats) {
6899                 next if $stat->{type} ne 'mirror';
6900                 $running_mirror_jobs->{$stat->{device}} = $stat;
6901             }
6902
6903             my $readycounter = 0;
6904
6905             foreach my $job (keys %$jobs) {
6906
6907                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6908                     print "$job : finished\n";
6909                     delete $jobs->{$job};
6910                     next;
6911                 }
6912
6913                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6914
6915                 my $busy = $running_mirror_jobs->{$job}->{busy};
6916                 my $ready = $running_mirror_jobs->{$job}->{ready};
6917                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6918                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6919                     my $remaining = $total - $transferred;
6920                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6921
6922                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6923                 }
6924
6925                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6926             }
6927
6928             last if scalar(keys %$jobs) == 0;
6929
6930             if ($readycounter == scalar(keys %$jobs)) {
6931                 print "all mirroring jobs are ready \n";
6932                 last if $skipcomplete; #do the complete later
6933
6934                 if ($vmiddst && $vmiddst != $vmid) {
6935                     my $agent_running = $qga && qga_check_running($vmid);
6936                     if ($agent_running) {
6937                         print "freeze filesystem\n";
6938                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6939                     } else {
6940                         print "suspend vm\n";
6941                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6942                     }
6943
6944                     # if we clone a disk for a new target vm, we don't switch the disk
6945                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6946
6947                     if ($agent_running) {
6948                         print "unfreeze filesystem\n";
6949                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6950                     } else {
6951                         print "resume vm\n";
6952                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6953                     }
6954
6955                     last;
6956                 } else {
6957
6958                     foreach my $job (keys %$jobs) {
6959                         # try to switch the disk if source and destination are on the same guest
6960                         print "$job: Completing block job...\n";
6961
6962                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6963                         if ($@ =~ m/cannot be completed/) {
6964                             print "$job: Block job cannot be completed, try again.\n";
6965                             $err_complete++;
6966                         }else {
6967                             print "$job: Completed successfully.\n";
6968                             $jobs->{$job}->{complete} = 1;
6969                         }
6970                     }
6971                 }
6972             }
6973             sleep 1;
6974         }
6975     };
6976     my $err = $@;
6977
6978     if ($err) {
6979         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6980         die "mirroring error: $err";
6981     }
6982
6983 }
6984
6985 sub qemu_blockjobs_cancel {
6986     my ($vmid, $jobs) = @_;
6987
6988     foreach my $job (keys %$jobs) {
6989         print "$job: Cancelling block job\n";
6990         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6991         $jobs->{$job}->{cancel} = 1;
6992     }
6993
6994     while (1) {
6995         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6996
6997         my $running_jobs = {};
6998         foreach my $stat (@$stats) {
6999             $running_jobs->{$stat->{device}} = $stat;
7000         }
7001
7002         foreach my $job (keys %$jobs) {
7003
7004             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7005                 print "$job: Done.\n";
7006                 delete $jobs->{$job};
7007             }
7008         }
7009
7010         last if scalar(keys %$jobs) == 0;
7011
7012         sleep 1;
7013     }
7014 }
7015
7016 sub clone_disk {
7017     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7018         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7019
7020     my $newvolid;
7021
7022     if (!$full) {
7023         print "create linked clone of drive $drivename ($drive->{file})\n";
7024         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7025         push @$newvollist, $newvolid;
7026     } else {
7027
7028         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7029         $storeid = $storage if $storage;
7030
7031         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7032         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7033
7034         print "create full clone of drive $drivename ($drive->{file})\n";
7035         my $name = undef;
7036         if (drive_is_cloudinit($drive)) {
7037             $name = "vm-$newvmid-cloudinit";
7038             $snapname = undef;
7039             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7040             if ($dst_format ne 'raw') {
7041                 $name .= ".$dst_format";
7042             }
7043         }
7044         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7045         push @$newvollist, $newvolid;
7046
7047         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7048
7049         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7050         if (!$running || $snapname) {
7051             # TODO: handle bwlimits
7052             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7053         } else {
7054
7055             my $kvmver = get_running_qemu_version ($vmid);
7056             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7057                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7058                     if $drive->{iothread};
7059             }
7060
7061             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7062         }
7063     }
7064
7065     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7066
7067     my $disk = $drive;
7068     $disk->{format} = undef;
7069     $disk->{file} = $newvolid;
7070     $disk->{size} = $size;
7071
7072     return $disk;
7073 }
7074
7075 # this only works if VM is running
7076 sub get_current_qemu_machine {
7077     my ($vmid) = @_;
7078
7079     my $cmd = { execute => 'query-machines', arguments => {} };
7080     my $res = vm_qmp_command($vmid, $cmd);
7081
7082     my ($current, $default);
7083     foreach my $e (@$res) {
7084         $default = $e->{name} if $e->{'is-default'};
7085         $current = $e->{name} if $e->{'is-current'};
7086     }
7087
7088     # fallback to the default machine if current is not supported by qemu
7089     return $current || $default || 'pc';
7090 }
7091
7092 sub get_running_qemu_version {
7093     my ($vmid) = @_;
7094     my $cmd = { execute => 'query-version', arguments => {} };
7095     my $res = vm_qmp_command($vmid, $cmd);
7096     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7097 }
7098
7099 sub qemu_machine_feature_enabled {
7100     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7101
7102     my $current_major;
7103     my $current_minor;
7104
7105     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7106
7107         $current_major = $3;
7108         $current_minor = $4;
7109
7110     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7111
7112         $current_major = $1;
7113         $current_minor = $2;
7114     }
7115
7116     return 1 if $current_major > $version_major ||
7117                 ($current_major == $version_major &&
7118                  $current_minor >= $version_minor);
7119 }
7120
7121 sub qemu_machine_pxe {
7122     my ($vmid, $conf, $machine) = @_;
7123
7124     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7125
7126     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7127         $machine .= '.pxe';
7128     }
7129
7130     return $machine;
7131 }
7132
7133 sub qemu_use_old_bios_files {
7134     my ($machine_type) = @_;
7135
7136     return if !$machine_type;
7137
7138     my $use_old_bios_files = undef;
7139
7140     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7141         $machine_type = $1;
7142         $use_old_bios_files = 1;
7143     } else {
7144         my $kvmver = kvm_user_version();
7145         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7146         # load new efi bios files on migration. So this hack is required to allow
7147         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7148         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7149         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7150     }
7151
7152     return ($use_old_bios_files, $machine_type);
7153 }
7154
7155 sub create_efidisk($$$$$) {
7156     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7157
7158     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7159     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7160
7161     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7162     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7163     PVE::Storage::activate_volumes($storecfg, [$volid]);
7164
7165     my $path = PVE::Storage::path($storecfg, $volid);
7166     eval {
7167         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7168     };
7169     die "Copying EFI vars image failed: $@" if $@;
7170
7171     return ($volid, $vars_size);
7172 }
7173
7174 sub vm_iothreads_list {
7175     my ($vmid) = @_;
7176
7177     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7178
7179     my $iothreads = {};
7180     foreach my $iothread (@$res) {
7181         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7182     }
7183
7184     return $iothreads;
7185 }
7186
7187 sub scsihw_infos {
7188     my ($conf, $drive) = @_;
7189
7190     my $maxdev = 0;
7191
7192     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7193         $maxdev = 7;
7194     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7195         $maxdev = 1;
7196     } else {
7197         $maxdev = 256;
7198     }
7199
7200     my $controller = int($drive->{index} / $maxdev);
7201     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7202
7203     return ($maxdev, $controller, $controller_prefix);
7204 }
7205
7206 sub add_hyperv_enlightenments {
7207     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7208
7209     return if $winversion < 6;
7210     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7211
7212     if ($gpu_passthrough || defined($hv_vendor_id)) {
7213         $hv_vendor_id //= 'proxmox';
7214         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7215     }
7216
7217     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7218         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7219         push @$cpuFlags , 'hv_vapic';
7220         push @$cpuFlags , 'hv_time';
7221     } else {
7222         push @$cpuFlags , 'hv_spinlocks=0xffff';
7223     }
7224
7225     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7226         push @$cpuFlags , 'hv_reset';
7227         push @$cpuFlags , 'hv_vpindex';
7228         push @$cpuFlags , 'hv_runtime';
7229     }
7230
7231     if ($winversion >= 7) {
7232         push @$cpuFlags , 'hv_relaxed';
7233
7234         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7235             push @$cpuFlags , 'hv_synic';
7236             push @$cpuFlags , 'hv_stimer';
7237         }
7238
7239         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7240             push @$cpuFlags , 'hv_tlbflush';
7241             push @$cpuFlags , 'hv_ipi';
7242             # FIXME: AMD does not supports this currently, only add with special flag??
7243             #push @$cpuFlags , 'hv_evmcs';
7244         }
7245     }
7246 }
7247
7248 sub windows_version {
7249     my ($ostype) = @_;
7250
7251     return 0 if !$ostype;
7252
7253     my $winversion = 0;
7254
7255     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7256         $winversion = 5;
7257     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7258         $winversion = 6;
7259     } elsif ($ostype =~ m/^win(\d+)$/) {
7260         $winversion = $1;
7261     }
7262
7263     return $winversion;
7264 }
7265
7266 sub resolve_dst_disk_format {
7267         my ($storecfg, $storeid, $src_volname, $format) = @_;
7268         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7269
7270         if (!$format) {
7271             # if no target format is specified, use the source disk format as hint
7272             if ($src_volname) {
7273                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7274                 $format = qemu_img_format($scfg, $src_volname);
7275             } else {
7276                 return $defFormat;
7277             }
7278         }
7279
7280         # test if requested format is supported - else use default
7281         my $supported = grep { $_ eq $format } @$validFormats;
7282         $format = $defFormat if !$supported;
7283         return $format;
7284 }
7285
7286 sub resolve_first_disk {
7287     my $conf = shift;
7288     my @disks = PVE::QemuServer::valid_drive_names();
7289     my $firstdisk;
7290     foreach my $ds (reverse @disks) {
7291         next if !$conf->{$ds};
7292         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7293         next if PVE::QemuServer::drive_is_cdrom($disk);
7294         $firstdisk = $ds;
7295     }
7296     return $firstdisk;
7297 }
7298
7299 sub generate_uuid {
7300     my ($uuid, $uuid_str);
7301     UUID::generate($uuid);
7302     UUID::unparse($uuid, $uuid_str);
7303     return $uuid_str;
7304 }
7305
7306 sub generate_smbios1_uuid {
7307     return "uuid=".generate_uuid();
7308 }
7309
7310 sub nbd_stop {
7311     my ($vmid) = @_;
7312
7313     vm_mon_cmd($vmid, 'nbd-server-stop');
7314 }
7315
7316 # bash completion helper
7317
7318 sub complete_backup_archives {
7319     my ($cmdname, $pname, $cvalue) = @_;
7320
7321     my $cfg = PVE::Storage::config();
7322
7323     my $storeid;
7324
7325     if ($cvalue =~ m/^([^:]+):/) {
7326         $storeid = $1;
7327     }
7328
7329     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7330
7331     my $res = [];
7332     foreach my $id (keys %$data) {
7333         foreach my $item (@{$data->{$id}}) {
7334             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7335             push @$res, $item->{volid} if defined($item->{volid});
7336         }
7337     }
7338
7339     return $res;
7340 }
7341
7342 my $complete_vmid_full = sub {
7343     my ($running) = @_;
7344
7345     my $idlist = vmstatus();
7346
7347     my $res = [];
7348
7349     foreach my $id (keys %$idlist) {
7350         my $d = $idlist->{$id};
7351         if (defined($running)) {
7352             next if $d->{template};
7353             next if $running && $d->{status} ne 'running';
7354             next if !$running && $d->{status} eq 'running';
7355         }
7356         push @$res, $id;
7357
7358     }
7359     return $res;
7360 };
7361
7362 sub complete_vmid {
7363     return &$complete_vmid_full();
7364 }
7365
7366 sub complete_vmid_stopped {
7367     return &$complete_vmid_full(0);
7368 }
7369
7370 sub complete_vmid_running {
7371     return &$complete_vmid_full(1);
7372 }
7373
7374 sub complete_storage {
7375
7376     my $cfg = PVE::Storage::config();
7377     my $ids = $cfg->{ids};
7378
7379     my $res = [];
7380     foreach my $sid (keys %$ids) {
7381         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7382         next if !$ids->{$sid}->{content}->{images};
7383         push @$res, $sid;
7384     }
7385
7386     return $res;
7387 }
7388
7389 1;