PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use PVE::Exception qw(raise raise_param_exc);
  25 use PVE::Storage;
  26 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  27 use PVE::JSONSchema qw(get_standard_option);
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::INotify;
  30 use PVE::ProcFSTools;
  31 use PVE::QemuConfig;
  32 use PVE::QMPClient;
  33 use PVE::RPCEnvironment;
  34 use PVE::GuestHelpers;
  35 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  36 use PVE::QemuServer::Memory;
  37 use PVE::QemuServer::USB qw(parse_usb_device);
  38 use PVE::QemuServer::Cloudinit;
  39 use PVE::SysFSTools;
  40 use PVE::Systemd;
  41 use Time::HiRes qw(gettimeofday);
  42 use File::Copy qw(copy);
  43 use URI::Escape;
  44
  45 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  46 my $OVMF = {
  47     x86_64 => [
  48         "$EDK2_FW_BASE/OVMF_CODE.fd",
  49         "$EDK2_FW_BASE/OVMF_VARS.fd"
  50     ],
  51     aarch64 => [
  52         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  53         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  54     ],
  55 };
  56
  57 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  58
  59 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  60
  61 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  62
  63 # Note about locking: we use flock on the config file protect
  64 # against concurent actions.
  65 # Aditionaly, we have a 'lock' setting in the config file. This
  66 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  67 # allowed when such lock is set. But you can ignore this kind of
  68 # lock with the --skiplock flag.
  69
  70 cfs_register_file('/qemu-server/',
  71                   \&parse_vm_config,
  72                   \&write_vm_config);
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  75     description => "Some command save/restore state from this location.",
  76     type => 'string',
  77     maxLength => 128,
  78     optional => 1,
  79 });
  80
  81 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  82     type => 'string',
  83     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  84     description => "The drive's backing file's data format.",
  85     optional => 1,
  86 });
  87
  88 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  89         description => "Specifies the Qemu machine type.",
  90         type => 'string',
  91         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  92         maxLength => 40,
  93         optional => 1,
  94 });
  95
  96 #no warnings 'redefine';
  97
  98 sub cgroups_write {
  99    my ($controller, $vmid, $option, $value) = @_;
 100
 101    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 102    PVE::ProcFSTools::write_proc_entry($path, $value);
 103
 104 }
 105
 106 my $nodename = PVE::INotify::nodename();
 107
 108 mkdir "/etc/pve/nodes/$nodename";
 109 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 110 mkdir $confdir;
 111
 112 my $var_run_tmpdir = "/var/run/qemu-server";
 113 mkdir $var_run_tmpdir;
 114
 115 my $lock_dir = "/var/lock/qemu-server";
 116 mkdir $lock_dir;
 117
 118 my $cpu_vendor_list = {
 119     # Intel CPUs
 120     486 => 'GenuineIntel',
 121     pentium => 'GenuineIntel',
 122     pentium2  => 'GenuineIntel',
 123     pentium3  => 'GenuineIntel',
 124     coreduo => 'GenuineIntel',
 125     core2duo => 'GenuineIntel',
 126     Conroe  => 'GenuineIntel',
 127     Penryn  => 'GenuineIntel',
 128     Nehalem  => 'GenuineIntel',
 129     'Nehalem-IBRS'  => 'GenuineIntel',
 130     Westmere => 'GenuineIntel',
 131     'Westmere-IBRS' => 'GenuineIntel',
 132     SandyBridge => 'GenuineIntel',
 133     'SandyBridge-IBRS' => 'GenuineIntel',
 134     IvyBridge => 'GenuineIntel',
 135     'IvyBridge-IBRS' => 'GenuineIntel',
 136     Haswell => 'GenuineIntel',
 137     'Haswell-IBRS' => 'GenuineIntel',
 138     'Haswell-noTSX' => 'GenuineIntel',
 139     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 140     Broadwell => 'GenuineIntel',
 141     'Broadwell-IBRS' => 'GenuineIntel',
 142     'Broadwell-noTSX' => 'GenuineIntel',
 143     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 144     'Skylake-Client' => 'GenuineIntel',
 145     'Skylake-Client-IBRS' => 'GenuineIntel',
 146     'Skylake-Server' => 'GenuineIntel',
 147     'Skylake-Server-IBRS' => 'GenuineIntel',
 148
 149     # AMD CPUs
 150     athlon => 'AuthenticAMD',
 151     phenom  => 'AuthenticAMD',
 152     Opteron_G1  => 'AuthenticAMD',
 153     Opteron_G2  => 'AuthenticAMD',
 154     Opteron_G3  => 'AuthenticAMD',
 155     Opteron_G4  => 'AuthenticAMD',
 156     Opteron_G5  => 'AuthenticAMD',
 157     EPYC => 'AuthenticAMD',
 158     'EPYC-IBPB' => 'AuthenticAMD',
 159
 160     # generic types, use vendor from host node
 161     host => 'default',
 162     kvm32 => 'default',
 163     kvm64 => 'default',
 164     qemu32 => 'default',
 165     qemu64 => 'default',
 166     max => 'default',
 167 };
 168
 169 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 170
 171 my $cpu_fmt = {
 172     cputype => {
 173         description => "Emulated CPU type.",
 174         type => 'string',
 175         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 176         default => 'kvm64',
 177         default_key => 1,
 178     },
 179     hidden => {
 180         description => "Do not identify as a KVM virtual machine.",
 181         type => 'boolean',
 182         optional => 1,
 183         default => 0
 184     },
 185     'hv-vendor-id' => {
 186         type => 'string',
 187         pattern => qr/[a-zA-Z0-9]{1,12}/,
 188         format_description => 'vendor-id',
 189         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 190         optional => 1,
 191     },
 192     flags => {
 193         description => "List of additional CPU flags separated by ';'."
 194                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 195                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 196         format_description => '+FLAG[;-FLAG...]',
 197         type => 'string',
 198         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 199         optional => 1,
 200     },
 201 };
 202
 203 my $watchdog_fmt = {
 204     model => {
 205         default_key => 1,
 206         type => 'string',
 207         enum => [qw(i6300esb ib700)],
 208         description => "Watchdog type to emulate.",
 209         default => 'i6300esb',
 210         optional => 1,
 211     },
 212     action => {
 213         type => 'string',
 214         enum => [qw(reset shutdown poweroff pause debug none)],
 215         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 216         optional => 1,
 217     },
 218 };
 219 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 220
 221 my $agent_fmt = {
 222     enabled => {
 223         description => "Enable/disable Qemu GuestAgent.",
 224         type => 'boolean',
 225         default => 0,
 226         default_key => 1,
 227     },
 228     fstrim_cloned_disks => {
 229         description => "Run fstrim after cloning/moving a disk.",
 230         type => 'boolean',
 231         optional => 1,
 232         default => 0
 233     },
 234 };
 235
 236 my $vga_fmt = {
 237     type => {
 238         description => "Select the VGA type.",
 239         type => 'string',
 240         default => 'std',
 241         optional => 1,
 242         default_key => 1,
 243         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 244     },
 245     memory => {
 246         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 247         type => 'integer',
 248         optional => 1,
 249         minimum => 4,
 250         maximum => 512,
 251     },
 252 };
 253
 254 my $ivshmem_fmt = {
 255     size => {
 256         type => 'integer',
 257         minimum => 1,
 258         description => "The size of the file in MB.",
 259     },
 260     name => {
 261         type => 'string',
 262         pattern => '[a-zA-Z0-9\-]+',
 263         optional => 1,
 264         format_description => 'string',
 265         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 266     },
 267 };
 268
 269 my $confdesc = {
 270     onboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Specifies whether a VM will be started during system bootup.",
 274         default => 0,
 275     },
 276     autostart => {
 277         optional => 1,
 278         type => 'boolean',
 279         description => "Automatic restart after crash (currently ignored).",
 280         default => 0,
 281     },
 282     hotplug => {
 283         optional => 1,
 284         type => 'string', format => 'pve-hotplug-features',
 285         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 286         default => 'network,disk,usb',
 287     },
 288     reboot => {
 289         optional => 1,
 290         type => 'boolean',
 291         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 292         default => 1,
 293     },
 294     lock => {
 295         optional => 1,
 296         type => 'string',
 297         description => "Lock/unlock the VM.",
 298         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 299     },
 300     cpulimit => {
 301         optional => 1,
 302         type => 'number',
 303         description => "Limit of CPU usage.",
 304         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 305         minimum => 0,
 306         maximum => 128,
 307         default => 0,
 308     },
 309     cpuunits => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "CPU weight for a VM.",
 313         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 314         minimum => 2,
 315         maximum => 262144,
 316         default => 1024,
 317     },
 318     memory => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 322         minimum => 16,
 323         default => 512,
 324     },
 325     balloon => {
 326         optional => 1,
 327         type => 'integer',
 328         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 329         minimum => 0,
 330     },
 331     shares => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 335         minimum => 0,
 336         maximum => 50000,
 337         default => 1000,
 338     },
 339     keyboard => {
 340         optional => 1,
 341         type => 'string',
 342         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 343                        "It should not be necessary to set it.",
 344         enum => PVE::Tools::kvmkeymaplist(),
 345         default => undef,
 346     },
 347     name => {
 348         optional => 1,
 349         type => 'string', format => 'dns-name',
 350         description => "Set a name for the VM. Only used on the configuration web interface.",
 351     },
 352     scsihw => {
 353         optional => 1,
 354         type => 'string',
 355         description => "SCSI controller model",
 356         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 357         default => 'lsi',
 358     },
 359     description => {
 360         optional => 1,
 361         type => 'string',
 362         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 363     },
 364     ostype => {
 365         optional => 1,
 366         type => 'string',
 367         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 368         description => "Specify guest operating system.",
 369         verbose_description => <<EODESC,
 370 Specify guest operating system. This is used to enable special
 371 optimization/features for specific operating systems:
 372
 373 [horizontal]
 374 other;; unspecified OS
 375 wxp;; Microsoft Windows XP
 376 w2k;; Microsoft Windows 2000
 377 w2k3;; Microsoft Windows 2003
 378 w2k8;; Microsoft Windows 2008
 379 wvista;; Microsoft Windows Vista
 380 win7;; Microsoft Windows 7
 381 win8;; Microsoft Windows 8/2012/2012r2
 382 win10;; Microsoft Windows 10/2016
 383 l24;; Linux 2.4 Kernel
 384 l26;; Linux 2.6/3.X Kernel
 385 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 386 EODESC
 387     },
 388     boot => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 392         pattern => '[acdn]{1,4}',
 393         default => 'cdn',
 394     },
 395     bootdisk => {
 396         optional => 1,
 397         type => 'string', format => 'pve-qm-bootdisk',
 398         description => "Enable booting from specified disk.",
 399         pattern => '(ide|sata|scsi|virtio)\d+',
 400     },
 401     smp => {
 402         optional => 1,
 403         type => 'integer',
 404         description => "The number of CPUs. Please use option -sockets instead.",
 405         minimum => 1,
 406         default => 1,
 407     },
 408     sockets => {
 409         optional => 1,
 410         type => 'integer',
 411         description => "The number of CPU sockets.",
 412         minimum => 1,
 413         default => 1,
 414     },
 415     cores => {
 416         optional => 1,
 417         type => 'integer',
 418         description => "The number of cores per socket.",
 419         minimum => 1,
 420         default => 1,
 421     },
 422     numa => {
 423         optional => 1,
 424         type => 'boolean',
 425         description => "Enable/disable NUMA.",
 426         default => 0,
 427     },
 428     hugepages => {
 429         optional => 1,
 430         type => 'string',
 431         description => "Enable/disable hugepages memory.",
 432         enum => [qw(any 2 1024)],
 433     },
 434     vcpus => {
 435         optional => 1,
 436         type => 'integer',
 437         description => "Number of hotplugged vcpus.",
 438         minimum => 1,
 439         default => 0,
 440     },
 441     acpi => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable ACPI.",
 445         default => 1,
 446     },
 447     agent => {
 448         optional => 1,
 449         description => "Enable/disable Qemu GuestAgent and its properties.",
 450         type => 'string',
 451         format => $agent_fmt,
 452     },
 453     kvm => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable KVM hardware virtualization.",
 457         default => 1,
 458     },
 459     tdf => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable time drift fix.",
 463         default => 0,
 464     },
 465     localtime => {
 466         optional => 1,
 467         type => 'boolean',
 468         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 469     },
 470     freeze => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 474     },
 475     vga => {
 476         optional => 1,
 477         type => 'string', format => $vga_fmt,
 478         description => "Configure the VGA hardware.",
 479         verbose_description => "Configure the VGA Hardware. If you want to use ".
 480             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 481             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 482             "is 'std' for all OS types besides some Windows versions (XP and " .
 483             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 484             "display server. For win* OS you can select how many independent " .
 485             "displays you want, Linux guests can add displays them self.\n".
 486             "You can also run without any graphic card, using a serial device as terminal.",
 487     },
 488     watchdog => {
 489         optional => 1,
 490         type => 'string', format => 'pve-qm-watchdog',
 491         description => "Create a virtual hardware watchdog device.",
 492         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 493             " (by a guest action), the watchdog must be periodically polled " .
 494             "by an agent inside the guest or else the watchdog will reset " .
 495             "the guest (or execute the respective action specified)",
 496     },
 497     startdate => {
 498         optional => 1,
 499         type => 'string',
 500         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 501         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 502         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 503         default => 'now',
 504     },
 505     startup =>  get_standard_option('pve-startup-order'),
 506     template => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Enable/disable Template.",
 510         default => 0,
 511     },
 512     args => {
 513         optional => 1,
 514         type => 'string',
 515         description => "Arbitrary arguments passed to kvm.",
 516         verbose_description => <<EODESCR,
 517 Arbitrary arguments passed to kvm, for example:
 518
 519 args: -no-reboot -no-hpet
 520
 521 NOTE: this option is for experts only.
 522 EODESCR
 523     },
 524     tablet => {
 525         optional => 1,
 526         type => 'boolean',
 527         default => 1,
 528         description => "Enable/disable the USB tablet device.",
 529         verbose_description => "Enable/disable the USB tablet device. This device is " .
 530             "usually needed to allow absolute mouse positioning with VNC. " .
 531             "Else the mouse runs out of sync with normal VNC clients. " .
 532             "If you're running lots of console-only guests on one host, " .
 533             "you may consider disabling this to save some context switches. " .
 534             "This is turned off by default if you use spice (-vga=qxl).",
 535     },
 536     migrate_speed => {
 537         optional => 1,
 538         type => 'integer',
 539         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 540         minimum => 0,
 541         default => 0,
 542     },
 543     migrate_downtime => {
 544         optional => 1,
 545         type => 'number',
 546         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 547         minimum => 0,
 548         default => 0.1,
 549     },
 550     cdrom => {
 551         optional => 1,
 552         type => 'string', format => 'pve-qm-ide',
 553         typetext => '<volume>',
 554         description => "This is an alias for option -ide2",
 555     },
 556     cpu => {
 557         optional => 1,
 558         description => "Emulated CPU type.",
 559         type => 'string',
 560         format => $cpu_fmt,
 561     },
 562     parent => get_standard_option('pve-snapshot-name', {
 563         optional => 1,
 564         description => "Parent snapshot name. This is used internally, and should not be modified.",
 565     }),
 566     snaptime => {
 567         optional => 1,
 568         description => "Timestamp for snapshots.",
 569         type => 'integer',
 570         minimum => 0,
 571     },
 572     vmstate => {
 573         optional => 1,
 574         type => 'string', format => 'pve-volume-id',
 575         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 576     },
 577     vmstatestorage => get_standard_option('pve-storage-id', {
 578         description => "Default storage for VM state volumes/files.",
 579         optional => 1,
 580     }),
 581     runningmachine => get_standard_option('pve-qemu-machine', {
 582         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 583     }),
 584     machine => get_standard_option('pve-qemu-machine'),
 585     arch => {
 586         description => "Virtual processor architecture. Defaults to the host.",
 587         optional => 1,
 588         type => 'string',
 589         enum => [qw(x86_64 aarch64)],
 590     },
 591     smbios1 => {
 592         description => "Specify SMBIOS type 1 fields.",
 593         type => 'string', format => 'pve-qm-smbios1',
 594         maxLength => 256,
 595         optional => 1,
 596     },
 597     protection => {
 598         optional => 1,
 599         type => 'boolean',
 600         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 601         default => 0,
 602     },
 603     bios => {
 604         optional => 1,
 605         type => 'string',
 606         enum => [ qw(seabios ovmf) ],
 607         description => "Select BIOS implementation.",
 608         default => 'seabios',
 609     },
 610     vmgenid => {
 611         type => 'string',
 612         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 613         format_description => 'UUID',
 614         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 615         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 616             " 128-bit integer value identifier to the guest OS. This allows to".
 617             " notify the guest operating system when the virtual machine is".
 618             " executed with a different configuration (e.g. snapshot execution".
 619             " or creation from a template). The guest operating system notices".
 620             " the change, and is then able to react as appropriate by marking".
 621             " its copies of distributed databases as dirty, re-initializing its".
 622             " random number generator, etc.\n".
 623             "Note that auto-creation only works when done throug API/CLI create".
 624             " or update methods, but not when manually editing the config file.",
 625         default => "1 (autogenerated)",
 626         optional => 1,
 627     },
 628     hookscript => {
 629         type => 'string',
 630         format => 'pve-volume-id',
 631         optional => 1,
 632         description => "Script that will be executed during various steps in the vms lifetime.",
 633     },
 634     ivshmem => {
 635         type => 'string',
 636         format => $ivshmem_fmt,
 637         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 638         optional => 1,
 639     }
 640 };
 641
 642 my $cicustom_fmt = {
 643     meta => {
 644         type => 'string',
 645         optional => 1,
 646         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 647         format => 'pve-volume-id',
 648         format_description => 'volume',
 649     },
 650     network => {
 651         type => 'string',
 652         optional => 1,
 653         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 654         format => 'pve-volume-id',
 655         format_description => 'volume',
 656     },
 657     user => {
 658         type => 'string',
 659         optional => 1,
 660         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 661         format => 'pve-volume-id',
 662         format_description => 'volume',
 663     },
 664 };
 665 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 666
 667 my $confdesc_cloudinit = {
 668     citype => {
 669         optional => 1,
 670         type => 'string',
 671         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 672         enum => ['configdrive2', 'nocloud'],
 673     },
 674     ciuser => {
 675         optional => 1,
 676         type => 'string',
 677         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 678     },
 679     cipassword => {
 680         optional => 1,
 681         type => 'string',
 682         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 683     },
 684     cicustom => {
 685         optional => 1,
 686         type => 'string',
 687         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 688         format => 'pve-qm-cicustom',
 689     },
 690     searchdomain => {
 691         optional => 1,
 692         type => 'string',
 693         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 694     },
 695     nameserver => {
 696         optional => 1,
 697         type => 'string', format => 'address-list',
 698         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 699     },
 700     sshkeys => {
 701         optional => 1,
 702         type => 'string',
 703         format => 'urlencoded',
 704         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 705     },
 706 };
 707
 708 # what about other qemu settings ?
 709 #cpu => 'string',
 710 #machine => 'string',
 711 #fda => 'file',
 712 #fdb => 'file',
 713 #mtdblock => 'file',
 714 #sd => 'file',
 715 #pflash => 'file',
 716 #snapshot => 'bool',
 717 #bootp => 'file',
 718 ##tftp => 'dir',
 719 ##smb => 'dir',
 720 #kernel => 'file',
 721 #append => 'string',
 722 #initrd => 'file',
 723 ##soundhw => 'string',
 724
 725 while (my ($k, $v) = each %$confdesc) {
 726     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 727 }
 728
 729 my $MAX_IDE_DISKS = 4;
 730 my $MAX_SCSI_DISKS = 14;
 731 my $MAX_VIRTIO_DISKS = 16;
 732 my $MAX_SATA_DISKS = 6;
 733 my $MAX_USB_DEVICES = 5;
 734 my $MAX_NETS = 32;
 735 my $MAX_UNUSED_DISKS = 256;
 736 my $MAX_HOSTPCI_DEVICES = 4;
 737 my $MAX_SERIAL_PORTS = 4;
 738 my $MAX_PARALLEL_PORTS = 3;
 739 my $MAX_NUMA = 8;
 740
 741 my $numa_fmt = {
 742     cpus => {
 743         type => "string",
 744         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 745         description => "CPUs accessing this NUMA node.",
 746         format_description => "id[-id];...",
 747     },
 748     memory => {
 749         type => "number",
 750         description => "Amount of memory this NUMA node provides.",
 751         optional => 1,
 752     },
 753     hostnodes => {
 754         type => "string",
 755         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 756         description => "Host NUMA nodes to use.",
 757         format_description => "id[-id];...",
 758         optional => 1,
 759     },
 760     policy => {
 761         type => 'string',
 762         enum => [qw(preferred bind interleave)],
 763         description => "NUMA allocation policy.",
 764         optional => 1,
 765     },
 766 };
 767 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 768 my $numadesc = {
 769     optional => 1,
 770     type => 'string', format => $numa_fmt,
 771     description => "NUMA topology.",
 772 };
 773 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 774
 775 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 776     $confdesc->{"numa$i"} = $numadesc;
 777 }
 778
 779 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 780                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 781                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 782 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 783
 784 my $net_fmt_bridge_descr = <<__EOD__;
 785 Bridge to attach the network device to. The Proxmox VE standard bridge
 786 is called 'vmbr0'.
 787
 788 If you do not specify a bridge, we create a kvm user (NATed) network
 789 device, which provides DHCP and DNS services. The following addresses
 790 are used:
 791
 792  10.0.2.2   Gateway
 793  10.0.2.3   DNS Server
 794  10.0.2.4   SMB Server
 795
 796 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 797 __EOD__
 798
 799 my $net_fmt = {
 800     macaddr  => get_standard_option('mac-addr', {
 801         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 802     }),
 803     model => {
 804         type => 'string',
 805         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 806         enum => $nic_model_list,
 807         default_key => 1,
 808     },
 809     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 810     bridge => {
 811         type => 'string',
 812         description => $net_fmt_bridge_descr,
 813         format_description => 'bridge',
 814         optional => 1,
 815     },
 816     queues => {
 817         type => 'integer',
 818         minimum => 0, maximum => 16,
 819         description => 'Number of packet queues to be used on the device.',
 820         optional => 1,
 821     },
 822     rate => {
 823         type => 'number',
 824         minimum => 0,
 825         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 826         optional => 1,
 827     },
 828     tag => {
 829         type => 'integer',
 830         minimum => 1, maximum => 4094,
 831         description => 'VLAN tag to apply to packets on this interface.',
 832         optional => 1,
 833     },
 834     trunks => {
 835         type => 'string',
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => 'VLAN trunks to pass through this interface.',
 838         format_description => 'vlanid[;vlanid...]',
 839         optional => 1,
 840     },
 841     firewall => {
 842         type => 'boolean',
 843         description => 'Whether this interface should be protected by the firewall.',
 844         optional => 1,
 845     },
 846     link_down => {
 847         type => 'boolean',
 848         description => 'Whether this interface should be disconnected (like pulling the plug).',
 849         optional => 1,
 850     },
 851 };
 852
 853 my $netdesc = {
 854     optional => 1,
 855     type => 'string', format => $net_fmt,
 856     description => "Specify network devices.",
 857 };
 858
 859 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 860
 861 my $ipconfig_fmt = {
 862     ip => {
 863         type => 'string',
 864         format => 'pve-ipv4-config',
 865         format_description => 'IPv4Format/CIDR',
 866         description => 'IPv4 address in CIDR format.',
 867         optional => 1,
 868         default => 'dhcp',
 869     },
 870     gw => {
 871         type => 'string',
 872         format => 'ipv4',
 873         format_description => 'GatewayIPv4',
 874         description => 'Default gateway for IPv4 traffic.',
 875         optional => 1,
 876         requires => 'ip',
 877     },
 878     ip6 => {
 879         type => 'string',
 880         format => 'pve-ipv6-config',
 881         format_description => 'IPv6Format/CIDR',
 882         description => 'IPv6 address in CIDR format.',
 883         optional => 1,
 884         default => 'dhcp',
 885     },
 886     gw6 => {
 887         type => 'string',
 888         format => 'ipv6',
 889         format_description => 'GatewayIPv6',
 890         description => 'Default gateway for IPv6 traffic.',
 891         optional => 1,
 892         requires => 'ip6',
 893     },
 894 };
 895 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 896 my $ipconfigdesc = {
 897     optional => 1,
 898     type => 'string', format => 'pve-qm-ipconfig',
 899     description => <<'EODESCR',
 900 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 901
 902 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 903
 904 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 905 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 906
 907 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 908 EODESCR
 909 };
 910 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 911
 912 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 913     $confdesc->{"net$i"} = $netdesc;
 914     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 915 }
 916
 917 foreach my $key (keys %$confdesc_cloudinit) {
 918     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 919 }
 920
 921 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 922 sub verify_volume_id_or_qm_path {
 923     my ($volid, $noerr) = @_;
 924
 925     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 926         return $volid;
 927     }
 928
 929     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 930     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 931     if ($@) {
 932         return undef if $noerr;
 933         die $@;
 934     }
 935     return $volid;
 936 }
 937
 938 my $drivename_hash;
 939
 940 my %drivedesc_base = (
 941     volume => { alias => 'file' },
 942     file => {
 943         type => 'string',
 944         format => 'pve-volume-id-or-qm-path',
 945         default_key => 1,
 946         format_description => 'volume',
 947         description => "The drive's backing volume.",
 948     },
 949     media => {
 950         type => 'string',
 951         enum => [qw(cdrom disk)],
 952         description => "The drive's media type.",
 953         default => 'disk',
 954         optional => 1
 955     },
 956     cyls => {
 957         type => 'integer',
 958         description => "Force the drive's physical geometry to have a specific cylinder count.",
 959         optional => 1
 960     },
 961     heads => {
 962         type => 'integer',
 963         description => "Force the drive's physical geometry to have a specific head count.",
 964         optional => 1
 965     },
 966     secs => {
 967         type => 'integer',
 968         description => "Force the drive's physical geometry to have a specific sector count.",
 969         optional => 1
 970     },
 971     trans => {
 972         type => 'string',
 973         enum => [qw(none lba auto)],
 974         description => "Force disk geometry bios translation mode.",
 975         optional => 1,
 976     },
 977     snapshot => {
 978         type => 'boolean',
 979         description => "Controls qemu's snapshot mode feature."
 980             . " If activated, changes made to the disk are temporary and will"
 981             . " be discarded when the VM is shutdown.",
 982         optional => 1,
 983     },
 984     cache => {
 985         type => 'string',
 986         enum => [qw(none writethrough writeback unsafe directsync)],
 987         description => "The drive's cache mode",
 988         optional => 1,
 989     },
 990     format => get_standard_option('pve-qm-image-format'),
 991     size => {
 992         type => 'string',
 993         format => 'disk-size',
 994         format_description => 'DiskSize',
 995         description => "Disk size. This is purely informational and has no effect.",
 996         optional => 1,
 997     },
 998     backup => {
 999         type => 'boolean',
1000         description => "Whether the drive should be included when making backups.",
1001         optional => 1,
1002     },
1003     replicate => {
1004         type => 'boolean',
1005         description => 'Whether the drive should considered for replication jobs.',
1006         optional => 1,
1007         default => 1,
1008     },
1009     rerror => {
1010         type => 'string',
1011         enum => [qw(ignore report stop)],
1012         description => 'Read error action.',
1013         optional => 1,
1014     },
1015     werror => {
1016         type => 'string',
1017         enum => [qw(enospc ignore report stop)],
1018         description => 'Write error action.',
1019         optional => 1,
1020     },
1021     aio => {
1022         type => 'string',
1023         enum => [qw(native threads)],
1024         description => 'AIO type to use.',
1025         optional => 1,
1026     },
1027     discard => {
1028         type => 'string',
1029         enum => [qw(ignore on)],
1030         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1031         optional => 1,
1032     },
1033     detect_zeroes => {
1034         type => 'boolean',
1035         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1036         optional => 1,
1037     },
1038     serial => {
1039         type => 'string',
1040         format => 'urlencoded',
1041         format_description => 'serial',
1042         maxLength => 20*3, # *3 since it's %xx url enoded
1043         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1044         optional => 1,
1045     },
1046     shared => {
1047         type => 'boolean',
1048         description => 'Mark this locally-managed volume as available on all nodes',
1049         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1050         optional => 1,
1051         default => 0,
1052     }
1053 );
1054
1055 my %iothread_fmt = ( iothread => {
1056         type => 'boolean',
1057         description => "Whether to use iothreads for this drive",
1058         optional => 1,
1059 });
1060
1061 my %model_fmt = (
1062     model => {
1063         type => 'string',
1064         format => 'urlencoded',
1065         format_description => 'model',
1066         maxLength => 40*3, # *3 since it's %xx url enoded
1067         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1068         optional => 1,
1069     },
1070 );
1071
1072 my %queues_fmt = (
1073     queues => {
1074         type => 'integer',
1075         description => "Number of queues.",
1076         minimum => 2,
1077         optional => 1
1078     }
1079 );
1080
1081 my %scsiblock_fmt = (
1082     scsiblock => {
1083         type => 'boolean',
1084         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1085         optional => 1,
1086         default => 0,
1087     },
1088 );
1089
1090 my %ssd_fmt = (
1091     ssd => {
1092         type => 'boolean',
1093         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1094         optional => 1,
1095     },
1096 );
1097
1098 my %wwn_fmt = (
1099     wwn => {
1100         type => 'string',
1101         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1102         format_description => 'wwn',
1103         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1104         optional => 1,
1105     },
1106 );
1107
1108 my $add_throttle_desc = sub {
1109     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1110     my $d = {
1111         type => $type,
1112         format_description => $unit,
1113         description => "Maximum $what in $longunit.",
1114         optional => 1,
1115     };
1116     $d->{minimum} = $minimum if defined($minimum);
1117     $drivedesc_base{$key} = $d;
1118 };
1119 # throughput: (leaky bucket)
1120 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1121 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1122 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1123 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1124 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1126 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1127 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1128 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1129
1130 # pools: (pool of IO before throttling starts taking effect)
1131 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1132 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1134 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1135 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1136 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1137
1138 # burst lengths
1139 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1140 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1142 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1145
1146 # legacy support
1147 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1148 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1149 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1150 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1151
1152 my $ide_fmt = {
1153     %drivedesc_base,
1154     %model_fmt,
1155     %ssd_fmt,
1156     %wwn_fmt,
1157 };
1158 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1159
1160 my $idedesc = {
1161     optional => 1,
1162     type => 'string', format => $ide_fmt,
1163     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1164 };
1165 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1166
1167 my $scsi_fmt = {
1168     %drivedesc_base,
1169     %iothread_fmt,
1170     %queues_fmt,
1171     %scsiblock_fmt,
1172     %ssd_fmt,
1173     %wwn_fmt,
1174 };
1175 my $scsidesc = {
1176     optional => 1,
1177     type => 'string', format => $scsi_fmt,
1178     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1179 };
1180 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1181
1182 my $sata_fmt = {
1183     %drivedesc_base,
1184     %ssd_fmt,
1185     %wwn_fmt,
1186 };
1187 my $satadesc = {
1188     optional => 1,
1189     type => 'string', format => $sata_fmt,
1190     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1191 };
1192 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1193
1194 my $virtio_fmt = {
1195     %drivedesc_base,
1196     %iothread_fmt,
1197 };
1198 my $virtiodesc = {
1199     optional => 1,
1200     type => 'string', format => $virtio_fmt,
1201     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1204
1205 my $alldrive_fmt = {
1206     %drivedesc_base,
1207     %iothread_fmt,
1208     %model_fmt,
1209     %queues_fmt,
1210     %scsiblock_fmt,
1211     %ssd_fmt,
1212     %wwn_fmt,
1213 };
1214
1215 my $efidisk_fmt = {
1216     volume => { alias => 'file' },
1217     file => {
1218         type => 'string',
1219         format => 'pve-volume-id-or-qm-path',
1220         default_key => 1,
1221         format_description => 'volume',
1222         description => "The drive's backing volume.",
1223     },
1224     format => get_standard_option('pve-qm-image-format'),
1225     size => {
1226         type => 'string',
1227         format => 'disk-size',
1228         format_description => 'DiskSize',
1229         description => "Disk size. This is purely informational and has no effect.",
1230         optional => 1,
1231     },
1232 };
1233
1234 my $efidisk_desc = {
1235     optional => 1,
1236     type => 'string', format => $efidisk_fmt,
1237     description => "Configure a Disk for storing EFI vars",
1238 };
1239
1240 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1241
1242 my $usb_fmt = {
1243     host => {
1244         default_key => 1,
1245         type => 'string', format => 'pve-qm-usb-device',
1246         format_description => 'HOSTUSBDEVICE|spice',
1247         description => <<EODESCR,
1248 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1249
1250  'bus-port(.port)*' (decimal numbers) or
1251  'vendor_id:product_id' (hexadeciaml numbers) or
1252  'spice'
1253
1254 You can use the 'lsusb -t' command to list existing usb devices.
1255
1256 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1257
1258 The value 'spice' can be used to add a usb redirection devices for spice.
1259 EODESCR
1260     },
1261     usb3 => {
1262         optional => 1,
1263         type => 'boolean',
1264         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1265         default => 0,
1266     },
1267 };
1268
1269 my $usbdesc = {
1270     optional => 1,
1271     type => 'string', format => $usb_fmt,
1272     description => "Configure an USB device (n is 0 to 4).",
1273 };
1274 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1275
1276 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1277 my $hostpci_fmt = {
1278     host => {
1279         default_key => 1,
1280         type => 'string',
1281         pattern => qr/$PCIRE(;$PCIRE)*/,
1282         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1283         description => <<EODESCR,
1284 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1285 of PCI virtual functions of the host. HOSTPCIID syntax is:
1286
1287 'bus:dev.func' (hexadecimal numbers)
1288
1289 You can us the 'lspci' command to list existing PCI devices.
1290 EODESCR
1291     },
1292     rombar => {
1293         type => 'boolean',
1294         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1295         optional => 1,
1296         default => 1,
1297     },
1298     romfile => {
1299         type => 'string',
1300         pattern => '[^,;]+',
1301         format_description => 'string',
1302         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1303         optional => 1,
1304     },
1305     pcie => {
1306         type => 'boolean',
1307         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1308         optional => 1,
1309         default => 0,
1310     },
1311     'x-vga' => {
1312         type => 'boolean',
1313         description =>  "Enable vfio-vga device support.",
1314         optional => 1,
1315         default => 0,
1316     },
1317     'mdev' => {
1318         type => 'string',
1319         format_description => 'string',
1320         pattern => '[^/\.:]+',
1321         optional => 1,
1322         description => <<EODESCR
1323 The type of mediated device to use.
1324 An instance of this type will be created on startup of the VM and
1325 will be cleaned up when the VM stops.
1326 EODESCR
1327     }
1328 };
1329 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1330
1331 my $hostpcidesc = {
1332         optional => 1,
1333         type => 'string', format => 'pve-qm-hostpci',
1334         description => "Map host PCI devices into guest.",
1335         verbose_description =>  <<EODESCR,
1336 Map host PCI devices into guest.
1337
1338 NOTE: This option allows direct access to host hardware. So it is no longer
1339 possible to migrate such machines - use with special care.
1340
1341 CAUTION: Experimental! User reported problems with this option.
1342 EODESCR
1343 };
1344 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1345
1346 my $serialdesc = {
1347         optional => 1,
1348         type => 'string',
1349         pattern => '(/dev/.+|socket)',
1350         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1351         verbose_description =>  <<EODESCR,
1352 Create a serial device inside the VM (n is 0 to 3), and pass through a
1353 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1354 host side (use 'qm terminal' to open a terminal connection).
1355
1356 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1357
1358 CAUTION: Experimental! User reported problems with this option.
1359 EODESCR
1360 };
1361
1362 my $paralleldesc= {
1363         optional => 1,
1364         type => 'string',
1365         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1366         description =>  "Map host parallel devices (n is 0 to 2).",
1367         verbose_description =>  <<EODESCR,
1368 Map host parallel devices (n is 0 to 2).
1369
1370 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1371
1372 CAUTION: Experimental! User reported problems with this option.
1373 EODESCR
1374 };
1375
1376 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1377     $confdesc->{"parallel$i"} = $paralleldesc;
1378 }
1379
1380 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1381     $confdesc->{"serial$i"} = $serialdesc;
1382 }
1383
1384 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1385     $confdesc->{"hostpci$i"} = $hostpcidesc;
1386 }
1387
1388 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1389     $drivename_hash->{"ide$i"} = 1;
1390     $confdesc->{"ide$i"} = $idedesc;
1391 }
1392
1393 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1394     $drivename_hash->{"sata$i"} = 1;
1395     $confdesc->{"sata$i"} = $satadesc;
1396 }
1397
1398 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1399     $drivename_hash->{"scsi$i"} = 1;
1400     $confdesc->{"scsi$i"} = $scsidesc ;
1401 }
1402
1403 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1404     $drivename_hash->{"virtio$i"} = 1;
1405     $confdesc->{"virtio$i"} = $virtiodesc;
1406 }
1407
1408 $drivename_hash->{efidisk0} = 1;
1409 $confdesc->{efidisk0} = $efidisk_desc;
1410
1411 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1412     $confdesc->{"usb$i"} = $usbdesc;
1413 }
1414
1415 my $unuseddesc = {
1416     optional => 1,
1417     type => 'string', format => 'pve-volume-id',
1418     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1419 };
1420
1421 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1422     $confdesc->{"unused$i"} = $unuseddesc;
1423 }
1424
1425 my $kvm_api_version = 0;
1426
1427 sub kvm_version {
1428     return $kvm_api_version if $kvm_api_version;
1429
1430     open my $fh, '<', '/dev/kvm'
1431         or return undef;
1432
1433     # 0xae00 => KVM_GET_API_VERSION
1434     $kvm_api_version = ioctl($fh, 0xae00, 0);
1435
1436     return $kvm_api_version;
1437 }
1438
1439 my $kvm_user_version;
1440
1441 sub kvm_user_version {
1442
1443     return $kvm_user_version if $kvm_user_version;
1444
1445     $kvm_user_version = 'unknown';
1446
1447     my $code = sub {
1448         my $line = shift;
1449         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1450             $kvm_user_version = $2;
1451         }
1452     };
1453
1454     eval { run_command("kvm -version", outfunc => $code); };
1455     warn $@ if $@;
1456
1457     return $kvm_user_version;
1458
1459 }
1460
1461 sub kernel_has_vhost_net {
1462     return -c '/dev/vhost-net';
1463 }
1464
1465 sub valid_drive_names {
1466     # order is important - used to autoselect boot disk
1467     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1468             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1469             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1470             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1471             'efidisk0');
1472 }
1473
1474 sub is_valid_drivename {
1475     my $dev = shift;
1476
1477     return defined($drivename_hash->{$dev});
1478 }
1479
1480 sub option_exists {
1481     my $key = shift;
1482     return defined($confdesc->{$key});
1483 }
1484
1485 sub nic_models {
1486     return $nic_model_list;
1487 }
1488
1489 sub os_list_description {
1490
1491     return {
1492         other => 'Other',
1493         wxp => 'Windows XP',
1494         w2k => 'Windows 2000',
1495         w2k3 =>, 'Windows 2003',
1496         w2k8 => 'Windows 2008',
1497         wvista => 'Windows Vista',
1498         win7 => 'Windows 7',
1499         win8 => 'Windows 8/2012',
1500         win10 => 'Windows 10/2016',
1501         l24 => 'Linux 2.4',
1502         l26 => 'Linux 2.6',
1503     };
1504 }
1505
1506 my $cdrom_path;
1507
1508 sub get_cdrom_path {
1509
1510     return  $cdrom_path if $cdrom_path;
1511
1512     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1513     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1514     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1515 }
1516
1517 sub get_iso_path {
1518     my ($storecfg, $vmid, $cdrom) = @_;
1519
1520     if ($cdrom eq 'cdrom') {
1521         return get_cdrom_path();
1522     } elsif ($cdrom eq 'none') {
1523         return '';
1524     } elsif ($cdrom =~ m|^/|) {
1525         return $cdrom;
1526     } else {
1527         return PVE::Storage::path($storecfg, $cdrom);
1528     }
1529 }
1530
1531 # try to convert old style file names to volume IDs
1532 sub filename_to_volume_id {
1533     my ($vmid, $file, $media) = @_;
1534
1535      if (!($file eq 'none' || $file eq 'cdrom' ||
1536           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1537
1538         return undef if $file =~ m|/|;
1539
1540         if ($media && $media eq 'cdrom') {
1541             $file = "local:iso/$file";
1542         } else {
1543             $file = "local:$vmid/$file";
1544         }
1545     }
1546
1547     return $file;
1548 }
1549
1550 sub verify_media_type {
1551     my ($opt, $vtype, $media) = @_;
1552
1553     return if !$media;
1554
1555     my $etype;
1556     if ($media eq 'disk') {
1557         $etype = 'images';
1558     } elsif ($media eq 'cdrom') {
1559         $etype = 'iso';
1560     } else {
1561         die "internal error";
1562     }
1563
1564     return if ($vtype eq $etype);
1565
1566     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1567 }
1568
1569 sub cleanup_drive_path {
1570     my ($opt, $storecfg, $drive) = @_;
1571
1572     # try to convert filesystem paths to volume IDs
1573
1574     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1575         ($drive->{file} !~ m|^/dev/.+|) &&
1576         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1577         ($drive->{file} !~ m/^\d+$/)) {
1578         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1579         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1580         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1581         verify_media_type($opt, $vtype, $drive->{media});
1582         $drive->{file} = $volid;
1583     }
1584
1585     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1586 }
1587
1588 sub parse_hotplug_features {
1589     my ($data) = @_;
1590
1591     my $res = {};
1592
1593     return $res if $data eq '0';
1594
1595     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1596
1597     foreach my $feature (PVE::Tools::split_list($data)) {
1598         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1599             $res->{$1} = 1;
1600         } else {
1601             die "invalid hotplug feature '$feature'\n";
1602         }
1603     }
1604     return $res;
1605 }
1606
1607 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1608 sub pve_verify_hotplug_features {
1609     my ($value, $noerr) = @_;
1610
1611     return $value if parse_hotplug_features($value);
1612
1613     return undef if $noerr;
1614
1615     die "unable to parse hotplug option\n";
1616 }
1617
1618 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1619 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1620 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1621 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1622 #        [,iothread=on][,serial=serial][,model=model]
1623
1624 sub parse_drive {
1625     my ($key, $data) = @_;
1626
1627     my ($interface, $index);
1628
1629     if ($key =~ m/^([^\d]+)(\d+)$/) {
1630         $interface = $1;
1631         $index = $2;
1632     } else {
1633         return undef;
1634     }
1635
1636     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1637                                      : $confdesc->{$key}->{format};
1638     if (!$desc) {
1639         warn "invalid drive key: $key\n";
1640         return undef;
1641     }
1642     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1643     return undef if !$res;
1644     $res->{interface} = $interface;
1645     $res->{index} = $index;
1646
1647     my $error = 0;
1648     foreach my $opt (qw(bps bps_rd bps_wr)) {
1649         if (my $bps = defined(delete $res->{$opt})) {
1650             if (defined($res->{"m$opt"})) {
1651                 warn "both $opt and m$opt specified\n";
1652                 ++$error;
1653                 next;
1654             }
1655             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1656         }
1657     }
1658
1659     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1660     for my $requirement (
1661         [mbps_max => 'mbps'],
1662         [mbps_rd_max => 'mbps_rd'],
1663         [mbps_wr_max => 'mbps_wr'],
1664         [miops_max => 'miops'],
1665         [miops_rd_max => 'miops_rd'],
1666         [miops_wr_max => 'miops_wr'],
1667         [bps_max_length => 'mbps_max'],
1668         [bps_rd_max_length => 'mbps_rd_max'],
1669         [bps_wr_max_length => 'mbps_wr_max'],
1670         [iops_max_length => 'iops_max'],
1671         [iops_rd_max_length => 'iops_rd_max'],
1672         [iops_wr_max_length => 'iops_wr_max']) {
1673         my ($option, $requires) = @$requirement;
1674         if ($res->{$option} && !$res->{$requires}) {
1675             warn "$option requires $requires\n";
1676             ++$error;
1677         }
1678     }
1679
1680     return undef if $error;
1681
1682     return undef if $res->{mbps_rd} && $res->{mbps};
1683     return undef if $res->{mbps_wr} && $res->{mbps};
1684     return undef if $res->{iops_rd} && $res->{iops};
1685     return undef if $res->{iops_wr} && $res->{iops};
1686
1687     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1688         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1689         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1690         return undef if $res->{interface} eq 'virtio';
1691     }
1692
1693     if (my $size = $res->{size}) {
1694         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1695     }
1696
1697     return $res;
1698 }
1699
1700 sub print_drive {
1701     my ($vmid, $drive) = @_;
1702     my $data = { %$drive };
1703     delete $data->{$_} for qw(index interface);
1704     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1705 }
1706
1707 sub scsi_inquiry {
1708     my($fh, $noerr) = @_;
1709
1710     my $SG_IO = 0x2285;
1711     my $SG_GET_VERSION_NUM = 0x2282;
1712
1713     my $versionbuf = "\x00" x 8;
1714     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1715     if (!$ret) {
1716         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1717         return undef;
1718     }
1719     my $version = unpack("I", $versionbuf);
1720     if ($version < 30000) {
1721         die "scsi generic interface too old\n"  if !$noerr;
1722         return undef;
1723     }
1724
1725     my $buf = "\x00" x 36;
1726     my $sensebuf = "\x00" x 8;
1727     my $cmd = pack("C x3 C x1", 0x12, 36);
1728
1729     # see /usr/include/scsi/sg.h
1730     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1731
1732     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1733                       length($sensebuf), 0, length($buf), $buf,
1734                       $cmd, $sensebuf, 6000);
1735
1736     $ret = ioctl($fh, $SG_IO, $packet);
1737     if (!$ret) {
1738         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1739         return undef;
1740     }
1741
1742     my @res = unpack($sg_io_hdr_t, $packet);
1743     if ($res[17] || $res[18]) {
1744         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1745         return undef;
1746     }
1747
1748     my $res = {};
1749     (my $byte0, my $byte1, $res->{vendor},
1750      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1751
1752     $res->{removable} = $byte1 & 128 ? 1 : 0;
1753     $res->{type} = $byte0 & 31;
1754
1755     return $res;
1756 }
1757
1758 sub path_is_scsi {
1759     my ($path) = @_;
1760
1761     my $fh = IO::File->new("+<$path") || return undef;
1762     my $res = scsi_inquiry($fh, 1);
1763     close($fh);
1764
1765     return $res;
1766 }
1767
1768 sub machine_type_is_q35 {
1769     my ($conf) = @_;
1770
1771     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1772 }
1773
1774 sub print_tabletdevice_full {
1775     my ($conf, $arch) = @_;
1776
1777     my $q35 = machine_type_is_q35($conf);
1778
1779     # we use uhci for old VMs because tablet driver was buggy in older qemu
1780     my $usbbus;
1781     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1782         $usbbus = 'ehci';
1783     } else {
1784         $usbbus = 'uhci';
1785     }
1786
1787     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1788 }
1789
1790 sub print_keyboarddevice_full {
1791     my ($conf, $arch, $machine) = @_;
1792
1793     return undef if $arch ne 'aarch64';
1794
1795     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1796 }
1797
1798 sub print_drivedevice_full {
1799     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1800
1801     my $device = '';
1802     my $maxdev = 0;
1803
1804     if ($drive->{interface} eq 'virtio') {
1805         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1806         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1807         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1808     } elsif ($drive->{interface} eq 'scsi') {
1809
1810         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1811         my $unit = $drive->{index} % $maxdev;
1812         my $devicetype = 'hd';
1813         my $path = '';
1814         if (drive_is_cdrom($drive)) {
1815             $devicetype = 'cd';
1816         } else {
1817             if ($drive->{file} =~ m|^/|) {
1818                 $path = $drive->{file};
1819                 if (my $info = path_is_scsi($path)) {
1820                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1821                         $devicetype = 'block';
1822                     } elsif ($info->{type} == 1) { # tape
1823                         $devicetype = 'generic';
1824                     }
1825                 }
1826             } else {
1827                  $path = PVE::Storage::path($storecfg, $drive->{file});
1828             }
1829
1830             if($path =~ m/^iscsi\:\/\//){
1831                 $devicetype = 'generic';
1832             }
1833         }
1834
1835         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1836             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1837         } else {
1838             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1839         }
1840
1841         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1842             $device .= ",rotation_rate=1";
1843         }
1844         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1845
1846     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1847         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1848         my $controller = int($drive->{index} / $maxdev);
1849         my $unit = $drive->{index} % $maxdev;
1850         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1851
1852         $device = "ide-$devicetype";
1853         if ($drive->{interface} eq 'ide') {
1854             $device .= ",bus=ide.$controller,unit=$unit";
1855         } else {
1856             $device .= ",bus=ahci$controller.$unit";
1857         }
1858         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1859
1860         if ($devicetype eq 'hd') {
1861             if (my $model = $drive->{model}) {
1862                 $model = URI::Escape::uri_unescape($model);
1863                 $device .= ",model=$model";
1864             }
1865             if ($drive->{ssd}) {
1866                 $device .= ",rotation_rate=1";
1867             }
1868         }
1869         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1870     } elsif ($drive->{interface} eq 'usb') {
1871         die "implement me";
1872         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1873     } else {
1874         die "unsupported interface type";
1875     }
1876
1877     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1878
1879     if (my $serial = $drive->{serial}) {
1880         $serial = URI::Escape::uri_unescape($serial);
1881         $device .= ",serial=$serial";
1882     }
1883
1884
1885     return $device;
1886 }
1887
1888 sub get_initiator_name {
1889     my $initiator;
1890
1891     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1892     while (defined(my $line = <$fh>)) {
1893         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1894         $initiator = $1;
1895         last;
1896     }
1897     $fh->close();
1898
1899     return $initiator;
1900 }
1901
1902 sub print_drive_full {
1903     my ($storecfg, $vmid, $drive) = @_;
1904
1905     my $path;
1906     my $volid = $drive->{file};
1907     my $format;
1908
1909     if (drive_is_cdrom($drive)) {
1910         $path = get_iso_path($storecfg, $vmid, $volid);
1911     } else {
1912         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1913         if ($storeid) {
1914             $path = PVE::Storage::path($storecfg, $volid);
1915             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1916             $format = qemu_img_format($scfg, $volname);
1917         } else {
1918             $path = $volid;
1919             $format = "raw";
1920         }
1921    }
1922
1923     my $opts = '';
1924     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1925     foreach my $o (@qemu_drive_options) {
1926         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1927     }
1928
1929     # snapshot only accepts on|off
1930     if (defined($drive->{snapshot})) {
1931         my $v = $drive->{snapshot} ? 'on' : 'off';
1932         $opts .= ",snapshot=$v";
1933     }
1934
1935     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1936         my ($dir, $qmpname) = @$type;
1937         if (my $v = $drive->{"mbps$dir"}) {
1938             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1939         }
1940         if (my $v = $drive->{"mbps${dir}_max"}) {
1941             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1942         }
1943         if (my $v = $drive->{"bps${dir}_max_length"}) {
1944             $opts .= ",throttling.bps$qmpname-max-length=$v";
1945         }
1946         if (my $v = $drive->{"iops${dir}"}) {
1947             $opts .= ",throttling.iops$qmpname=$v";
1948         }
1949         if (my $v = $drive->{"iops${dir}_max"}) {
1950             $opts .= ",throttling.iops$qmpname-max=$v";
1951         }
1952         if (my $v = $drive->{"iops${dir}_max_length"}) {
1953             $opts .= ",throttling.iops$qmpname-max-length=$v";
1954         }
1955     }
1956
1957     $opts .= ",format=$format" if $format && !$drive->{format};
1958
1959     my $cache_direct = 0;
1960
1961     if (my $cache = $drive->{cache}) {
1962         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1963     } elsif (!drive_is_cdrom($drive)) {
1964         $opts .= ",cache=none";
1965         $cache_direct = 1;
1966     }
1967
1968     # aio native works only with O_DIRECT
1969     if (!$drive->{aio}) {
1970         if($cache_direct) {
1971             $opts .= ",aio=native";
1972         } else {
1973             $opts .= ",aio=threads";
1974         }
1975     }
1976
1977     if (!drive_is_cdrom($drive)) {
1978         my $detectzeroes;
1979         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1980             $detectzeroes = 'off';
1981         } elsif ($drive->{discard}) {
1982             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1983         } else {
1984             # This used to be our default with discard not being specified:
1985             $detectzeroes = 'on';
1986         }
1987         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1988     }
1989
1990     my $pathinfo = $path ? "file=$path," : '';
1991
1992     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1993 }
1994
1995 sub print_netdevice_full {
1996     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1997
1998     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1999
2000     my $device = $net->{model};
2001     if ($net->{model} eq 'virtio') {
2002          $device = 'virtio-net-pci';
2003      };
2004
2005     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2006     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2007     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2008         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2009         my $vectors = $net->{queues} * 2 + 2;
2010         $tmpstr .= ",vectors=$vectors,mq=on";
2011     }
2012     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2013
2014     if ($use_old_bios_files) {
2015         my $romfile;
2016         if ($device eq 'virtio-net-pci') {
2017             $romfile = 'pxe-virtio.rom';
2018         } elsif ($device eq 'e1000') {
2019             $romfile = 'pxe-e1000.rom';
2020         } elsif ($device eq 'ne2k') {
2021             $romfile = 'pxe-ne2k_pci.rom';
2022         } elsif ($device eq 'pcnet') {
2023             $romfile = 'pxe-pcnet.rom';
2024         } elsif ($device eq 'rtl8139') {
2025             $romfile = 'pxe-rtl8139.rom';
2026         }
2027         $tmpstr .= ",romfile=$romfile" if $romfile;
2028     }
2029
2030     return $tmpstr;
2031 }
2032
2033 sub print_netdev_full {
2034     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2035
2036     my $i = '';
2037     if ($netid =~ m/^net(\d+)$/) {
2038         $i = int($1);
2039     }
2040
2041     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2042
2043     my $ifname = "tap${vmid}i$i";
2044
2045     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2046     die "interface name '$ifname' is too long (max 15 character)\n"
2047         if length($ifname) >= 16;
2048
2049     my $vhostparam = '';
2050     if (is_native($arch)) {
2051         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2052     }
2053
2054     my $vmname = $conf->{name} || "vm$vmid";
2055
2056     my $netdev = "";
2057     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2058
2059     if ($net->{bridge}) {
2060         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2061     } else {
2062         $netdev = "type=user,id=$netid,hostname=$vmname";
2063     }
2064
2065     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2066
2067     return $netdev;
2068 }
2069
2070
2071 sub print_cpu_device {
2072     my ($conf, $id) = @_;
2073
2074     my $kvm = $conf->{kvm} // 1;
2075     my $cpu = $kvm ? "kvm64" : "qemu64";
2076     if (my $cputype = $conf->{cpu}) {
2077         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2078             or die "Cannot parse cpu description: $cputype\n";
2079         $cpu = $cpuconf->{cputype};
2080     }
2081
2082     my $cores = $conf->{cores} || 1;
2083
2084     my $current_core = ($id - 1) % $cores;
2085     my $current_socket = int(($id - 1 - $current_core)/$cores);
2086
2087     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2088 }
2089
2090 my $vga_map = {
2091     'cirrus' => 'cirrus-vga',
2092     'std' => 'VGA',
2093     'vmware' => 'vmware-svga',
2094     'virtio' => 'virtio-vga',
2095 };
2096
2097 sub print_vga_device {
2098     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2099
2100     my $type = $vga_map->{$vga->{type}};
2101     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2102         $type = 'virtio-gpu';
2103     }
2104     my $vgamem_mb = $vga->{memory};
2105     if ($qxlnum) {
2106         $type = $id ? 'qxl' : 'qxl-vga';
2107     }
2108     die "no devicetype for $vga->{type}\n" if !$type;
2109
2110     my $memory = "";
2111     if ($vgamem_mb) {
2112         if ($vga->{type} eq 'virtio') {
2113             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2114             $memory = ",max_hostmem=$bytes";
2115         } elsif ($qxlnum) {
2116             # from https://www.spice-space.org/multiple-monitors.html
2117             $memory = ",vgamem_mb=$vga->{memory}";
2118             my $ram = $vgamem_mb * 4;
2119             my $vram = $vgamem_mb * 2;
2120             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2121         } else {
2122             $memory = ",vgamem_mb=$vga->{memory}";
2123         }
2124     } elsif ($qxlnum && $id) {
2125         $memory = ",ram_size=67108864,vram_size=33554432";
2126     }
2127
2128     my $q35 = machine_type_is_q35($conf);
2129     my $vgaid = "vga" . ($id // '');
2130     my $pciaddr;
2131
2132     if ($q35 && $vgaid eq 'vga') {
2133         # the first display uses pcie.0 bus on q35 machines
2134         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2135     } else {
2136         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2137     }
2138
2139     return "$type,id=${vgaid}${memory}${pciaddr}";
2140 }
2141
2142 sub drive_is_cloudinit {
2143     my ($drive) = @_;
2144     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2145 }
2146
2147 sub drive_is_cdrom {
2148     my ($drive, $exclude_cloudinit) = @_;
2149
2150     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2151
2152     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2153
2154 }
2155
2156 sub parse_number_sets {
2157     my ($set) = @_;
2158     my $res = [];
2159     foreach my $part (split(/;/, $set)) {
2160         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2161             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2162             push @$res, [ $1, $2 ];
2163         } else {
2164             die "invalid range: $part\n";
2165         }
2166     }
2167     return $res;
2168 }
2169
2170 sub parse_numa {
2171     my ($data) = @_;
2172
2173     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2174     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2175     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2176     return $res;
2177 }
2178
2179 sub parse_hostpci {
2180     my ($value) = @_;
2181
2182     return undef if !$value;
2183
2184     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2185
2186     my @idlist = split(/;/, $res->{host});
2187     delete $res->{host};
2188     foreach my $id (@idlist) {
2189         if ($id =~ m/\./) { # full id 00:00.1
2190             push @{$res->{pciid}}, {
2191                 id => $id,
2192             };
2193         } else { # partial id 00:00
2194             $res->{pciid} = PVE::SysFSTools::lspci($id);
2195         }
2196     }
2197     return $res;
2198 }
2199
2200 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2201 sub parse_net {
2202     my ($data) = @_;
2203
2204     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2205     if ($@) {
2206         warn $@;
2207         return undef;
2208     }
2209     if (!defined($res->{macaddr})) {
2210         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2211         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2212     }
2213     return $res;
2214 }
2215
2216 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2217 sub parse_ipconfig {
2218     my ($data) = @_;
2219
2220     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2221     if ($@) {
2222         warn $@;
2223         return undef;
2224     }
2225
2226     if ($res->{gw} && !$res->{ip}) {
2227         warn 'gateway specified without specifying an IP address';
2228         return undef;
2229     }
2230     if ($res->{gw6} && !$res->{ip6}) {
2231         warn 'IPv6 gateway specified without specifying an IPv6 address';
2232         return undef;
2233     }
2234     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2235         warn 'gateway specified together with DHCP';
2236         return undef;
2237     }
2238     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2239         # gw6 + auto/dhcp
2240         warn "IPv6 gateway specified together with $res->{ip6} address";
2241         return undef;
2242     }
2243
2244     if (!$res->{ip} && !$res->{ip6}) {
2245         return { ip => 'dhcp', ip6 => 'dhcp' };
2246     }
2247
2248     return $res;
2249 }
2250
2251 sub print_net {
2252     my $net = shift;
2253
2254     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2255 }
2256
2257 sub add_random_macs {
2258     my ($settings) = @_;
2259
2260     foreach my $opt (keys %$settings) {
2261         next if $opt !~ m/^net(\d+)$/;
2262         my $net = parse_net($settings->{$opt});
2263         next if !$net;
2264         $settings->{$opt} = print_net($net);
2265     }
2266 }
2267
2268 sub vm_is_volid_owner {
2269     my ($storecfg, $vmid, $volid) = @_;
2270
2271     if ($volid !~  m|^/|) {
2272         my ($path, $owner);
2273         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2274         if ($owner && ($owner == $vmid)) {
2275             return 1;
2276         }
2277     }
2278
2279     return undef;
2280 }
2281
2282 sub split_flagged_list {
2283     my $text = shift || '';
2284     $text =~ s/[,;]/ /g;
2285     $text =~ s/^\s+//;
2286     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2287 }
2288
2289 sub join_flagged_list {
2290     my ($how, $lst) = @_;
2291     join $how, map { $lst->{$_} . $_ } keys %$lst;
2292 }
2293
2294 sub vmconfig_delete_pending_option {
2295     my ($conf, $key, $force) = @_;
2296
2297     delete $conf->{pending}->{$key};
2298     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2299     $pending_delete_hash->{$key} = $force ? '!' : '';
2300     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2301 }
2302
2303 sub vmconfig_undelete_pending_option {
2304     my ($conf, $key) = @_;
2305
2306     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2307     delete $pending_delete_hash->{$key};
2308
2309     if (%$pending_delete_hash) {
2310         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2311     } else {
2312         delete $conf->{pending}->{delete};
2313     }
2314 }
2315
2316 sub vmconfig_register_unused_drive {
2317     my ($storecfg, $vmid, $conf, $drive) = @_;
2318
2319     if (drive_is_cloudinit($drive)) {
2320         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2321         warn $@ if $@;
2322     } elsif (!drive_is_cdrom($drive)) {
2323         my $volid = $drive->{file};
2324         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2325             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2326         }
2327     }
2328 }
2329
2330 sub vmconfig_cleanup_pending {
2331     my ($conf) = @_;
2332
2333     # remove pending changes when nothing changed
2334     my $changes;
2335     foreach my $opt (keys %{$conf->{pending}}) {
2336         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2337             $changes = 1;
2338             delete $conf->{pending}->{$opt};
2339         }
2340     }
2341
2342     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2343     my $pending_delete_hash = {};
2344     while (my ($opt, $force) = each %$current_delete_hash) {
2345         if (defined($conf->{$opt})) {
2346             $pending_delete_hash->{$opt} = $force;
2347         } else {
2348             $changes = 1;
2349         }
2350     }
2351
2352     if (%$pending_delete_hash) {
2353         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2354     } else {
2355         delete $conf->{pending}->{delete};
2356     }
2357
2358     return $changes;
2359 }
2360
2361 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2362 my $smbios1_fmt = {
2363     uuid => {
2364         type => 'string',
2365         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2366         format_description => 'UUID',
2367         description => "Set SMBIOS1 UUID.",
2368         optional => 1,
2369     },
2370     version => {
2371         type => 'string',
2372         pattern => '\S+',
2373         format_description => 'string',
2374         description => "Set SMBIOS1 version.",
2375         optional => 1,
2376     },
2377     serial => {
2378         type => 'string',
2379         pattern => '\S+',
2380         format_description => 'string',
2381         description => "Set SMBIOS1 serial number.",
2382         optional => 1,
2383     },
2384     manufacturer => {
2385         type => 'string',
2386         pattern => '\S+',
2387         format_description => 'string',
2388         description => "Set SMBIOS1 manufacturer.",
2389         optional => 1,
2390     },
2391     product => {
2392         type => 'string',
2393         pattern => '\S+',
2394         format_description => 'string',
2395         description => "Set SMBIOS1 product ID.",
2396         optional => 1,
2397     },
2398     sku => {
2399         type => 'string',
2400         pattern => '\S+',
2401         format_description => 'string',
2402         description => "Set SMBIOS1 SKU string.",
2403         optional => 1,
2404     },
2405     family => {
2406         type => 'string',
2407         pattern => '\S+',
2408         format_description => 'string',
2409         description => "Set SMBIOS1 family string.",
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub touch_config {
2539     my ($vmid) = @_;
2540
2541     my $conf = PVE::QemuConfig->config_file($vmid);
2542     utime undef, undef, $conf;
2543 }
2544
2545 sub destroy_vm {
2546     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2547
2548     my $conffile = PVE::QemuConfig->config_file($vmid);
2549
2550     my $conf = PVE::QemuConfig->load_config($vmid);
2551
2552     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2553
2554     if ($conf->{template}) {
2555         # check if any base image is still used by a linked clone
2556         foreach_drive($conf, sub {
2557                 my ($ds, $drive) = @_;
2558
2559                 return if drive_is_cdrom($drive);
2560
2561                 my $volid = $drive->{file};
2562
2563                 return if !$volid || $volid =~ m|^/|;
2564
2565                 die "base volume '$volid' is still in use by linked cloned\n"
2566                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2567
2568         });
2569     }
2570
2571     # only remove disks owned by this VM
2572     foreach_drive($conf, sub {
2573         my ($ds, $drive) = @_;
2574
2575         return if drive_is_cdrom($drive, 1);
2576
2577         my $volid = $drive->{file};
2578
2579         return if !$volid || $volid =~ m|^/|;
2580
2581         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2582         return if !$path || !$owner || ($owner != $vmid);
2583
2584         eval {
2585             PVE::Storage::vdisk_free($storecfg, $volid);
2586         };
2587         warn "Could not remove disk '$volid', check manually: $@" if $@;
2588
2589     });
2590
2591     if ($keep_empty_config) {
2592         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2593     } else {
2594         unlink $conffile;
2595     }
2596
2597     # also remove unused disk
2598     eval {
2599         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2600
2601         eval {
2602             PVE::Storage::foreach_volid($dl, sub {
2603                 my ($volid, $sid, $volname, $d) = @_;
2604                 PVE::Storage::vdisk_free($storecfg, $volid);
2605             });
2606         };
2607         warn $@ if $@;
2608
2609     };
2610     warn $@ if $@;
2611 }
2612
2613 sub parse_vm_config {
2614     my ($filename, $raw) = @_;
2615
2616     return undef if !defined($raw);
2617
2618     my $res = {
2619         digest => Digest::SHA::sha1_hex($raw),
2620         snapshots => {},
2621         pending => {},
2622     };
2623
2624     $filename =~ m|/qemu-server/(\d+)\.conf$|
2625         || die "got strange filename '$filename'";
2626
2627     my $vmid = $1;
2628
2629     my $conf = $res;
2630     my $descr;
2631     my $section = '';
2632
2633     my @lines = split(/\n/, $raw);
2634     foreach my $line (@lines) {
2635         next if $line =~ m/^\s*$/;
2636
2637         if ($line =~ m/^\[PENDING\]\s*$/i) {
2638             $section = 'pending';
2639             if (defined($descr)) {
2640                 $descr =~ s/\s+$//;
2641                 $conf->{description} = $descr;
2642             }
2643             $descr = undef;
2644             $conf = $res->{$section} = {};
2645             next;
2646
2647         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2648             $section = $1;
2649             if (defined($descr)) {
2650                 $descr =~ s/\s+$//;
2651                 $conf->{description} = $descr;
2652             }
2653             $descr = undef;
2654             $conf = $res->{snapshots}->{$section} = {};
2655             next;
2656         }
2657
2658         if ($line =~ m/^\#(.*)\s*$/) {
2659             $descr = '' if !defined($descr);
2660             $descr .= PVE::Tools::decode_text($1) . "\n";
2661             next;
2662         }
2663
2664         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2665             $descr = '' if !defined($descr);
2666             $descr .= PVE::Tools::decode_text($2);
2667         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2668             $conf->{snapstate} = $1;
2669         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2670             my $key = $1;
2671             my $value = $2;
2672             $conf->{$key} = $value;
2673         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2674             my $value = $1;
2675             if ($section eq 'pending') {
2676                 $conf->{delete} = $value; # we parse this later
2677             } else {
2678                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2679             }
2680         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2681             my $key = $1;
2682             my $value = $2;
2683             eval { $value = check_type($key, $value); };
2684             if ($@) {
2685                 warn "vm $vmid - unable to parse value of '$key' - $@";
2686             } else {
2687                 $key = 'ide2' if $key eq 'cdrom';
2688                 my $fmt = $confdesc->{$key}->{format};
2689                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2690                     my $v = parse_drive($key, $value);
2691                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2692                         $v->{file} = $volid;
2693                         $value = print_drive($vmid, $v);
2694                     } else {
2695                         warn "vm $vmid - unable to parse value of '$key'\n";
2696                         next;
2697                     }
2698                 }
2699
2700                 $conf->{$key} = $value;
2701             }
2702         }
2703     }
2704
2705     if (defined($descr)) {
2706         $descr =~ s/\s+$//;
2707         $conf->{description} = $descr;
2708     }
2709     delete $res->{snapstate}; # just to be sure
2710
2711     return $res;
2712 }
2713
2714 sub write_vm_config {
2715     my ($filename, $conf) = @_;
2716
2717     delete $conf->{snapstate}; # just to be sure
2718
2719     if ($conf->{cdrom}) {
2720         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2721         $conf->{ide2} = $conf->{cdrom};
2722         delete $conf->{cdrom};
2723     }
2724
2725     # we do not use 'smp' any longer
2726     if ($conf->{sockets}) {
2727         delete $conf->{smp};
2728     } elsif ($conf->{smp}) {
2729         $conf->{sockets} = $conf->{smp};
2730         delete $conf->{cores};
2731         delete $conf->{smp};
2732     }
2733
2734     my $used_volids = {};
2735
2736     my $cleanup_config = sub {
2737         my ($cref, $pending, $snapname) = @_;
2738
2739         foreach my $key (keys %$cref) {
2740             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2741                 $key eq 'snapstate' || $key eq 'pending';
2742             my $value = $cref->{$key};
2743             if ($key eq 'delete') {
2744                 die "propertry 'delete' is only allowed in [PENDING]\n"
2745                     if !$pending;
2746                 # fixme: check syntax?
2747                 next;
2748             }
2749             eval { $value = check_type($key, $value); };
2750             die "unable to parse value of '$key' - $@" if $@;
2751
2752             $cref->{$key} = $value;
2753
2754             if (!$snapname && is_valid_drivename($key)) {
2755                 my $drive = parse_drive($key, $value);
2756                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2757             }
2758         }
2759     };
2760
2761     &$cleanup_config($conf);
2762
2763     &$cleanup_config($conf->{pending}, 1);
2764
2765     foreach my $snapname (keys %{$conf->{snapshots}}) {
2766         die "internal error" if $snapname eq 'pending';
2767         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2768     }
2769
2770     # remove 'unusedX' settings if we re-add a volume
2771     foreach my $key (keys %$conf) {
2772         my $value = $conf->{$key};
2773         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2774             delete $conf->{$key};
2775         }
2776     }
2777
2778     my $generate_raw_config = sub {
2779         my ($conf, $pending) = @_;
2780
2781         my $raw = '';
2782
2783         # add description as comment to top of file
2784         if (defined(my $descr = $conf->{description})) {
2785             if ($descr) {
2786                 foreach my $cl (split(/\n/, $descr)) {
2787                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2788                 }
2789             } else {
2790                 $raw .= "#\n" if $pending;
2791             }
2792         }
2793
2794         foreach my $key (sort keys %$conf) {
2795             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2796             $raw .= "$key: $conf->{$key}\n";
2797         }
2798         return $raw;
2799     };
2800
2801     my $raw = &$generate_raw_config($conf);
2802
2803     if (scalar(keys %{$conf->{pending}})){
2804         $raw .= "\n[PENDING]\n";
2805         $raw .= &$generate_raw_config($conf->{pending}, 1);
2806     }
2807
2808     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2809         $raw .= "\n[$snapname]\n";
2810         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2811     }
2812
2813     return $raw;
2814 }
2815
2816 sub load_defaults {
2817
2818     my $res = {};
2819
2820     # we use static defaults from our JSON schema configuration
2821     foreach my $key (keys %$confdesc) {
2822         if (defined(my $default = $confdesc->{$key}->{default})) {
2823             $res->{$key} = $default;
2824         }
2825     }
2826
2827     return $res;
2828 }
2829
2830 sub config_list {
2831     my $vmlist = PVE::Cluster::get_vmlist();
2832     my $res = {};
2833     return $res if !$vmlist || !$vmlist->{ids};
2834     my $ids = $vmlist->{ids};
2835
2836     foreach my $vmid (keys %$ids) {
2837         my $d = $ids->{$vmid};
2838         next if !$d->{node} || $d->{node} ne $nodename;
2839         next if !$d->{type} || $d->{type} ne 'qemu';
2840         $res->{$vmid}->{exists} = 1;
2841     }
2842     return $res;
2843 }
2844
2845 # test if VM uses local resources (to prevent migration)
2846 sub check_local_resources {
2847     my ($conf, $noerr) = @_;
2848
2849     my $loc_res = 0;
2850
2851     $loc_res = 1 if $conf->{hostusb}; # old syntax
2852     $loc_res = 1 if $conf->{hostpci}; # old syntax
2853
2854     $loc_res = 1 if $conf->{ivshmem};
2855
2856     foreach my $k (keys %$conf) {
2857         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2858         # sockets are safe: they will recreated be on the target side post-migrate
2859         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2860         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2861     }
2862
2863     die "VM uses local resources\n" if $loc_res && !$noerr;
2864
2865     return $loc_res;
2866 }
2867
2868 # check if used storages are available on all nodes (use by migrate)
2869 sub check_storage_availability {
2870     my ($storecfg, $conf, $node) = @_;
2871
2872     foreach_drive($conf, sub {
2873         my ($ds, $drive) = @_;
2874
2875         my $volid = $drive->{file};
2876         return if !$volid;
2877
2878         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2879         return if !$sid;
2880
2881         # check if storage is available on both nodes
2882         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2883         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2884    });
2885 }
2886
2887 # list nodes where all VM images are available (used by has_feature API)
2888 sub shared_nodes {
2889     my ($conf, $storecfg) = @_;
2890
2891     my $nodelist = PVE::Cluster::get_nodelist();
2892     my $nodehash = { map { $_ => 1 } @$nodelist };
2893     my $nodename = PVE::INotify::nodename();
2894
2895     foreach_drive($conf, sub {
2896         my ($ds, $drive) = @_;
2897
2898         my $volid = $drive->{file};
2899         return if !$volid;
2900
2901         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2902         if ($storeid) {
2903             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2904             if ($scfg->{disable}) {
2905                 $nodehash = {};
2906             } elsif (my $avail = $scfg->{nodes}) {
2907                 foreach my $node (keys %$nodehash) {
2908                     delete $nodehash->{$node} if !$avail->{$node};
2909                 }
2910             } elsif (!$scfg->{shared}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     delete $nodehash->{$node} if $node ne $nodename
2913                 }
2914             }
2915         }
2916     });
2917
2918     return $nodehash
2919 }
2920
2921 sub check_cmdline {
2922     my ($pidfile, $pid) = @_;
2923
2924     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2925     if (defined($fh)) {
2926         my $line = <$fh>;
2927         $fh->close;
2928         return undef if !$line;
2929         my @param = split(/\0/, $line);
2930
2931         my $cmd = $param[0];
2932         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2933
2934         for (my $i = 0; $i < scalar (@param); $i++) {
2935             my $p = $param[$i];
2936             next if !$p;
2937             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2938                 my $p = $param[$i+1];
2939                 return 1 if $p && ($p eq $pidfile);
2940                 return undef;
2941             }
2942         }
2943     }
2944     return undef;
2945 }
2946
2947 sub check_running {
2948     my ($vmid, $nocheck, $node) = @_;
2949
2950     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2951
2952     die "unable to find configuration file for VM $vmid - no such machine\n"
2953         if !$nocheck && ! -f $filename;
2954
2955     my $pidfile = pidfile_name($vmid);
2956
2957     if (my $fd = IO::File->new("<$pidfile")) {
2958         my $st = stat($fd);
2959         my $line = <$fd>;
2960         close($fd);
2961
2962         my $mtime = $st->mtime;
2963         if ($mtime > time()) {
2964             warn "file '$filename' modified in future\n";
2965         }
2966
2967         if ($line =~ m/^(\d+)$/) {
2968             my $pid = $1;
2969             if (check_cmdline($pidfile, $pid)) {
2970                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2971                     return $pid;
2972                 }
2973             }
2974         }
2975     }
2976
2977     return undef;
2978 }
2979
2980 sub vzlist {
2981
2982     my $vzlist = config_list();
2983
2984     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2985
2986     while (defined(my $de = $fd->read)) {
2987         next if $de !~ m/^(\d+)\.pid$/;
2988         my $vmid = $1;
2989         next if !defined($vzlist->{$vmid});
2990         if (my $pid = check_running($vmid)) {
2991             $vzlist->{$vmid}->{pid} = $pid;
2992         }
2993     }
2994
2995     return $vzlist;
2996 }
2997
2998 sub disksize {
2999     my ($storecfg, $conf) = @_;
3000
3001     my $bootdisk = $conf->{bootdisk};
3002     return undef if !$bootdisk;
3003     return undef if !is_valid_drivename($bootdisk);
3004
3005     return undef if !$conf->{$bootdisk};
3006
3007     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3008     return undef if !defined($drive);
3009
3010     return undef if drive_is_cdrom($drive);
3011
3012     my $volid = $drive->{file};
3013     return undef if !$volid;
3014
3015     return $drive->{size};
3016 }
3017
3018 our $vmstatus_return_properties = {
3019     vmid => get_standard_option('pve-vmid'),
3020     status => {
3021         description => "Qemu process status.",
3022         type => 'string',
3023         enum => ['stopped', 'running'],
3024     },
3025     maxmem => {
3026         description => "Maximum memory in bytes.",
3027         type => 'integer',
3028         optional => 1,
3029         renderer => 'bytes',
3030     },
3031     maxdisk => {
3032         description => "Root disk size in bytes.",
3033         type => 'integer',
3034         optional => 1,
3035         renderer => 'bytes',
3036     },
3037     name => {
3038         description => "VM name.",
3039         type => 'string',
3040         optional => 1,
3041     },
3042     qmpstatus => {
3043         description => "Qemu QMP agent status.",
3044         type => 'string',
3045         optional => 1,
3046     },
3047     pid => {
3048         description => "PID of running qemu process.",
3049         type => 'integer',
3050         optional => 1,
3051     },
3052     uptime => {
3053         description => "Uptime.",
3054         type => 'integer',
3055         optional => 1,
3056         renderer => 'duration',
3057     },
3058     cpus => {
3059         description => "Maximum usable CPUs.",
3060         type => 'number',
3061         optional => 1,
3062     },
3063     lock => {
3064         description => "The current config lock, if any.",
3065         type => 'string',
3066         optional => 1,
3067     }
3068 };
3069
3070 my $last_proc_pid_stat;
3071
3072 # get VM status information
3073 # This must be fast and should not block ($full == false)
3074 # We only query KVM using QMP if $full == true (this can be slow)
3075 sub vmstatus {
3076     my ($opt_vmid, $full) = @_;
3077
3078     my $res = {};
3079
3080     my $storecfg = PVE::Storage::config();
3081
3082     my $list = vzlist();
3083     my $defaults = load_defaults();
3084
3085     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3086
3087     my $cpucount = $cpuinfo->{cpus} || 1;
3088
3089     foreach my $vmid (keys %$list) {
3090         next if $opt_vmid && ($vmid ne $opt_vmid);
3091
3092         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3093         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3094
3095         my $d = { vmid => $vmid };
3096         $d->{pid} = $list->{$vmid}->{pid};
3097
3098         # fixme: better status?
3099         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3100
3101         my $size = disksize($storecfg, $conf);
3102         if (defined($size)) {
3103             $d->{disk} = 0; # no info available
3104             $d->{maxdisk} = $size;
3105         } else {
3106             $d->{disk} = 0;
3107             $d->{maxdisk} = 0;
3108         }
3109
3110         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3111             * ($conf->{cores} || $defaults->{cores});
3112         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3113         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3114
3115         $d->{name} = $conf->{name} || "VM $vmid";
3116         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3117             : $defaults->{memory}*(1024*1024);
3118
3119         if ($conf->{balloon}) {
3120             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3121             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3122                 : $defaults->{shares};
3123         }
3124
3125         $d->{uptime} = 0;
3126         $d->{cpu} = 0;
3127         $d->{mem} = 0;
3128
3129         $d->{netout} = 0;
3130         $d->{netin} = 0;
3131
3132         $d->{diskread} = 0;
3133         $d->{diskwrite} = 0;
3134
3135         $d->{template} = PVE::QemuConfig->is_template($conf);
3136
3137         $d->{serial} = 1 if conf_has_serial($conf);
3138         $d->{lock} = $conf->{lock} if $conf->{lock};
3139
3140         $res->{$vmid} = $d;
3141     }
3142
3143     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3144     foreach my $dev (keys %$netdev) {
3145         next if $dev !~ m/^tap([1-9]\d*)i/;
3146         my $vmid = $1;
3147         my $d = $res->{$vmid};
3148         next if !$d;
3149
3150         $d->{netout} += $netdev->{$dev}->{receive};
3151         $d->{netin} += $netdev->{$dev}->{transmit};
3152
3153         if ($full) {
3154             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3155             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3156         }
3157
3158     }
3159
3160     my $ctime = gettimeofday;
3161
3162     foreach my $vmid (keys %$list) {
3163
3164         my $d = $res->{$vmid};
3165         my $pid = $d->{pid};
3166         next if !$pid;
3167
3168         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3169         next if !$pstat; # not running
3170
3171         my $used = $pstat->{utime} + $pstat->{stime};
3172
3173         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3174
3175         if ($pstat->{vsize}) {
3176             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3177         }
3178
3179         my $old = $last_proc_pid_stat->{$pid};
3180         if (!$old) {
3181             $last_proc_pid_stat->{$pid} = {
3182                 time => $ctime,
3183                 used => $used,
3184                 cpu => 0,
3185             };
3186             next;
3187         }
3188
3189         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3190
3191         if ($dtime > 1000) {
3192             my $dutime = $used -  $old->{used};
3193
3194             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3195             $last_proc_pid_stat->{$pid} = {
3196                 time => $ctime,
3197                 used => $used,
3198                 cpu => $d->{cpu},
3199             };
3200         } else {
3201             $d->{cpu} = $old->{cpu};
3202         }
3203     }
3204
3205     return $res if !$full;
3206
3207     my $qmpclient = PVE::QMPClient->new();
3208
3209     my $ballooncb = sub {
3210         my ($vmid, $resp) = @_;
3211
3212         my $info = $resp->{'return'};
3213         return if !$info->{max_mem};
3214
3215         my $d = $res->{$vmid};
3216
3217         # use memory assigned to VM
3218         $d->{maxmem} = $info->{max_mem};
3219         $d->{balloon} = $info->{actual};
3220
3221         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3222             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3223             $d->{freemem} = $info->{free_mem};
3224         }
3225
3226         $d->{ballooninfo} = $info;
3227     };
3228
3229     my $blockstatscb = sub {
3230         my ($vmid, $resp) = @_;
3231         my $data = $resp->{'return'} || [];
3232         my $totalrdbytes = 0;
3233         my $totalwrbytes = 0;
3234
3235         for my $blockstat (@$data) {
3236             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3237             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3238
3239             $blockstat->{device} =~ s/drive-//;
3240             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3241         }
3242         $res->{$vmid}->{diskread} = $totalrdbytes;
3243         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3244     };
3245
3246     my $statuscb = sub {
3247         my ($vmid, $resp) = @_;
3248
3249         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3250         # this fails if ballon driver is not loaded, so this must be
3251         # the last commnand (following command are aborted if this fails).
3252         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3253
3254         my $status = 'unknown';
3255         if (!defined($status = $resp->{'return'}->{status})) {
3256             warn "unable to get VM status\n";
3257             return;
3258         }
3259
3260         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3261     };
3262
3263     foreach my $vmid (keys %$list) {
3264         next if $opt_vmid && ($vmid ne $opt_vmid);
3265         next if !$res->{$vmid}->{pid}; # not running
3266         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3267     }
3268
3269     $qmpclient->queue_execute(undef, 2);
3270
3271     foreach my $vmid (keys %$list) {
3272         next if $opt_vmid && ($vmid ne $opt_vmid);
3273         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3274     }
3275
3276     return $res;
3277 }
3278
3279 sub foreach_drive {
3280     my ($conf, $func, @param) = @_;
3281
3282     foreach my $ds (valid_drive_names()) {
3283         next if !defined($conf->{$ds});
3284
3285         my $drive = parse_drive($ds, $conf->{$ds});
3286         next if !$drive;
3287
3288         &$func($ds, $drive, @param);
3289     }
3290 }
3291
3292 sub foreach_volid {
3293     my ($conf, $func, @param) = @_;
3294
3295     my $volhash = {};
3296
3297     my $test_volid = sub {
3298         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3299
3300         return if !$volid;
3301
3302         $volhash->{$volid}->{cdrom} //= 1;
3303         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3304
3305         $volhash->{$volid}->{replicate} //= 0;
3306         $volhash->{$volid}->{replicate} = 1 if $replicate;
3307
3308         $volhash->{$volid}->{shared} //= 0;
3309         $volhash->{$volid}->{shared} = 1 if $shared;
3310
3311         $volhash->{$volid}->{referenced_in_config} //= 0;
3312         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3313
3314         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3315             if defined($snapname);
3316     };
3317
3318     foreach_drive($conf, sub {
3319         my ($ds, $drive) = @_;
3320         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3321     });
3322
3323     foreach my $snapname (keys %{$conf->{snapshots}}) {
3324         my $snap = $conf->{snapshots}->{$snapname};
3325         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3326         foreach_drive($snap, sub {
3327             my ($ds, $drive) = @_;
3328             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3329         });
3330     }
3331
3332     foreach my $volid (keys %$volhash) {
3333         &$func($volid, $volhash->{$volid}, @param);
3334     }
3335 }
3336
3337 sub conf_has_serial {
3338     my ($conf) = @_;
3339
3340     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3341         if ($conf->{"serial$i"}) {
3342             return 1;
3343         }
3344     }
3345
3346     return 0;
3347 }
3348
3349 sub vga_conf_has_spice {
3350     my ($vga) = @_;
3351
3352     my $vgaconf = parse_vga($vga);
3353     my $vgatype = $vgaconf->{type};
3354     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3355
3356     return $1 || 1;
3357 }
3358
3359 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3360 sub get_host_arch() {
3361     $host_arch = (POSIX::uname())[4] if !$host_arch;
3362     return $host_arch;
3363 }
3364
3365 sub is_native($) {
3366     my ($arch) = @_;
3367     return get_host_arch() eq $arch;
3368 }
3369
3370 my $default_machines = {
3371     x86_64 => 'pc',
3372     aarch64 => 'virt',
3373 };
3374
3375 sub get_basic_machine_info {
3376     my ($conf, $forcemachine) = @_;
3377
3378     my $arch = $conf->{arch} // get_host_arch();
3379     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3380     return ($arch, $machine);
3381 }
3382
3383 sub get_ovmf_files($) {
3384     my ($arch) = @_;
3385
3386     my $ovmf = $OVMF->{$arch}
3387         or die "no OVMF images known for architecture '$arch'\n";
3388
3389     return @$ovmf;
3390 }
3391
3392 my $Arch2Qemu = {
3393     aarch64 => '/usr/bin/qemu-system-aarch64',
3394     x86_64 => '/usr/bin/qemu-system-x86_64',
3395 };
3396 sub get_command_for_arch($) {
3397     my ($arch) = @_;
3398     return '/usr/bin/kvm' if is_native($arch);
3399
3400     my $cmd = $Arch2Qemu->{$arch}
3401         or die "don't know how to emulate architecture '$arch'\n";
3402     return $cmd;
3403 }
3404
3405 sub get_cpu_options {
3406     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3407
3408     my $cpuFlags = [];
3409     my $ostype = $conf->{ostype};
3410
3411     my $cpu = $kvm ? "kvm64" : "qemu64";
3412     if ($arch eq 'aarch64') {
3413         $cpu = 'cortex-a57';
3414     }
3415     my $hv_vendor_id;
3416     if (my $cputype = $conf->{cpu}) {
3417         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3418             or die "Cannot parse cpu description: $cputype\n";
3419         $cpu = $cpuconf->{cputype};
3420         $kvm_off = 1 if $cpuconf->{hidden};
3421         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3422
3423         if (defined(my $flags = $cpuconf->{flags})) {
3424             push @$cpuFlags, split(";", $flags);
3425         }
3426     }
3427
3428     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3429
3430     push @$cpuFlags , '-x2apic'
3431         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3432
3433     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3434
3435     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3436
3437     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3438
3439         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3440         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3441     }
3442
3443     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3444
3445     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3446
3447     push @$cpuFlags, 'kvm=off' if $kvm_off;
3448
3449     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3450         push @$cpuFlags, "vendor=${cpu_vendor}"
3451             if $cpu_vendor ne 'default';
3452     } elsif ($arch ne 'aarch64') {
3453         die "internal error"; # should not happen
3454     }
3455
3456     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3457
3458     return ('-cpu', $cpu);
3459 }
3460
3461 sub config_to_command {
3462     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3463
3464     my $cmd = [];
3465     my $globalFlags = [];
3466     my $machineFlags = [];
3467     my $rtcFlags = [];
3468     my $devices = [];
3469     my $pciaddr = '';
3470     my $bridges = {};
3471     my $kvmver = kvm_user_version();
3472     my $vernum = 0; # unknown
3473     my $ostype = $conf->{ostype};
3474     my $winversion = windows_version($ostype);
3475     my $kvm = $conf->{kvm};
3476
3477     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3478     $kvm //= 1 if is_native($arch);
3479
3480     if ($kvm) {
3481         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3482             if !defined kvm_version();
3483     }
3484
3485     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3486         $vernum = $1*1000000+$2*1000;
3487     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3488         $vernum = $1*1000000+$2*1000+$3;
3489     }
3490
3491     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3492
3493     my $have_ovz = -f '/proc/vz/vestat';
3494
3495     my $q35 = machine_type_is_q35($conf);
3496     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3497     my $use_old_bios_files = undef;
3498     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3499
3500     my $cpuunits = defined($conf->{cpuunits}) ?
3501             $conf->{cpuunits} : $defaults->{cpuunits};
3502
3503     push @$cmd, get_command_for_arch($arch);
3504
3505     push @$cmd, '-id', $vmid;
3506
3507     my $vmname = $conf->{name} || "vm$vmid";
3508
3509     push @$cmd, '-name', $vmname;
3510
3511     my $use_virtio = 0;
3512
3513     my $qmpsocket = qmp_socket($vmid);
3514     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3515     push @$cmd, '-mon', "chardev=qmp,mode=control";
3516
3517     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3518         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3519         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3520     }
3521
3522     push @$cmd, '-pidfile' , pidfile_name($vmid);
3523
3524     push @$cmd, '-daemonize';
3525
3526     if ($conf->{smbios1}) {
3527         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3528     }
3529
3530     if ($conf->{vmgenid}) {
3531         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3532     }
3533
3534     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3535     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3536         die "uefi base image not found\n" if ! -f $ovmf_code;
3537
3538         my $path;
3539         my $format;
3540         if (my $efidisk = $conf->{efidisk0}) {
3541             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3542             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3543             $format = $d->{format};
3544             if ($storeid) {
3545                 $path = PVE::Storage::path($storecfg, $d->{file});
3546                 if (!defined($format)) {
3547                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3548                     $format = qemu_img_format($scfg, $volname);
3549                 }
3550             } else {
3551                 $path = $d->{file};
3552                 die "efidisk format must be specified\n"
3553                     if !defined($format);
3554             }
3555         } else {
3556             warn "no efidisk configured! Using temporary efivars disk.\n";
3557             $path = "/tmp/$vmid-ovmf.fd";
3558             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3559             $format = 'raw';
3560         }
3561
3562         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3563         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3564     }
3565
3566
3567     # add usb controllers
3568     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3569     push @$devices, @usbcontrollers if @usbcontrollers;
3570     my $vga = parse_vga($conf->{vga});
3571
3572     my $qxlnum = vga_conf_has_spice($conf->{vga});
3573     $vga->{type} = 'qxl' if $qxlnum;
3574
3575     if (!$vga->{type}) {
3576         if ($arch eq 'aarch64') {
3577             $vga->{type} = 'virtio';
3578         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3579             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3580         } else {
3581             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3582         }
3583     }
3584
3585     # enable absolute mouse coordinates (needed by vnc)
3586     my $tablet;
3587     if (defined($conf->{tablet})) {
3588         $tablet = $conf->{tablet};
3589     } else {
3590         $tablet = $defaults->{tablet};
3591         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3592         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3593     }
3594
3595     if ($tablet) {
3596         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3597         my $kbd = print_keyboarddevice_full($conf, $arch);
3598         push @$devices, '-device', $kbd if defined($kbd);
3599     }
3600
3601     my $kvm_off = 0;
3602     my $gpu_passthrough;
3603
3604     # host pci devices
3605     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3606         my $d = parse_hostpci($conf->{"hostpci$i"});
3607         next if !$d;
3608
3609         my $pcie = $d->{pcie};
3610         if ($pcie) {
3611             die "q35 machine model is not enabled" if !$q35;
3612             # win7 wants to have the pcie devices directly on the pcie bus
3613             # instead of in the root port
3614             if ($winversion == 7) {
3615                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3616             } else {
3617                 $pciaddr = print_pcie_addr("hostpci$i");
3618             }
3619         } else {
3620             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3621         }
3622
3623         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3624         my $romfile = $d->{romfile};
3625
3626         my $xvga = '';
3627         if ($d->{'x-vga'}) {
3628             $xvga = ',x-vga=on';
3629             $kvm_off = 1;
3630             $vga->{type} = 'none' if !defined($conf->{vga});
3631             $gpu_passthrough = 1;
3632
3633             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3634                 $xvga = "";
3635             }
3636         }
3637         my $pcidevices = $d->{pciid};
3638         my $multifunction = 1 if @$pcidevices > 1;
3639         my $sysfspath;
3640         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3641             my $id = $pcidevices->[0]->{id};
3642             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3643             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3644         } elsif ($d->{mdev}) {
3645             warn "ignoring mediated device with multifunction device\n";
3646         }
3647
3648         my $j=0;
3649         foreach my $pcidevice (@$pcidevices) {
3650
3651             my $id = "hostpci$i";
3652             $id .= ".$j" if $multifunction;
3653             my $addr = $pciaddr;
3654             $addr .= ".$j" if $multifunction;
3655             my $devicestr = "vfio-pci";
3656             if ($sysfspath) {
3657                 $devicestr .= ",sysfsdev=$sysfspath";
3658             } else {
3659                 $devicestr .= ",host=$pcidevice->{id}";
3660             }
3661             $devicestr .= ",id=$id$addr";
3662
3663             if($j == 0){
3664                 $devicestr .= "$rombar$xvga";
3665                 $devicestr .= ",multifunction=on" if $multifunction;
3666                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3667             }
3668
3669             push @$devices, '-device', $devicestr;
3670             $j++;
3671         }
3672     }
3673
3674     # usb devices
3675     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3676     push @$devices, @usbdevices if @usbdevices;
3677     # serial devices
3678     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3679         if (my $path = $conf->{"serial$i"}) {
3680             if ($path eq 'socket') {
3681                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3682                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3683                 # On aarch64, serial0 is the UART device. Qemu only allows
3684                 # connecting UART devices via the '-serial' command line, as
3685                 # the device has a fixed slot on the hardware...
3686                 if ($arch eq 'aarch64' && $i == 0) {
3687                     push @$devices, '-serial', "chardev:serial$i";
3688                 } else {
3689                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3690                 }
3691             } else {
3692                 die "no such serial device\n" if ! -c $path;
3693                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3694                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3695             }
3696         }
3697     }
3698
3699     # parallel devices
3700     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3701         if (my $path = $conf->{"parallel$i"}) {
3702             die "no such parallel device\n" if ! -c $path;
3703             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3704             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3705             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3706         }
3707     }
3708
3709
3710     my $sockets = 1;
3711     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3712     $sockets = $conf->{sockets} if  $conf->{sockets};
3713
3714     my $cores = $conf->{cores} || 1;
3715
3716     my $maxcpus = $sockets * $cores;
3717
3718     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3719
3720     my $allowed_vcpus = $cpuinfo->{cpus};
3721
3722     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3723         if ($allowed_vcpus < $maxcpus);
3724
3725     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3726
3727         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3728         for (my $i = 2; $i <= $vcpus; $i++)  {
3729             my $cpustr = print_cpu_device($conf,$i);
3730             push @$cmd, '-device', $cpustr;
3731         }
3732
3733     } else {
3734
3735         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3736     }
3737     push @$cmd, '-nodefaults';
3738
3739     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3740
3741     my $bootindex_hash = {};
3742     my $i = 1;
3743     foreach my $o (split(//, $bootorder)) {
3744         $bootindex_hash->{$o} = $i*100;
3745         $i++;
3746     }
3747
3748     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3749
3750     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3751
3752     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3753
3754     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3755         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3756         my $socket = vnc_socket($vmid);
3757         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3758     } else {
3759         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3760         push @$cmd, '-nographic';
3761     }
3762
3763     # time drift fix
3764     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3765
3766     my $useLocaltime = $conf->{localtime};
3767
3768     if ($winversion >= 5) { # windows
3769         $useLocaltime = 1 if !defined($conf->{localtime});
3770
3771         # use time drift fix when acpi is enabled
3772         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3773             $tdf = 1 if !defined($conf->{tdf});
3774         }
3775     }
3776
3777     if ($winversion >= 6) {
3778         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3779         push @$cmd, '-no-hpet';
3780     }
3781
3782     push @$rtcFlags, 'driftfix=slew' if $tdf;
3783
3784     if (!$kvm) {
3785         push @$machineFlags, 'accel=tcg';
3786     }
3787
3788     if ($machine_type) {
3789         push @$machineFlags, "type=${machine_type}";
3790     }
3791
3792     if ($conf->{startdate}) {
3793         push @$rtcFlags, "base=$conf->{startdate}";
3794     } elsif ($useLocaltime) {
3795         push @$rtcFlags, 'base=localtime';
3796     }
3797
3798     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3799
3800     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3801
3802     push @$cmd, '-S' if $conf->{freeze};
3803
3804     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3805
3806     # enable sound
3807     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3808     #push @$cmd, '-soundhw', 'es1370';
3809     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3810
3811     if (parse_guest_agent($conf)->{enabled}) {
3812         my $qgasocket = qmp_socket($vmid, 1);
3813         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3814         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3815         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3816         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3817     }
3818
3819     my $spice_port;
3820
3821     if ($qxlnum) {
3822         if ($qxlnum > 1) {
3823             if ($winversion){
3824                 for(my $i = 1; $i < $qxlnum; $i++){
3825                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3826                 }
3827             } else {
3828                 # assume other OS works like Linux
3829                 my ($ram, $vram) = ("134217728", "67108864");
3830                 if ($vga->{memory}) {
3831                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3832                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3833                 }
3834                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3835                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3836             }
3837         }
3838
3839         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3840
3841         my $nodename = PVE::INotify::nodename();
3842         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3843         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3844         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3845         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3846         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3847
3848         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3849
3850         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3851         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3852         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3853     }
3854
3855     # enable balloon by default, unless explicitly disabled
3856     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3857         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3858         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3859     }
3860
3861     if ($conf->{watchdog}) {
3862         my $wdopts = parse_watchdog($conf->{watchdog});
3863         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3864         my $watchdog = $wdopts->{model} || 'i6300esb';
3865         push @$devices, '-device', "$watchdog$pciaddr";
3866         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3867     }
3868
3869     my $vollist = [];
3870     my $scsicontroller = {};
3871     my $ahcicontroller = {};
3872     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3873
3874     # Add iscsi initiator name if available
3875     if (my $initiator = get_initiator_name()) {
3876         push @$devices, '-iscsi', "initiator-name=$initiator";
3877     }
3878
3879     foreach_drive($conf, sub {
3880         my ($ds, $drive) = @_;
3881
3882         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3883             push @$vollist, $drive->{file};
3884         }
3885
3886         # ignore efidisk here, already added in bios/fw handling code above
3887         return if $drive->{interface} eq 'efidisk';
3888
3889         $use_virtio = 1 if $ds =~ m/^virtio/;
3890
3891         if (drive_is_cdrom ($drive)) {
3892             if ($bootindex_hash->{d}) {
3893                 $drive->{bootindex} = $bootindex_hash->{d};
3894                 $bootindex_hash->{d} += 1;
3895             }
3896         } else {
3897             if ($bootindex_hash->{c}) {
3898                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3899                 $bootindex_hash->{c} += 1;
3900             }
3901         }
3902
3903         if($drive->{interface} eq 'virtio'){
3904            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3905         }
3906
3907         if ($drive->{interface} eq 'scsi') {
3908
3909             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3910
3911             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3912             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3913
3914             my $iothread = '';
3915             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3916                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3917                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3918             } elsif ($drive->{iothread}) {
3919                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3920             }
3921
3922             my $queues = '';
3923             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3924                 $queues = ",num_queues=$drive->{queues}";
3925             }
3926
3927             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3928             $scsicontroller->{$controller}=1;
3929         }
3930
3931         if ($drive->{interface} eq 'sata') {
3932            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3933            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3934            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3935            $ahcicontroller->{$controller}=1;
3936         }
3937
3938         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3939         push @$devices, '-drive',$drive_cmd;
3940         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3941     });
3942
3943     for (my $i = 0; $i < $MAX_NETS; $i++) {
3944          next if !$conf->{"net$i"};
3945          my $d = parse_net($conf->{"net$i"});
3946          next if !$d;
3947
3948          $use_virtio = 1 if $d->{model} eq 'virtio';
3949
3950          if ($bootindex_hash->{n}) {
3951             $d->{bootindex} = $bootindex_hash->{n};
3952             $bootindex_hash->{n} += 1;
3953          }
3954
3955          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3956          push @$devices, '-netdev', $netdevfull;
3957
3958          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3959          push @$devices, '-device', $netdevicefull;
3960     }
3961
3962     if ($conf->{ivshmem}) {
3963         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3964
3965         my $bus;
3966         if ($q35) {
3967             $bus = print_pcie_addr("ivshmem");
3968         } else {
3969             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3970         }
3971
3972         my $ivshmem_name = $ivshmem->{name} // $vmid;
3973         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3974
3975         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3976         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3977     }
3978
3979     if (!$q35) {
3980         # add pci bridges
3981         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3982            $bridges->{1} = 1;
3983            $bridges->{2} = 1;
3984         }
3985
3986         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3987
3988         while (my ($k, $v) = each %$bridges) {
3989             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3990             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3991         }
3992     }
3993
3994     push @$cmd, @$devices;
3995     push @$cmd, '-rtc', join(',', @$rtcFlags)
3996         if scalar(@$rtcFlags);
3997     push @$cmd, '-machine', join(',', @$machineFlags)
3998         if scalar(@$machineFlags);
3999     push @$cmd, '-global', join(',', @$globalFlags)
4000         if scalar(@$globalFlags);
4001
4002     if (my $vmstate = $conf->{vmstate}) {
4003         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4004         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4005         push @$cmd, '-loadstate', $statepath;
4006     }
4007
4008     # add custom args
4009     if ($conf->{args}) {
4010         my $aa = PVE::Tools::split_args($conf->{args});
4011         push @$cmd, @$aa;
4012     }
4013
4014     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4015 }
4016
4017 sub vnc_socket {
4018     my ($vmid) = @_;
4019     return "${var_run_tmpdir}/$vmid.vnc";
4020 }
4021
4022 sub spice_port {
4023     my ($vmid) = @_;
4024
4025     my $res = vm_mon_cmd($vmid, 'query-spice');
4026
4027     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4028 }
4029
4030 sub qmp_socket {
4031     my ($vmid, $qga, $name) = @_;
4032     my $sockettype = $qga ? 'qga' : 'qmp';
4033     my $ext = $name ? '-'.$name : '';
4034     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4035 }
4036
4037 sub pidfile_name {
4038     my ($vmid) = @_;
4039     return "${var_run_tmpdir}/$vmid.pid";
4040 }
4041
4042 sub vm_devices_list {
4043     my ($vmid) = @_;
4044
4045     my $res = vm_mon_cmd($vmid, 'query-pci');
4046     my $devices_to_check = [];
4047     my $devices = {};
4048     foreach my $pcibus (@$res) {
4049         push @$devices_to_check, @{$pcibus->{devices}},
4050     }
4051
4052     while (@$devices_to_check) {
4053         my $to_check = [];
4054         for my $d (@$devices_to_check) {
4055             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4056             next if !$d->{'pci_bridge'};
4057
4058             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4059             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4060         }
4061         $devices_to_check = $to_check;
4062     }
4063
4064     my $resblock = vm_mon_cmd($vmid, 'query-block');
4065     foreach my $block (@$resblock) {
4066         if($block->{device} =~ m/^drive-(\S+)/){
4067                 $devices->{$1} = 1;
4068         }
4069     }
4070
4071     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4072     foreach my $mice (@$resmice) {
4073         if ($mice->{name} eq 'QEMU HID Tablet') {
4074             $devices->{tablet} = 1;
4075             last;
4076         }
4077     }
4078
4079     # for usb devices there is no query-usb
4080     # but we can iterate over the entries in
4081     # qom-list path=/machine/peripheral
4082     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4083     foreach my $per (@$resperipheral) {
4084         if ($per->{name} =~ m/^usb\d+$/) {
4085             $devices->{$per->{name}} = 1;
4086         }
4087     }
4088
4089     return $devices;
4090 }
4091
4092 sub vm_deviceplug {
4093     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4094
4095     my $q35 = machine_type_is_q35($conf);
4096
4097     my $devices_list = vm_devices_list($vmid);
4098     return 1 if defined($devices_list->{$deviceid});
4099
4100     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4101
4102     if ($deviceid eq 'tablet') {
4103
4104         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4105
4106     } elsif ($deviceid eq 'keyboard') {
4107
4108         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4109
4110     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4111
4112         die "usb hotplug currently not reliable\n";
4113         # since we can't reliably hot unplug all added usb devices
4114         # and usb passthrough disables live migration
4115         # we disable usb hotplugging for now
4116         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4117
4118     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4119
4120         qemu_iothread_add($vmid, $deviceid, $device);
4121
4122         qemu_driveadd($storecfg, $vmid, $device);
4123         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4124
4125         qemu_deviceadd($vmid, $devicefull);
4126         eval { qemu_deviceaddverify($vmid, $deviceid); };
4127         if (my $err = $@) {
4128             eval { qemu_drivedel($vmid, $deviceid); };
4129             warn $@ if $@;
4130             die $err;
4131         }
4132
4133     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4134
4135
4136         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4137         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4138         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4139
4140         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4141
4142         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4143             qemu_iothread_add($vmid, $deviceid, $device);
4144             $devicefull .= ",iothread=iothread-$deviceid";
4145         }
4146
4147         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4148             $devicefull .= ",num_queues=$device->{queues}";
4149         }
4150
4151         qemu_deviceadd($vmid, $devicefull);
4152         qemu_deviceaddverify($vmid, $deviceid);
4153
4154     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4155
4156         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4157         qemu_driveadd($storecfg, $vmid, $device);
4158
4159         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4160         eval { qemu_deviceadd($vmid, $devicefull); };
4161         if (my $err = $@) {
4162             eval { qemu_drivedel($vmid, $deviceid); };
4163             warn $@ if $@;
4164             die $err;
4165         }
4166
4167     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4168
4169         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4170
4171         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4172         my $use_old_bios_files = undef;
4173         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4174
4175         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4176         qemu_deviceadd($vmid, $netdevicefull);
4177         eval {
4178             qemu_deviceaddverify($vmid, $deviceid);
4179             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4180         };
4181         if (my $err = $@) {
4182             eval { qemu_netdevdel($vmid, $deviceid); };
4183             warn $@ if $@;
4184             die $err;
4185         }
4186
4187     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4188
4189         my $bridgeid = $2;
4190         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4191         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4192
4193         qemu_deviceadd($vmid, $devicefull);
4194         qemu_deviceaddverify($vmid, $deviceid);
4195
4196     } else {
4197         die "can't hotplug device '$deviceid'\n";
4198     }
4199
4200     return 1;
4201 }
4202
4203 # fixme: this should raise exceptions on error!
4204 sub vm_deviceunplug {
4205     my ($vmid, $conf, $deviceid) = @_;
4206
4207     my $devices_list = vm_devices_list($vmid);
4208     return 1 if !defined($devices_list->{$deviceid});
4209
4210     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4211
4212     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4213
4214         qemu_devicedel($vmid, $deviceid);
4215
4216     } elsif ($deviceid =~ m/^usb\d+$/) {
4217
4218         die "usb hotplug currently not reliable\n";
4219         # when unplugging usb devices this way,
4220         # there may be remaining usb controllers/hubs
4221         # so we disable it for now
4222         qemu_devicedel($vmid, $deviceid);
4223         qemu_devicedelverify($vmid, $deviceid);
4224
4225     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4226
4227         qemu_devicedel($vmid, $deviceid);
4228         qemu_devicedelverify($vmid, $deviceid);
4229         qemu_drivedel($vmid, $deviceid);
4230         qemu_iothread_del($conf, $vmid, $deviceid);
4231
4232     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4233
4234         qemu_devicedel($vmid, $deviceid);
4235         qemu_devicedelverify($vmid, $deviceid);
4236         qemu_iothread_del($conf, $vmid, $deviceid);
4237
4238     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4239
4240         qemu_devicedel($vmid, $deviceid);
4241         qemu_drivedel($vmid, $deviceid);
4242         qemu_deletescsihw($conf, $vmid, $deviceid);
4243
4244     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4245
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248         qemu_netdevdel($vmid, $deviceid);
4249
4250     } else {
4251         die "can't unplug device '$deviceid'\n";
4252     }
4253
4254     return 1;
4255 }
4256
4257 sub qemu_deviceadd {
4258     my ($vmid, $devicefull) = @_;
4259
4260     $devicefull = "driver=".$devicefull;
4261     my %options =  split(/[=,]/, $devicefull);
4262
4263     vm_mon_cmd($vmid, "device_add" , %options);
4264 }
4265
4266 sub qemu_devicedel {
4267     my ($vmid, $deviceid) = @_;
4268
4269     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4270 }
4271
4272 sub qemu_iothread_add {
4273     my($vmid, $deviceid, $device) = @_;
4274
4275     if ($device->{iothread}) {
4276         my $iothreads = vm_iothreads_list($vmid);
4277         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4278     }
4279 }
4280
4281 sub qemu_iothread_del {
4282     my($conf, $vmid, $deviceid) = @_;
4283
4284     my $confid = $deviceid;
4285     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4286         $confid = 'scsi' . $1;
4287     }
4288     my $device = parse_drive($confid, $conf->{$confid});
4289     if ($device->{iothread}) {
4290         my $iothreads = vm_iothreads_list($vmid);
4291         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4292     }
4293 }
4294
4295 sub qemu_objectadd {
4296     my($vmid, $objectid, $qomtype) = @_;
4297
4298     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4299
4300     return 1;
4301 }
4302
4303 sub qemu_objectdel {
4304     my($vmid, $objectid) = @_;
4305
4306     vm_mon_cmd($vmid, "object-del", id => $objectid);
4307
4308     return 1;
4309 }
4310
4311 sub qemu_driveadd {
4312     my ($storecfg, $vmid, $device) = @_;
4313
4314     my $drive = print_drive_full($storecfg, $vmid, $device);
4315     $drive =~ s/\\/\\\\/g;
4316     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4317
4318     # If the command succeeds qemu prints: "OK"
4319     return 1 if $ret =~ m/OK/s;
4320
4321     die "adding drive failed: $ret\n";
4322 }
4323
4324 sub qemu_drivedel {
4325     my($vmid, $deviceid) = @_;
4326
4327     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4328     $ret =~ s/^\s+//;
4329
4330     return 1 if $ret eq "";
4331
4332     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4333     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4334
4335     die "deleting drive $deviceid failed : $ret\n";
4336 }
4337
4338 sub qemu_deviceaddverify {
4339     my ($vmid, $deviceid) = @_;
4340
4341     for (my $i = 0; $i <= 5; $i++) {
4342          my $devices_list = vm_devices_list($vmid);
4343          return 1 if defined($devices_list->{$deviceid});
4344          sleep 1;
4345     }
4346
4347     die "error on hotplug device '$deviceid'\n";
4348 }
4349
4350
4351 sub qemu_devicedelverify {
4352     my ($vmid, $deviceid) = @_;
4353
4354     # need to verify that the device is correctly removed as device_del
4355     # is async and empty return is not reliable
4356
4357     for (my $i = 0; $i <= 5; $i++) {
4358          my $devices_list = vm_devices_list($vmid);
4359          return 1 if !defined($devices_list->{$deviceid});
4360          sleep 1;
4361     }
4362
4363     die "error on hot-unplugging device '$deviceid'\n";
4364 }
4365
4366 sub qemu_findorcreatescsihw {
4367     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4368
4369     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4370
4371     my $scsihwid="$controller_prefix$controller";
4372     my $devices_list = vm_devices_list($vmid);
4373
4374     if(!defined($devices_list->{$scsihwid})) {
4375         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4376     }
4377
4378     return 1;
4379 }
4380
4381 sub qemu_deletescsihw {
4382     my ($conf, $vmid, $opt) = @_;
4383
4384     my $device = parse_drive($opt, $conf->{$opt});
4385
4386     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4387         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4388         return 1;
4389     }
4390
4391     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4392
4393     my $devices_list = vm_devices_list($vmid);
4394     foreach my $opt (keys %{$devices_list}) {
4395         if (PVE::QemuServer::is_valid_drivename($opt)) {
4396             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4397             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4398                 return 1;
4399             }
4400         }
4401     }
4402
4403     my $scsihwid="scsihw$controller";
4404
4405     vm_deviceunplug($vmid, $conf, $scsihwid);
4406
4407     return 1;
4408 }
4409
4410 sub qemu_add_pci_bridge {
4411     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4412
4413     my $bridges = {};
4414
4415     my $bridgeid;
4416
4417     print_pci_addr($device, $bridges, $arch, $machine_type);
4418
4419     while (my ($k, $v) = each %$bridges) {
4420         $bridgeid = $k;
4421     }
4422     return 1 if !defined($bridgeid) || $bridgeid < 1;
4423
4424     my $bridge = "pci.$bridgeid";
4425     my $devices_list = vm_devices_list($vmid);
4426
4427     if (!defined($devices_list->{$bridge})) {
4428         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4429     }
4430
4431     return 1;
4432 }
4433
4434 sub qemu_set_link_status {
4435     my ($vmid, $device, $up) = @_;
4436
4437     vm_mon_cmd($vmid, "set_link", name => $device,
4438                up => $up ? JSON::true : JSON::false);
4439 }
4440
4441 sub qemu_netdevadd {
4442     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4443
4444     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4445     my %options =  split(/[=,]/, $netdev);
4446
4447     vm_mon_cmd($vmid, "netdev_add",  %options);
4448     return 1;
4449 }
4450
4451 sub qemu_netdevdel {
4452     my ($vmid, $deviceid) = @_;
4453
4454     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4455 }
4456
4457 sub qemu_usb_hotplug {
4458     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4459
4460     return if !$device;
4461
4462     # remove the old one first
4463     vm_deviceunplug($vmid, $conf, $deviceid);
4464
4465     # check if xhci controller is necessary and available
4466     if ($device->{usb3}) {
4467
4468         my $devicelist = vm_devices_list($vmid);
4469
4470         if (!$devicelist->{xhci}) {
4471             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4472             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4473         }
4474     }
4475     my $d = parse_usb_device($device->{host});
4476     $d->{usb3} = $device->{usb3};
4477
4478     # add the new one
4479     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4480 }
4481
4482 sub qemu_cpu_hotplug {
4483     my ($vmid, $conf, $vcpus) = @_;
4484
4485     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4486
4487     my $sockets = 1;
4488     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4489     $sockets = $conf->{sockets} if  $conf->{sockets};
4490     my $cores = $conf->{cores} || 1;
4491     my $maxcpus = $sockets * $cores;
4492
4493     $vcpus = $maxcpus if !$vcpus;
4494
4495     die "you can't add more vcpus than maxcpus\n"
4496         if $vcpus > $maxcpus;
4497
4498     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4499
4500     if ($vcpus < $currentvcpus) {
4501
4502         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4503
4504             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4505                 qemu_devicedel($vmid, "cpu$i");
4506                 my $retry = 0;
4507                 my $currentrunningvcpus = undef;
4508                 while (1) {
4509                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4510                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4511                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4512                     $retry++;
4513                     sleep 1;
4514                 }
4515                 #update conf after each succesfull cpu unplug
4516                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4517                 PVE::QemuConfig->write_config($vmid, $conf);
4518             }
4519         } else {
4520             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4521         }
4522
4523         return;
4524     }
4525
4526     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4527     die "vcpus in running vm does not match its configuration\n"
4528         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4529
4530     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4531
4532         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4533             my $cpustr = print_cpu_device($conf, $i);
4534             qemu_deviceadd($vmid, $cpustr);
4535
4536             my $retry = 0;
4537             my $currentrunningvcpus = undef;
4538             while (1) {
4539                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4540                 last if scalar(@{$currentrunningvcpus}) == $i;
4541                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4542                 sleep 1;
4543                 $retry++;
4544             }
4545             #update conf after each succesfull cpu hotplug
4546             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4547             PVE::QemuConfig->write_config($vmid, $conf);
4548         }
4549     } else {
4550
4551         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4552             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4553         }
4554     }
4555 }
4556
4557 sub qemu_block_set_io_throttle {
4558     my ($vmid, $deviceid,
4559         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4560         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4561         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4562         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4563
4564     return if !check_running($vmid) ;
4565
4566     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4567         bps => int($bps),
4568         bps_rd => int($bps_rd),
4569         bps_wr => int($bps_wr),
4570         iops => int($iops),
4571         iops_rd => int($iops_rd),
4572         iops_wr => int($iops_wr),
4573         bps_max => int($bps_max),
4574         bps_rd_max => int($bps_rd_max),
4575         bps_wr_max => int($bps_wr_max),
4576         iops_max => int($iops_max),
4577         iops_rd_max => int($iops_rd_max),
4578         iops_wr_max => int($iops_wr_max),
4579         bps_max_length => int($bps_max_length),
4580         bps_rd_max_length => int($bps_rd_max_length),
4581         bps_wr_max_length => int($bps_wr_max_length),
4582         iops_max_length => int($iops_max_length),
4583         iops_rd_max_length => int($iops_rd_max_length),
4584         iops_wr_max_length => int($iops_wr_max_length),
4585     );
4586
4587 }
4588
4589 # old code, only used to shutdown old VM after update
4590 sub __read_avail {
4591     my ($fh, $timeout) = @_;
4592
4593     my $sel = new IO::Select;
4594     $sel->add($fh);
4595
4596     my $res = '';
4597     my $buf;
4598
4599     my @ready;
4600     while (scalar (@ready = $sel->can_read($timeout))) {
4601         my $count;
4602         if ($count = $fh->sysread($buf, 8192)) {
4603             if ($buf =~ /^(.*)\(qemu\) $/s) {
4604                 $res .= $1;
4605                 last;
4606             } else {
4607                 $res .= $buf;
4608             }
4609         } else {
4610             if (!defined($count)) {
4611                 die "$!\n";
4612             }
4613             last;
4614         }
4615     }
4616
4617     die "monitor read timeout\n" if !scalar(@ready);
4618
4619     return $res;
4620 }
4621
4622 sub qemu_block_resize {
4623     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4624
4625     my $running = check_running($vmid);
4626
4627     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4628
4629     return if !$running;
4630
4631     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4632
4633 }
4634
4635 sub qemu_volume_snapshot {
4636     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4637
4638     my $running = check_running($vmid);
4639
4640     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4641         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4642     } else {
4643         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4644     }
4645 }
4646
4647 sub qemu_volume_snapshot_delete {
4648     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4649
4650     my $running = check_running($vmid);
4651
4652     if($running) {
4653
4654         $running = undef;
4655         my $conf = PVE::QemuConfig->load_config($vmid);
4656         foreach_drive($conf, sub {
4657             my ($ds, $drive) = @_;
4658             $running = 1 if $drive->{file} eq $volid;
4659         });
4660     }
4661
4662     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4663         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4664     } else {
4665         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4666     }
4667 }
4668
4669 sub set_migration_caps {
4670     my ($vmid) = @_;
4671
4672     my $cap_ref = [];
4673
4674     my $enabled_cap = {
4675         "auto-converge" => 1,
4676         "xbzrle" => 1,
4677         "x-rdma-pin-all" => 0,
4678         "zero-blocks" => 0,
4679         "compress" => 0
4680     };
4681
4682     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4683
4684     for my $supported_capability (@$supported_capabilities) {
4685         push @$cap_ref, {
4686             capability => $supported_capability->{capability},
4687             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4688         };
4689     }
4690
4691     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4692 }
4693
4694 my $fast_plug_option = {
4695     'lock' => 1,
4696     'name' => 1,
4697     'onboot' => 1,
4698     'shares' => 1,
4699     'startup' => 1,
4700     'description' => 1,
4701     'protection' => 1,
4702     'vmstatestorage' => 1,
4703     'hookscript' => 1,
4704 };
4705
4706 # hotplug changes in [PENDING]
4707 # $selection hash can be used to only apply specified options, for
4708 # example: { cores => 1 } (only apply changed 'cores')
4709 # $errors ref is used to return error messages
4710 sub vmconfig_hotplug_pending {
4711     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4712
4713     my $defaults = load_defaults();
4714     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4715
4716     # commit values which do not have any impact on running VM first
4717     # Note: those option cannot raise errors, we we do not care about
4718     # $selection and always apply them.
4719
4720     my $add_error = sub {
4721         my ($opt, $msg) = @_;
4722         $errors->{$opt} = "hotplug problem - $msg";
4723     };
4724
4725     my $changes = 0;
4726     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4727         if ($fast_plug_option->{$opt}) {
4728             $conf->{$opt} = $conf->{pending}->{$opt};
4729             delete $conf->{pending}->{$opt};
4730             $changes = 1;
4731         }
4732     }
4733
4734     if ($changes) {
4735         PVE::QemuConfig->write_config($vmid, $conf);
4736         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4737     }
4738
4739     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4740
4741     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4742     while (my ($opt, $force) = each %$pending_delete_hash) {
4743         next if $selection && !$selection->{$opt};
4744         eval {
4745             if ($opt eq 'hotplug') {
4746                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4747             } elsif ($opt eq 'tablet') {
4748                 die "skip\n" if !$hotplug_features->{usb};
4749                 if ($defaults->{tablet}) {
4750                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4751                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4752                         if $arch eq 'aarch64';
4753                 } else {
4754                     vm_deviceunplug($vmid, $conf, 'tablet');
4755                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4756                 }
4757             } elsif ($opt =~ m/^usb\d+/) {
4758                 die "skip\n";
4759                 # since we cannot reliably hot unplug usb devices
4760                 # we are disabling it
4761                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4762                 vm_deviceunplug($vmid, $conf, $opt);
4763             } elsif ($opt eq 'vcpus') {
4764                 die "skip\n" if !$hotplug_features->{cpu};
4765                 qemu_cpu_hotplug($vmid, $conf, undef);
4766             } elsif ($opt eq 'balloon') {
4767                 # enable balloon device is not hotpluggable
4768                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4769                 # here we reset the ballooning value to memory
4770                 my $balloon = $conf->{memory} || $defaults->{memory};
4771                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4772             } elsif ($fast_plug_option->{$opt}) {
4773                 # do nothing
4774             } elsif ($opt =~ m/^net(\d+)$/) {
4775                 die "skip\n" if !$hotplug_features->{network};
4776                 vm_deviceunplug($vmid, $conf, $opt);
4777             } elsif (is_valid_drivename($opt)) {
4778                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4779                 vm_deviceunplug($vmid, $conf, $opt);
4780                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4781             } elsif ($opt =~ m/^memory$/) {
4782                 die "skip\n" if !$hotplug_features->{memory};
4783                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4784             } elsif ($opt eq 'cpuunits') {
4785                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4786             } elsif ($opt eq 'cpulimit') {
4787                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4788             } else {
4789                 die "skip\n";
4790             }
4791         };
4792         if (my $err = $@) {
4793             &$add_error($opt, $err) if $err ne "skip\n";
4794         } else {
4795             # save new config if hotplug was successful
4796             delete $conf->{$opt};
4797             vmconfig_undelete_pending_option($conf, $opt);
4798             PVE::QemuConfig->write_config($vmid, $conf);
4799             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4800         }
4801     }
4802
4803     my $apply_pending_cloudinit;
4804     $apply_pending_cloudinit = sub {
4805         my ($key, $value) = @_;
4806         $apply_pending_cloudinit = sub {}; # once is enough
4807
4808         my @cloudinit_opts = keys %$confdesc_cloudinit;
4809         foreach my $opt (keys %{$conf->{pending}}) {
4810             next if !grep { $_ eq $opt } @cloudinit_opts;
4811             $conf->{$opt} = delete $conf->{pending}->{$opt};
4812         }
4813
4814         my $new_conf = { %$conf };
4815         $new_conf->{$key} = $value;
4816         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4817     };
4818
4819     foreach my $opt (keys %{$conf->{pending}}) {
4820         next if $selection && !$selection->{$opt};
4821         my $value = $conf->{pending}->{$opt};
4822         eval {
4823             if ($opt eq 'hotplug') {
4824                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4825             } elsif ($opt eq 'tablet') {
4826                 die "skip\n" if !$hotplug_features->{usb};
4827                 if ($value == 1) {
4828                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4829                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4830                         if $arch eq 'aarch64';
4831                 } elsif ($value == 0) {
4832                     vm_deviceunplug($vmid, $conf, 'tablet');
4833                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4834                 }
4835             } elsif ($opt =~ m/^usb\d+$/) {
4836                 die "skip\n";
4837                 # since we cannot reliably hot unplug usb devices
4838                 # we are disabling it
4839                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4840                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4841                 die "skip\n" if !$d;
4842                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4843             } elsif ($opt eq 'vcpus') {
4844                 die "skip\n" if !$hotplug_features->{cpu};
4845                 qemu_cpu_hotplug($vmid, $conf, $value);
4846             } elsif ($opt eq 'balloon') {
4847                 # enable/disable balloning device is not hotpluggable
4848                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4849                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4850                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4851
4852                 # allow manual ballooning if shares is set to zero
4853                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4854                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4855                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4856                 }
4857             } elsif ($opt =~ m/^net(\d+)$/) {
4858                 # some changes can be done without hotplug
4859                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4860                                     $vmid, $opt, $value, $arch, $machine_type);
4861             } elsif (is_valid_drivename($opt)) {
4862                 # some changes can be done without hotplug
4863                 my $drive = parse_drive($opt, $value);
4864                 if (drive_is_cloudinit($drive)) {
4865                     &$apply_pending_cloudinit($opt, $value);
4866                 }
4867                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4868                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4869             } elsif ($opt =~ m/^memory$/) { #dimms
4870                 die "skip\n" if !$hotplug_features->{memory};
4871                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4872             } elsif ($opt eq 'cpuunits') {
4873                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4874             } elsif ($opt eq 'cpulimit') {
4875                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4876                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4877             } else {
4878                 die "skip\n";  # skip non-hot-pluggable options
4879             }
4880         };
4881         if (my $err = $@) {
4882             &$add_error($opt, $err) if $err ne "skip\n";
4883         } else {
4884             # save new config if hotplug was successful
4885             $conf->{$opt} = $value;
4886             delete $conf->{pending}->{$opt};
4887             PVE::QemuConfig->write_config($vmid, $conf);
4888             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4889         }
4890     }
4891 }
4892
4893 sub try_deallocate_drive {
4894     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4895
4896     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4897         my $volid = $drive->{file};
4898         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4899             my $sid = PVE::Storage::parse_volume_id($volid);
4900             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4901
4902             # check if the disk is really unused
4903             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4904                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4905             PVE::Storage::vdisk_free($storecfg, $volid);
4906             return 1;
4907         } else {
4908             # If vm is not owner of this disk remove from config
4909             return 1;
4910         }
4911     }
4912
4913     return undef;
4914 }
4915
4916 sub vmconfig_delete_or_detach_drive {
4917     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4918
4919     my $drive = parse_drive($opt, $conf->{$opt});
4920
4921     my $rpcenv = PVE::RPCEnvironment::get();
4922     my $authuser = $rpcenv->get_user();
4923
4924     if ($force) {
4925         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4926         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4927     } else {
4928         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4929     }
4930 }
4931
4932 sub vmconfig_apply_pending {
4933     my ($vmid, $conf, $storecfg) = @_;
4934
4935     # cold plug
4936
4937     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4938     while (my ($opt, $force) = each %$pending_delete_hash) {
4939         die "internal error" if $opt =~ m/^unused/;
4940         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4941         if (!defined($conf->{$opt})) {
4942             vmconfig_undelete_pending_option($conf, $opt);
4943             PVE::QemuConfig->write_config($vmid, $conf);
4944         } elsif (is_valid_drivename($opt)) {
4945             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4946             vmconfig_undelete_pending_option($conf, $opt);
4947             delete $conf->{$opt};
4948             PVE::QemuConfig->write_config($vmid, $conf);
4949         } else {
4950             vmconfig_undelete_pending_option($conf, $opt);
4951             delete $conf->{$opt};
4952             PVE::QemuConfig->write_config($vmid, $conf);
4953         }
4954     }
4955
4956     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4957
4958     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4959         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4960
4961         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4962             # skip if nothing changed
4963         } elsif (is_valid_drivename($opt)) {
4964             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4965                 if defined($conf->{$opt});
4966             $conf->{$opt} = $conf->{pending}->{$opt};
4967         } else {
4968             $conf->{$opt} = $conf->{pending}->{$opt};
4969         }
4970
4971         delete $conf->{pending}->{$opt};
4972         PVE::QemuConfig->write_config($vmid, $conf);
4973     }
4974 }
4975
4976 my $safe_num_ne = sub {
4977     my ($a, $b) = @_;
4978
4979     return 0 if !defined($a) && !defined($b);
4980     return 1 if !defined($a);
4981     return 1 if !defined($b);
4982
4983     return $a != $b;
4984 };
4985
4986 my $safe_string_ne = sub {
4987     my ($a, $b) = @_;
4988
4989     return 0 if !defined($a) && !defined($b);
4990     return 1 if !defined($a);
4991     return 1 if !defined($b);
4992
4993     return $a ne $b;
4994 };
4995
4996 sub vmconfig_update_net {
4997     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4998
4999     my $newnet = parse_net($value);
5000
5001     if ($conf->{$opt}) {
5002         my $oldnet = parse_net($conf->{$opt});
5003
5004         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5005             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5006             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5007             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5008
5009             # for non online change, we try to hot-unplug
5010             die "skip\n" if !$hotplug;
5011             vm_deviceunplug($vmid, $conf, $opt);
5012         } else {
5013
5014             die "internal error" if $opt !~ m/net(\d+)/;
5015             my $iface = "tap${vmid}i$1";
5016
5017             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5018                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5019                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5020                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5021                 PVE::Network::tap_unplug($iface);
5022                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5023             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5024                 # Rate can be applied on its own but any change above needs to
5025                 # include the rate in tap_plug since OVS resets everything.
5026                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5027             }
5028
5029             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5030                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5031             }
5032
5033             return 1;
5034         }
5035     }
5036
5037     if ($hotplug) {
5038         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5039     } else {
5040         die "skip\n";
5041     }
5042 }
5043
5044 sub vmconfig_update_disk {
5045     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5046
5047     # fixme: do we need force?
5048
5049     my $drive = parse_drive($opt, $value);
5050
5051     if ($conf->{$opt}) {
5052
5053         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5054
5055             my $media = $drive->{media} || 'disk';
5056             my $oldmedia = $old_drive->{media} || 'disk';
5057             die "unable to change media type\n" if $media ne $oldmedia;
5058
5059             if (!drive_is_cdrom($old_drive)) {
5060
5061                 if ($drive->{file} ne $old_drive->{file}) {
5062
5063                     die "skip\n" if !$hotplug;
5064
5065                     # unplug and register as unused
5066                     vm_deviceunplug($vmid, $conf, $opt);
5067                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5068
5069                 } else {
5070                     # update existing disk
5071
5072                     # skip non hotpluggable value
5073                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5074                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5075                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5076                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5077                         die "skip\n";
5078                     }
5079
5080                     # apply throttle
5081                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5082                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5083                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5084                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5085                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5086                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5087                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5088                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5089                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5090                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5091                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5092                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5093                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5094                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5095                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5096                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5097                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5098                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5099
5100                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5101                                                    ($drive->{mbps} || 0)*1024*1024,
5102                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5103                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5104                                                    $drive->{iops} || 0,
5105                                                    $drive->{iops_rd} || 0,
5106                                                    $drive->{iops_wr} || 0,
5107                                                    ($drive->{mbps_max} || 0)*1024*1024,
5108                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5109                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5110                                                    $drive->{iops_max} || 0,
5111                                                    $drive->{iops_rd_max} || 0,
5112                                                    $drive->{iops_wr_max} || 0,
5113                                                    $drive->{bps_max_length} || 1,
5114                                                    $drive->{bps_rd_max_length} || 1,
5115                                                    $drive->{bps_wr_max_length} || 1,
5116                                                    $drive->{iops_max_length} || 1,
5117                                                    $drive->{iops_rd_max_length} || 1,
5118                                                    $drive->{iops_wr_max_length} || 1);
5119
5120                     }
5121
5122                     return 1;
5123                 }
5124
5125             } else { # cdrom
5126
5127                 if ($drive->{file} eq 'none') {
5128                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5129                     if (drive_is_cloudinit($old_drive)) {
5130                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5131                     }
5132                 } else {
5133                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5134                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5135                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5136                 }
5137
5138                 return 1;
5139             }
5140         }
5141     }
5142
5143     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5144     # hotplug new disks
5145     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5146     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5147 }
5148
5149 sub vm_start {
5150     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5151         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5152
5153     PVE::QemuConfig->lock_config($vmid, sub {
5154         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5155
5156         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5157
5158         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5159
5160         PVE::QemuConfig->check_lock($conf)
5161             if !($skiplock || $is_suspended);
5162
5163         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5164
5165         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5166             vmconfig_apply_pending($vmid, $conf, $storecfg);
5167             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5168         }
5169
5170         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5171
5172         my $defaults = load_defaults();
5173
5174         # set environment variable useful inside network script
5175         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5176
5177         my $local_volumes = {};
5178
5179         if ($targetstorage) {
5180             foreach_drive($conf, sub {
5181                 my ($ds, $drive) = @_;
5182
5183                 return if drive_is_cdrom($drive);
5184
5185                 my $volid = $drive->{file};
5186
5187                 return if !$volid;
5188
5189                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5190
5191                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5192                 return if $scfg->{shared};
5193                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5194             });
5195
5196             my $format = undef;
5197
5198             foreach my $opt (sort keys %$local_volumes) {
5199
5200                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5201                 my $drive = parse_drive($opt, $conf->{$opt});
5202
5203                 #if remote storage is specified, use default format
5204                 if ($targetstorage && $targetstorage ne "1") {
5205                     $storeid = $targetstorage;
5206                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5207                     $format = $defFormat;
5208                 } else {
5209                     #else we use same format than original
5210                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5211                     $format = qemu_img_format($scfg, $volid);
5212                 }
5213
5214                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5215                 my $newdrive = $drive;
5216                 $newdrive->{format} = $format;
5217                 $newdrive->{file} = $newvolid;
5218                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5219                 $local_volumes->{$opt} = $drivestr;
5220                 #pass drive to conf for command line
5221                 $conf->{$opt} = $drivestr;
5222             }
5223         }
5224
5225         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5226
5227         if ($is_suspended) {
5228             # enforce machine type on suspended vm to ensure HW compatibility
5229             $forcemachine = $conf->{runningmachine};
5230             print "Resuming suspended VM\n";
5231         }
5232
5233         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5234
5235         my $migrate_port = 0;
5236         my $migrate_uri;
5237         if ($statefile) {
5238             if ($statefile eq 'tcp') {
5239                 my $localip = "localhost";
5240                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5241                 my $nodename = PVE::INotify::nodename();
5242
5243                 if (!defined($migration_type)) {
5244                     if (defined($datacenterconf->{migration}->{type})) {
5245                         $migration_type = $datacenterconf->{migration}->{type};
5246                     } else {
5247                         $migration_type = 'secure';
5248                     }
5249                 }
5250
5251                 if ($migration_type eq 'insecure') {
5252                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5253                     if ($migrate_network_addr) {
5254                         $localip = $migrate_network_addr;
5255                     } else {
5256                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5257                     }
5258
5259                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5260                 }
5261
5262                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5263                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5264                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5265                 push @$cmd, '-incoming', $migrate_uri;
5266                 push @$cmd, '-S';
5267
5268             } elsif ($statefile eq 'unix') {
5269                 # should be default for secure migrations as a ssh TCP forward
5270                 # tunnel is not deterministic reliable ready and fails regurarly
5271                 # to set up in time, so use UNIX socket forwards
5272                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5273                 unlink $socket_addr;
5274
5275                 $migrate_uri = "unix:$socket_addr";
5276
5277                 push @$cmd, '-incoming', $migrate_uri;
5278                 push @$cmd, '-S';
5279
5280             } else {
5281                 push @$cmd, '-loadstate', $statefile;
5282             }
5283         } elsif ($paused) {
5284             push @$cmd, '-S';
5285         }
5286
5287         # host pci devices
5288         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5289           my $d = parse_hostpci($conf->{"hostpci$i"});
5290           next if !$d;
5291           my $pcidevices = $d->{pciid};
5292           foreach my $pcidevice (@$pcidevices) {
5293                 my $pciid = $pcidevice->{id};
5294
5295                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5296                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5297                 die "no pci device info for device '$pciid'\n" if !$info;
5298
5299                 if ($d->{mdev}) {
5300                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5301                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5302                 } else {
5303                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5304                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5305                     die "can't reset pci device '$pciid'\n"
5306                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5307                 }
5308           }
5309         }
5310
5311         PVE::Storage::activate_volumes($storecfg, $vollist);
5312
5313         if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5314             eval {
5315                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5316                     outfunc => sub {}, errfunc => sub {});
5317             };
5318         }
5319
5320         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5321                                                   : $defaults->{cpuunits};
5322
5323         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5324         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5325
5326         my %properties = (
5327             Slice => 'qemu.slice',
5328             KillMode => 'none',
5329             CPUShares => $cpuunits
5330         );
5331
5332         if (my $cpulimit = $conf->{cpulimit}) {
5333             $properties{CPUQuota} = int($cpulimit * 100);
5334         }
5335         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5336
5337         my $run_qemu = sub {
5338             PVE::Tools::run_fork sub {
5339                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5340                 run_command($cmd, %run_params);
5341             };
5342         };
5343
5344         if ($conf->{hugepages}) {
5345
5346             my $code = sub {
5347                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5348                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5349
5350                 PVE::QemuServer::Memory::hugepages_mount();
5351                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5352
5353                 eval { $run_qemu->() };
5354                 if (my $err = $@) {
5355                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5356                     die $err;
5357                 }
5358
5359                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5360             };
5361             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5362
5363         } else {
5364             eval { $run_qemu->() };
5365         }
5366
5367         if (my $err = $@) {
5368             # deactivate volumes if start fails
5369             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5370             die "start failed: $err";
5371         }
5372
5373         print "migration listens on $migrate_uri\n" if $migrate_uri;
5374
5375         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5376             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5377             warn $@ if $@;
5378         }
5379
5380         #start nbd server for storage migration
5381         if ($targetstorage) {
5382             my $nodename = PVE::INotify::nodename();
5383             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5384             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5385             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5386             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5387
5388             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5389
5390             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5391
5392             foreach my $opt (sort keys %$local_volumes) {
5393                 my $volid = $local_volumes->{$opt};
5394                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5395                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5396                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5397             }
5398         }
5399
5400         if ($migratedfrom) {
5401             eval {
5402                 set_migration_caps($vmid);
5403             };
5404             warn $@ if $@;
5405
5406             if ($spice_port) {
5407                 print "spice listens on port $spice_port\n";
5408                 if ($spice_ticket) {
5409                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5410                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5411                 }
5412             }
5413
5414         } else {
5415             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5416                 if !$statefile && $conf->{balloon};
5417
5418             foreach my $opt (keys %$conf) {
5419                 next if $opt !~  m/^net\d+$/;
5420                 my $nicconf = parse_net($conf->{$opt});
5421                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5422             }
5423         }
5424
5425         vm_mon_cmd_nocheck($vmid, 'qom-set',
5426                     path => "machine/peripheral/balloon0",
5427                     property => "guest-stats-polling-interval",
5428                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5429
5430         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5431             print "Resumed VM, removing state\n";
5432             delete $conf->@{qw(lock vmstate runningmachine)};
5433             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5434             PVE::Storage::vdisk_free($storecfg, $vmstate);
5435             PVE::QemuConfig->write_config($vmid, $conf);
5436         }
5437
5438         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5439     });
5440 }
5441
5442 sub vm_mon_cmd {
5443     my ($vmid, $execute, %params) = @_;
5444
5445     my $cmd = { execute => $execute, arguments => \%params };
5446     vm_qmp_command($vmid, $cmd);
5447 }
5448
5449 sub vm_mon_cmd_nocheck {
5450     my ($vmid, $execute, %params) = @_;
5451
5452     my $cmd = { execute => $execute, arguments => \%params };
5453     vm_qmp_command($vmid, $cmd, 1);
5454 }
5455
5456 sub vm_qmp_command {
5457     my ($vmid, $cmd, $nocheck) = @_;
5458
5459     my $res;
5460
5461     my $timeout;
5462     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5463         $timeout = $cmd->{arguments}->{timeout};
5464         delete $cmd->{arguments}->{timeout};
5465     }
5466
5467     eval {
5468         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5469         my $sname = qmp_socket($vmid);
5470         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5471             my $qmpclient = PVE::QMPClient->new();
5472
5473             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5474         } else {
5475             die "unable to open monitor socket\n";
5476         }
5477     };
5478     if (my $err = $@) {
5479         syslog("err", "VM $vmid qmp command failed - $err");
5480         die $err;
5481     }
5482
5483     return $res;
5484 }
5485
5486 sub vm_human_monitor_command {
5487     my ($vmid, $cmdline) = @_;
5488
5489     my $res;
5490
5491     my $cmd = {
5492         execute => 'human-monitor-command',
5493         arguments => { 'command-line' => $cmdline},
5494     };
5495
5496     return vm_qmp_command($vmid, $cmd);
5497 }
5498
5499 sub vm_commandline {
5500     my ($storecfg, $vmid, $snapname) = @_;
5501
5502     my $conf = PVE::QemuConfig->load_config($vmid);
5503
5504     if ($snapname) {
5505         my $snapshot = $conf->{snapshots}->{$snapname};
5506         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5507
5508         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5509
5510         $conf = $snapshot;
5511     }
5512
5513     my $defaults = load_defaults();
5514
5515     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5516
5517     return PVE::Tools::cmd2string($cmd);
5518 }
5519
5520 sub vm_reset {
5521     my ($vmid, $skiplock) = @_;
5522
5523     PVE::QemuConfig->lock_config($vmid, sub {
5524
5525         my $conf = PVE::QemuConfig->load_config($vmid);
5526
5527         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5528
5529         vm_mon_cmd($vmid, "system_reset");
5530     });
5531 }
5532
5533 sub get_vm_volumes {
5534     my ($conf) = @_;
5535
5536     my $vollist = [];
5537     foreach_volid($conf, sub {
5538         my ($volid, $attr) = @_;
5539
5540         return if $volid =~ m|^/|;
5541
5542         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5543         return if !$sid;
5544
5545         push @$vollist, $volid;
5546     });
5547
5548     return $vollist;
5549 }
5550
5551 sub vm_stop_cleanup {
5552     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5553
5554     eval {
5555
5556         if (!$keepActive) {
5557             my $vollist = get_vm_volumes($conf);
5558             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5559         }
5560
5561         foreach my $ext (qw(mon qmp pid vnc qga)) {
5562             unlink "/var/run/qemu-server/${vmid}.$ext";
5563         }
5564
5565         if ($conf->{ivshmem}) {
5566             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5567             # just delete it for now, VMs which have this already open do not
5568             # are affected, but new VMs will get a separated one. If this
5569             # becomes an issue we either add some sort of ref-counting or just
5570             # add a "don't delete on stop" flag to the ivshmem format.
5571             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5572         }
5573
5574         foreach my $key (keys %$conf) {
5575             next if $key !~ m/^hostpci(\d+)$/;
5576             my $hostpciindex = $1;
5577             my $d = parse_hostpci($conf->{$key});
5578             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5579
5580             foreach my $pci (@{$d->{pciid}}) {
5581                 my $pciid = $pci->{id};
5582                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5583             }
5584         }
5585
5586         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5587     };
5588     warn $@ if $@; # avoid errors - just warn
5589 }
5590
5591 # Note: use $nockeck to skip tests if VM configuration file exists.
5592 # We need that when migration VMs to other nodes (files already moved)
5593 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5594 sub vm_stop {
5595     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5596
5597     $force = 1 if !defined($force) && !$shutdown;
5598
5599     if ($migratedfrom){
5600         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5601         kill 15, $pid if $pid;
5602         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5603         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5604         return;
5605     }
5606
5607     PVE::QemuConfig->lock_config($vmid, sub {
5608
5609         my $pid = check_running($vmid, $nocheck);
5610         return if !$pid;
5611
5612         my $conf;
5613         if (!$nocheck) {
5614             $conf = PVE::QemuConfig->load_config($vmid);
5615             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5616             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5617                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5618                 $timeout = $opts->{down} if $opts->{down};
5619             }
5620             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5621         }
5622
5623         $timeout = 60 if !defined($timeout);
5624
5625         eval {
5626             if ($shutdown) {
5627                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5628                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5629                 } else {
5630                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5631                 }
5632             } else {
5633                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5634             }
5635         };
5636         my $err = $@;
5637
5638         if (!$err) {
5639             my $count = 0;
5640             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5641                 $count++;
5642                 sleep 1;
5643             }
5644
5645             if ($count >= $timeout) {
5646                 if ($force) {
5647                     warn "VM still running - terminating now with SIGTERM\n";
5648                     kill 15, $pid;
5649                 } else {
5650                     die "VM quit/powerdown failed - got timeout\n";
5651                 }
5652             } else {
5653                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5654                 return;
5655             }
5656         } else {
5657             if ($force) {
5658                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5659                 kill 15, $pid;
5660             } else {
5661                 die "VM quit/powerdown failed\n";
5662             }
5663         }
5664
5665         # wait again
5666         $timeout = 10;
5667
5668         my $count = 0;
5669         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5670             $count++;
5671             sleep 1;
5672         }
5673
5674         if ($count >= $timeout) {
5675             warn "VM still running - terminating now with SIGKILL\n";
5676             kill 9, $pid;
5677             sleep 1;
5678         }
5679
5680         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5681    });
5682 }
5683
5684 sub vm_suspend {
5685     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5686
5687     my $conf;
5688     my $path;
5689     my $storecfg;
5690     my $vmstate;
5691
5692     PVE::QemuConfig->lock_config($vmid, sub {
5693
5694         $conf = PVE::QemuConfig->load_config($vmid);
5695
5696         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5697         PVE::QemuConfig->check_lock($conf)
5698             if !($skiplock || $is_backing_up);
5699
5700         die "cannot suspend to disk during backup\n"
5701             if $is_backing_up && $includestate;
5702
5703         if ($includestate) {
5704             $conf->{lock} = 'suspending';
5705             my $date = strftime("%Y-%m-%d", localtime(time()));
5706             $storecfg = PVE::Storage::config();
5707             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5708             $path = PVE::Storage::path($storecfg, $vmstate);
5709             PVE::QemuConfig->write_config($vmid, $conf);
5710         } else {
5711             vm_mon_cmd($vmid, "stop");
5712         }
5713     });
5714
5715     if ($includestate) {
5716         # save vm state
5717         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5718
5719         eval {
5720             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5721             for(;;) {
5722                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5723                 if (!$state->{status}) {
5724                     die "savevm not active\n";
5725                 } elsif ($state->{status} eq 'active') {
5726                     sleep(1);
5727                     next;
5728                 } elsif ($state->{status} eq 'completed') {
5729                     print "State saved, quitting\n";
5730                     last;
5731                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5732                     die "query-savevm failed with error '$state->{error}'\n"
5733                 } else {
5734                     die "query-savevm returned status '$state->{status}'\n";
5735                 }
5736             }
5737         };
5738         my $err = $@;
5739
5740         PVE::QemuConfig->lock_config($vmid, sub {
5741             $conf = PVE::QemuConfig->load_config($vmid);
5742             if ($err) {
5743                 # cleanup, but leave suspending lock, to indicate something went wrong
5744                 eval {
5745                     vm_mon_cmd($vmid, "savevm-end");
5746                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5747                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5748                     delete $conf->@{qw(vmstate runningmachine)};
5749                     PVE::QemuConfig->write_config($vmid, $conf);
5750                 };
5751                 warn $@ if $@;
5752                 die $err;
5753             }
5754
5755             die "lock changed unexpectedly\n"
5756                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5757
5758             vm_qmp_command($vmid, { execute => "quit" });
5759             $conf->{lock} = 'suspended';
5760             PVE::QemuConfig->write_config($vmid, $conf);
5761         });
5762     }
5763 }
5764
5765 sub vm_resume {
5766     my ($vmid, $skiplock, $nocheck) = @_;
5767
5768     PVE::QemuConfig->lock_config($vmid, sub {
5769
5770         my $res = vm_mon_cmd($vmid, 'query-status');
5771         my $resume_cmd = 'cont';
5772
5773         if ($res->{status} && $res->{status} eq 'suspended') {
5774             $resume_cmd = 'system_wakeup';
5775         }
5776
5777         if (!$nocheck) {
5778
5779             my $conf = PVE::QemuConfig->load_config($vmid);
5780
5781             PVE::QemuConfig->check_lock($conf)
5782                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5783
5784             vm_mon_cmd($vmid, $resume_cmd);
5785
5786         } else {
5787             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5788         }
5789     });
5790 }
5791
5792 sub vm_sendkey {
5793     my ($vmid, $skiplock, $key) = @_;
5794
5795     PVE::QemuConfig->lock_config($vmid, sub {
5796
5797         my $conf = PVE::QemuConfig->load_config($vmid);
5798
5799         # there is no qmp command, so we use the human monitor command
5800         vm_human_monitor_command($vmid, "sendkey $key");
5801     });
5802 }
5803
5804 sub vm_destroy {
5805     my ($storecfg, $vmid, $skiplock) = @_;
5806
5807     PVE::QemuConfig->lock_config($vmid, sub {
5808
5809         my $conf = PVE::QemuConfig->load_config($vmid);
5810
5811         if (!check_running($vmid)) {
5812             destroy_vm($storecfg, $vmid, undef, $skiplock);
5813         } else {
5814             die "VM $vmid is running - destroy failed\n";
5815         }
5816     });
5817 }
5818
5819 # vzdump restore implementaion
5820
5821 sub tar_archive_read_firstfile {
5822     my $archive = shift;
5823
5824     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5825
5826     # try to detect archive type first
5827     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5828         die "unable to open file '$archive'\n";
5829     my $firstfile = <$fh>;
5830     kill 15, $pid;
5831     close $fh;
5832
5833     die "ERROR: archive contaions no data\n" if !$firstfile;
5834     chomp $firstfile;
5835
5836     return $firstfile;
5837 }
5838
5839 sub tar_restore_cleanup {
5840     my ($storecfg, $statfile) = @_;
5841
5842     print STDERR "starting cleanup\n";
5843
5844     if (my $fd = IO::File->new($statfile, "r")) {
5845         while (defined(my $line = <$fd>)) {
5846             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5847                 my $volid = $2;
5848                 eval {
5849                     if ($volid =~ m|^/|) {
5850                         unlink $volid || die 'unlink failed\n';
5851                     } else {
5852                         PVE::Storage::vdisk_free($storecfg, $volid);
5853                     }
5854                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5855                 };
5856                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5857             } else {
5858                 print STDERR "unable to parse line in statfile - $line";
5859             }
5860         }
5861         $fd->close();
5862     }
5863 }
5864
5865 sub restore_archive {
5866     my ($archive, $vmid, $user, $opts) = @_;
5867
5868     my $format = $opts->{format};
5869     my $comp;
5870
5871     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5872         $format = 'tar' if !$format;
5873         $comp = 'gzip';
5874     } elsif ($archive =~ m/\.tar$/) {
5875         $format = 'tar' if !$format;
5876     } elsif ($archive =~ m/.tar.lzo$/) {
5877         $format = 'tar' if !$format;
5878         $comp = 'lzop';
5879     } elsif ($archive =~ m/\.vma$/) {
5880         $format = 'vma' if !$format;
5881     } elsif ($archive =~ m/\.vma\.gz$/) {
5882         $format = 'vma' if !$format;
5883         $comp = 'gzip';
5884     } elsif ($archive =~ m/\.vma\.lzo$/) {
5885         $format = 'vma' if !$format;
5886         $comp = 'lzop';
5887     } else {
5888         $format = 'vma' if !$format; # default
5889     }
5890
5891     # try to detect archive format
5892     if ($format eq 'tar') {
5893         return restore_tar_archive($archive, $vmid, $user, $opts);
5894     } else {
5895         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5896     }
5897 }
5898
5899 sub restore_update_config_line {
5900     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5901
5902     return if $line =~ m/^\#qmdump\#/;
5903     return if $line =~ m/^\#vzdump\#/;
5904     return if $line =~ m/^lock:/;
5905     return if $line =~ m/^unused\d+:/;
5906     return if $line =~ m/^parent:/;
5907     return if $line =~ m/^template:/; # restored VM is never a template
5908
5909     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5910     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5911         # try to convert old 1.X settings
5912         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5913         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5914             my ($model, $macaddr) = split(/\=/, $devconfig);
5915             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5916             my $net = {
5917                 model => $model,
5918                 bridge => "vmbr$ind",
5919                 macaddr => $macaddr,
5920             };
5921             my $netstr = print_net($net);
5922
5923             print $outfd "net$cookie->{netcount}: $netstr\n";
5924             $cookie->{netcount}++;
5925         }
5926     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5927         my ($id, $netstr) = ($1, $2);
5928         my $net = parse_net($netstr);
5929         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5930         $netstr = print_net($net);
5931         print $outfd "$id: $netstr\n";
5932     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5933         my $virtdev = $1;
5934         my $value = $3;
5935         my $di = parse_drive($virtdev, $value);
5936         if (defined($di->{backup}) && !$di->{backup}) {
5937             print $outfd "#$line";
5938         } elsif ($map->{$virtdev}) {
5939             delete $di->{format}; # format can change on restore
5940             $di->{file} = $map->{$virtdev};
5941             $value = print_drive($vmid, $di);
5942             print $outfd "$virtdev: $value\n";
5943         } else {
5944             print $outfd $line;
5945         }
5946     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5947         my $vmgenid = $1;
5948         if ($vmgenid ne '0') {
5949             # always generate a new vmgenid if there was a valid one setup
5950             $vmgenid = generate_uuid();
5951         }
5952         print $outfd "vmgenid: $vmgenid\n";
5953     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5954         my ($uuid, $uuid_str);
5955         UUID::generate($uuid);
5956         UUID::unparse($uuid, $uuid_str);
5957         my $smbios1 = parse_smbios1($2);
5958         $smbios1->{uuid} = $uuid_str;
5959         print $outfd $1.print_smbios1($smbios1)."\n";
5960     } else {
5961         print $outfd $line;
5962     }
5963 }
5964
5965 sub scan_volids {
5966     my ($cfg, $vmid) = @_;
5967
5968     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5969
5970     my $volid_hash = {};
5971     foreach my $storeid (keys %$info) {
5972         foreach my $item (@{$info->{$storeid}}) {
5973             next if !($item->{volid} && $item->{size});
5974             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5975             $volid_hash->{$item->{volid}} = $item;
5976         }
5977     }
5978
5979     return $volid_hash;
5980 }
5981
5982 sub is_volume_in_use {
5983     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5984
5985     my $path = PVE::Storage::path($storecfg, $volid);
5986
5987     my $scan_config = sub {
5988         my ($cref, $snapname) = @_;
5989
5990         foreach my $key (keys %$cref) {
5991             my $value = $cref->{$key};
5992             if (is_valid_drivename($key)) {
5993                 next if $skip_drive && $key eq $skip_drive;
5994                 my $drive = parse_drive($key, $value);
5995                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5996                 return 1 if $volid eq $drive->{file};
5997                 if ($drive->{file} =~ m!^/!) {
5998                     return 1 if $drive->{file} eq $path;
5999                 } else {
6000                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6001                     next if !$storeid;
6002                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6003                     next if !$scfg;
6004                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6005                 }
6006             }
6007         }
6008
6009         return 0;
6010     };
6011
6012     return 1 if &$scan_config($conf);
6013
6014     undef $skip_drive;
6015
6016     foreach my $snapname (keys %{$conf->{snapshots}}) {
6017         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6018     }
6019
6020     return 0;
6021 }
6022
6023 sub update_disksize {
6024     my ($vmid, $conf, $volid_hash) = @_;
6025
6026     my $changes;
6027     my $prefix = "VM $vmid:";
6028
6029     # used and unused disks
6030     my $referenced = {};
6031
6032     # Note: it is allowed to define multiple storages with same path (alias), so
6033     # we need to check both 'volid' and real 'path' (two different volid can point
6034     # to the same path).
6035
6036     my $referencedpath = {};
6037
6038     # update size info
6039     foreach my $opt (keys %$conf) {
6040         if (is_valid_drivename($opt)) {
6041             my $drive = parse_drive($opt, $conf->{$opt});
6042             my $volid = $drive->{file};
6043             next if !$volid;
6044
6045             $referenced->{$volid} = 1;
6046             if ($volid_hash->{$volid} &&
6047                 (my $path = $volid_hash->{$volid}->{path})) {
6048                 $referencedpath->{$path} = 1;
6049             }
6050
6051             next if drive_is_cdrom($drive);
6052             next if !$volid_hash->{$volid};
6053
6054             $drive->{size} = $volid_hash->{$volid}->{size};
6055             my $new = print_drive($vmid, $drive);
6056             if ($new ne $conf->{$opt}) {
6057                 $changes = 1;
6058                 $conf->{$opt} = $new;
6059                 print "$prefix update disk '$opt' information.\n";
6060             }
6061         }
6062     }
6063
6064     # remove 'unusedX' entry if volume is used
6065     foreach my $opt (keys %$conf) {
6066         next if $opt !~ m/^unused\d+$/;
6067         my $volid = $conf->{$opt};
6068         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6069         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6070             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6071             $changes = 1;
6072             delete $conf->{$opt};
6073         }
6074
6075         $referenced->{$volid} = 1;
6076         $referencedpath->{$path} = 1 if $path;
6077     }
6078
6079     foreach my $volid (sort keys %$volid_hash) {
6080         next if $volid =~ m/vm-$vmid-state-/;
6081         next if $referenced->{$volid};
6082         my $path = $volid_hash->{$volid}->{path};
6083         next if !$path; # just to be sure
6084         next if $referencedpath->{$path};
6085         $changes = 1;
6086         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6087         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6088         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6089     }
6090
6091     return $changes;
6092 }
6093
6094 sub rescan {
6095     my ($vmid, $nolock, $dryrun) = @_;
6096
6097     my $cfg = PVE::Storage::config();
6098
6099     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6100     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6101     foreach my $stor (keys %{$cfg->{ids}}) {
6102         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6103     }
6104
6105     print "rescan volumes...\n";
6106     my $volid_hash = scan_volids($cfg, $vmid);
6107
6108     my $updatefn =  sub {
6109         my ($vmid) = @_;
6110
6111         my $conf = PVE::QemuConfig->load_config($vmid);
6112
6113         PVE::QemuConfig->check_lock($conf);
6114
6115         my $vm_volids = {};
6116         foreach my $volid (keys %$volid_hash) {
6117             my $info = $volid_hash->{$volid};
6118             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6119         }
6120
6121         my $changes = update_disksize($vmid, $conf, $vm_volids);
6122
6123         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6124     };
6125
6126     if (defined($vmid)) {
6127         if ($nolock) {
6128             &$updatefn($vmid);
6129         } else {
6130             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6131         }
6132     } else {
6133         my $vmlist = config_list();
6134         foreach my $vmid (keys %$vmlist) {
6135             if ($nolock) {
6136                 &$updatefn($vmid);
6137             } else {
6138                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6139             }
6140         }
6141     }
6142 }
6143
6144 sub restore_vma_archive {
6145     my ($archive, $vmid, $user, $opts, $comp) = @_;
6146
6147     my $readfrom = $archive;
6148
6149     my $cfg = PVE::Storage::config();
6150     my $commands = [];
6151     my $bwlimit = $opts->{bwlimit};
6152
6153     my $dbg_cmdstring = '';
6154     my $add_pipe = sub {
6155         my ($cmd) = @_;
6156         push @$commands, $cmd;
6157         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6158         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6159         $readfrom = '-';
6160     };
6161
6162     my $input = undef;
6163     if ($archive eq '-') {
6164         $input = '<&STDIN';
6165     } else {
6166         # If we use a backup from a PVE defined storage we also consider that
6167         # storage's rate limit:
6168         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6169         if (defined($volid)) {
6170             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6171             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6172             if ($readlimit) {
6173                 print STDERR "applying read rate limit: $readlimit\n";
6174                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6175                 $add_pipe->($cstream);
6176             }
6177         }
6178     }
6179
6180     if ($comp) {
6181         my $cmd;
6182         if ($comp eq 'gzip') {
6183             $cmd = ['zcat', $readfrom];
6184         } elsif ($comp eq 'lzop') {
6185             $cmd = ['lzop', '-d', '-c', $readfrom];
6186         } else {
6187             die "unknown compression method '$comp'\n";
6188         }
6189         $add_pipe->($cmd);
6190     }
6191
6192     my $tmpdir = "/var/tmp/vzdumptmp$$";
6193     rmtree $tmpdir;
6194
6195     # disable interrupts (always do cleanups)
6196     local $SIG{INT} =
6197         local $SIG{TERM} =
6198         local $SIG{QUIT} =
6199         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6200
6201     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6202     POSIX::mkfifo($mapfifo, 0600);
6203     my $fifofh;
6204
6205     my $openfifo = sub {
6206         open($fifofh, '>', $mapfifo) || die $!;
6207     };
6208
6209     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6210
6211     my $oldtimeout;
6212     my $timeout = 5;
6213
6214     my $devinfo = {};
6215
6216     my $rpcenv = PVE::RPCEnvironment::get();
6217
6218     my $conffile = PVE::QemuConfig->config_file($vmid);
6219     my $tmpfn = "$conffile.$$.tmp";
6220
6221     # Note: $oldconf is undef if VM does not exists
6222     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6223     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6224
6225     my %storage_limits;
6226
6227     my $print_devmap = sub {
6228         my $virtdev_hash = {};
6229
6230         my $cfgfn = "$tmpdir/qemu-server.conf";
6231
6232         # we can read the config - that is already extracted
6233         my $fh = IO::File->new($cfgfn, "r") ||
6234             "unable to read qemu-server.conf - $!\n";
6235
6236         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6237         if (-f $fwcfgfn) {
6238             my $pve_firewall_dir = '/etc/pve/firewall';
6239             mkdir $pve_firewall_dir; # make sure the dir exists
6240             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6241         }
6242
6243         while (defined(my $line = <$fh>)) {
6244             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6245                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6246                 die "archive does not contain data for drive '$virtdev'\n"
6247                     if !$devinfo->{$devname};
6248                 if (defined($opts->{storage})) {
6249                     $storeid = $opts->{storage} || 'local';
6250                 } elsif (!$storeid) {
6251                     $storeid = 'local';
6252                 }
6253                 $format = 'raw' if !$format;
6254                 $devinfo->{$devname}->{devname} = $devname;
6255                 $devinfo->{$devname}->{virtdev} = $virtdev;
6256                 $devinfo->{$devname}->{format} = $format;
6257                 $devinfo->{$devname}->{storeid} = $storeid;
6258
6259                 # check permission on storage
6260                 my $pool = $opts->{pool}; # todo: do we need that?
6261                 if ($user ne 'root@pam') {
6262                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6263                 }
6264
6265                 $storage_limits{$storeid} = $bwlimit;
6266
6267                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6268             }
6269         }
6270
6271         foreach my $key (keys %storage_limits) {
6272             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6273             next if !$limit;
6274             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6275             $storage_limits{$key} = $limit * 1024;
6276         }
6277
6278         foreach my $devname (keys %$devinfo) {
6279             die "found no device mapping information for device '$devname'\n"
6280                 if !$devinfo->{$devname}->{virtdev};
6281         }
6282
6283         # create empty/temp config
6284         if ($oldconf) {
6285             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6286             foreach_drive($oldconf, sub {
6287                 my ($ds, $drive) = @_;
6288
6289                 return if drive_is_cdrom($drive);
6290
6291                 my $volid = $drive->{file};
6292
6293                 return if !$volid || $volid =~ m|^/|;
6294
6295                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6296                 return if !$path || !$owner || ($owner != $vmid);
6297
6298                 # Note: only delete disk we want to restore
6299                 # other volumes will become unused
6300                 if ($virtdev_hash->{$ds}) {
6301                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6302                     if (my $err = $@) {
6303                         warn $err;
6304                     }
6305                 }
6306             });
6307
6308             # delete vmstate files
6309             # since after the restore we have no snapshots anymore
6310             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6311                 my $snap = $oldconf->{snapshots}->{$snapname};
6312                 if ($snap->{vmstate}) {
6313                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6314                     if (my $err = $@) {
6315                         warn $err;
6316                     }
6317                 }
6318             }
6319         }
6320
6321         my $map = {};
6322         foreach my $virtdev (sort keys %$virtdev_hash) {
6323             my $d = $virtdev_hash->{$virtdev};
6324             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6325             my $storeid = $d->{storeid};
6326             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6327
6328             my $map_opts = '';
6329             if (my $limit = $storage_limits{$storeid}) {
6330                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6331             }
6332
6333             # test if requested format is supported
6334             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6335             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6336             $d->{format} = $defFormat if !$supported;
6337
6338             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6339                                                   $d->{format}, undef, $alloc_size);
6340             print STDERR "new volume ID is '$volid'\n";
6341             $d->{volid} = $volid;
6342             my $path = PVE::Storage::path($cfg, $volid);
6343
6344             PVE::Storage::activate_volumes($cfg,[$volid]);
6345
6346             my $write_zeros = 1;
6347             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6348                 $write_zeros = 0;
6349             }
6350
6351             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6352
6353             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6354             $map->{$virtdev} = $volid;
6355         }
6356
6357         $fh->seek(0, 0) || die "seek failed - $!\n";
6358
6359         my $outfd = new IO::File ($tmpfn, "w") ||
6360             die "unable to write config for VM $vmid\n";
6361
6362         my $cookie = { netcount => 0 };
6363         while (defined(my $line = <$fh>)) {
6364             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6365         }
6366
6367         $fh->close();
6368         $outfd->close();
6369     };
6370
6371     eval {
6372         # enable interrupts
6373         local $SIG{INT} =
6374             local $SIG{TERM} =
6375             local $SIG{QUIT} =
6376             local $SIG{HUP} =
6377             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6378         local $SIG{ALRM} = sub { die "got timeout\n"; };
6379
6380         $oldtimeout = alarm($timeout);
6381
6382         my $parser = sub {
6383             my $line = shift;
6384
6385             print "$line\n";
6386
6387             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6388                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6389                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6390             } elsif ($line =~ m/^CTIME: /) {
6391                 # we correctly received the vma config, so we can disable
6392                 # the timeout now for disk allocation (set to 10 minutes, so
6393                 # that we always timeout if something goes wrong)
6394                 alarm(600);
6395                 &$print_devmap();
6396                 print $fifofh "done\n";
6397                 my $tmp = $oldtimeout || 0;
6398                 $oldtimeout = undef;
6399                 alarm($tmp);
6400                 close($fifofh);
6401             }
6402         };
6403
6404         print "restore vma archive: $dbg_cmdstring\n";
6405         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6406     };
6407     my $err = $@;
6408
6409     alarm($oldtimeout) if $oldtimeout;
6410
6411     my $vollist = [];
6412     foreach my $devname (keys %$devinfo) {
6413         my $volid = $devinfo->{$devname}->{volid};
6414         push @$vollist, $volid if $volid;
6415     }
6416
6417     PVE::Storage::deactivate_volumes($cfg, $vollist);
6418
6419     unlink $mapfifo;
6420
6421     if ($err) {
6422         rmtree $tmpdir;
6423         unlink $tmpfn;
6424
6425         foreach my $devname (keys %$devinfo) {
6426             my $volid = $devinfo->{$devname}->{volid};
6427             next if !$volid;
6428             eval {
6429                 if ($volid =~ m|^/|) {
6430                     unlink $volid || die 'unlink failed\n';
6431                 } else {
6432                     PVE::Storage::vdisk_free($cfg, $volid);
6433                 }
6434                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6435             };
6436             print STDERR "unable to cleanup '$volid' - $@" if $@;
6437         }
6438         die $err;
6439     }
6440
6441     rmtree $tmpdir;
6442
6443     rename($tmpfn, $conffile) ||
6444         die "unable to commit configuration file '$conffile'\n";
6445
6446     PVE::Cluster::cfs_update(); # make sure we read new file
6447
6448     eval { rescan($vmid, 1); };
6449     warn $@ if $@;
6450 }
6451
6452 sub restore_tar_archive {
6453     my ($archive, $vmid, $user, $opts) = @_;
6454
6455     if ($archive ne '-') {
6456         my $firstfile = tar_archive_read_firstfile($archive);
6457         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6458             if $firstfile ne 'qemu-server.conf';
6459     }
6460
6461     my $storecfg = PVE::Storage::config();
6462
6463     # destroy existing data - keep empty config
6464     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6465     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6466
6467     my $tocmd = "/usr/lib/qemu-server/qmextract";
6468
6469     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6470     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6471     $tocmd .= ' --prealloc' if $opts->{prealloc};
6472     $tocmd .= ' --info' if $opts->{info};
6473
6474     # tar option "xf" does not autodetect compression when read from STDIN,
6475     # so we pipe to zcat
6476     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6477         PVE::Tools::shellquote("--to-command=$tocmd");
6478
6479     my $tmpdir = "/var/tmp/vzdumptmp$$";
6480     mkpath $tmpdir;
6481
6482     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6483     local $ENV{VZDUMP_VMID} = $vmid;
6484     local $ENV{VZDUMP_USER} = $user;
6485
6486     my $conffile = PVE::QemuConfig->config_file($vmid);
6487     my $tmpfn = "$conffile.$$.tmp";
6488
6489     # disable interrupts (always do cleanups)
6490     local $SIG{INT} =
6491         local $SIG{TERM} =
6492         local $SIG{QUIT} =
6493         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6494
6495     eval {
6496         # enable interrupts
6497         local $SIG{INT} =
6498             local $SIG{TERM} =
6499             local $SIG{QUIT} =
6500             local $SIG{HUP} =
6501             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6502
6503         if ($archive eq '-') {
6504             print "extracting archive from STDIN\n";
6505             run_command($cmd, input => "<&STDIN");
6506         } else {
6507             print "extracting archive '$archive'\n";
6508             run_command($cmd);
6509         }
6510
6511         return if $opts->{info};
6512
6513         # read new mapping
6514         my $map = {};
6515         my $statfile = "$tmpdir/qmrestore.stat";
6516         if (my $fd = IO::File->new($statfile, "r")) {
6517             while (defined (my $line = <$fd>)) {
6518                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6519                     $map->{$1} = $2 if $1;
6520                 } else {
6521                     print STDERR "unable to parse line in statfile - $line\n";
6522                 }
6523             }
6524             $fd->close();
6525         }
6526
6527         my $confsrc = "$tmpdir/qemu-server.conf";
6528
6529         my $srcfd = new IO::File($confsrc, "r") ||
6530             die "unable to open file '$confsrc'\n";
6531
6532         my $outfd = new IO::File ($tmpfn, "w") ||
6533             die "unable to write config for VM $vmid\n";
6534
6535         my $cookie = { netcount => 0 };
6536         while (defined (my $line = <$srcfd>)) {
6537             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6538         }
6539
6540         $srcfd->close();
6541         $outfd->close();
6542     };
6543     my $err = $@;
6544
6545     if ($err) {
6546
6547         unlink $tmpfn;
6548
6549         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6550
6551         die $err;
6552     }
6553
6554     rmtree $tmpdir;
6555
6556     rename $tmpfn, $conffile ||
6557         die "unable to commit configuration file '$conffile'\n";
6558
6559     PVE::Cluster::cfs_update(); # make sure we read new file
6560
6561     eval { rescan($vmid, 1); };
6562     warn $@ if $@;
6563 };
6564
6565 sub foreach_storage_used_by_vm {
6566     my ($conf, $func) = @_;
6567
6568     my $sidhash = {};
6569
6570     foreach_drive($conf, sub {
6571         my ($ds, $drive) = @_;
6572         return if drive_is_cdrom($drive);
6573
6574         my $volid = $drive->{file};
6575
6576         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6577         $sidhash->{$sid} = $sid if $sid;
6578     });
6579
6580     foreach my $sid (sort keys %$sidhash) {
6581         &$func($sid);
6582     }
6583 }
6584
6585 sub do_snapshots_with_qemu {
6586     my ($storecfg, $volid) = @_;
6587
6588     my $storage_name = PVE::Storage::parse_volume_id($volid);
6589
6590     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6591         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6592         return 1;
6593     }
6594
6595     if ($volid =~ m/\.(qcow2|qed)$/){
6596         return 1;
6597     }
6598
6599     return undef;
6600 }
6601
6602 sub qga_check_running {
6603     my ($vmid, $nowarn) = @_;
6604
6605     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6606     if ($@) {
6607         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6608         return 0;
6609     }
6610     return 1;
6611 }
6612
6613 sub template_create {
6614     my ($vmid, $conf, $disk) = @_;
6615
6616     my $storecfg = PVE::Storage::config();
6617
6618     foreach_drive($conf, sub {
6619         my ($ds, $drive) = @_;
6620
6621         return if drive_is_cdrom($drive);
6622         return if $disk && $ds ne $disk;
6623
6624         my $volid = $drive->{file};
6625         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6626
6627         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6628         $drive->{file} = $voliddst;
6629         $conf->{$ds} = print_drive($vmid, $drive);
6630         PVE::QemuConfig->write_config($vmid, $conf);
6631     });
6632 }
6633
6634 sub convert_iscsi_path {
6635     my ($path) = @_;
6636
6637     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6638         my $portal = $1;
6639         my $target = $2;
6640         my $lun = $3;
6641
6642         my $initiator_name = get_initiator_name();
6643
6644         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6645                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6646     }
6647
6648     die "cannot convert iscsi path '$path', unkown format\n";
6649 }
6650
6651 sub qemu_img_convert {
6652     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6653
6654     my $storecfg = PVE::Storage::config();
6655     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6656     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6657
6658     if ($src_storeid && $dst_storeid) {
6659
6660         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6661
6662         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6663         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6664
6665         my $src_format = qemu_img_format($src_scfg, $src_volname);
6666         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6667
6668         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6669         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6670
6671         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6672         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6673
6674         my $cmd = [];
6675         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6676         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6677         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6678         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6679
6680         if ($src_is_iscsi) {
6681             push @$cmd, '--image-opts';
6682             $src_path = convert_iscsi_path($src_path);
6683         } else {
6684             push @$cmd, '-f', $src_format;
6685         }
6686
6687         if ($dst_is_iscsi) {
6688             push @$cmd, '--target-image-opts';
6689             $dst_path = convert_iscsi_path($dst_path);
6690         } else {
6691             push @$cmd, '-O', $dst_format;
6692         }
6693
6694         push @$cmd, $src_path;
6695
6696         if (!$dst_is_iscsi && $is_zero_initialized) {
6697             push @$cmd, "zeroinit:$dst_path";
6698         } else {
6699             push @$cmd, $dst_path;
6700         }
6701
6702         my $parser = sub {
6703             my $line = shift;
6704             if($line =~ m/\((\S+)\/100\%\)/){
6705                 my $percent = $1;
6706                 my $transferred = int($size * $percent / 100);
6707                 my $remaining = $size - $transferred;
6708
6709                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6710             }
6711
6712         };
6713
6714         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6715         my $err = $@;
6716         die "copy failed: $err" if $err;
6717     }
6718 }
6719
6720 sub qemu_img_format {
6721     my ($scfg, $volname) = @_;
6722
6723     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6724         return $1;
6725     } else {
6726         return "raw";
6727     }
6728 }
6729
6730 sub qemu_drive_mirror {
6731     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6732
6733     $jobs = {} if !$jobs;
6734
6735     my $qemu_target;
6736     my $format;
6737     $jobs->{"drive-$drive"} = {};
6738
6739     if ($dst_volid =~ /^nbd:/) {
6740         $qemu_target = $dst_volid;
6741         $format = "nbd";
6742     } else {
6743         my $storecfg = PVE::Storage::config();
6744         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6745
6746         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6747
6748         $format = qemu_img_format($dst_scfg, $dst_volname);
6749
6750         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6751
6752         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6753     }
6754
6755     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6756     $opts->{format} = $format if $format;
6757
6758     print "drive mirror is starting for drive-$drive\n";
6759
6760     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6761
6762     if (my $err = $@) {
6763         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6764         die "mirroring error: $err";
6765     }
6766
6767     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6768 }
6769
6770 sub qemu_drive_mirror_monitor {
6771     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6772
6773     eval {
6774         my $err_complete = 0;
6775
6776         while (1) {
6777             die "storage migration timed out\n" if $err_complete > 300;
6778
6779             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6780
6781             my $running_mirror_jobs = {};
6782             foreach my $stat (@$stats) {
6783                 next if $stat->{type} ne 'mirror';
6784                 $running_mirror_jobs->{$stat->{device}} = $stat;
6785             }
6786
6787             my $readycounter = 0;
6788
6789             foreach my $job (keys %$jobs) {
6790
6791                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6792                     print "$job : finished\n";
6793                     delete $jobs->{$job};
6794                     next;
6795                 }
6796
6797                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6798
6799                 my $busy = $running_mirror_jobs->{$job}->{busy};
6800                 my $ready = $running_mirror_jobs->{$job}->{ready};
6801                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6802                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6803                     my $remaining = $total - $transferred;
6804                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6805
6806                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6807                 }
6808
6809                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6810             }
6811
6812             last if scalar(keys %$jobs) == 0;
6813
6814             if ($readycounter == scalar(keys %$jobs)) {
6815                 print "all mirroring jobs are ready \n";
6816                 last if $skipcomplete; #do the complete later
6817
6818                 if ($vmiddst && $vmiddst != $vmid) {
6819                     my $agent_running = $qga && qga_check_running($vmid);
6820                     if ($agent_running) {
6821                         print "freeze filesystem\n";
6822                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6823                     } else {
6824                         print "suspend vm\n";
6825                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6826                     }
6827
6828                     # if we clone a disk for a new target vm, we don't switch the disk
6829                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6830
6831                     if ($agent_running) {
6832                         print "unfreeze filesystem\n";
6833                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6834                     } else {
6835                         print "resume vm\n";
6836                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6837                     }
6838
6839                     last;
6840                 } else {
6841
6842                     foreach my $job (keys %$jobs) {
6843                         # try to switch the disk if source and destination are on the same guest
6844                         print "$job: Completing block job...\n";
6845
6846                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6847                         if ($@ =~ m/cannot be completed/) {
6848                             print "$job: Block job cannot be completed, try again.\n";
6849                             $err_complete++;
6850                         }else {
6851                             print "$job: Completed successfully.\n";
6852                             $jobs->{$job}->{complete} = 1;
6853                         }
6854                     }
6855                 }
6856             }
6857             sleep 1;
6858         }
6859     };
6860     my $err = $@;
6861
6862     if ($err) {
6863         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6864         die "mirroring error: $err";
6865     }
6866
6867 }
6868
6869 sub qemu_blockjobs_cancel {
6870     my ($vmid, $jobs) = @_;
6871
6872     foreach my $job (keys %$jobs) {
6873         print "$job: Cancelling block job\n";
6874         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6875         $jobs->{$job}->{cancel} = 1;
6876     }
6877
6878     while (1) {
6879         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6880
6881         my $running_jobs = {};
6882         foreach my $stat (@$stats) {
6883             $running_jobs->{$stat->{device}} = $stat;
6884         }
6885
6886         foreach my $job (keys %$jobs) {
6887
6888             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6889                 print "$job: Done.\n";
6890                 delete $jobs->{$job};
6891             }
6892         }
6893
6894         last if scalar(keys %$jobs) == 0;
6895
6896         sleep 1;
6897     }
6898 }
6899
6900 sub clone_disk {
6901     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6902         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6903
6904     my $newvolid;
6905
6906     if (!$full) {
6907         print "create linked clone of drive $drivename ($drive->{file})\n";
6908         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6909         push @$newvollist, $newvolid;
6910     } else {
6911
6912         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6913         $storeid = $storage if $storage;
6914
6915         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6916         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6917
6918         print "create full clone of drive $drivename ($drive->{file})\n";
6919         my $name = undef;
6920         if (drive_is_cloudinit($drive)) {
6921             $name = "vm-$newvmid-cloudinit";
6922             $snapname = undef;
6923             # cloudinit only supports raw and qcow2 atm:
6924             if ($dst_format eq 'qcow2') {
6925                 $name .= '.qcow2';
6926             } elsif ($dst_format ne 'raw') {
6927                 die "clone: unhandled format for cloudinit image\n";
6928             }
6929         }
6930         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6931         push @$newvollist, $newvolid;
6932
6933         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6934
6935         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6936         if (!$running || $snapname) {
6937             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6938         } else {
6939
6940             my $kvmver = get_running_qemu_version ($vmid);
6941             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6942                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6943                     if $drive->{iothread};
6944             }
6945
6946             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6947         }
6948     }
6949
6950     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6951
6952     my $disk = $drive;
6953     $disk->{format} = undef;
6954     $disk->{file} = $newvolid;
6955     $disk->{size} = $size;
6956
6957     return $disk;
6958 }
6959
6960 # this only works if VM is running
6961 sub get_current_qemu_machine {
6962     my ($vmid) = @_;
6963
6964     my $cmd = { execute => 'query-machines', arguments => {} };
6965     my $res = vm_qmp_command($vmid, $cmd);
6966
6967     my ($current, $default);
6968     foreach my $e (@$res) {
6969         $default = $e->{name} if $e->{'is-default'};
6970         $current = $e->{name} if $e->{'is-current'};
6971     }
6972
6973     # fallback to the default machine if current is not supported by qemu
6974     return $current || $default || 'pc';
6975 }
6976
6977 sub get_running_qemu_version {
6978     my ($vmid) = @_;
6979     my $cmd = { execute => 'query-version', arguments => {} };
6980     my $res = vm_qmp_command($vmid, $cmd);
6981     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6982 }
6983
6984 sub qemu_machine_feature_enabled {
6985     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6986
6987     my $current_major;
6988     my $current_minor;
6989
6990     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6991
6992         $current_major = $3;
6993         $current_minor = $4;
6994
6995     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6996
6997         $current_major = $1;
6998         $current_minor = $2;
6999     }
7000
7001     return 1 if $current_major > $version_major ||
7002                 ($current_major == $version_major &&
7003                  $current_minor >= $version_minor);
7004 }
7005
7006 sub qemu_machine_pxe {
7007     my ($vmid, $conf, $machine) = @_;
7008
7009     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7010
7011     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7012         $machine .= '.pxe';
7013     }
7014
7015     return $machine;
7016 }
7017
7018 sub qemu_use_old_bios_files {
7019     my ($machine_type) = @_;
7020
7021     return if !$machine_type;
7022
7023     my $use_old_bios_files = undef;
7024
7025     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7026         $machine_type = $1;
7027         $use_old_bios_files = 1;
7028     } else {
7029         my $kvmver = kvm_user_version();
7030         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7031         # load new efi bios files on migration. So this hack is required to allow
7032         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7033         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7034         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7035     }
7036
7037     return ($use_old_bios_files, $machine_type);
7038 }
7039
7040 sub create_efidisk($$$$$) {
7041     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7042
7043     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7044     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7045
7046     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7047     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7048     PVE::Storage::activate_volumes($storecfg, [$volid]);
7049
7050     my $path = PVE::Storage::path($storecfg, $volid);
7051     eval {
7052         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7053     };
7054     die "Copying EFI vars image failed: $@" if $@;
7055
7056     return ($volid, $vars_size);
7057 }
7058
7059 sub vm_iothreads_list {
7060     my ($vmid) = @_;
7061
7062     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7063
7064     my $iothreads = {};
7065     foreach my $iothread (@$res) {
7066         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7067     }
7068
7069     return $iothreads;
7070 }
7071
7072 sub scsihw_infos {
7073     my ($conf, $drive) = @_;
7074
7075     my $maxdev = 0;
7076
7077     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7078         $maxdev = 7;
7079     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7080         $maxdev = 1;
7081     } else {
7082         $maxdev = 256;
7083     }
7084
7085     my $controller = int($drive->{index} / $maxdev);
7086     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7087
7088     return ($maxdev, $controller, $controller_prefix);
7089 }
7090
7091 sub add_hyperv_enlightenments {
7092     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7093
7094     return if $winversion < 6;
7095     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7096
7097     if ($gpu_passthrough || defined($hv_vendor_id)) {
7098         $hv_vendor_id //= 'proxmox';
7099         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7100     }
7101
7102     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7103         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7104         push @$cpuFlags , 'hv_vapic';
7105         push @$cpuFlags , 'hv_time';
7106     } else {
7107         push @$cpuFlags , 'hv_spinlocks=0xffff';
7108     }
7109
7110     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7111         push @$cpuFlags , 'hv_reset';
7112         push @$cpuFlags , 'hv_vpindex';
7113         push @$cpuFlags , 'hv_runtime';
7114     }
7115
7116     if ($winversion >= 7) {
7117         push @$cpuFlags , 'hv_relaxed';
7118
7119         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7120             push @$cpuFlags , 'hv_synic';
7121             push @$cpuFlags , 'hv_stimer';
7122         }
7123     }
7124 }
7125
7126 sub windows_version {
7127     my ($ostype) = @_;
7128
7129     return 0 if !$ostype;
7130
7131     my $winversion = 0;
7132
7133     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7134         $winversion = 5;
7135     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7136         $winversion = 6;
7137     } elsif ($ostype =~ m/^win(\d+)$/) {
7138         $winversion = $1;
7139     }
7140
7141     return $winversion;
7142 }
7143
7144 sub resolve_dst_disk_format {
7145         my ($storecfg, $storeid, $src_volname, $format) = @_;
7146         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7147
7148         if (!$format) {
7149             # if no target format is specified, use the source disk format as hint
7150             if ($src_volname) {
7151                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7152                 $format = qemu_img_format($scfg, $src_volname);
7153             } else {
7154                 return $defFormat;
7155             }
7156         }
7157
7158         # test if requested format is supported - else use default
7159         my $supported = grep { $_ eq $format } @$validFormats;
7160         $format = $defFormat if !$supported;
7161         return $format;
7162 }
7163
7164 sub resolve_first_disk {
7165     my $conf = shift;
7166     my @disks = PVE::QemuServer::valid_drive_names();
7167     my $firstdisk;
7168     foreach my $ds (reverse @disks) {
7169         next if !$conf->{$ds};
7170         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7171         next if PVE::QemuServer::drive_is_cdrom($disk);
7172         $firstdisk = $ds;
7173     }
7174     return $firstdisk;
7175 }
7176
7177 sub generate_uuid {
7178     my ($uuid, $uuid_str);
7179     UUID::generate($uuid);
7180     UUID::unparse($uuid, $uuid_str);
7181     return $uuid_str;
7182 }
7183
7184 sub generate_smbios1_uuid {
7185     return "uuid=".generate_uuid();
7186 }
7187
7188 sub nbd_stop {
7189     my ($vmid) = @_;
7190
7191     vm_mon_cmd($vmid, 'nbd-server-stop');
7192 }
7193
7194 # bash completion helper
7195
7196 sub complete_backup_archives {
7197     my ($cmdname, $pname, $cvalue) = @_;
7198
7199     my $cfg = PVE::Storage::config();
7200
7201     my $storeid;
7202
7203     if ($cvalue =~ m/^([^:]+):/) {
7204         $storeid = $1;
7205     }
7206
7207     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7208
7209     my $res = [];
7210     foreach my $id (keys %$data) {
7211         foreach my $item (@{$data->{$id}}) {
7212             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7213             push @$res, $item->{volid} if defined($item->{volid});
7214         }
7215     }
7216
7217     return $res;
7218 }
7219
7220 my $complete_vmid_full = sub {
7221     my ($running) = @_;
7222
7223     my $idlist = vmstatus();
7224
7225     my $res = [];
7226
7227     foreach my $id (keys %$idlist) {
7228         my $d = $idlist->{$id};
7229         if (defined($running)) {
7230             next if $d->{template};
7231             next if $running && $d->{status} ne 'running';
7232             next if !$running && $d->{status} eq 'running';
7233         }
7234         push @$res, $id;
7235
7236     }
7237     return $res;
7238 };
7239
7240 sub complete_vmid {
7241     return &$complete_vmid_full();
7242 }
7243
7244 sub complete_vmid_stopped {
7245     return &$complete_vmid_full(0);
7246 }
7247
7248 sub complete_vmid_running {
7249     return &$complete_vmid_full(1);
7250 }
7251
7252 sub complete_storage {
7253
7254     my $cfg = PVE::Storage::config();
7255     my $ids = $cfg->{ids};
7256
7257     my $res = [];
7258     foreach my $sid (keys %$ids) {
7259         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7260         next if !$ids->{$sid}->{content}->{images};
7261         push @$res, $sid;
7262     }
7263
7264     return $res;
7265 }
7266
7267 1;