PVE/API2/Qemu.pm

   1 package PVE::API2::Qemu;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd 'abs_path';
   6 use Net::SSLeay;
   7 use POSIX;
   8 use IO::Socket::IP;
   9 use URI::Escape;
  10 use Crypt::OpenSSL::Random;
  11
  12 use PVE::Cluster qw (cfs_read_file cfs_write_file);;
  13 use PVE::RRD;
  14 use PVE::SafeSyslog;
  15 use PVE::Tools qw(extract_param);
  16 use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
  17 use PVE::Storage;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::RESTHandler;
  20 use PVE::ReplicationConfig;
  21 use PVE::GuestHelpers;
  22 use PVE::QemuConfig;
  23 use PVE::QemuServer;
  24 use PVE::QemuServer::CPUConfig;
  25 use PVE::QemuServer::Drive;
  26 use PVE::QemuServer::ImportDisk;
  27 use PVE::QemuServer::Monitor qw(mon_cmd);
  28 use PVE::QemuServer::Machine;
  29 use PVE::QemuMigrate;
  30 use PVE::RPCEnvironment;
  31 use PVE::AccessControl;
  32 use PVE::INotify;
  33 use PVE::Network;
  34 use PVE::Firewall;
  35 use PVE::API2::Firewall::VM;
  36 use PVE::API2::Qemu::Agent;
  37 use PVE::VZDump::Plugin;
  38 use PVE::DataCenterConfig;
  39 use PVE::SSHInfo;
  40 use PVE::Replication;
  41
  42 BEGIN {
  43     if (!$ENV{PVE_GENERATING_DOCS}) {
  44         require PVE::HA::Env::PVE2;
  45         import PVE::HA::Env::PVE2;
  46         require PVE::HA::Config;
  47         import PVE::HA::Config;
  48     }
  49 }
  50
  51 use Data::Dumper; # fixme: remove
  52
  53 use base qw(PVE::RESTHandler);
  54
  55 my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal.";
  56
  57 my $resolve_cdrom_alias = sub {
  58     my $param = shift;
  59
  60     if (my $value = $param->{cdrom}) {
  61         $value .= ",media=cdrom" if $value !~ m/media=/;
  62         $param->{ide2} = $value;
  63         delete $param->{cdrom};
  64     }
  65 };
  66
  67 # Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema.
  68 my $foreach_volume_with_alloc = sub {
  69     my ($param, $func) = @_;
  70
  71     for my $opt (sort keys $param->%*) {
  72         next if !PVE::QemuServer::is_valid_drivename($opt);
  73
  74         my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1);
  75         next if !$drive;
  76
  77         $func->($opt, $drive);
  78     }
  79 };
  80
  81 my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
  82
  83 my $check_drive_param = sub {
  84     my ($param, $storecfg, $extra_checks) = @_;
  85
  86     for my $opt (sort keys $param->%*) {
  87         next if !PVE::QemuServer::is_valid_drivename($opt);
  88
  89         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
  90         raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
  91
  92         if ($drive->{'import-from'}) {
  93             if ($drive->{file} !~ $NEW_DISK_RE || $3 != 0) {
  94                 raise_param_exc({
  95                     $opt => "'import-from' requires special syntax - ".
  96                         "use <storage ID>:0,import-from=<source>",
  97                 });
  98             }
  99
 100             if ($opt eq 'efidisk0') {
 101                 for my $required (qw(efitype pre-enrolled-keys)) {
 102                     if (!defined($drive->{$required})) {
 103                         raise_param_exc({
 104                             $opt => "need to specify '$required' when using 'import-from'",
 105                         });
 106                     }
 107                 }
 108             } elsif ($opt eq 'tpmstate0') {
 109                 raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" })
 110                     if !defined($drive->{version});
 111             }
 112         }
 113
 114         PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 115
 116         $extra_checks->($drive) if $extra_checks;
 117
 118         $param->{$opt} = PVE::QemuServer::print_drive($drive, 1);
 119     }
 120 };
 121
 122 my $check_storage_access = sub {
 123    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 124
 125    $foreach_volume_with_alloc->($settings, sub {
 126         my ($ds, $drive) = @_;
 127
 128         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 129
 130         my $volid = $drive->{file};
 131         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 132
 133         if (!$volid || ($volid eq 'none' || $volid eq 'cloudinit' || (defined($volname) && $volname eq 'cloudinit'))) {
 134             # nothing to check
 135         } elsif ($isCDROM && ($volid eq 'cdrom')) {
 136             $rpcenv->check($authuser, "/", ['Sys.Console']);
 137         } elsif (!$isCDROM && ($volid =~ $NEW_DISK_RE)) {
 138             my ($storeid, $size) = ($2 || $default_storage, $3);
 139             die "no storage ID specified (and no default storage)\n" if !$storeid;
 140             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
 141             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 142             raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
 143                 if !$scfg->{content}->{images};
 144         } else {
 145             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 146             if ($storeid) {
 147                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 148                 raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" })
 149                     if $vtype ne 'images' && $vtype ne 'iso';
 150             }
 151         }
 152
 153         if (my $src_image = $drive->{'import-from'}) {
 154             my $src_vmid;
 155             if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume
 156                 (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image);
 157                 raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" })
 158                     if $vtype ne 'images';
 159             }
 160
 161             if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk()
 162                 $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']);
 163             } else {
 164                 PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image);
 165             }
 166         }
 167     });
 168
 169    $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
 170        if defined($settings->{vmstatestorage});
 171 };
 172
 173 my $check_storage_access_clone = sub {
 174    my ($rpcenv, $authuser, $storecfg, $conf, $storage) = @_;
 175
 176    my $sharedvm = 1;
 177
 178    PVE::QemuConfig->foreach_volume($conf, sub {
 179         my ($ds, $drive) = @_;
 180
 181         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 182
 183         my $volid = $drive->{file};
 184
 185         return if !$volid || $volid eq 'none';
 186
 187         if ($isCDROM) {
 188             if ($volid eq 'cdrom') {
 189                 $rpcenv->check($authuser, "/", ['Sys.Console']);
 190             } else {
 191                 # we simply allow access
 192                 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 193                 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 194                 $sharedvm = 0 if !$scfg->{shared};
 195
 196             }
 197         } else {
 198             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 199             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 200             $sharedvm = 0 if !$scfg->{shared};
 201
 202             $sid = $storage if $storage;
 203             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
 204         }
 205     });
 206
 207    $rpcenv->check($authuser, "/storage/$conf->{vmstatestorage}", ['Datastore.AllocateSpace'])
 208        if defined($conf->{vmstatestorage});
 209
 210    return $sharedvm;
 211 };
 212
 213 my $check_storage_access_migrate = sub {
 214     my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
 215
 216     PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
 217
 218     $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
 219
 220     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
 221     die "storage '$storage' does not support vm images\n"
 222         if !$scfg->{content}->{images};
 223 };
 224
 225 my $import_from_volid = sub {
 226     my ($storecfg, $src_volid, $dest_info, $vollist) = @_;
 227
 228     die "could not get size of $src_volid\n"
 229         if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10);
 230
 231     die "cannot import from cloudinit disk\n"
 232         if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid });
 233
 234     my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2];
 235
 236     my $src_vm_state = sub {
 237         my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0;
 238
 239         my $runs = 0;
 240         if ($exists) {
 241             eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); };
 242             die "owner VM $src_vmid not on local node\n" if $@;
 243             $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0;
 244         }
 245
 246         return ($exists, $runs);
 247     };
 248
 249     my ($src_vm_exists, $running) = $src_vm_state->();
 250
 251     die "cannot import from '$src_volid' - full clone feature is not supported\n"
 252         if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running);
 253
 254     my $clonefn = sub {
 255         my ($src_vm_exists_now, $running_now) = $src_vm_state->();
 256
 257         die "owner VM $src_vmid changed state unexpectedly\n"
 258             if $src_vm_exists_now != $src_vm_exists || $running_now != $running;
 259
 260         my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {};
 261
 262         my $src_drive = { file => $src_volid };
 263         my $src_drivename;
 264         PVE::QemuConfig->foreach_volume($src_conf, sub {
 265             my ($ds, $drive) = @_;
 266
 267             return if $src_drivename;
 268
 269             if ($drive->{file} eq $src_volid) {
 270                 $src_drive = $drive;
 271                 $src_drivename = $ds;
 272             }
 273         });
 274
 275         my $source_info = {
 276             vmid => $src_vmid,
 277             running => $running_now,
 278             drivename => $src_drivename,
 279             drive => $src_drive,
 280             snapname => undef,
 281         };
 282
 283         my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid);
 284
 285         return PVE::QemuServer::clone_disk(
 286             $storecfg,
 287             $source_info,
 288             $dest_info,
 289             1,
 290             $vollist,
 291             undef,
 292             undef,
 293             $src_conf->{agent},
 294             PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]),
 295         );
 296     };
 297
 298     my $cloned;
 299     if ($running) {
 300         $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn);
 301     } elsif ($src_vmid) {
 302         $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn);
 303     } else {
 304         $cloned = $clonefn->();
 305     }
 306
 307     return $cloned->@{qw(file size)};
 308 };
 309
 310 # Note: $pool is only needed when creating a VM, because pool permissions
 311 # are automatically inherited if VM already exists inside a pool.
 312 my $create_disks = sub {
 313     my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
 314
 315     my $vollist = [];
 316
 317     my $res = {};
 318
 319     my $code = sub {
 320         my ($ds, $disk) = @_;
 321
 322         my $volid = $disk->{file};
 323         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 324
 325         if (!$volid || $volid eq 'none' || $volid eq 'cdrom') {
 326             delete $disk->{size};
 327             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 328         } elsif (defined($volname) && $volname eq 'cloudinit') {
 329             $storeid = $storeid // $default_storage;
 330             die "no storage ID specified (and no default storage)\n" if !$storeid;
 331             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 332             my $name = "vm-$vmid-cloudinit";
 333
 334             my $fmt = undef;
 335             if ($scfg->{path}) {
 336                 $fmt = $disk->{format} // "qcow2";
 337                 $name .= ".$fmt";
 338             } else {
 339                 $fmt = $disk->{format} // "raw";
 340             }
 341
 342             # Initial disk created with 4 MB and aligned to 4MB on regeneration
 343             my $ci_size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 344             my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, $name, $ci_size/1024);
 345             $disk->{file} = $volid;
 346             $disk->{media} = 'cdrom';
 347             push @$vollist, $volid;
 348             delete $disk->{format}; # no longer needed
 349             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 350             print "$ds: successfully created disk '$res->{$ds}'\n";
 351         } elsif ($volid =~ $NEW_DISK_RE) {
 352             my ($storeid, $size) = ($2 || $default_storage, $3);
 353             die "no storage ID specified (and no default storage)\n" if !$storeid;
 354
 355             if (my $source = delete $disk->{'import-from'}) {
 356                 my $dst_volid;
 357
 358                 if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume
 359                     my $dest_info = {
 360                         vmid => $vmid,
 361                         drivename => $ds,
 362                         storage => $storeid,
 363                         format => $disk->{format},
 364                     };
 365
 366                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk)
 367                         if $ds eq 'efidisk0';
 368
 369                     ($dst_volid, $size) = eval {
 370                         $import_from_volid->($storecfg, $source, $dest_info, $vollist);
 371                     };
 372                     die "cannot import from '$source' - $@" if $@;
 373                 } else {
 374                     $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
 375                     $size = PVE::Storage::file_size_info($source);
 376                     die "could not get file size of $source\n" if !$size;
 377
 378                     (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import(
 379                         $source,
 380                         $vmid,
 381                         $storeid,
 382                         {
 383                             drive_name => $ds,
 384                             format => $disk->{format},
 385                             'skip-config-update' => 1,
 386                         },
 387                     );
 388                     push @$vollist, $dst_volid;
 389                 }
 390
 391                 $disk->{file} = $dst_volid;
 392                 $disk->{size} = $size;
 393                 delete $disk->{format}; # no longer needed
 394                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 395             } else {
 396                 my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
 397                 my $fmt = $disk->{format} || $defformat;
 398
 399                 $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb
 400
 401                 my $volid;
 402                 if ($ds eq 'efidisk0') {
 403                     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 404                     ($volid, $size) = PVE::QemuServer::create_efidisk(
 405                         $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm);
 406                 } elsif ($ds eq 'tpmstate0') {
 407                     # swtpm can only use raw volumes, and uses a fixed size
 408                     $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb');
 409                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size);
 410                 } else {
 411                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
 412                 }
 413                 push @$vollist, $volid;
 414                 $disk->{file} = $volid;
 415                 $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
 416                 delete $disk->{format}; # no longer needed
 417                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 418             }
 419
 420             print "$ds: successfully created disk '$res->{$ds}'\n";
 421         } else {
 422             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 423             if ($storeid) {
 424                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 425                 die "cannot use volume $volid - content type needs to be 'images' or 'iso'"
 426                     if $vtype ne 'images' && $vtype ne 'iso';
 427             }
 428
 429             PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
 430
 431             my $size = PVE::Storage::volume_size_info($storecfg, $volid);
 432             die "volume $volid does not exist\n" if !$size;
 433             $disk->{size} = $size;
 434
 435             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 436         }
 437     };
 438
 439     eval { $foreach_volume_with_alloc->($settings, $code); };
 440
 441     # free allocated images on error
 442     if (my $err = $@) {
 443         syslog('err', "VM $vmid creating disks failed");
 444         foreach my $volid (@$vollist) {
 445             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 446             warn $@ if $@;
 447         }
 448         die $err;
 449     }
 450
 451     return ($vollist, $res);
 452 };
 453
 454 my $check_cpu_model_access = sub {
 455     my ($rpcenv, $authuser, $new, $existing) = @_;
 456
 457     return if !defined($new->{cpu});
 458
 459     my $cpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $new->{cpu});
 460     return if !$cpu || !$cpu->{cputype}; # always allow default
 461     my $cputype = $cpu->{cputype};
 462
 463     if ($existing && $existing->{cpu}) {
 464         # changing only other settings doesn't require permissions for CPU model
 465         my $existingCpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $existing->{cpu});
 466         return if $existingCpu->{cputype} eq $cputype;
 467     }
 468
 469     if (PVE::QemuServer::CPUConfig::is_custom_model($cputype)) {
 470         $rpcenv->check($authuser, "/nodes", ['Sys.Audit']);
 471     }
 472 };
 473
 474 my $cpuoptions = {
 475     'cores' => 1,
 476     'cpu' => 1,
 477     'cpulimit' => 1,
 478     'cpuunits' => 1,
 479     'numa' => 1,
 480     'smp' => 1,
 481     'sockets' => 1,
 482     'vcpus' => 1,
 483 };
 484
 485 my $memoryoptions = {
 486     'memory' => 1,
 487     'balloon' => 1,
 488     'shares' => 1,
 489 };
 490
 491 my $hwtypeoptions = {
 492     'acpi' => 1,
 493     'hotplug' => 1,
 494     'kvm' => 1,
 495     'machine' => 1,
 496     'scsihw' => 1,
 497     'smbios1' => 1,
 498     'tablet' => 1,
 499     'vga' => 1,
 500     'watchdog' => 1,
 501     'audio0' => 1,
 502 };
 503
 504 my $generaloptions = {
 505     'agent' => 1,
 506     'autostart' => 1,
 507     'bios' => 1,
 508     'description' => 1,
 509     'keyboard' => 1,
 510     'localtime' => 1,
 511     'migrate_downtime' => 1,
 512     'migrate_speed' => 1,
 513     'name' => 1,
 514     'onboot' => 1,
 515     'ostype' => 1,
 516     'protection' => 1,
 517     'reboot' => 1,
 518     'startdate' => 1,
 519     'startup' => 1,
 520     'tdf' => 1,
 521     'template' => 1,
 522     'tags' => 1,
 523 };
 524
 525 my $vmpoweroptions = {
 526     'freeze' => 1,
 527 };
 528
 529 my $diskoptions = {
 530     'boot' => 1,
 531     'bootdisk' => 1,
 532     'vmstatestorage' => 1,
 533 };
 534
 535 my $cloudinitoptions = {
 536     cicustom => 1,
 537     cipassword => 1,
 538     citype => 1,
 539     ciuser => 1,
 540     nameserver => 1,
 541     searchdomain => 1,
 542     sshkeys => 1,
 543 };
 544
 545 my $check_vm_create_serial_perm = sub {
 546     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 547
 548     return 1 if $authuser eq 'root@pam';
 549
 550     foreach my $opt (keys %{$param}) {
 551         next if $opt !~ m/^serial\d+$/;
 552
 553         if ($param->{$opt} eq 'socket') {
 554             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 555         } else {
 556             die "only root can set '$opt' config for real devices\n";
 557         }
 558     }
 559
 560     return 1;
 561 };
 562
 563 my $check_vm_create_usb_perm = sub {
 564     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 565
 566     return 1 if $authuser eq 'root@pam';
 567
 568     foreach my $opt (keys %{$param}) {
 569         next if $opt !~ m/^usb\d+$/;
 570
 571         if ($param->{$opt} =~ m/spice/) {
 572             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 573         } else {
 574             die "only root can set '$opt' config for real devices\n";
 575         }
 576     }
 577
 578     return 1;
 579 };
 580
 581 my $check_vm_modify_config_perm = sub {
 582     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
 583
 584     return 1 if $authuser eq 'root@pam';
 585
 586     foreach my $opt (@$key_list) {
 587         # some checks (e.g., disk, serial port, usb) need to be done somewhere
 588         # else, as there the permission can be value dependend
 589         next if PVE::QemuServer::is_valid_drivename($opt);
 590         next if $opt eq 'cdrom';
 591         next if $opt =~ m/^(?:unused|serial|usb)\d+$/;
 592
 593
 594         if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) {
 595             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
 596         } elsif ($memoryoptions->{$opt}) {
 597             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
 598         } elsif ($hwtypeoptions->{$opt}) {
 599             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 600         } elsif ($generaloptions->{$opt}) {
 601             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
 602             # special case for startup since it changes host behaviour
 603             if ($opt eq 'startup') {
 604                 $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
 605             }
 606         } elsif ($vmpoweroptions->{$opt}) {
 607             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']);
 608         } elsif ($diskoptions->{$opt}) {
 609             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
 610         } elsif ($opt =~ m/^(?:net|ipconfig)\d+$/) {
 611             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
 612         } elsif ($cloudinitoptions->{$opt}) {
 613             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1);
 614         } elsif ($opt eq 'vmstate') {
 615             # the user needs Disk and PowerMgmt privileges to change the vmstate
 616             # also needs privileges on the storage, that will be checked later
 617             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]);
 618         } else {
 619             # catches hostpci\d+, args, lock, etc.
 620             # new options will be checked here
 621             die "only root can set '$opt' config\n";
 622         }
 623     }
 624
 625     return 1;
 626 };
 627
 628 __PACKAGE__->register_method({
 629     name => 'vmlist',
 630     path => '',
 631     method => 'GET',
 632     description => "Virtual machine index (per node).",
 633     permissions => {
 634         description => "Only list VMs where you have VM.Audit permissons on /vms/<vmid>.",
 635         user => 'all',
 636     },
 637     proxyto => 'node',
 638     protected => 1, # qemu pid files are only readable by root
 639     parameters => {
 640         additionalProperties => 0,
 641         properties => {
 642             node => get_standard_option('pve-node'),
 643             full => {
 644                 type => 'boolean',
 645                 optional => 1,
 646                 description => "Determine the full status of active VMs.",
 647             },
 648         },
 649     },
 650     returns => {
 651         type => 'array',
 652         items => {
 653             type => "object",
 654             properties => $PVE::QemuServer::vmstatus_return_properties,
 655         },
 656         links => [ { rel => 'child', href => "{vmid}" } ],
 657     },
 658     code => sub {
 659         my ($param) = @_;
 660
 661         my $rpcenv = PVE::RPCEnvironment::get();
 662         my $authuser = $rpcenv->get_user();
 663
 664         my $vmstatus = PVE::QemuServer::vmstatus(undef, $param->{full});
 665
 666         my $res = [];
 667         foreach my $vmid (keys %$vmstatus) {
 668             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
 669
 670             my $data = $vmstatus->{$vmid};
 671             push @$res, $data;
 672         }
 673
 674         return $res;
 675     }});
 676
 677 my $parse_restore_archive = sub {
 678     my ($storecfg, $archive) = @_;
 679
 680     my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1);
 681
 682     if (defined($archive_storeid)) {
 683         my $scfg =  PVE::Storage::storage_config($storecfg, $archive_storeid);
 684         if ($scfg->{type} eq 'pbs') {
 685             return {
 686                 type => 'pbs',
 687                 volid => $archive,
 688             };
 689         }
 690     }
 691     my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive);
 692     return {
 693         type => 'file',
 694         path => $path,
 695     };
 696 };
 697
 698
 699 __PACKAGE__->register_method({
 700     name => 'create_vm',
 701     path => '',
 702     method => 'POST',
 703     description => "Create or restore a virtual machine.",
 704     permissions => {
 705         description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
 706             "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
 707             "If you create disks you need 'Datastore.AllocateSpace' on any used storage.",
 708         user => 'all', # check inside
 709     },
 710     protected => 1,
 711     proxyto => 'node',
 712     parameters => {
 713         additionalProperties => 0,
 714         properties => PVE::QemuServer::json_config_properties(
 715             {
 716                 node => get_standard_option('pve-node'),
 717                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
 718                 archive => {
 719                     description => "The backup archive. Either the file system path to a .tar or .vma file (use '-' to pipe data from stdin) or a proxmox storage backup volume identifier.",
 720                     type => 'string',
 721                     optional => 1,
 722                     maxLength => 255,
 723                     completion => \&PVE::QemuServer::complete_backup_archives,
 724                 },
 725                 storage => get_standard_option('pve-storage-id', {
 726                     description => "Default storage.",
 727                     optional => 1,
 728                     completion => \&PVE::QemuServer::complete_storage,
 729                 }),
 730                 force => {
 731                     optional => 1,
 732                     type => 'boolean',
 733                     description => "Allow to overwrite existing VM.",
 734                     requires => 'archive',
 735                 },
 736                 unique => {
 737                     optional => 1,
 738                     type => 'boolean',
 739                     description => "Assign a unique random ethernet address.",
 740                     requires => 'archive',
 741                 },
 742                 'live-restore' => {
 743                     optional => 1,
 744                     type => 'boolean',
 745                     description => "Start the VM immediately from the backup and restore in background. PBS only.",
 746                     requires => 'archive',
 747                 },
 748                 pool => {
 749                     optional => 1,
 750                     type => 'string', format => 'pve-poolid',
 751                     description => "Add the VM to the specified pool.",
 752                 },
 753                 bwlimit => {
 754                     description => "Override I/O bandwidth limit (in KiB/s).",
 755                     optional => 1,
 756                     type => 'integer',
 757                     minimum => '0',
 758                     default => 'restore limit from datacenter or storage config',
 759                 },
 760                 start => {
 761                     optional => 1,
 762                     type => 'boolean',
 763                     default => 0,
 764                     description => "Start VM after it was created successfully.",
 765                 },
 766             },
 767             1, # with_disk_alloc
 768         ),
 769     },
 770     returns => {
 771         type => 'string',
 772     },
 773     code => sub {
 774         my ($param) = @_;
 775
 776         my $rpcenv = PVE::RPCEnvironment::get();
 777         my $authuser = $rpcenv->get_user();
 778
 779         my $node = extract_param($param, 'node');
 780         my $vmid = extract_param($param, 'vmid');
 781
 782         my $archive = extract_param($param, 'archive');
 783         my $is_restore = !!$archive;
 784
 785         my $bwlimit = extract_param($param, 'bwlimit');
 786         my $force = extract_param($param, 'force');
 787         my $pool = extract_param($param, 'pool');
 788         my $start_after_create = extract_param($param, 'start');
 789         my $storage = extract_param($param, 'storage');
 790         my $unique = extract_param($param, 'unique');
 791         my $live_restore = extract_param($param, 'live-restore');
 792
 793         if (defined(my $ssh_keys = $param->{sshkeys})) {
 794                 $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
 795                 PVE::Tools::validate_ssh_public_keys($ssh_keys);
 796         }
 797
 798         PVE::Cluster::check_cfs_quorum();
 799
 800         my $filename = PVE::QemuConfig->config_file($vmid);
 801         my $storecfg = PVE::Storage::config();
 802
 803         if (defined($pool)) {
 804             $rpcenv->check_pool_exist($pool);
 805         }
 806
 807         $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace'])
 808             if defined($storage);
 809
 810         if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
 811             # OK
 812         } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
 813             # OK
 814         } elsif ($archive && $force && (-f $filename) &&
 815                  $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
 816             # OK: user has VM.Backup permissions, and want to restore an existing VM
 817         } else {
 818             raise_perm_exc();
 819         }
 820
 821         if (!$archive) {
 822             &$resolve_cdrom_alias($param);
 823
 824             &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage);
 825
 826             &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]);
 827
 828             &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param);
 829             &$check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param);
 830
 831             &$check_cpu_model_access($rpcenv, $authuser, $param);
 832
 833             $check_drive_param->($param, $storecfg);
 834
 835             PVE::QemuServer::add_random_macs($param);
 836         } else {
 837             my $keystr = join(' ', keys %$param);
 838             raise_param_exc({ archive => "option conflicts with other options ($keystr)"}) if $keystr;
 839
 840             if ($archive eq '-') {
 841                 die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli';
 842                 $archive = { type => 'pipe' };
 843             } else {
 844                 PVE::Storage::check_volume_access(
 845                     $rpcenv,
 846                     $authuser,
 847                     $storecfg,
 848                     $vmid,
 849                     $archive,
 850                     'backup',
 851                 );
 852
 853                 $archive = $parse_restore_archive->($storecfg, $archive);
 854             }
 855         }
 856
 857         my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -";
 858
 859         eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) };
 860         die "$emsg $@" if $@;
 861
 862         my $restored_data = 0;
 863         my $restorefn = sub {
 864             my $conf = PVE::QemuConfig->load_config($vmid);
 865
 866             PVE::QemuConfig->check_protection($conf, $emsg);
 867
 868             die "$emsg vm is running\n" if PVE::QemuServer::check_running($vmid);
 869
 870             my $realcmd = sub {
 871                 my $restore_options = {
 872                     storage => $storage,
 873                     pool => $pool,
 874                     unique => $unique,
 875                     bwlimit => $bwlimit,
 876                     live => $live_restore,
 877                 };
 878                 if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') {
 879                     die "live-restore is only compatible with backup images from a Proxmox Backup Server\n"
 880                         if $live_restore;
 881                     PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options);
 882                 } elsif ($archive->{type} eq 'pbs') {
 883                     PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options);
 884                 } else {
 885                     die "unknown backup archive type\n";
 886                 }
 887                 $restored_data = 1;
 888
 889                 my $restored_conf = PVE::QemuConfig->load_config($vmid);
 890                 # Convert restored VM to template if backup was VM template
 891                 if (PVE::QemuConfig->is_template($restored_conf)) {
 892                     warn "Convert to template.\n";
 893                     eval { PVE::QemuServer::template_create($vmid, $restored_conf) };
 894                     warn $@ if $@;
 895                 }
 896             };
 897
 898             # ensure no old replication state are exists
 899             PVE::ReplicationState::delete_guest_states($vmid);
 900
 901             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
 902
 903             if ($start_after_create && !$live_restore) {
 904                 print "Execute autostart\n";
 905                 eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) };
 906                 warn $@ if $@;
 907             }
 908         };
 909
 910         my $createfn = sub {
 911             # ensure no old replication state are exists
 912             PVE::ReplicationState::delete_guest_states($vmid);
 913
 914             my $realcmd = sub {
 915                 my $conf = $param;
 916                 my $arch = PVE::QemuServer::get_vm_arch($conf);
 917
 918                 $conf->{meta} = PVE::QemuServer::new_meta_info_string();
 919
 920                 my $vollist = [];
 921                 eval {
 922                     ($vollist, my $created_opts) = $create_disks->(
 923                         $rpcenv,
 924                         $authuser,
 925                         $conf,
 926                         $arch,
 927                         $storecfg,
 928                         $vmid,
 929                         $pool,
 930                         $param,
 931                         $storage,
 932                     );
 933                     $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*;
 934
 935                     if (!$conf->{boot}) {
 936                         my $devs = PVE::QemuServer::get_default_bootdevices($conf);
 937                         $conf->{boot} = PVE::QemuServer::print_bootorder($devs);
 938                     }
 939
 940                     # auto generate uuid if user did not specify smbios1 option
 941                     if (!$conf->{smbios1}) {
 942                         $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid();
 943                     }
 944
 945                     if ((!defined($conf->{vmgenid}) || $conf->{vmgenid} eq '1') && $arch ne 'aarch64') {
 946                         $conf->{vmgenid} = PVE::QemuServer::generate_uuid();
 947                     }
 948
 949                     my $machine = $conf->{machine};
 950                     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
 951                         # always pin Windows' machine version on create, they get to easily confused
 952                         if (PVE::QemuServer::windows_version($conf->{ostype})) {
 953                             $conf->{machine} = PVE::QemuServer::windows_get_pinned_machine_version($machine);
 954                         }
 955                     }
 956
 957                     PVE::QemuConfig->write_config($vmid, $conf);
 958
 959                 };
 960                 my $err = $@;
 961
 962                 if ($err) {
 963                     foreach my $volid (@$vollist) {
 964                         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 965                         warn $@ if $@;
 966                     }
 967                     die "$emsg $err";
 968                 }
 969
 970                 PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
 971             };
 972
 973             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
 974
 975             if ($start_after_create) {
 976                 print "Execute autostart\n";
 977                 eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) };
 978                 warn $@ if $@;
 979             }
 980         };
 981
 982         my ($code, $worker_name);
 983         if ($is_restore) {
 984             $worker_name = 'qmrestore';
 985             $code = sub {
 986                 eval { $restorefn->() };
 987                 if (my $err = $@) {
 988                     eval { PVE::QemuConfig->remove_lock($vmid, 'create') };
 989                     warn $@ if $@;
 990                     if ($restored_data) {
 991                         warn "error after data was restored, VM disks should be OK but config may "
 992                             ."require adaptions. VM $vmid state is NOT cleaned up.\n";
 993                     } else {
 994                         warn "error before or during data restore, some or all disks were not "
 995                             ."completely restored. VM $vmid state is NOT cleaned up.\n";
 996                     }
 997                     die $err;
 998                 }
 999             };
1000         } else {
1001             $worker_name = 'qmcreate';
1002             $code = sub {
1003                 eval { $createfn->() };
1004                 if (my $err = $@) {
1005                     eval {
1006                         my $conffile = PVE::QemuConfig->config_file($vmid);
1007                         unlink($conffile) or die "failed to remove config file: $!\n";
1008                     };
1009                     warn $@ if $@;
1010                     die $err;
1011                 }
1012             };
1013         }
1014
1015         return $rpcenv->fork_worker($worker_name, $vmid, $authuser, $code);
1016     }});
1017
1018 __PACKAGE__->register_method({
1019     name => 'vmdiridx',
1020     path => '{vmid}',
1021     method => 'GET',
1022     proxyto => 'node',
1023     description => "Directory index",
1024     permissions => {
1025         user => 'all',
1026     },
1027     parameters => {
1028         additionalProperties => 0,
1029         properties => {
1030             node => get_standard_option('pve-node'),
1031             vmid => get_standard_option('pve-vmid'),
1032         },
1033     },
1034     returns => {
1035         type => 'array',
1036         items => {
1037             type => "object",
1038             properties => {
1039                 subdir => { type => 'string' },
1040             },
1041         },
1042         links => [ { rel => 'child', href => "{subdir}" } ],
1043     },
1044     code => sub {
1045         my ($param) = @_;
1046
1047         my $res = [
1048             { subdir => 'config' },
1049             { subdir => 'pending' },
1050             { subdir => 'status' },
1051             { subdir => 'unlink' },
1052             { subdir => 'vncproxy' },
1053             { subdir => 'termproxy' },
1054             { subdir => 'migrate' },
1055             { subdir => 'resize' },
1056             { subdir => 'move' },
1057             { subdir => 'rrd' },
1058             { subdir => 'rrddata' },
1059             { subdir => 'monitor' },
1060             { subdir => 'agent' },
1061             { subdir => 'snapshot' },
1062             { subdir => 'spiceproxy' },
1063             { subdir => 'sendkey' },
1064             { subdir => 'firewall' },
1065             ];
1066
1067         return $res;
1068     }});
1069
1070 __PACKAGE__->register_method ({
1071     subclass => "PVE::API2::Firewall::VM",
1072     path => '{vmid}/firewall',
1073 });
1074
1075 __PACKAGE__->register_method ({
1076     subclass => "PVE::API2::Qemu::Agent",
1077     path => '{vmid}/agent',
1078 });
1079
1080 __PACKAGE__->register_method({
1081     name => 'rrd',
1082     path => '{vmid}/rrd',
1083     method => 'GET',
1084     protected => 1, # fixme: can we avoid that?
1085     permissions => {
1086         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1087     },
1088     description => "Read VM RRD statistics (returns PNG)",
1089     parameters => {
1090         additionalProperties => 0,
1091         properties => {
1092             node => get_standard_option('pve-node'),
1093             vmid => get_standard_option('pve-vmid'),
1094             timeframe => {
1095                 description => "Specify the time frame you are interested in.",
1096                 type => 'string',
1097                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1098             },
1099             ds => {
1100                 description => "The list of datasources you want to display.",
1101                 type => 'string', format => 'pve-configid-list',
1102             },
1103             cf => {
1104                 description => "The RRD consolidation function",
1105                 type => 'string',
1106                 enum => [ 'AVERAGE', 'MAX' ],
1107                 optional => 1,
1108             },
1109         },
1110     },
1111     returns => {
1112         type => "object",
1113         properties => {
1114             filename => { type => 'string' },
1115         },
1116     },
1117     code => sub {
1118         my ($param) = @_;
1119
1120         return PVE::RRD::create_rrd_graph(
1121             "pve2-vm/$param->{vmid}", $param->{timeframe},
1122             $param->{ds}, $param->{cf});
1123
1124     }});
1125
1126 __PACKAGE__->register_method({
1127     name => 'rrddata',
1128     path => '{vmid}/rrddata',
1129     method => 'GET',
1130     protected => 1, # fixme: can we avoid that?
1131     permissions => {
1132         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1133     },
1134     description => "Read VM RRD statistics",
1135     parameters => {
1136         additionalProperties => 0,
1137         properties => {
1138             node => get_standard_option('pve-node'),
1139             vmid => get_standard_option('pve-vmid'),
1140             timeframe => {
1141                 description => "Specify the time frame you are interested in.",
1142                 type => 'string',
1143                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1144             },
1145             cf => {
1146                 description => "The RRD consolidation function",
1147                 type => 'string',
1148                 enum => [ 'AVERAGE', 'MAX' ],
1149                 optional => 1,
1150             },
1151         },
1152     },
1153     returns => {
1154         type => "array",
1155         items => {
1156             type => "object",
1157             properties => {},
1158         },
1159     },
1160     code => sub {
1161         my ($param) = @_;
1162
1163         return PVE::RRD::create_rrd_data(
1164             "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
1165     }});
1166
1167
1168 __PACKAGE__->register_method({
1169     name => 'vm_config',
1170     path => '{vmid}/config',
1171     method => 'GET',
1172     proxyto => 'node',
1173     description => "Get the virtual machine configuration with pending configuration " .
1174         "changes applied. Set the 'current' parameter to get the current configuration instead.",
1175     permissions => {
1176         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1177     },
1178     parameters => {
1179         additionalProperties => 0,
1180         properties => {
1181             node => get_standard_option('pve-node'),
1182             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1183             current => {
1184                 description => "Get current values (instead of pending values).",
1185                 optional => 1,
1186                 default => 0,
1187                 type => 'boolean',
1188             },
1189             snapshot => get_standard_option('pve-snapshot-name', {
1190                 description => "Fetch config values from given snapshot.",
1191                 optional => 1,
1192                 completion => sub {
1193                     my ($cmd, $pname, $cur, $args) = @_;
1194                     PVE::QemuConfig->snapshot_list($args->[0]);
1195                 },
1196             }),
1197         },
1198     },
1199     returns => {
1200         description => "The VM configuration.",
1201         type => "object",
1202         properties => PVE::QemuServer::json_config_properties({
1203             digest => {
1204                 type => 'string',
1205                 description => 'SHA1 digest of configuration file. This can be used to prevent concurrent modifications.',
1206             }
1207         }),
1208     },
1209     code => sub {
1210         my ($param) = @_;
1211
1212         raise_param_exc({ snapshot => "cannot use 'snapshot' parameter with 'current'",
1213                           current => "cannot use 'snapshot' parameter with 'current'"})
1214             if ($param->{snapshot} && $param->{current});
1215
1216         my $conf;
1217         if ($param->{snapshot}) {
1218             $conf = PVE::QemuConfig->load_snapshot_config($param->{vmid}, $param->{snapshot});
1219         } else {
1220             $conf = PVE::QemuConfig->load_current_config($param->{vmid}, $param->{current});
1221         }
1222         $conf->{cipassword} = '**********' if $conf->{cipassword};
1223         return $conf;
1224
1225     }});
1226
1227 __PACKAGE__->register_method({
1228     name => 'vm_pending',
1229     path => '{vmid}/pending',
1230     method => 'GET',
1231     proxyto => 'node',
1232     description => "Get the virtual machine configuration with both current and pending values.",
1233     permissions => {
1234         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1235     },
1236     parameters => {
1237         additionalProperties => 0,
1238         properties => {
1239             node => get_standard_option('pve-node'),
1240             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1241         },
1242     },
1243     returns => {
1244         type => "array",
1245         items => {
1246             type => "object",
1247             properties => {
1248                 key => {
1249                     description => "Configuration option name.",
1250                     type => 'string',
1251                 },
1252                 value => {
1253                     description => "Current value.",
1254                     type => 'string',
1255                     optional => 1,
1256                 },
1257                 pending => {
1258                     description => "Pending value.",
1259                     type => 'string',
1260                     optional => 1,
1261                 },
1262                 delete => {
1263                     description => "Indicates a pending delete request if present and not 0. " .
1264                                    "The value 2 indicates a force-delete request.",
1265                     type => 'integer',
1266                     minimum => 0,
1267                     maximum => 2,
1268                     optional => 1,
1269                 },
1270             },
1271         },
1272     },
1273     code => sub {
1274         my ($param) = @_;
1275
1276         my $conf = PVE::QemuConfig->load_config($param->{vmid});
1277
1278         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
1279
1280         $conf->{cipassword} = '**********' if defined($conf->{cipassword});
1281         $conf->{pending}->{cipassword} = '********** ' if defined($conf->{pending}->{cipassword});
1282
1283         return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
1284    }});
1285
1286 # POST/PUT {vmid}/config implementation
1287 #
1288 # The original API used PUT (idempotent) an we assumed that all operations
1289 # are fast. But it turned out that almost any configuration change can
1290 # involve hot-plug actions, or disk alloc/free. Such actions can take long
1291 # time to complete and have side effects (not idempotent).
1292 #
1293 # The new implementation uses POST and forks a worker process. We added
1294 # a new option 'background_delay'. If specified we wait up to
1295 # 'background_delay' second for the worker task to complete. It returns null
1296 # if the task is finished within that time, else we return the UPID.
1297
1298 my $update_vm_api  = sub {
1299     my ($param, $sync) = @_;
1300
1301     my $rpcenv = PVE::RPCEnvironment::get();
1302
1303     my $authuser = $rpcenv->get_user();
1304
1305     my $node = extract_param($param, 'node');
1306
1307     my $vmid = extract_param($param, 'vmid');
1308
1309     my $digest = extract_param($param, 'digest');
1310
1311     my $background_delay = extract_param($param, 'background_delay');
1312
1313     if (defined(my $cipassword = $param->{cipassword})) {
1314         # Same logic as in cloud-init (but with the regex fixed...)
1315         $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
1316             if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
1317     }
1318
1319     my @paramarr = (); # used for log message
1320     foreach my $key (sort keys %$param) {
1321         my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
1322         push @paramarr, "-$key", $value;
1323     }
1324
1325     my $skiplock = extract_param($param, 'skiplock');
1326     raise_param_exc({ skiplock => "Only root may use this option." })
1327         if $skiplock && $authuser ne 'root@pam';
1328
1329     my $delete_str = extract_param($param, 'delete');
1330
1331     my $revert_str = extract_param($param, 'revert');
1332
1333     my $force = extract_param($param, 'force');
1334
1335     if (defined(my $ssh_keys = $param->{sshkeys})) {
1336         $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
1337         PVE::Tools::validate_ssh_public_keys($ssh_keys);
1338     }
1339
1340     die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param);
1341
1342     my $storecfg = PVE::Storage::config();
1343
1344     my $defaults = PVE::QemuServer::load_defaults();
1345
1346     &$resolve_cdrom_alias($param);
1347
1348     # now try to verify all parameters
1349
1350     my $revert = {};
1351     foreach my $opt (PVE::Tools::split_list($revert_str)) {
1352         if (!PVE::QemuServer::option_exists($opt)) {
1353             raise_param_exc({ revert => "unknown option '$opt'" });
1354         }
1355
1356         raise_param_exc({ delete => "you can't use '-$opt' and " .
1357                               "-revert $opt' at the same time" })
1358             if defined($param->{$opt});
1359
1360         $revert->{$opt} = 1;
1361     }
1362
1363     my @delete = ();
1364     foreach my $opt (PVE::Tools::split_list($delete_str)) {
1365         $opt = 'ide2' if $opt eq 'cdrom';
1366
1367         raise_param_exc({ delete => "you can't use '-$opt' and " .
1368                               "-delete $opt' at the same time" })
1369             if defined($param->{$opt});
1370
1371         raise_param_exc({ revert => "you can't use '-delete $opt' and " .
1372                               "-revert $opt' at the same time" })
1373             if $revert->{$opt};
1374
1375         if (!PVE::QemuServer::option_exists($opt)) {
1376             raise_param_exc({ delete => "unknown option '$opt'" });
1377         }
1378
1379         push @delete, $opt;
1380     }
1381
1382     my $repl_conf = PVE::ReplicationConfig->new();
1383     my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
1384     my $check_replication = sub {
1385         my ($drive) = @_;
1386         return if !$is_replicated;
1387         my $volid = $drive->{file};
1388         return if !$volid || !($drive->{replicate}//1);
1389         return if PVE::QemuServer::drive_is_cdrom($drive);
1390
1391         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1392         die "cannot add non-managed/pass-through volume to a replicated VM\n"
1393             if !defined($storeid);
1394
1395         return if defined($volname) && $volname eq 'cloudinit';
1396
1397         my $format;
1398         if ($volid =~ $NEW_DISK_RE) {
1399             $storeid = $2;
1400             $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid);
1401         } else {
1402             $format = (PVE::Storage::parse_volname($storecfg, $volid))[6];
1403         }
1404         return if PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
1405         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1406         return if $scfg->{shared};
1407         die "cannot add non-replicatable volume to a replicated VM\n";
1408     };
1409
1410     $check_drive_param->($param, $storecfg, $check_replication);
1411
1412     foreach my $opt (keys %$param) {
1413         if ($opt =~ m/^net(\d+)$/) {
1414             # add macaddr
1415             my $net = PVE::QemuServer::parse_net($param->{$opt});
1416             $param->{$opt} = PVE::QemuServer::print_net($net);
1417         } elsif ($opt eq 'vmgenid') {
1418             if ($param->{$opt} eq '1') {
1419                 $param->{$opt} = PVE::QemuServer::generate_uuid();
1420             }
1421         } elsif ($opt eq 'hookscript') {
1422             eval { PVE::GuestHelpers::check_hookscript($param->{$opt}, $storecfg); };
1423             raise_param_exc({ $opt => $@ }) if $@;
1424         }
1425     }
1426
1427     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
1428
1429     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
1430
1431     &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
1432
1433     my $updatefn =  sub {
1434
1435         my $conf = PVE::QemuConfig->load_config($vmid);
1436
1437         die "checksum missmatch (file change by other user?)\n"
1438             if $digest && $digest ne $conf->{digest};
1439
1440         &$check_cpu_model_access($rpcenv, $authuser, $param, $conf);
1441
1442         # FIXME: 'suspended' lock should probabyl be a state or "weak" lock?!
1443         if (scalar(@delete) && grep { $_ eq 'vmstate'} @delete) {
1444             if (defined($conf->{lock}) && $conf->{lock} eq 'suspended') {
1445                 delete $conf->{lock}; # for check lock check, not written out
1446                 push @delete, 'lock'; # this is the real deal to write it out
1447             }
1448             push @delete, 'runningmachine' if $conf->{runningmachine};
1449             push @delete, 'runningcpu' if $conf->{runningcpu};
1450         }
1451
1452         PVE::QemuConfig->check_lock($conf) if !$skiplock;
1453
1454         foreach my $opt (keys %$revert) {
1455             if (defined($conf->{$opt})) {
1456                 $param->{$opt} = $conf->{$opt};
1457             } elsif (defined($conf->{pending}->{$opt})) {
1458                 push @delete, $opt;
1459             }
1460         }
1461
1462         if ($param->{memory} || defined($param->{balloon})) {
1463             my $maxmem = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory} || $defaults->{memory};
1464             my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon};
1465
1466             die "balloon value too large (must be smaller than assigned memory)\n"
1467                 if $balloon && $balloon > $maxmem;
1468         }
1469
1470         PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: " . join (' ', @paramarr));
1471
1472         my $worker = sub {
1473
1474             print "update VM $vmid: " . join (' ', @paramarr) . "\n";
1475
1476             # write updates to pending section
1477
1478             my $modified = {}; # record what $option we modify
1479
1480             my @bootorder;
1481             if (my $boot = $conf->{boot}) {
1482                 my $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot);
1483                 @bootorder = PVE::Tools::split_list($bootcfg->{order}) if $bootcfg && $bootcfg->{order};
1484             }
1485             my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete;
1486
1487             my $check_drive_perms = sub {
1488                 my ($opt, $val) = @_;
1489                 my $drive = PVE::QemuServer::parse_drive($opt, $val, 1);
1490                 # FIXME: cloudinit: CDROM or Disk?
1491                 if (PVE::QemuServer::drive_is_cdrom($drive)) { # CDROM
1492                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
1493                 } else {
1494                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1495                 }
1496             };
1497
1498             foreach my $opt (@delete) {
1499                 $modified->{$opt} = 1;
1500                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1501
1502                 # value of what we want to delete, independent if pending or not
1503                 my $val = $conf->{$opt} // $conf->{pending}->{$opt};
1504                 if (!defined($val)) {
1505                     warn "cannot delete '$opt' - not set in current configuration!\n";
1506                     $modified->{$opt} = 0;
1507                     next;
1508                 }
1509                 my $is_pending_val = defined($conf->{pending}->{$opt});
1510                 delete $conf->{pending}->{$opt};
1511
1512                 # remove from bootorder if necessary
1513                 if (!$bootorder_deleted && @bootorder && grep {$_ eq $opt} @bootorder) {
1514                     @bootorder = grep {$_ ne $opt} @bootorder;
1515                     $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1516                     $modified->{boot} = 1;
1517                 }
1518
1519                 if ($opt =~ m/^unused/) {
1520                     my $drive = PVE::QemuServer::parse_drive($opt, $val);
1521                     PVE::QemuConfig->check_protection($conf, "can't remove unused disk '$drive->{file}'");
1522                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1523                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser)) {
1524                         delete $conf->{$opt};
1525                         PVE::QemuConfig->write_config($vmid, $conf);
1526                     }
1527                 } elsif ($opt eq 'vmstate') {
1528                     PVE::QemuConfig->check_protection($conf, "can't remove vmstate '$val'");
1529                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, { file => $val }, $rpcenv, $authuser, 1)) {
1530                         delete $conf->{$opt};
1531                         PVE::QemuConfig->write_config($vmid, $conf);
1532                     }
1533                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
1534                     PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'");
1535                     $check_drive_perms->($opt, $val);
1536                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val))
1537                         if $is_pending_val;
1538                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1539                     PVE::QemuConfig->write_config($vmid, $conf);
1540                 } elsif ($opt =~ m/^serial\d+$/) {
1541                     if ($val eq 'socket') {
1542                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1543                     } elsif ($authuser ne 'root@pam') {
1544                         die "only root can delete '$opt' config for real devices\n";
1545                     }
1546                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1547                     PVE::QemuConfig->write_config($vmid, $conf);
1548                 } elsif ($opt =~ m/^usb\d+$/) {
1549                     if ($val =~ m/spice/) {
1550                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1551                     } elsif ($authuser ne 'root@pam') {
1552                         die "only root can delete '$opt' config for real devices\n";
1553                     }
1554                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1555                     PVE::QemuConfig->write_config($vmid, $conf);
1556                 } else {
1557                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1558                     PVE::QemuConfig->write_config($vmid, $conf);
1559                 }
1560             }
1561
1562             foreach my $opt (keys %$param) { # add/change
1563                 $modified->{$opt} = 1;
1564                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1565                 next if defined($conf->{pending}->{$opt}) && ($param->{$opt} eq $conf->{pending}->{$opt}); # skip if nothing changed
1566
1567                 my $arch = PVE::QemuServer::get_vm_arch($conf);
1568
1569                 if (PVE::QemuServer::is_valid_drivename($opt)) {
1570                     # old drive
1571                     if ($conf->{$opt}) {
1572                         $check_drive_perms->($opt, $conf->{$opt});
1573                     }
1574
1575                     # new drive
1576                     $check_drive_perms->($opt, $param->{$opt});
1577                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
1578                         if defined($conf->{pending}->{$opt});
1579
1580                     my (undef, $created_opts) = $create_disks->(
1581                         $rpcenv,
1582                         $authuser,
1583                         $conf,
1584                         $arch,
1585                         $storecfg,
1586                         $vmid,
1587                         undef,
1588                         {$opt => $param->{$opt}},
1589                     );
1590                     $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*;
1591
1592                     # default legacy boot order implies all cdroms anyway
1593                     if (@bootorder) {
1594                         # append new CD drives to bootorder to mark them bootable
1595                         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
1596                         if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) {
1597                             push @bootorder, $opt;
1598                             $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1599                             $modified->{boot} = 1;
1600                         }
1601                     }
1602                 } elsif ($opt =~ m/^serial\d+/) {
1603                     if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
1604                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1605                     } elsif ($authuser ne 'root@pam') {
1606                         die "only root can modify '$opt' config for real devices\n";
1607                     }
1608                     $conf->{pending}->{$opt} = $param->{$opt};
1609                 } elsif ($opt =~ m/^usb\d+/) {
1610                     if ((!defined($conf->{$opt}) || $conf->{$opt} =~ m/spice/) && $param->{$opt} =~ m/spice/) {
1611                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1612                     } elsif ($authuser ne 'root@pam') {
1613                         die "only root can modify '$opt' config for real devices\n";
1614                     }
1615                     $conf->{pending}->{$opt} = $param->{$opt};
1616                 } else {
1617                     $conf->{pending}->{$opt} = $param->{$opt};
1618
1619                     if ($opt eq 'boot') {
1620                         my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $param->{$opt});
1621                         if ($new_bootcfg->{order}) {
1622                             my @devs = PVE::Tools::split_list($new_bootcfg->{order});
1623                             for my $dev (@devs) {
1624                                 my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev};
1625                                 my $deleted = grep {$_ eq $dev} @delete;
1626                                 die "invalid bootorder: device '$dev' does not exist'\n"
1627                                     if !$exists || $deleted;
1628                             }
1629
1630                             # remove legacy boot order settings if new one set
1631                             $conf->{pending}->{$opt} = PVE::QemuServer::print_bootorder(\@devs);
1632                             PVE::QemuConfig->add_to_pending_delete($conf, "bootdisk")
1633                                 if $conf->{bootdisk};
1634                         }
1635                     }
1636                 }
1637                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
1638                 PVE::QemuConfig->write_config($vmid, $conf);
1639             }
1640
1641             # remove pending changes when nothing changed
1642             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1643             my $changes = PVE::QemuConfig->cleanup_pending($conf);
1644             PVE::QemuConfig->write_config($vmid, $conf) if $changes;
1645
1646             return if !scalar(keys %{$conf->{pending}});
1647
1648             my $running = PVE::QemuServer::check_running($vmid);
1649
1650             # apply pending changes
1651
1652             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1653
1654             my $errors = {};
1655             if ($running) {
1656                 PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors);
1657             } else {
1658                 PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors);
1659             }
1660             raise_param_exc($errors) if scalar(keys %$errors);
1661
1662             return;
1663         };
1664
1665         if ($sync) {
1666             &$worker();
1667             return;
1668         } else {
1669             my $upid = $rpcenv->fork_worker('qmconfig', $vmid, $authuser, $worker);
1670
1671             if ($background_delay) {
1672
1673                 # Note: It would be better to do that in the Event based HTTPServer
1674                 # to avoid blocking call to sleep.
1675
1676                 my $end_time = time() + $background_delay;
1677
1678                 my $task = PVE::Tools::upid_decode($upid);
1679
1680                 my $running = 1;
1681                 while (time() < $end_time) {
1682                     $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
1683                     last if !$running;
1684                     sleep(1); # this gets interrupted when child process ends
1685                 }
1686
1687                 if (!$running) {
1688                     my $status = PVE::Tools::upid_read_status($upid);
1689                     return if !PVE::Tools::upid_status_is_error($status);
1690                     die "failed to update VM $vmid: $status\n";
1691                 }
1692             }
1693
1694             return $upid;
1695         }
1696     };
1697
1698     return PVE::QemuConfig->lock_config($vmid, $updatefn);
1699 };
1700
1701 my $vm_config_perm_list = [
1702             'VM.Config.Disk',
1703             'VM.Config.CDROM',
1704             'VM.Config.CPU',
1705             'VM.Config.Memory',
1706             'VM.Config.Network',
1707             'VM.Config.HWType',
1708             'VM.Config.Options',
1709             'VM.Config.Cloudinit',
1710     ];
1711
1712 __PACKAGE__->register_method({
1713     name => 'update_vm_async',
1714     path => '{vmid}/config',
1715     method => 'POST',
1716     protected => 1,
1717     proxyto => 'node',
1718     description => "Set virtual machine options (asynchrounous API).",
1719     permissions => {
1720         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1721     },
1722     parameters => {
1723         additionalProperties => 0,
1724         properties => PVE::QemuServer::json_config_properties(
1725             {
1726                 node => get_standard_option('pve-node'),
1727                 vmid => get_standard_option('pve-vmid'),
1728                 skiplock => get_standard_option('skiplock'),
1729                 delete => {
1730                     type => 'string', format => 'pve-configid-list',
1731                     description => "A list of settings you want to delete.",
1732                     optional => 1,
1733                 },
1734                 revert => {
1735                     type => 'string', format => 'pve-configid-list',
1736                     description => "Revert a pending change.",
1737                     optional => 1,
1738                 },
1739                 force => {
1740                     type => 'boolean',
1741                     description => $opt_force_description,
1742                     optional => 1,
1743                     requires => 'delete',
1744                 },
1745                 digest => {
1746                     type => 'string',
1747                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1748                     maxLength => 40,
1749                     optional => 1,
1750                 },
1751                 background_delay => {
1752                     type => 'integer',
1753                     description => "Time to wait for the task to finish. We return 'null' if the task finish within that time.",
1754                     minimum => 1,
1755                     maximum => 30,
1756                     optional => 1,
1757                 },
1758             },
1759             1, # with_disk_alloc
1760         ),
1761     },
1762     returns => {
1763         type => 'string',
1764         optional => 1,
1765     },
1766     code => $update_vm_api,
1767 });
1768
1769 __PACKAGE__->register_method({
1770     name => 'update_vm',
1771     path => '{vmid}/config',
1772     method => 'PUT',
1773     protected => 1,
1774     proxyto => 'node',
1775     description => "Set virtual machine options (synchrounous API) - You should consider using the POST method instead for any actions involving hotplug or storage allocation.",
1776     permissions => {
1777         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1778     },
1779     parameters => {
1780         additionalProperties => 0,
1781         properties => PVE::QemuServer::json_config_properties(
1782             {
1783                 node => get_standard_option('pve-node'),
1784                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1785                 skiplock => get_standard_option('skiplock'),
1786                 delete => {
1787                     type => 'string', format => 'pve-configid-list',
1788                     description => "A list of settings you want to delete.",
1789                     optional => 1,
1790                 },
1791                 revert => {
1792                     type => 'string', format => 'pve-configid-list',
1793                     description => "Revert a pending change.",
1794                     optional => 1,
1795                 },
1796                 force => {
1797                     type => 'boolean',
1798                     description => $opt_force_description,
1799                     optional => 1,
1800                     requires => 'delete',
1801                 },
1802                 digest => {
1803                     type => 'string',
1804                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1805                     maxLength => 40,
1806                     optional => 1,
1807                 },
1808             },
1809             1, # with_disk_alloc
1810         ),
1811     },
1812     returns => { type => 'null' },
1813     code => sub {
1814         my ($param) = @_;
1815         &$update_vm_api($param, 1);
1816         return;
1817     }
1818 });
1819
1820 __PACKAGE__->register_method({
1821     name => 'destroy_vm',
1822     path => '{vmid}',
1823     method => 'DELETE',
1824     protected => 1,
1825     proxyto => 'node',
1826     description => "Destroy the VM and  all used/owned volumes. Removes any VM specific permissions"
1827         ." and firewall rules",
1828     permissions => {
1829         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
1830     },
1831     parameters => {
1832         additionalProperties => 0,
1833         properties => {
1834             node => get_standard_option('pve-node'),
1835             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
1836             skiplock => get_standard_option('skiplock'),
1837             purge => {
1838                 type => 'boolean',
1839                 description => "Remove VMID from configurations, like backup & replication jobs and HA.",
1840                 optional => 1,
1841             },
1842             'destroy-unreferenced-disks' => {
1843                 type => 'boolean',
1844                 description => "If set, destroy additionally all disks not referenced in the config"
1845                  ." but with a matching VMID from all enabled storages.",
1846                 optional => 1,
1847                 default => 0,
1848             },
1849         },
1850     },
1851     returns => {
1852         type => 'string',
1853     },
1854     code => sub {
1855         my ($param) = @_;
1856
1857         my $rpcenv = PVE::RPCEnvironment::get();
1858         my $authuser = $rpcenv->get_user();
1859         my $vmid = $param->{vmid};
1860
1861         my $skiplock = $param->{skiplock};
1862         raise_param_exc({ skiplock => "Only root may use this option." })
1863             if $skiplock && $authuser ne 'root@pam';
1864
1865         my $early_checks = sub {
1866             # test if VM exists
1867             my $conf = PVE::QemuConfig->load_config($vmid);
1868             PVE::QemuConfig->check_protection($conf, "can't remove VM $vmid");
1869
1870             my $ha_managed = PVE::HA::Config::service_is_configured("vm:$vmid");
1871
1872             if (!$param->{purge}) {
1873                 die "unable to remove VM $vmid - used in HA resources and purge parameter not set.\n"
1874                     if $ha_managed;
1875                 # don't allow destroy if with replication jobs but no purge param
1876                 my $repl_conf = PVE::ReplicationConfig->new();
1877                 $repl_conf->check_for_existing_jobs($vmid);
1878             }
1879
1880             die "VM $vmid is running - destroy failed\n"
1881                 if PVE::QemuServer::check_running($vmid);
1882
1883             return $ha_managed;
1884         };
1885
1886         $early_checks->();
1887
1888         my $realcmd = sub {
1889             my $upid = shift;
1890
1891             my $storecfg = PVE::Storage::config();
1892
1893             syslog('info', "destroy VM $vmid: $upid\n");
1894             PVE::QemuConfig->lock_config($vmid, sub {
1895                 # repeat, config might have changed
1896                 my $ha_managed = $early_checks->();
1897
1898                 my $purge_unreferenced = $param->{'destroy-unreferenced-disks'};
1899
1900                 PVE::QemuServer::destroy_vm(
1901                     $storecfg,
1902                     $vmid,
1903                     $skiplock, { lock => 'destroyed' },
1904                     $purge_unreferenced,
1905                 );
1906
1907                 PVE::AccessControl::remove_vm_access($vmid);
1908                 PVE::Firewall::remove_vmfw_conf($vmid);
1909                 if ($param->{purge}) {
1910                     print "purging VM $vmid from related configurations..\n";
1911                     PVE::ReplicationConfig::remove_vmid_jobs($vmid);
1912                     PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);
1913
1914                     if ($ha_managed) {
1915                         PVE::HA::Config::delete_service_from_config("vm:$vmid");
1916                         print "NOTE: removed VM $vmid from HA resource configuration.\n";
1917                     }
1918                 }
1919
1920                 # only now remove the zombie config, else we can have reuse race
1921                 PVE::QemuConfig->destroy_config($vmid);
1922             });
1923         };
1924
1925         return $rpcenv->fork_worker('qmdestroy', $vmid, $authuser, $realcmd);
1926     }});
1927
1928 __PACKAGE__->register_method({
1929     name => 'unlink',
1930     path => '{vmid}/unlink',
1931     method => 'PUT',
1932     protected => 1,
1933     proxyto => 'node',
1934     description => "Unlink/delete disk images.",
1935     permissions => {
1936         check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']],
1937     },
1938     parameters => {
1939         additionalProperties => 0,
1940         properties => {
1941             node => get_standard_option('pve-node'),
1942             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1943             idlist => {
1944                 type => 'string', format => 'pve-configid-list',
1945                 description => "A list of disk IDs you want to delete.",
1946             },
1947             force => {
1948                 type => 'boolean',
1949                 description => $opt_force_description,
1950                 optional => 1,
1951             },
1952         },
1953     },
1954     returns => { type => 'null'},
1955     code => sub {
1956         my ($param) = @_;
1957
1958         $param->{delete} = extract_param($param, 'idlist');
1959
1960         __PACKAGE__->update_vm($param);
1961
1962         return;
1963     }});
1964
1965 # uses good entropy, each char is limited to 6 bit to get printable chars simply
1966 my $gen_rand_chars = sub {
1967     my ($length) = @_;
1968
1969     die "invalid length $length" if $length < 1;
1970
1971     my $min = ord('!'); # first printable ascii
1972
1973     my $rand_bytes = Crypt::OpenSSL::Random::random_bytes($length);
1974     die "failed to generate random bytes!\n"
1975       if !$rand_bytes;
1976
1977     my $str = join('', map { chr((ord($_) & 0x3F) + $min) } split('', $rand_bytes));
1978
1979     return $str;
1980 };
1981
1982 my $sslcert;
1983
1984 __PACKAGE__->register_method({
1985     name => 'vncproxy',
1986     path => '{vmid}/vncproxy',
1987     method => 'POST',
1988     protected => 1,
1989     permissions => {
1990         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
1991     },
1992     description => "Creates a TCP VNC proxy connections.",
1993     parameters => {
1994         additionalProperties => 0,
1995         properties => {
1996             node => get_standard_option('pve-node'),
1997             vmid => get_standard_option('pve-vmid'),
1998             websocket => {
1999                 optional => 1,
2000                 type => 'boolean',
2001                 description => "starts websockify instead of vncproxy",
2002             },
2003             'generate-password' => {
2004                 optional => 1,
2005                 type => 'boolean',
2006                 default => 0,
2007                 description => "Generates a random password to be used as ticket instead of the API ticket.",
2008             },
2009         },
2010     },
2011     returns => {
2012         additionalProperties => 0,
2013         properties => {
2014             user => { type => 'string' },
2015             ticket => { type => 'string' },
2016             password => {
2017                 optional => 1,
2018                 description => "Returned if requested with 'generate-password' param."
2019                     ." Consists of printable ASCII characters ('!' .. '~').",
2020                 type => 'string',
2021             },
2022             cert => { type => 'string' },
2023             port => { type => 'integer' },
2024             upid => { type => 'string' },
2025         },
2026     },
2027     code => sub {
2028         my ($param) = @_;
2029
2030         my $rpcenv = PVE::RPCEnvironment::get();
2031
2032         my $authuser = $rpcenv->get_user();
2033
2034         my $vmid = $param->{vmid};
2035         my $node = $param->{node};
2036         my $websocket = $param->{websocket};
2037
2038         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2039
2040         my $serial;
2041         if ($conf->{vga}) {
2042             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2043             $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2044         }
2045
2046         my $authpath = "/vms/$vmid";
2047
2048         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2049         my $password = $ticket;
2050         if ($param->{'generate-password'}) {
2051             $password = $gen_rand_chars->(8);
2052         }
2053
2054         $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
2055             if !$sslcert;
2056
2057         my $family;
2058         my $remcmd = [];
2059
2060         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2061             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2062             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2063             # NOTE: kvm VNC traffic is already TLS encrypted or is known unsecure
2064             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, defined($serial) ? '-t' : '-T');
2065         } else {
2066             $family = PVE::Tools::get_host_address_family($node);
2067         }
2068
2069         my $port = PVE::Tools::next_vnc_port($family);
2070
2071         my $timeout = 10;
2072
2073         my $realcmd = sub {
2074             my $upid = shift;
2075
2076             syslog('info', "starting vnc proxy $upid\n");
2077
2078             my $cmd;
2079
2080             if (defined($serial)) {
2081
2082                 my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-iface', $serial, '-escape', '0' ];
2083
2084                 $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
2085                         '-timeout', $timeout, '-authpath', $authpath,
2086                         '-perm', 'Sys.Console'];
2087
2088                 if ($param->{websocket}) {
2089                     $ENV{PVE_VNC_TICKET} = $password; # pass ticket to vncterm
2090                     push @$cmd, '-notls', '-listen', 'localhost';
2091                 }
2092
2093                 push @$cmd, '-c', @$remcmd, @$termcmd;
2094
2095                 PVE::Tools::run_command($cmd);
2096
2097             } else {
2098
2099                 $ENV{LC_PVE_TICKET} = $password if $websocket; # set ticket with "qm vncproxy"
2100
2101                 $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid];
2102
2103                 my $sock = IO::Socket::IP->new(
2104                     ReuseAddr => 1,
2105                     Listen => 1,
2106                     LocalPort => $port,
2107                     Proto => 'tcp',
2108                     GetAddrInfoFlags => 0,
2109                 ) or die "failed to create socket: $!\n";
2110                 # Inside the worker we shouldn't have any previous alarms
2111                 # running anyway...:
2112                 alarm(0);
2113                 local $SIG{ALRM} = sub { die "connection timed out\n" };
2114                 alarm $timeout;
2115                 accept(my $cli, $sock) or die "connection failed: $!\n";
2116                 alarm(0);
2117                 close($sock);
2118                 if (PVE::Tools::run_command($cmd,
2119                     output => '>&'.fileno($cli),
2120                     input => '<&'.fileno($cli),
2121                     noerr => 1) != 0)
2122                 {
2123                     die "Failed to run vncproxy.\n";
2124                 }
2125             }
2126
2127             return;
2128         };
2129
2130         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2131
2132         PVE::Tools::wait_for_vnc_port($port);
2133
2134         my $res = {
2135             user => $authuser,
2136             ticket => $ticket,
2137             port => $port,
2138             upid => $upid,
2139             cert => $sslcert,
2140         };
2141         $res->{password} = $password if $param->{'generate-password'};
2142
2143         return $res;
2144     }});
2145
2146 __PACKAGE__->register_method({
2147     name => 'termproxy',
2148     path => '{vmid}/termproxy',
2149     method => 'POST',
2150     protected => 1,
2151     permissions => {
2152         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2153     },
2154     description => "Creates a TCP proxy connections.",
2155     parameters => {
2156         additionalProperties => 0,
2157         properties => {
2158             node => get_standard_option('pve-node'),
2159             vmid => get_standard_option('pve-vmid'),
2160             serial=> {
2161                 optional => 1,
2162                 type => 'string',
2163                 enum => [qw(serial0 serial1 serial2 serial3)],
2164                 description => "opens a serial terminal (defaults to display)",
2165             },
2166         },
2167     },
2168     returns => {
2169         additionalProperties => 0,
2170         properties => {
2171             user => { type => 'string' },
2172             ticket => { type => 'string' },
2173             port => { type => 'integer' },
2174             upid => { type => 'string' },
2175         },
2176     },
2177     code => sub {
2178         my ($param) = @_;
2179
2180         my $rpcenv = PVE::RPCEnvironment::get();
2181
2182         my $authuser = $rpcenv->get_user();
2183
2184         my $vmid = $param->{vmid};
2185         my $node = $param->{node};
2186         my $serial = $param->{serial};
2187
2188         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2189
2190         if (!defined($serial)) {
2191             if ($conf->{vga}) {
2192                 my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2193                 $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2194             }
2195         }
2196
2197         my $authpath = "/vms/$vmid";
2198
2199         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2200
2201         my $family;
2202         my $remcmd = [];
2203
2204         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2205             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2206             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2207             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, '-t');
2208             push @$remcmd, '--';
2209         } else {
2210             $family = PVE::Tools::get_host_address_family($node);
2211         }
2212
2213         my $port = PVE::Tools::next_vnc_port($family);
2214
2215         my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-escape', '0'];
2216         push @$termcmd, '-iface', $serial if $serial;
2217
2218         my $realcmd = sub {
2219             my $upid = shift;
2220
2221             syslog('info', "starting qemu termproxy $upid\n");
2222
2223             my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
2224                        '--perm', 'VM.Console', '--'];
2225             push @$cmd, @$remcmd, @$termcmd;
2226
2227             PVE::Tools::run_command($cmd);
2228         };
2229
2230         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2231
2232         PVE::Tools::wait_for_vnc_port($port);
2233
2234         return {
2235             user => $authuser,
2236             ticket => $ticket,
2237             port => $port,
2238             upid => $upid,
2239         };
2240     }});
2241
2242 __PACKAGE__->register_method({
2243     name => 'vncwebsocket',
2244     path => '{vmid}/vncwebsocket',
2245     method => 'GET',
2246     permissions => {
2247         description => "You also need to pass a valid ticket (vncticket).",
2248         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2249     },
2250     description => "Opens a weksocket for VNC traffic.",
2251     parameters => {
2252         additionalProperties => 0,
2253         properties => {
2254             node => get_standard_option('pve-node'),
2255             vmid => get_standard_option('pve-vmid'),
2256             vncticket => {
2257                 description => "Ticket from previous call to vncproxy.",
2258                 type => 'string',
2259                 maxLength => 512,
2260             },
2261             port => {
2262                 description => "Port number returned by previous vncproxy call.",
2263                 type => 'integer',
2264                 minimum => 5900,
2265                 maximum => 5999,
2266             },
2267         },
2268     },
2269     returns => {
2270         type => "object",
2271         properties => {
2272             port => { type => 'string' },
2273         },
2274     },
2275     code => sub {
2276         my ($param) = @_;
2277
2278         my $rpcenv = PVE::RPCEnvironment::get();
2279
2280         my $authuser = $rpcenv->get_user();
2281
2282         my $vmid = $param->{vmid};
2283         my $node = $param->{node};
2284
2285         my $authpath = "/vms/$vmid";
2286
2287         PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);
2288
2289         my $conf = PVE::QemuConfig->load_config($vmid, $node); # VM exists ?
2290
2291         # Note: VNC ports are acessible from outside, so we do not gain any
2292         # security if we verify that $param->{port} belongs to VM $vmid. This
2293         # check is done by verifying the VNC ticket (inside VNC protocol).
2294
2295         my $port = $param->{port};
2296
2297         return { port => $port };
2298     }});
2299
2300 __PACKAGE__->register_method({
2301     name => 'spiceproxy',
2302     path => '{vmid}/spiceproxy',
2303     method => 'POST',
2304     protected => 1,
2305     proxyto => 'node',
2306     permissions => {
2307         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2308     },
2309     description => "Returns a SPICE configuration to connect to the VM.",
2310     parameters => {
2311         additionalProperties => 0,
2312         properties => {
2313             node => get_standard_option('pve-node'),
2314             vmid => get_standard_option('pve-vmid'),
2315             proxy => get_standard_option('spice-proxy', { optional => 1 }),
2316         },
2317     },
2318     returns => get_standard_option('remote-viewer-config'),
2319     code => sub {
2320         my ($param) = @_;
2321
2322         my $rpcenv = PVE::RPCEnvironment::get();
2323
2324         my $authuser = $rpcenv->get_user();
2325
2326         my $vmid = $param->{vmid};
2327         my $node = $param->{node};
2328         my $proxy = $param->{proxy};
2329
2330         my $conf = PVE::QemuConfig->load_config($vmid, $node);
2331         my $title = "VM $vmid";
2332         $title .= " - ". $conf->{name} if $conf->{name};
2333
2334         my $port = PVE::QemuServer::spice_port($vmid);
2335
2336         my ($ticket, undef, $remote_viewer_config) =
2337             PVE::AccessControl::remote_viewer_config($authuser, $vmid, $node, $proxy, $title, $port);
2338
2339         mon_cmd($vmid, "set_password", protocol => 'spice', password => $ticket);
2340         mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
2341
2342         return $remote_viewer_config;
2343     }});
2344
2345 __PACKAGE__->register_method({
2346     name => 'vmcmdidx',
2347     path => '{vmid}/status',
2348     method => 'GET',
2349     proxyto => 'node',
2350     description => "Directory index",
2351     permissions => {
2352         user => 'all',
2353     },
2354     parameters => {
2355         additionalProperties => 0,
2356         properties => {
2357             node => get_standard_option('pve-node'),
2358             vmid => get_standard_option('pve-vmid'),
2359         },
2360     },
2361     returns => {
2362         type => 'array',
2363         items => {
2364             type => "object",
2365             properties => {
2366                 subdir => { type => 'string' },
2367             },
2368         },
2369         links => [ { rel => 'child', href => "{subdir}" } ],
2370     },
2371     code => sub {
2372         my ($param) = @_;
2373
2374         # test if VM exists
2375         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2376
2377         my $res = [
2378             { subdir => 'current' },
2379             { subdir => 'start' },
2380             { subdir => 'stop' },
2381             { subdir => 'reset' },
2382             { subdir => 'shutdown' },
2383             { subdir => 'suspend' },
2384             { subdir => 'reboot' },
2385             ];
2386
2387         return $res;
2388     }});
2389
2390 __PACKAGE__->register_method({
2391     name => 'vm_status',
2392     path => '{vmid}/status/current',
2393     method => 'GET',
2394     proxyto => 'node',
2395     protected => 1, # qemu pid files are only readable by root
2396     description => "Get virtual machine status.",
2397     permissions => {
2398         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
2399     },
2400     parameters => {
2401         additionalProperties => 0,
2402         properties => {
2403             node => get_standard_option('pve-node'),
2404             vmid => get_standard_option('pve-vmid'),
2405         },
2406     },
2407     returns => {
2408         type => 'object',
2409         properties => {
2410             %$PVE::QemuServer::vmstatus_return_properties,
2411             ha => {
2412                 description => "HA manager service status.",
2413                 type => 'object',
2414             },
2415             spice => {
2416                 description => "Qemu VGA configuration supports spice.",
2417                 type => 'boolean',
2418                 optional => 1,
2419             },
2420             agent => {
2421                 description => "Qemu GuestAgent enabled in config.",
2422                 type => 'boolean',
2423                 optional => 1,
2424             },
2425         },
2426     },
2427     code => sub {
2428         my ($param) = @_;
2429
2430         # test if VM exists
2431         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2432
2433         my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1);
2434         my $status = $vmstatus->{$param->{vmid}};
2435
2436         $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}");
2437
2438         if ($conf->{vga}) {
2439             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2440             $status->{spice} = 1
2441                 if $vga->{type} eq 'virtio-gl' || PVE::QemuServer::vga_conf_has_spice($conf->{vga});
2442         }
2443         $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled');
2444
2445         return $status;
2446     }});
2447
2448 __PACKAGE__->register_method({
2449     name => 'vm_start',
2450     path => '{vmid}/status/start',
2451     method => 'POST',
2452     protected => 1,
2453     proxyto => 'node',
2454     description => "Start virtual machine.",
2455     permissions => {
2456         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2457     },
2458     parameters => {
2459         additionalProperties => 0,
2460         properties => {
2461             node => get_standard_option('pve-node'),
2462             vmid => get_standard_option('pve-vmid',
2463                                         { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2464             skiplock => get_standard_option('skiplock'),
2465             stateuri => get_standard_option('pve-qm-stateuri'),
2466             migratedfrom => get_standard_option('pve-node',{ optional => 1 }),
2467             migration_type => {
2468                 type => 'string',
2469                 enum => ['secure', 'insecure'],
2470                 description => "Migration traffic is encrypted using an SSH " .
2471                   "tunnel by default. On secure, completely private networks " .
2472                   "this can be disabled to increase performance.",
2473                 optional => 1,
2474             },
2475             migration_network => {
2476                 type => 'string', format => 'CIDR',
2477                 description => "CIDR of the (sub) network that is used for migration.",
2478                 optional => 1,
2479             },
2480             machine => get_standard_option('pve-qemu-machine'),
2481             'force-cpu' => {
2482                 description => "Override QEMU's -cpu argument with the given string.",
2483                 type => 'string',
2484                 optional => 1,
2485             },
2486             targetstorage => get_standard_option('pve-targetstorage'),
2487             timeout => {
2488                 description => "Wait maximal timeout seconds.",
2489                 type => 'integer',
2490                 minimum => 0,
2491                 default => 'max(30, vm memory in GiB)',
2492                 optional => 1,
2493             },
2494         },
2495     },
2496     returns => {
2497         type => 'string',
2498     },
2499     code => sub {
2500         my ($param) = @_;
2501
2502         my $rpcenv = PVE::RPCEnvironment::get();
2503         my $authuser = $rpcenv->get_user();
2504
2505         my $node = extract_param($param, 'node');
2506         my $vmid = extract_param($param, 'vmid');
2507         my $timeout = extract_param($param, 'timeout');
2508         my $machine = extract_param($param, 'machine');
2509
2510         my $get_root_param = sub {
2511             my $value = extract_param($param, $_[0]);
2512             raise_param_exc({ "$_[0]" => "Only root may use this option." })
2513                 if $value && $authuser ne 'root@pam';
2514             return $value;
2515         };
2516
2517         my $stateuri = $get_root_param->('stateuri');
2518         my $skiplock = $get_root_param->('skiplock');
2519         my $migratedfrom = $get_root_param->('migratedfrom');
2520         my $migration_type = $get_root_param->('migration_type');
2521         my $migration_network = $get_root_param->('migration_network');
2522         my $targetstorage = $get_root_param->('targetstorage');
2523         my $force_cpu = $get_root_param->('force-cpu');
2524
2525         my $storagemap;
2526
2527         if ($targetstorage) {
2528             raise_param_exc({ targetstorage => "targetstorage can only by used with migratedfrom." })
2529                 if !$migratedfrom;
2530             $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
2531             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
2532                 if $@;
2533         }
2534
2535         # read spice ticket from STDIN
2536         my $spice_ticket;
2537         my $nbd_protocol_version = 0;
2538         my $replicated_volumes = {};
2539         my $tpmstate_vol;
2540         if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) {
2541             while (defined(my $line = <STDIN>)) {
2542                 chomp $line;
2543                 if ($line =~ m/^spice_ticket: (.+)$/) {
2544                     $spice_ticket = $1;
2545                 } elsif ($line =~ m/^nbd_protocol_version: (\d+)$/) {
2546                     $nbd_protocol_version = $1;
2547                 } elsif ($line =~ m/^replicated_volume: (.*)$/) {
2548                     $replicated_volumes->{$1} = 1;
2549                 } elsif ($line =~ m/^tpmstate0: (.*)$/) {
2550                     $tpmstate_vol = $1;
2551                 } elsif (!$spice_ticket) {
2552                     # fallback for old source node
2553                     $spice_ticket = $line;
2554                 } else {
2555                     warn "unknown 'start' parameter on STDIN: '$line'\n";
2556                 }
2557             }
2558         }
2559
2560         PVE::Cluster::check_cfs_quorum();
2561
2562         my $storecfg = PVE::Storage::config();
2563
2564         if (PVE::HA::Config::vm_is_ha_managed($vmid) && !$stateuri &&  $rpcenv->{type} ne 'ha') {
2565             my $hacmd = sub {
2566                 my $upid = shift;
2567
2568                 print "Requesting HA start for VM $vmid\n";
2569
2570                 my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
2571                 PVE::Tools::run_command($cmd);
2572                 return;
2573             };
2574
2575             return $rpcenv->fork_worker('hastart', $vmid, $authuser, $hacmd);
2576
2577         } else {
2578
2579             my $realcmd = sub {
2580                 my $upid = shift;
2581
2582                 syslog('info', "start VM $vmid: $upid\n");
2583
2584                 my $migrate_opts = {
2585                     migratedfrom => $migratedfrom,
2586                     spice_ticket => $spice_ticket,
2587                     network => $migration_network,
2588                     type => $migration_type,
2589                     storagemap => $storagemap,
2590                     nbd_proto_version => $nbd_protocol_version,
2591                     replicated_volumes => $replicated_volumes,
2592                     tpmstate_vol => $tpmstate_vol,
2593                 };
2594
2595                 my $params = {
2596                     statefile => $stateuri,
2597                     skiplock => $skiplock,
2598                     forcemachine => $machine,
2599                     timeout => $timeout,
2600                     forcecpu => $force_cpu,
2601                 };
2602
2603                 PVE::QemuServer::vm_start($storecfg, $vmid, $params, $migrate_opts);
2604                 return;
2605             };
2606
2607             return $rpcenv->fork_worker('qmstart', $vmid, $authuser, $realcmd);
2608         }
2609     }});
2610
2611 __PACKAGE__->register_method({
2612     name => 'vm_stop',
2613     path => '{vmid}/status/stop',
2614     method => 'POST',
2615     protected => 1,
2616     proxyto => 'node',
2617     description => "Stop virtual machine. The qemu process will exit immediately. This" .
2618         "is akin to pulling the power plug of a running computer and may damage the VM data",
2619     permissions => {
2620         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2621     },
2622     parameters => {
2623         additionalProperties => 0,
2624         properties => {
2625             node => get_standard_option('pve-node'),
2626             vmid => get_standard_option('pve-vmid',
2627                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2628             skiplock => get_standard_option('skiplock'),
2629             migratedfrom => get_standard_option('pve-node', { optional => 1 }),
2630             timeout => {
2631                 description => "Wait maximal timeout seconds.",
2632                 type => 'integer',
2633                 minimum => 0,
2634                 optional => 1,
2635             },
2636             keepActive => {
2637                 description => "Do not deactivate storage volumes.",
2638                 type => 'boolean',
2639                 optional => 1,
2640                 default => 0,
2641             }
2642         },
2643     },
2644     returns => {
2645         type => 'string',
2646     },
2647     code => sub {
2648         my ($param) = @_;
2649
2650         my $rpcenv = PVE::RPCEnvironment::get();
2651         my $authuser = $rpcenv->get_user();
2652
2653         my $node = extract_param($param, 'node');
2654         my $vmid = extract_param($param, 'vmid');
2655
2656         my $skiplock = extract_param($param, 'skiplock');
2657         raise_param_exc({ skiplock => "Only root may use this option." })
2658             if $skiplock && $authuser ne 'root@pam';
2659
2660         my $keepActive = extract_param($param, 'keepActive');
2661         raise_param_exc({ keepActive => "Only root may use this option." })
2662             if $keepActive && $authuser ne 'root@pam';
2663
2664         my $migratedfrom = extract_param($param, 'migratedfrom');
2665         raise_param_exc({ migratedfrom => "Only root may use this option." })
2666             if $migratedfrom && $authuser ne 'root@pam';
2667
2668
2669         my $storecfg = PVE::Storage::config();
2670
2671         if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) {
2672
2673             my $hacmd = sub {
2674                 my $upid = shift;
2675
2676                 print "Requesting HA stop for VM $vmid\n";
2677
2678                 my $cmd = ['ha-manager', 'crm-command', 'stop',  "vm:$vmid", '0'];
2679                 PVE::Tools::run_command($cmd);
2680                 return;
2681             };
2682
2683             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
2684
2685         } else {
2686             my $realcmd = sub {
2687                 my $upid = shift;
2688
2689                 syslog('info', "stop VM $vmid: $upid\n");
2690
2691                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0,
2692                                          $param->{timeout}, 0, 1, $keepActive, $migratedfrom);
2693                 return;
2694             };
2695
2696             return $rpcenv->fork_worker('qmstop', $vmid, $authuser, $realcmd);
2697         }
2698     }});
2699
2700 __PACKAGE__->register_method({
2701     name => 'vm_reset',
2702     path => '{vmid}/status/reset',
2703     method => 'POST',
2704     protected => 1,
2705     proxyto => 'node',
2706     description => "Reset virtual machine.",
2707     permissions => {
2708         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2709     },
2710     parameters => {
2711         additionalProperties => 0,
2712         properties => {
2713             node => get_standard_option('pve-node'),
2714             vmid => get_standard_option('pve-vmid',
2715                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2716             skiplock => get_standard_option('skiplock'),
2717         },
2718     },
2719     returns => {
2720         type => 'string',
2721     },
2722     code => sub {
2723         my ($param) = @_;
2724
2725         my $rpcenv = PVE::RPCEnvironment::get();
2726
2727         my $authuser = $rpcenv->get_user();
2728
2729         my $node = extract_param($param, 'node');
2730
2731         my $vmid = extract_param($param, 'vmid');
2732
2733         my $skiplock = extract_param($param, 'skiplock');
2734         raise_param_exc({ skiplock => "Only root may use this option." })
2735             if $skiplock && $authuser ne 'root@pam';
2736
2737         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
2738
2739         my $realcmd = sub {
2740             my $upid = shift;
2741
2742             PVE::QemuServer::vm_reset($vmid, $skiplock);
2743
2744             return;
2745         };
2746
2747         return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd);
2748     }});
2749
2750 __PACKAGE__->register_method({
2751     name => 'vm_shutdown',
2752     path => '{vmid}/status/shutdown',
2753     method => 'POST',
2754     protected => 1,
2755     proxyto => 'node',
2756     description => "Shutdown virtual machine. This is similar to pressing the power button on a physical machine." .
2757         "This will send an ACPI event for the guest OS, which should then proceed to a clean shutdown.",
2758     permissions => {
2759         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2760     },
2761     parameters => {
2762         additionalProperties => 0,
2763         properties => {
2764             node => get_standard_option('pve-node'),
2765             vmid => get_standard_option('pve-vmid',
2766                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2767             skiplock => get_standard_option('skiplock'),
2768             timeout => {
2769                 description => "Wait maximal timeout seconds.",
2770                 type => 'integer',
2771                 minimum => 0,
2772                 optional => 1,
2773             },
2774             forceStop => {
2775                 description => "Make sure the VM stops.",
2776                 type => 'boolean',
2777                 optional => 1,
2778                 default => 0,
2779             },
2780             keepActive => {
2781                 description => "Do not deactivate storage volumes.",
2782                 type => 'boolean',
2783                 optional => 1,
2784                 default => 0,
2785             }
2786         },
2787     },
2788     returns => {
2789         type => 'string',
2790     },
2791     code => sub {
2792         my ($param) = @_;
2793
2794         my $rpcenv = PVE::RPCEnvironment::get();
2795         my $authuser = $rpcenv->get_user();
2796
2797         my $node = extract_param($param, 'node');
2798         my $vmid = extract_param($param, 'vmid');
2799
2800         my $skiplock = extract_param($param, 'skiplock');
2801         raise_param_exc({ skiplock => "Only root may use this option." })
2802             if $skiplock && $authuser ne 'root@pam';
2803
2804         my $keepActive = extract_param($param, 'keepActive');
2805         raise_param_exc({ keepActive => "Only root may use this option." })
2806             if $keepActive && $authuser ne 'root@pam';
2807
2808         my $storecfg = PVE::Storage::config();
2809
2810         my $shutdown = 1;
2811
2812         # if vm is paused, do not shutdown (but stop if forceStop = 1)
2813         # otherwise, we will infer a shutdown command, but run into the timeout,
2814         # then when the vm is resumed, it will instantly shutdown
2815         #
2816         # checking the qmp status here to get feedback to the gui/cli/api
2817         # and the status query should not take too long
2818         if (PVE::QemuServer::vm_is_paused($vmid)) {
2819             if ($param->{forceStop}) {
2820                 warn "VM is paused - stop instead of shutdown\n";
2821                 $shutdown = 0;
2822             } else {
2823                 die "VM is paused - cannot shutdown\n";
2824             }
2825         }
2826
2827         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
2828
2829             my $timeout = $param->{timeout} // 60;
2830             my $hacmd = sub {
2831                 my $upid = shift;
2832
2833                 print "Requesting HA stop for VM $vmid\n";
2834
2835                 my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", "$timeout"];
2836                 PVE::Tools::run_command($cmd);
2837                 return;
2838             };
2839
2840             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
2841
2842         } else {
2843
2844             my $realcmd = sub {
2845                 my $upid = shift;
2846
2847                 syslog('info', "shutdown VM $vmid: $upid\n");
2848
2849                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0, $param->{timeout},
2850                                          $shutdown, $param->{forceStop}, $keepActive);
2851                 return;
2852             };
2853
2854             return $rpcenv->fork_worker('qmshutdown', $vmid, $authuser, $realcmd);
2855         }
2856     }});
2857
2858 __PACKAGE__->register_method({
2859     name => 'vm_reboot',
2860     path => '{vmid}/status/reboot',
2861     method => 'POST',
2862     protected => 1,
2863     proxyto => 'node',
2864     description => "Reboot the VM by shutting it down, and starting it again. Applies pending changes.",
2865     permissions => {
2866         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2867     },
2868     parameters => {
2869         additionalProperties => 0,
2870         properties => {
2871             node => get_standard_option('pve-node'),
2872             vmid => get_standard_option('pve-vmid',
2873                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2874             timeout => {
2875                 description => "Wait maximal timeout seconds for the shutdown.",
2876                 type => 'integer',
2877                 minimum => 0,
2878                 optional => 1,
2879             },
2880         },
2881     },
2882     returns => {
2883         type => 'string',
2884     },
2885     code => sub {
2886         my ($param) = @_;
2887
2888         my $rpcenv = PVE::RPCEnvironment::get();
2889         my $authuser = $rpcenv->get_user();
2890
2891         my $node = extract_param($param, 'node');
2892         my $vmid = extract_param($param, 'vmid');
2893
2894         die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid);
2895
2896         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
2897
2898         my $realcmd = sub {
2899             my $upid = shift;
2900
2901             syslog('info', "requesting reboot of VM $vmid: $upid\n");
2902             PVE::QemuServer::vm_reboot($vmid, $param->{timeout});
2903             return;
2904         };
2905
2906         return $rpcenv->fork_worker('qmreboot', $vmid, $authuser, $realcmd);
2907     }});
2908
2909 __PACKAGE__->register_method({
2910     name => 'vm_suspend',
2911     path => '{vmid}/status/suspend',
2912     method => 'POST',
2913     protected => 1,
2914     proxyto => 'node',
2915     description => "Suspend virtual machine.",
2916     permissions => {
2917         description => "You need 'VM.PowerMgmt' on /vms/{vmid}, and if you have set 'todisk',".
2918             " you need also 'VM.Config.Disk' on /vms/{vmid} and 'Datastore.AllocateSpace'".
2919             " on the storage for the vmstate.",
2920         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2921     },
2922     parameters => {
2923         additionalProperties => 0,
2924         properties => {
2925             node => get_standard_option('pve-node'),
2926             vmid => get_standard_option('pve-vmid',
2927                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2928             skiplock => get_standard_option('skiplock'),
2929             todisk => {
2930                 type => 'boolean',
2931                 default => 0,
2932                 optional => 1,
2933                 description => 'If set, suspends the VM to disk. Will be resumed on next VM start.',
2934             },
2935             statestorage => get_standard_option('pve-storage-id', {
2936                 description => "The storage for the VM state",
2937                 requires => 'todisk',
2938                 optional => 1,
2939                 completion => \&PVE::Storage::complete_storage_enabled,
2940             }),
2941         },
2942     },
2943     returns => {
2944         type => 'string',
2945     },
2946     code => sub {
2947         my ($param) = @_;
2948
2949         my $rpcenv = PVE::RPCEnvironment::get();
2950         my $authuser = $rpcenv->get_user();
2951
2952         my $node = extract_param($param, 'node');
2953         my $vmid = extract_param($param, 'vmid');
2954
2955         my $todisk = extract_param($param, 'todisk') // 0;
2956
2957         my $statestorage = extract_param($param, 'statestorage');
2958
2959         my $skiplock = extract_param($param, 'skiplock');
2960         raise_param_exc({ skiplock => "Only root may use this option." })
2961             if $skiplock && $authuser ne 'root@pam';
2962
2963         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
2964
2965         die "Cannot suspend HA managed VM to disk\n"
2966             if $todisk && PVE::HA::Config::vm_is_ha_managed($vmid);
2967
2968         # early check for storage permission, for better user feedback
2969         if ($todisk) {
2970             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
2971
2972             if (!$statestorage) {
2973                 # get statestorage from config if none is given
2974                 my $conf = PVE::QemuConfig->load_config($vmid);
2975                 my $storecfg = PVE::Storage::config();
2976                 $statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
2977             }
2978
2979             $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
2980         }
2981
2982         my $realcmd = sub {
2983             my $upid = shift;
2984
2985             syslog('info', "suspend VM $vmid: $upid\n");
2986
2987             PVE::QemuServer::vm_suspend($vmid, $skiplock, $todisk, $statestorage);
2988
2989             return;
2990         };
2991
2992         my $taskname = $todisk ? 'qmsuspend' : 'qmpause';
2993         return $rpcenv->fork_worker($taskname, $vmid, $authuser, $realcmd);
2994     }});
2995
2996 __PACKAGE__->register_method({
2997     name => 'vm_resume',
2998     path => '{vmid}/status/resume',
2999     method => 'POST',
3000     protected => 1,
3001     proxyto => 'node',
3002     description => "Resume virtual machine.",
3003     permissions => {
3004         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3005     },
3006     parameters => {
3007         additionalProperties => 0,
3008         properties => {
3009             node => get_standard_option('pve-node'),
3010             vmid => get_standard_option('pve-vmid',
3011                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3012             skiplock => get_standard_option('skiplock'),
3013             nocheck => { type => 'boolean', optional => 1 },
3014
3015         },
3016     },
3017     returns => {
3018         type => 'string',
3019     },
3020     code => sub {
3021         my ($param) = @_;
3022
3023         my $rpcenv = PVE::RPCEnvironment::get();
3024
3025         my $authuser = $rpcenv->get_user();
3026
3027         my $node = extract_param($param, 'node');
3028
3029         my $vmid = extract_param($param, 'vmid');
3030
3031         my $skiplock = extract_param($param, 'skiplock');
3032         raise_param_exc({ skiplock => "Only root may use this option." })
3033             if $skiplock && $authuser ne 'root@pam';
3034
3035         my $nocheck = extract_param($param, 'nocheck');
3036         raise_param_exc({ nocheck => "Only root may use this option." })
3037             if $nocheck && $authuser ne 'root@pam';
3038
3039         my $to_disk_suspended;
3040         eval {
3041             PVE::QemuConfig->lock_config($vmid, sub {
3042                 my $conf = PVE::QemuConfig->load_config($vmid);
3043                 $to_disk_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
3044             });
3045         };
3046
3047         die "VM $vmid not running\n"
3048             if !$to_disk_suspended && !PVE::QemuServer::check_running($vmid, $nocheck);
3049
3050         my $realcmd = sub {
3051             my $upid = shift;
3052
3053             syslog('info', "resume VM $vmid: $upid\n");
3054
3055             if (!$to_disk_suspended) {
3056                 PVE::QemuServer::vm_resume($vmid, $skiplock, $nocheck);
3057             } else {
3058                 my $storecfg = PVE::Storage::config();
3059                 PVE::QemuServer::vm_start($storecfg, $vmid, { skiplock => $skiplock });
3060             }
3061
3062             return;
3063         };
3064
3065         return $rpcenv->fork_worker('qmresume', $vmid, $authuser, $realcmd);
3066     }});
3067
3068 __PACKAGE__->register_method({
3069     name => 'vm_sendkey',
3070     path => '{vmid}/sendkey',
3071     method => 'PUT',
3072     protected => 1,
3073     proxyto => 'node',
3074     description => "Send key event to virtual machine.",
3075     permissions => {
3076         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
3077     },
3078     parameters => {
3079         additionalProperties => 0,
3080         properties => {
3081             node => get_standard_option('pve-node'),
3082             vmid => get_standard_option('pve-vmid',
3083                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3084             skiplock => get_standard_option('skiplock'),
3085             key => {
3086                 description => "The key (qemu monitor encoding).",
3087                 type => 'string'
3088             }
3089         },
3090     },
3091     returns => { type => 'null'},
3092     code => sub {
3093         my ($param) = @_;
3094
3095         my $rpcenv = PVE::RPCEnvironment::get();
3096
3097         my $authuser = $rpcenv->get_user();
3098
3099         my $node = extract_param($param, 'node');
3100
3101         my $vmid = extract_param($param, 'vmid');
3102
3103         my $skiplock = extract_param($param, 'skiplock');
3104         raise_param_exc({ skiplock => "Only root may use this option." })
3105             if $skiplock && $authuser ne 'root@pam';
3106
3107         PVE::QemuServer::vm_sendkey($vmid, $skiplock, $param->{key});
3108
3109         return;
3110     }});
3111
3112 __PACKAGE__->register_method({
3113     name => 'vm_feature',
3114     path => '{vmid}/feature',
3115     method => 'GET',
3116     proxyto => 'node',
3117     protected => 1,
3118     description => "Check if feature for virtual machine is available.",
3119     permissions => {
3120         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
3121     },
3122     parameters => {
3123         additionalProperties => 0,
3124         properties => {
3125             node => get_standard_option('pve-node'),
3126             vmid => get_standard_option('pve-vmid'),
3127             feature => {
3128                 description => "Feature to check.",
3129                 type => 'string',
3130                 enum => [ 'snapshot', 'clone', 'copy' ],
3131             },
3132             snapname => get_standard_option('pve-snapshot-name', {
3133                 optional => 1,
3134             }),
3135         },
3136     },
3137     returns => {
3138         type => "object",
3139         properties => {
3140             hasFeature => { type => 'boolean' },
3141             nodes => {
3142                 type => 'array',
3143                 items => { type => 'string' },
3144             }
3145         },
3146     },
3147     code => sub {
3148         my ($param) = @_;
3149
3150         my $node = extract_param($param, 'node');
3151
3152         my $vmid = extract_param($param, 'vmid');
3153
3154         my $snapname = extract_param($param, 'snapname');
3155
3156         my $feature = extract_param($param, 'feature');
3157
3158         my $running = PVE::QemuServer::check_running($vmid);
3159
3160         my $conf = PVE::QemuConfig->load_config($vmid);
3161
3162         if($snapname){
3163             my $snap = $conf->{snapshots}->{$snapname};
3164             die "snapshot '$snapname' does not exist\n" if !defined($snap);
3165             $conf = $snap;
3166         }
3167         my $storecfg = PVE::Storage::config();
3168
3169         my $nodelist = PVE::QemuServer::shared_nodes($conf, $storecfg);
3170         my $hasFeature = PVE::QemuConfig->has_feature($feature, $conf, $storecfg, $snapname, $running);
3171
3172         return {
3173             hasFeature => $hasFeature,
3174             nodes => [ keys %$nodelist ],
3175         };
3176     }});
3177
3178 __PACKAGE__->register_method({
3179     name => 'clone_vm',
3180     path => '{vmid}/clone',
3181     method => 'POST',
3182     protected => 1,
3183     proxyto => 'node',
3184     description => "Create a copy of virtual machine/template.",
3185     permissions => {
3186         description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " .
3187             "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
3188             "'Datastore.AllocateSpace' on any used storage.",
3189         check =>
3190         [ 'and',
3191           ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
3192           [ 'or',
3193             [ 'perm', '/vms/{newid}', ['VM.Allocate']],
3194             [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
3195           ],
3196         ]
3197     },
3198     parameters => {
3199         additionalProperties => 0,
3200         properties => {
3201             node => get_standard_option('pve-node'),
3202             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3203             newid => get_standard_option('pve-vmid', {
3204                 completion => \&PVE::Cluster::complete_next_vmid,
3205                 description => 'VMID for the clone.' }),
3206             name => {
3207                 optional => 1,
3208                 type => 'string', format => 'dns-name',
3209                 description => "Set a name for the new VM.",
3210             },
3211             description => {
3212                 optional => 1,
3213                 type => 'string',
3214                 description => "Description for the new VM.",
3215             },
3216             pool => {
3217                 optional => 1,
3218                 type => 'string', format => 'pve-poolid',
3219                 description => "Add the new VM to the specified pool.",
3220             },
3221             snapname => get_standard_option('pve-snapshot-name', {
3222                 optional => 1,
3223             }),
3224             storage => get_standard_option('pve-storage-id', {
3225                 description => "Target storage for full clone.",
3226                 optional => 1,
3227             }),
3228             'format' => {
3229                 description => "Target format for file storage. Only valid for full clone.",
3230                 type => 'string',
3231                 optional => 1,
3232                 enum => [ 'raw', 'qcow2', 'vmdk'],
3233             },
3234             full => {
3235                 optional => 1,
3236                 type => 'boolean',
3237                 description => "Create a full copy of all disks. This is always done when " .
3238                     "you clone a normal VM. For VM templates, we try to create a linked clone by default.",
3239             },
3240             target => get_standard_option('pve-node', {
3241                 description => "Target node. Only allowed if the original VM is on shared storage.",
3242                 optional => 1,
3243             }),
3244             bwlimit => {
3245                 description => "Override I/O bandwidth limit (in KiB/s).",
3246                 optional => 1,
3247                 type => 'integer',
3248                 minimum => '0',
3249                 default => 'clone limit from datacenter or storage config',
3250             },
3251         },
3252     },
3253     returns => {
3254         type => 'string',
3255     },
3256     code => sub {
3257         my ($param) = @_;
3258
3259         my $rpcenv = PVE::RPCEnvironment::get();
3260         my $authuser = $rpcenv->get_user();
3261
3262         my $node = extract_param($param, 'node');
3263         my $vmid = extract_param($param, 'vmid');
3264         my $newid = extract_param($param, 'newid');
3265         my $pool = extract_param($param, 'pool');
3266
3267         my $snapname = extract_param($param, 'snapname');
3268         my $storage = extract_param($param, 'storage');
3269         my $format = extract_param($param, 'format');
3270         my $target = extract_param($param, 'target');
3271
3272         my $localnode = PVE::INotify::nodename();
3273
3274         if ($target && ($target eq $localnode || $target eq 'localhost')) {
3275             undef $target;
3276         }
3277
3278         my $running = PVE::QemuServer::check_running($vmid) || 0;
3279
3280         my $load_and_check = sub {
3281             $rpcenv->check_pool_exist($pool) if defined($pool);
3282             PVE::Cluster::check_node_exists($target) if $target;
3283
3284             my $storecfg = PVE::Storage::config();
3285
3286             if ($storage) {
3287                 # check if storage is enabled on local node
3288                 PVE::Storage::storage_check_enabled($storecfg, $storage);
3289                 if ($target) {
3290                     # check if storage is available on target node
3291                     PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
3292                     # clone only works if target storage is shared
3293                     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
3294                     die "can't clone to non-shared storage '$storage'\n"
3295                         if !$scfg->{shared};
3296                 }
3297             }
3298
3299             PVE::Cluster::check_cfs_quorum();
3300
3301             my $conf = PVE::QemuConfig->load_config($vmid);
3302             PVE::QemuConfig->check_lock($conf);
3303
3304             my $verify_running = PVE::QemuServer::check_running($vmid) || 0;
3305             die "unexpected state change\n" if $verify_running != $running;
3306
3307             die "snapshot '$snapname' does not exist\n"
3308                 if $snapname && !defined( $conf->{snapshots}->{$snapname});
3309
3310             my $full = $param->{full} // !PVE::QemuConfig->is_template($conf);
3311
3312             die "parameter 'storage' not allowed for linked clones\n"
3313                 if defined($storage) && !$full;
3314
3315             die "parameter 'format' not allowed for linked clones\n"
3316                 if defined($format) && !$full;
3317
3318             my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf;
3319
3320             my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage);
3321
3322             die "can't clone VM to node '$target' (VM uses local storage)\n"
3323                 if $target && !$sharedvm;
3324
3325             my $conffile = PVE::QemuConfig->config_file($newid);
3326             die "unable to create VM $newid: config file already exists\n"
3327                 if -f $conffile;
3328
3329             my $newconf = { lock => 'clone' };
3330             my $drives = {};
3331             my $fullclone = {};
3332             my $vollist = [];
3333
3334             foreach my $opt (keys %$oldconf) {
3335                 my $value = $oldconf->{$opt};
3336
3337                 # do not copy snapshot related info
3338                 next if $opt eq 'snapshots' ||  $opt eq 'parent' || $opt eq 'snaptime' ||
3339                     $opt eq 'vmstate' || $opt eq 'snapstate';
3340
3341                 # no need to copy unused images, because VMID(owner) changes anyways
3342                 next if $opt =~ m/^unused\d+$/;
3343
3344                 die "cannot clone TPM state while VM is running\n"
3345                     if $full && $running && !$snapname && $opt eq 'tpmstate0';
3346
3347                 # always change MAC! address
3348                 if ($opt =~ m/^net(\d+)$/) {
3349                     my $net = PVE::QemuServer::parse_net($value);
3350                     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
3351                     $net->{macaddr} =  PVE::Tools::random_ether_addr($dc->{mac_prefix});
3352                     $newconf->{$opt} = PVE::QemuServer::print_net($net);
3353                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
3354                     my $drive = PVE::QemuServer::parse_drive($opt, $value);
3355                     die "unable to parse drive options for '$opt'\n" if !$drive;
3356                     if (PVE::QemuServer::drive_is_cdrom($drive, 1)) {
3357                         $newconf->{$opt} = $value; # simply copy configuration
3358                     } else {
3359                         if ($full || PVE::QemuServer::drive_is_cloudinit($drive)) {
3360                             die "Full clone feature is not supported for drive '$opt'\n"
3361                                 if !PVE::Storage::volume_has_feature($storecfg, 'copy', $drive->{file}, $snapname, $running);
3362                             $fullclone->{$opt} = 1;
3363                         } else {
3364                             # not full means clone instead of copy
3365                             die "Linked clone feature is not supported for drive '$opt'\n"
3366                                 if !PVE::Storage::volume_has_feature($storecfg, 'clone', $drive->{file}, $snapname, $running);
3367                         }
3368                         $drives->{$opt} = $drive;
3369                         next if PVE::QemuServer::drive_is_cloudinit($drive);
3370                         push @$vollist, $drive->{file};
3371                     }
3372                 } else {
3373                     # copy everything else
3374                     $newconf->{$opt} = $value;
3375                 }
3376             }
3377
3378             return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone);
3379         };
3380
3381         my $clonefn = sub {
3382             my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->();
3383             my $storecfg = PVE::Storage::config();
3384
3385             # auto generate a new uuid
3386             my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || '');
3387             $smbios1->{uuid} = PVE::QemuServer::generate_uuid();
3388             $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1);
3389             # auto generate a new vmgenid only if the option was set for template
3390             if ($newconf->{vmgenid}) {
3391                 $newconf->{vmgenid} = PVE::QemuServer::generate_uuid();
3392             }
3393
3394             delete $newconf->{template};
3395
3396             if ($param->{name}) {
3397                 $newconf->{name} = $param->{name};
3398             } else {
3399                 $newconf->{name} = "Copy-of-VM-" . ($oldconf->{name} // $vmid);
3400             }
3401
3402             if ($param->{description}) {
3403                 $newconf->{description} = $param->{description};
3404             }
3405
3406             # create empty/temp config - this fails if VM already exists on other node
3407             # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code
3408             PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n");
3409
3410             PVE::Firewall::clone_vmfw_conf($vmid, $newid);
3411
3412             my $newvollist = [];
3413             my $jobs = {};
3414
3415             eval {
3416                 local $SIG{INT} =
3417                     local $SIG{TERM} =
3418                     local $SIG{QUIT} =
3419                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3420
3421                 PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
3422
3423                 my $bwlimit = extract_param($param, 'bwlimit');
3424
3425                 my $total_jobs = scalar(keys %{$drives});
3426                 my $i = 1;
3427
3428                 foreach my $opt (sort keys %$drives) {
3429                     my $drive = $drives->{$opt};
3430                     my $skipcomplete = ($total_jobs != $i); # finish after last drive
3431                     my $completion = $skipcomplete ? 'skip' : 'complete';
3432
3433                     my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
3434                     my $storage_list = [ $src_sid ];
3435                     push @$storage_list, $storage if defined($storage);
3436                     my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
3437
3438                     my $source_info = {
3439                         vmid => $vmid,
3440                         running => $running,
3441                         drivename => $opt,
3442                         drive => $drive,
3443                         snapname => $snapname,
3444                     };
3445
3446                     my $dest_info = {
3447                         vmid => $newid,
3448                         drivename => $opt,
3449                         storage => $storage,
3450                         format => $format,
3451                     };
3452
3453                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf)
3454                         if $opt eq 'efidisk0';
3455
3456                     my $newdrive = PVE::QemuServer::clone_disk(
3457                         $storecfg,
3458                         $source_info,
3459                         $dest_info,
3460                         $fullclone->{$opt},
3461                         $newvollist,
3462                         $jobs,
3463                         $completion,
3464                         $oldconf->{agent},
3465                         $clonelimit,
3466                     );
3467
3468                     $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
3469
3470                     PVE::QemuConfig->write_config($newid, $newconf);
3471                     $i++;
3472                 }
3473
3474                 delete $newconf->{lock};
3475
3476                 # do not write pending changes
3477                 if (my @changes = keys %{$newconf->{pending}}) {
3478                     my $pending = join(',', @changes);
3479                     warn "found pending changes for '$pending', discarding for clone\n";
3480                     delete $newconf->{pending};
3481                 }
3482
3483                 PVE::QemuConfig->write_config($newid, $newconf);
3484
3485                 if ($target) {
3486                     # always deactivate volumes - avoid lvm LVs to be active on several nodes
3487                     PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
3488                     PVE::Storage::deactivate_volumes($storecfg, $newvollist);
3489
3490                     my $newconffile = PVE::QemuConfig->config_file($newid, $target);
3491                     die "Failed to move config to node '$target' - rename failed: $!\n"
3492                         if !rename($conffile, $newconffile);
3493                 }
3494
3495                 PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
3496             };
3497             if (my $err = $@) {
3498                 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
3499                 sleep 1; # some storage like rbd need to wait before release volume - really?
3500
3501                 foreach my $volid (@$newvollist) {
3502                     eval { PVE::Storage::vdisk_free($storecfg, $volid); };
3503                     warn $@ if $@;
3504                 }
3505
3506                 PVE::Firewall::remove_vmfw_conf($newid);
3507
3508                 unlink $conffile; # avoid races -> last thing before die
3509
3510                 die "clone failed: $err";
3511             }
3512
3513             return;
3514         };
3515
3516         # Aquire exclusive lock lock for $newid
3517         my $lock_target_vm = sub {
3518             return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn);
3519         };
3520
3521         my $lock_source_vm = sub {
3522             # exclusive lock if VM is running - else shared lock is enough;
3523             if ($running) {
3524                 return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
3525             } else {
3526                 return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
3527             }
3528         };
3529
3530         $load_and_check->(); # early checks before forking/locking
3531
3532         return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm);
3533     }});
3534
3535 __PACKAGE__->register_method({
3536     name => 'move_vm_disk',
3537     path => '{vmid}/move_disk',
3538     method => 'POST',
3539     protected => 1,
3540     proxyto => 'node',
3541     description => "Move volume to different storage or to a different VM.",
3542     permissions => {
3543         description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
3544             "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
3545             "a disk to another VM, you need the permissions on the target VM as well.",
3546         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
3547     },
3548     parameters => {
3549         additionalProperties => 0,
3550         properties => {
3551             node => get_standard_option('pve-node'),
3552             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3553             'target-vmid' => get_standard_option('pve-vmid', {
3554                 completion => \&PVE::QemuServer::complete_vmid,
3555                 optional => 1,
3556             }),
3557             disk => {
3558                 type => 'string',
3559                 description => "The disk you want to move.",
3560                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3561             },
3562             storage => get_standard_option('pve-storage-id', {
3563                 description => "Target storage.",
3564                 completion => \&PVE::QemuServer::complete_storage,
3565                 optional => 1,
3566             }),
3567             'format' => {
3568                 type => 'string',
3569                 description => "Target Format.",
3570                 enum => [ 'raw', 'qcow2', 'vmdk' ],
3571                 optional => 1,
3572             },
3573             delete => {
3574                 type => 'boolean',
3575                 description => "Delete the original disk after successful copy. By default the"
3576                     ." original disk is kept as unused disk.",
3577                 optional => 1,
3578                 default => 0,
3579             },
3580             digest => {
3581                 type => 'string',
3582                 description => 'Prevent changes if current configuration file has different SHA1"
3583                     ." digest. This can be used to prevent concurrent modifications.',
3584                 maxLength => 40,
3585                 optional => 1,
3586             },
3587             bwlimit => {
3588                 description => "Override I/O bandwidth limit (in KiB/s).",
3589                 optional => 1,
3590                 type => 'integer',
3591                 minimum => '0',
3592                 default => 'move limit from datacenter or storage config',
3593             },
3594             'target-disk' => {
3595                 type => 'string',
3596                 description => "The config key the disk will be moved to on the target VM"
3597                     ." (for example, ide0 or scsi1). Default is the source disk key.",
3598                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3599                 optional => 1,
3600             },
3601             'target-digest' => {
3602                 type => 'string',
3603                 description => 'Prevent changes if the current config file of the target VM has a"
3604                     ." different SHA1 digest. This can be used to detect concurrent modifications.',
3605                 maxLength => 40,
3606                 optional => 1,
3607             },
3608         },
3609     },
3610     returns => {
3611         type => 'string',
3612         description => "the task ID.",
3613     },
3614     code => sub {
3615         my ($param) = @_;
3616
3617         my $rpcenv = PVE::RPCEnvironment::get();
3618         my $authuser = $rpcenv->get_user();
3619
3620         my $node = extract_param($param, 'node');
3621         my $vmid = extract_param($param, 'vmid');
3622         my $target_vmid = extract_param($param, 'target-vmid');
3623         my $digest = extract_param($param, 'digest');
3624         my $target_digest = extract_param($param, 'target-digest');
3625         my $disk = extract_param($param, 'disk');
3626         my $target_disk = extract_param($param, 'target-disk') // $disk;
3627         my $storeid = extract_param($param, 'storage');
3628         my $format = extract_param($param, 'format');
3629
3630         my $storecfg = PVE::Storage::config();
3631
3632         my $load_and_check_move = sub {
3633             my $conf = PVE::QemuConfig->load_config($vmid);
3634             PVE::QemuConfig->check_lock($conf);
3635
3636             PVE::Tools::assert_if_modified($digest, $conf->{digest});
3637
3638             die "disk '$disk' does not exist\n" if !$conf->{$disk};
3639
3640             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
3641
3642             die "disk '$disk' has no associated volume\n" if !$drive->{file};
3643             die "you can't move a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive, 1);
3644
3645             my $old_volid = $drive->{file};
3646             my $oldfmt;
3647             my ($oldstoreid, $oldvolname) = PVE::Storage::parse_volume_id($old_volid);
3648             if ($oldvolname =~ m/\.(raw|qcow2|vmdk)$/){
3649                 $oldfmt = $1;
3650             }
3651
3652             die "you can't move to the same storage with same format\n"
3653                 if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format);
3654
3655             # this only checks snapshots because $disk is passed!
3656             my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use(
3657                 $storecfg,
3658                 $conf,
3659                 $disk,
3660                 $old_volid
3661             );
3662             die "you can't move a disk with snapshots and delete the source\n"
3663                 if $snapshotted && $param->{delete};
3664
3665             return ($conf, $drive, $oldstoreid, $snapshotted);
3666         };
3667
3668         my $move_updatefn = sub {
3669             my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->();
3670             my $old_volid = $drive->{file};
3671
3672             PVE::Cluster::log_msg(
3673                 'info',
3674                 $authuser,
3675                 "move disk VM $vmid: move --disk $disk --storage $storeid"
3676             );
3677
3678             my $running = PVE::QemuServer::check_running($vmid);
3679
3680             PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]);
3681
3682             my $newvollist = [];
3683
3684             eval {
3685                 local $SIG{INT} =
3686                     local $SIG{TERM} =
3687                     local $SIG{QUIT} =
3688                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3689
3690                 warn "moving disk with snapshots, snapshots will not be moved!\n"
3691                     if $snapshotted;
3692
3693                 my $bwlimit = extract_param($param, 'bwlimit');
3694                 my $movelimit = PVE::Storage::get_bandwidth_limit(
3695                     'move',
3696                     [$oldstoreid, $storeid],
3697                     $bwlimit
3698                 );
3699
3700                 my $source_info = {
3701                     vmid => $vmid,
3702                     running => $running,
3703                     drivename => $disk,
3704                     drive => $drive,
3705                     snapname => undef,
3706                 };
3707
3708                 my $dest_info = {
3709                     vmid => $vmid,
3710                     drivename => $disk,
3711                     storage => $storeid,
3712                     format => $format,
3713                 };
3714
3715                 $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf)
3716                     if $disk eq 'efidisk0';
3717
3718                 my $newdrive = PVE::QemuServer::clone_disk(
3719                     $storecfg,
3720                     $source_info,
3721                     $dest_info,
3722                     1,
3723                     $newvollist,
3724                     undef,
3725                     undef,
3726                     undef,
3727                     $movelimit,
3728                 );
3729                 $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
3730
3731                 PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
3732
3733                 # convert moved disk to base if part of template
3734                 PVE::QemuServer::template_create($vmid, $conf, $disk)
3735                     if PVE::QemuConfig->is_template($conf);
3736
3737                 PVE::QemuConfig->write_config($vmid, $conf);
3738
3739                 my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks');
3740                 if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
3741                     eval { mon_cmd($vmid, "guest-fstrim") };
3742                 }
3743
3744                 eval {
3745                     # try to deactivate volumes - avoid lvm LVs to be active on several nodes
3746                     PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
3747                         if !$running;
3748                 };
3749                 warn $@ if $@;
3750             };
3751             if (my $err = $@) {
3752                 foreach my $volid (@$newvollist) {
3753                     eval { PVE::Storage::vdisk_free($storecfg, $volid) };
3754                     warn $@ if $@;
3755                 }
3756                 die "storage migration failed: $err";
3757             }
3758
3759             if ($param->{delete}) {
3760                 eval {
3761                     PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
3762                     PVE::Storage::vdisk_free($storecfg, $old_volid);
3763                 };
3764                 warn $@ if $@;
3765             }
3766         };
3767
3768         my $load_and_check_reassign_configs = sub {
3769             my $vmlist = PVE::Cluster::get_vmlist()->{ids};
3770
3771             die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid});
3772             die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
3773
3774             my $source_node = $vmlist->{$vmid}->{node};
3775             my $target_node = $vmlist->{$target_vmid}->{node};
3776
3777             die "Both VMs need to be on the same node ($source_node != $target_node)\n"
3778                 if $source_node ne $target_node;
3779
3780             my $source_conf = PVE::QemuConfig->load_config($vmid);
3781             PVE::QemuConfig->check_lock($source_conf);
3782             my $target_conf = PVE::QemuConfig->load_config($target_vmid);
3783             PVE::QemuConfig->check_lock($target_conf);
3784
3785             die "Can't move disks from or to template VMs\n"
3786                 if ($source_conf->{template} || $target_conf->{template});
3787
3788             if ($digest) {
3789                 eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
3790                 die "VM ${vmid}: $@" if $@;
3791             }
3792
3793             if ($target_digest) {
3794                 eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
3795                 die "VM ${target_vmid}: $@" if $@;
3796             }
3797
3798             die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk});
3799
3800             die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n"
3801                 if $target_conf->{$target_disk};
3802
3803             my $drive = PVE::QemuServer::parse_drive(
3804                 $disk,
3805                 $source_conf->{$disk},
3806             );
3807             die "failed to parse source disk - $@\n" if !$drive;
3808
3809             my $source_volid = $drive->{file};
3810
3811             die "disk '${disk}' has no associated volume\n" if !$source_volid;
3812             die "CD drive contents can't be moved to another VM\n"
3813                 if PVE::QemuServer::drive_is_cdrom($drive, 1);
3814
3815             my $storeid = PVE::Storage::parse_volume_id($source_volid, 1);
3816             die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid);
3817
3818             die "Can't move disk used by a snapshot to another VM\n"
3819                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid);
3820             die "Storage does not support moving of this disk to another VM\n"
3821                 if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid));
3822             die "Cannot move disk to another VM while the source VM is running - detach first\n"
3823                 if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/;
3824
3825             # now re-parse using target disk slot format
3826             if ($target_disk =~ /^unused\d+$/) {
3827                 $drive = PVE::QemuServer::parse_drive(
3828                     $target_disk,
3829                     $source_volid,
3830                 );
3831             } else {
3832                 $drive = PVE::QemuServer::parse_drive(
3833                     $target_disk,
3834                     $source_conf->{$disk},
3835                 );
3836             }
3837             die "failed to parse source disk for target disk format - $@\n" if !$drive;
3838
3839             my $repl_conf = PVE::ReplicationConfig->new();
3840             if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
3841                 my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
3842                 die "Cannot move disk to a replicated VM. Storage does not support replication!\n"
3843                     if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
3844             }
3845
3846             return ($source_conf, $target_conf, $drive);
3847         };
3848
3849         my $logfunc = sub {
3850             my ($msg) = @_;
3851             print STDERR "$msg\n";
3852         };
3853
3854         my $disk_reassignfn = sub {
3855             return PVE::QemuConfig->lock_config($vmid, sub {
3856                 return PVE::QemuConfig->lock_config($target_vmid, sub {
3857                     my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs();
3858
3859                     my $source_volid = $drive->{file};
3860
3861                     print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n";
3862                     my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
3863
3864                     my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
3865
3866                     my $new_volid = PVE::Storage::rename_volume(
3867                         $storecfg,
3868                         $source_volid,
3869                         $target_vmid,
3870                     );
3871
3872                     $drive->{file} = $new_volid;
3873
3874                     delete $source_conf->{$disk};
3875                     print "removing disk '${disk}' from VM '${vmid}' config\n";
3876                     PVE::QemuConfig->write_config($vmid, $source_conf);
3877
3878                     my $drive_string = PVE::QemuServer::print_drive($drive);
3879
3880                     if ($target_disk =~ /^unused\d+$/) {
3881                         $target_conf->{$target_disk} = $drive_string;
3882                         PVE::QemuConfig->write_config($target_vmid, $target_conf);
3883                     } else {
3884                         &$update_vm_api(
3885                             {
3886                                 node => $node,
3887                                 vmid => $target_vmid,
3888                                 digest => $target_digest,
3889                                 $target_disk => $drive_string,
3890                             },
3891                             1,
3892                         );
3893                     }
3894
3895                     # remove possible replication snapshots
3896                     if (PVE::Storage::volume_has_feature(
3897                             $storecfg,
3898                             'replicate',
3899                             $source_volid),
3900                     ) {
3901                         eval {
3902                             PVE::Replication::prepare(
3903                                 $storecfg,
3904                                 [$new_volid],
3905                                 undef,
3906                                 1,
3907                                 undef,
3908                                 $logfunc,
3909                             )
3910                         };
3911                         if (my $err = $@) {
3912                             print "Failed to remove replication snapshots on moved disk " .
3913                                 "'$target_disk'. Manual cleanup could be necessary.\n";
3914                         }
3915                     }
3916                 });
3917             });
3918         };
3919
3920         if ($target_vmid && $storeid) {
3921             my $msg = "either set 'storage' or 'target-vmid', but not both";
3922             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
3923         } elsif ($target_vmid) {
3924             $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
3925                 if $authuser ne 'root@pam';
3926
3927             raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" })
3928                 if $vmid eq $target_vmid;
3929
3930             my (undef, undef, $drive) = &$load_and_check_reassign_configs();
3931             my $storage = PVE::Storage::parse_volume_id($drive->{file});
3932             $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
3933
3934             return $rpcenv->fork_worker(
3935                 'qmmove',
3936                 "${vmid}-${disk}>${target_vmid}-${target_disk}",
3937                 $authuser,
3938                 $disk_reassignfn
3939             );
3940         } elsif ($storeid) {
3941             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
3942
3943             die "cannot move disk '$disk', only configured disks can be moved to another storage\n"
3944                 if $disk =~ m/^unused\d+$/;
3945
3946             $load_and_check_move->(); # early checks before forking/locking
3947
3948             my $realcmd = sub {
3949                 PVE::QemuConfig->lock_config($vmid, $move_updatefn);
3950             };
3951
3952             return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
3953         } else {
3954             my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
3955             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
3956         }
3957     }});
3958
3959 my $check_vm_disks_local = sub {
3960     my ($storecfg, $vmconf, $vmid) = @_;
3961
3962     my $local_disks = {};
3963
3964     # add some more information to the disks e.g. cdrom
3965     PVE::QemuServer::foreach_volid($vmconf, sub {
3966         my ($volid, $attr) = @_;
3967
3968         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3969         if ($storeid) {
3970             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3971             return if $scfg->{shared};
3972         }
3973         # The shared attr here is just a special case where the vdisk
3974         # is marked as shared manually
3975         return if $attr->{shared};
3976         return if $attr->{cdrom} and $volid eq "none";
3977
3978         if (exists $local_disks->{$volid}) {
3979             @{$local_disks->{$volid}}{keys %$attr} = values %$attr
3980         } else {
3981             $local_disks->{$volid} = $attr;
3982             # ensure volid is present in case it's needed
3983             $local_disks->{$volid}->{volid} = $volid;
3984         }
3985     });
3986
3987     return $local_disks;
3988 };
3989
3990 __PACKAGE__->register_method({
3991     name => 'migrate_vm_precondition',
3992     path => '{vmid}/migrate',
3993     method => 'GET',
3994     protected => 1,
3995     proxyto => 'node',
3996     description => "Get preconditions for migration.",
3997     permissions => {
3998         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
3999     },
4000     parameters => {
4001         additionalProperties => 0,
4002         properties => {
4003             node => get_standard_option('pve-node'),
4004             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4005             target => get_standard_option('pve-node', {
4006                 description => "Target node.",
4007                 completion =>  \&PVE::Cluster::complete_migration_target,
4008                 optional => 1,
4009             }),
4010         },
4011     },
4012     returns => {
4013         type => "object",
4014         properties => {
4015             running => { type => 'boolean' },
4016             allowed_nodes => {
4017                 type => 'array',
4018                 optional => 1,
4019                 description => "List nodes allowed for offline migration, only passed if VM is offline"
4020             },
4021             not_allowed_nodes => {
4022                 type => 'object',
4023                 optional => 1,
4024                 description => "List not allowed nodes with additional informations, only passed if VM is offline"
4025             },
4026             local_disks => {
4027                 type => 'array',
4028                 description => "List local disks including CD-Rom, unsused and not referenced disks"
4029             },
4030             local_resources => {
4031                 type => 'array',
4032                 description => "List local resources e.g. pci, usb"
4033             }
4034         },
4035     },
4036     code => sub {
4037         my ($param) = @_;
4038
4039         my $rpcenv = PVE::RPCEnvironment::get();
4040
4041         my $authuser = $rpcenv->get_user();
4042
4043         PVE::Cluster::check_cfs_quorum();
4044
4045         my $res = {};
4046
4047         my $vmid = extract_param($param, 'vmid');
4048         my $target = extract_param($param, 'target');
4049         my $localnode = PVE::INotify::nodename();
4050
4051
4052         # test if VM exists
4053         my $vmconf = PVE::QemuConfig->load_config($vmid);
4054         my $storecfg = PVE::Storage::config();
4055
4056
4057         # try to detect errors early
4058         PVE::QemuConfig->check_lock($vmconf);
4059
4060         $res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0;
4061
4062         # if vm is not running, return target nodes where local storage is available
4063         # for offline migration
4064         if (!$res->{running}) {
4065             $res->{allowed_nodes} = [];
4066             my $checked_nodes = PVE::QemuServer::check_local_storage_availability($vmconf, $storecfg);
4067             delete $checked_nodes->{$localnode};
4068
4069             foreach my $node (keys %$checked_nodes) {
4070                 if (!defined $checked_nodes->{$node}->{unavailable_storages}) {
4071                     push @{$res->{allowed_nodes}}, $node;
4072                 }
4073
4074             }
4075             $res->{not_allowed_nodes} = $checked_nodes;
4076         }
4077
4078
4079         my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid);
4080         $res->{local_disks} = [ values %$local_disks ];;
4081
4082         my $local_resources =  PVE::QemuServer::check_local_resources($vmconf, 1);
4083
4084         $res->{local_resources} = $local_resources;
4085
4086         return $res;
4087
4088
4089     }});
4090
4091 __PACKAGE__->register_method({
4092     name => 'migrate_vm',
4093     path => '{vmid}/migrate',
4094     method => 'POST',
4095     protected => 1,
4096     proxyto => 'node',
4097     description => "Migrate virtual machine. Creates a new migration task.",
4098     permissions => {
4099         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4100     },
4101     parameters => {
4102         additionalProperties => 0,
4103         properties => {
4104             node => get_standard_option('pve-node'),
4105             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4106             target => get_standard_option('pve-node', {
4107                 description => "Target node.",
4108                 completion =>  \&PVE::Cluster::complete_migration_target,
4109             }),
4110             online => {
4111                 type => 'boolean',
4112                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4113                 optional => 1,
4114             },
4115             force => {
4116                 type => 'boolean',
4117                 description => "Allow to migrate VMs which use local devices. Only root may use this option.",
4118                 optional => 1,
4119             },
4120             migration_type => {
4121                 type => 'string',
4122                 enum => ['secure', 'insecure'],
4123                 description => "Migration traffic is encrypted using an SSH tunnel by default. On secure, completely private networks this can be disabled to increase performance.",
4124                 optional => 1,
4125             },
4126             migration_network => {
4127                 type => 'string', format => 'CIDR',
4128                 description => "CIDR of the (sub) network that is used for migration.",
4129                 optional => 1,
4130             },
4131             "with-local-disks" => {
4132                 type => 'boolean',
4133                 description => "Enable live storage migration for local disk",
4134                 optional => 1,
4135             },
4136             targetstorage => get_standard_option('pve-targetstorage', {
4137                 completion => \&PVE::QemuServer::complete_migration_storage,
4138             }),
4139             bwlimit => {
4140                 description => "Override I/O bandwidth limit (in KiB/s).",
4141                 optional => 1,
4142                 type => 'integer',
4143                 minimum => '0',
4144                 default => 'migrate limit from datacenter or storage config',
4145             },
4146         },
4147     },
4148     returns => {
4149         type => 'string',
4150         description => "the task ID.",
4151     },
4152     code => sub {
4153         my ($param) = @_;
4154
4155         my $rpcenv = PVE::RPCEnvironment::get();
4156         my $authuser = $rpcenv->get_user();
4157
4158         my $target = extract_param($param, 'target');
4159
4160         my $localnode = PVE::INotify::nodename();
4161         raise_param_exc({ target => "target is local node."}) if $target eq $localnode;
4162
4163         PVE::Cluster::check_cfs_quorum();
4164
4165         PVE::Cluster::check_node_exists($target);
4166
4167         my $targetip = PVE::Cluster::remote_node_ip($target);
4168
4169         my $vmid = extract_param($param, 'vmid');
4170
4171         raise_param_exc({ force => "Only root may use this option." })
4172             if $param->{force} && $authuser ne 'root@pam';
4173
4174         raise_param_exc({ migration_type => "Only root may use this option." })
4175             if $param->{migration_type} && $authuser ne 'root@pam';
4176
4177         # allow root only until better network permissions are available
4178         raise_param_exc({ migration_network => "Only root may use this option." })
4179             if $param->{migration_network} && $authuser ne 'root@pam';
4180
4181         # test if VM exists
4182         my $conf = PVE::QemuConfig->load_config($vmid);
4183
4184         # try to detect errors early
4185
4186         PVE::QemuConfig->check_lock($conf);
4187
4188         if (PVE::QemuServer::check_running($vmid)) {
4189             die "can't migrate running VM without --online\n" if !$param->{online};
4190
4191             my $repl_conf = PVE::ReplicationConfig->new();
4192             my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
4193             my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target));
4194             if (!$param->{force} && $is_replicated && !$is_replicated_to_target) {
4195                 die "Cannot live-migrate replicated VM to node '$target' - not a replication " .
4196                     "target. Use 'force' to override.\n";
4197             }
4198         } else {
4199             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4200             $param->{online} = 0;
4201         }
4202
4203         my $storecfg = PVE::Storage::config();
4204         if (my $targetstorage = $param->{targetstorage}) {
4205             my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
4206             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
4207                 if $@;
4208
4209             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
4210                 if !defined($storagemap->{identity});
4211
4212             foreach my $target_sid (values %{$storagemap->{entries}}) {
4213                 $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
4214             }
4215
4216             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
4217                 if $storagemap->{default};
4218
4219             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target)
4220                 if $storagemap->{identity};
4221
4222             $param->{storagemap} = $storagemap;
4223         } else {
4224             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target);
4225         }
4226
4227         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
4228
4229             my $hacmd = sub {
4230                 my $upid = shift;
4231
4232                 print "Requesting HA migration for VM $vmid to node $target\n";
4233
4234                 my $cmd = ['ha-manager', 'migrate', "vm:$vmid", $target];
4235                 PVE::Tools::run_command($cmd);
4236                 return;
4237             };
4238
4239             return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);
4240
4241         } else {
4242
4243             my $realcmd = sub {
4244                 PVE::QemuMigrate->migrate($target, $targetip, $vmid, $param);
4245             };
4246
4247             my $worker = sub {
4248                 return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
4249             };
4250
4251             return $rpcenv->fork_worker('qmigrate', $vmid, $authuser, $worker);
4252         }
4253
4254     }});
4255
4256 __PACKAGE__->register_method({
4257     name => 'monitor',
4258     path => '{vmid}/monitor',
4259     method => 'POST',
4260     protected => 1,
4261     proxyto => 'node',
4262     description => "Execute Qemu monitor commands.",
4263     permissions => {
4264         description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')",
4265         check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]],
4266     },
4267     parameters => {
4268         additionalProperties => 0,
4269         properties => {
4270             node => get_standard_option('pve-node'),
4271             vmid => get_standard_option('pve-vmid'),
4272             command => {
4273                 type => 'string',
4274                 description => "The monitor command.",
4275             }
4276         },
4277     },
4278     returns => { type => 'string'},
4279     code => sub {
4280         my ($param) = @_;
4281
4282         my $rpcenv = PVE::RPCEnvironment::get();
4283         my $authuser = $rpcenv->get_user();
4284
4285         my $is_ro = sub {
4286             my $command = shift;
4287             return $command =~ m/^\s*info(\s+|$)/
4288                 || $command =~ m/^\s*help\s*$/;
4289         };
4290
4291         $rpcenv->check_full($authuser, "/", ['Sys.Modify'])
4292             if !&$is_ro($param->{command});
4293
4294         my $vmid = $param->{vmid};
4295
4296         my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists
4297
4298         my $res = '';
4299         eval {
4300             $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, $param->{command});
4301         };
4302         $res = "ERROR: $@" if $@;
4303
4304         return $res;
4305     }});
4306
4307 __PACKAGE__->register_method({
4308     name => 'resize_vm',
4309     path => '{vmid}/resize',
4310     method => 'PUT',
4311     protected => 1,
4312     proxyto => 'node',
4313     description => "Extend volume size.",
4314     permissions => {
4315         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
4316     },
4317     parameters => {
4318         additionalProperties => 0,
4319         properties => {
4320             node => get_standard_option('pve-node'),
4321             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4322             skiplock => get_standard_option('skiplock'),
4323             disk => {
4324                 type => 'string',
4325                 description => "The disk you want to resize.",
4326                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
4327             },
4328             size => {
4329                 type => 'string',
4330                 pattern => '\+?\d+(\.\d+)?[KMGT]?',
4331                 description => "The new size. With the `+` sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
4332             },
4333             digest => {
4334                 type => 'string',
4335                 description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
4336                 maxLength => 40,
4337                 optional => 1,
4338             },
4339         },
4340     },
4341     returns => { type => 'null'},
4342     code => sub {
4343         my ($param) = @_;
4344
4345         my $rpcenv = PVE::RPCEnvironment::get();
4346
4347         my $authuser = $rpcenv->get_user();
4348
4349         my $node = extract_param($param, 'node');
4350
4351         my $vmid = extract_param($param, 'vmid');
4352
4353         my $digest = extract_param($param, 'digest');
4354
4355         my $disk = extract_param($param, 'disk');
4356
4357         my $sizestr = extract_param($param, 'size');
4358
4359         my $skiplock = extract_param($param, 'skiplock');
4360         raise_param_exc({ skiplock => "Only root may use this option." })
4361             if $skiplock && $authuser ne 'root@pam';
4362
4363         my $storecfg = PVE::Storage::config();
4364
4365         my $updatefn =  sub {
4366
4367             my $conf = PVE::QemuConfig->load_config($vmid);
4368
4369             die "checksum missmatch (file change by other user?)\n"
4370                 if $digest && $digest ne $conf->{digest};
4371             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4372
4373             die "disk '$disk' does not exist\n" if !$conf->{$disk};
4374
4375             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
4376
4377             my (undef, undef, undef, undef, undef, undef, $format) =
4378                 PVE::Storage::parse_volname($storecfg, $drive->{file});
4379
4380             die "can't resize volume: $disk if snapshot exists\n"
4381                 if %{$conf->{snapshots}} && $format eq 'qcow2';
4382
4383             my $volid = $drive->{file};
4384
4385             die "disk '$disk' has no associated volume\n" if !$volid;
4386
4387             die "you can't resize a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive);
4388
4389             my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4390
4391             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4392
4393             PVE::Storage::activate_volumes($storecfg, [$volid]);
4394             my $size = PVE::Storage::volume_size_info($storecfg, $volid, 5);
4395
4396             die "Could not determine current size of volume '$volid'\n" if !defined($size);
4397
4398             die "internal error" if $sizestr !~ m/^(\+)?(\d+(\.\d+)?)([KMGT])?$/;
4399             my ($ext, $newsize, $unit) = ($1, $2, $4);
4400             if ($unit) {
4401                 if ($unit eq 'K') {
4402                     $newsize = $newsize * 1024;
4403                 } elsif ($unit eq 'M') {
4404                     $newsize = $newsize * 1024 * 1024;
4405                 } elsif ($unit eq 'G') {
4406                     $newsize = $newsize * 1024 * 1024 * 1024;
4407                 } elsif ($unit eq 'T') {
4408                     $newsize = $newsize * 1024 * 1024 * 1024 * 1024;
4409                 }
4410             }
4411             $newsize += $size if $ext;
4412             $newsize = int($newsize);
4413
4414             die "shrinking disks is not supported\n" if $newsize < $size;
4415
4416             return if $size == $newsize;
4417
4418             PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: resize --disk $disk --size $sizestr");
4419
4420             PVE::QemuServer::qemu_block_resize($vmid, "drive-$disk", $storecfg, $volid, $newsize);
4421
4422             $drive->{size} = $newsize;
4423             $conf->{$disk} = PVE::QemuServer::print_drive($drive);
4424
4425             PVE::QemuConfig->write_config($vmid, $conf);
4426         };
4427
4428         PVE::QemuConfig->lock_config($vmid, $updatefn);
4429         return;
4430     }});
4431
4432 __PACKAGE__->register_method({
4433     name => 'snapshot_list',
4434     path => '{vmid}/snapshot',
4435     method => 'GET',
4436     description => "List all snapshots.",
4437     permissions => {
4438         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
4439     },
4440     proxyto => 'node',
4441     protected => 1, # qemu pid files are only readable by root
4442     parameters => {
4443         additionalProperties => 0,
4444         properties => {
4445             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4446             node => get_standard_option('pve-node'),
4447         },
4448     },
4449     returns => {
4450         type => 'array',
4451         items => {
4452             type => "object",
4453             properties => {
4454                 name => {
4455                     description => "Snapshot identifier. Value 'current' identifies the current VM.",
4456                     type => 'string',
4457                 },
4458                 vmstate => {
4459                     description => "Snapshot includes RAM.",
4460                     type => 'boolean',
4461                     optional => 1,
4462                 },
4463                 description => {
4464                     description => "Snapshot description.",
4465                     type => 'string',
4466                 },
4467                 snaptime => {
4468                     description => "Snapshot creation time",
4469                     type => 'integer',
4470                     renderer => 'timestamp',
4471                     optional => 1,
4472                 },
4473                 parent => {
4474                     description => "Parent snapshot identifier.",
4475                     type => 'string',
4476                     optional => 1,
4477                 },
4478             },
4479         },
4480         links => [ { rel => 'child', href => "{name}" } ],
4481     },
4482     code => sub {
4483         my ($param) = @_;
4484
4485         my $vmid = $param->{vmid};
4486
4487         my $conf = PVE::QemuConfig->load_config($vmid);
4488         my $snaphash = $conf->{snapshots} || {};
4489
4490         my $res = [];
4491
4492         foreach my $name (keys %$snaphash) {
4493             my $d = $snaphash->{$name};
4494             my $item = {
4495                 name => $name,
4496                 snaptime => $d->{snaptime} || 0,
4497                 vmstate => $d->{vmstate} ? 1 : 0,
4498                 description => $d->{description} || '',
4499             };
4500             $item->{parent} = $d->{parent} if $d->{parent};
4501             $item->{snapstate} = $d->{snapstate} if $d->{snapstate};
4502             push @$res, $item;
4503         }
4504
4505         my $running = PVE::QemuServer::check_running($vmid, 1) ? 1 : 0;
4506         my $current = {
4507             name => 'current',
4508             digest => $conf->{digest},
4509             running => $running,
4510             description => "You are here!",
4511         };
4512         $current->{parent} = $conf->{parent} if $conf->{parent};
4513
4514         push @$res, $current;
4515
4516         return $res;
4517     }});
4518
4519 __PACKAGE__->register_method({
4520     name => 'snapshot',
4521     path => '{vmid}/snapshot',
4522     method => 'POST',
4523     protected => 1,
4524     proxyto => 'node',
4525     description => "Snapshot a VM.",
4526     permissions => {
4527         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4528     },
4529     parameters => {
4530         additionalProperties => 0,
4531         properties => {
4532             node => get_standard_option('pve-node'),
4533             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4534             snapname => get_standard_option('pve-snapshot-name'),
4535             vmstate => {
4536                 optional => 1,
4537                 type => 'boolean',
4538                 description => "Save the vmstate",
4539             },
4540             description => {
4541                 optional => 1,
4542                 type => 'string',
4543                 description => "A textual description or comment.",
4544             },
4545         },
4546     },
4547     returns => {
4548         type => 'string',
4549         description => "the task ID.",
4550     },
4551     code => sub {
4552         my ($param) = @_;
4553
4554         my $rpcenv = PVE::RPCEnvironment::get();
4555
4556         my $authuser = $rpcenv->get_user();
4557
4558         my $node = extract_param($param, 'node');
4559
4560         my $vmid = extract_param($param, 'vmid');
4561
4562         my $snapname = extract_param($param, 'snapname');
4563
4564         die "unable to use snapshot name 'current' (reserved name)\n"
4565             if $snapname eq 'current';
4566
4567         die "unable to use snapshot name 'pending' (reserved name)\n"
4568             if lc($snapname) eq 'pending';
4569
4570         my $realcmd = sub {
4571             PVE::Cluster::log_msg('info', $authuser, "snapshot VM $vmid: $snapname");
4572             PVE::QemuConfig->snapshot_create($vmid, $snapname, $param->{vmstate},
4573                                              $param->{description});
4574         };
4575
4576         return $rpcenv->fork_worker('qmsnapshot', $vmid, $authuser, $realcmd);
4577     }});
4578
4579 __PACKAGE__->register_method({
4580     name => 'snapshot_cmd_idx',
4581     path => '{vmid}/snapshot/{snapname}',
4582     description => '',
4583     method => 'GET',
4584     permissions => {
4585         user => 'all',
4586     },
4587     parameters => {
4588         additionalProperties => 0,
4589         properties => {
4590             vmid => get_standard_option('pve-vmid'),
4591             node => get_standard_option('pve-node'),
4592             snapname => get_standard_option('pve-snapshot-name'),
4593         },
4594     },
4595     returns => {
4596         type => 'array',
4597         items => {
4598             type => "object",
4599             properties => {},
4600         },
4601         links => [ { rel => 'child', href => "{cmd}" } ],
4602     },
4603     code => sub {
4604         my ($param) = @_;
4605
4606         my $res = [];
4607
4608         push @$res, { cmd => 'rollback' };
4609         push @$res, { cmd => 'config' };
4610
4611         return $res;
4612     }});
4613
4614 __PACKAGE__->register_method({
4615     name => 'update_snapshot_config',
4616     path => '{vmid}/snapshot/{snapname}/config',
4617     method => 'PUT',
4618     protected => 1,
4619     proxyto => 'node',
4620     description => "Update snapshot metadata.",
4621     permissions => {
4622         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4623     },
4624     parameters => {
4625         additionalProperties => 0,
4626         properties => {
4627             node => get_standard_option('pve-node'),
4628             vmid => get_standard_option('pve-vmid'),
4629             snapname => get_standard_option('pve-snapshot-name'),
4630             description => {
4631                 optional => 1,
4632                 type => 'string',
4633                 description => "A textual description or comment.",
4634             },
4635         },
4636     },
4637     returns => { type => 'null' },
4638     code => sub {
4639         my ($param) = @_;
4640
4641         my $rpcenv = PVE::RPCEnvironment::get();
4642
4643         my $authuser = $rpcenv->get_user();
4644
4645         my $vmid = extract_param($param, 'vmid');
4646
4647         my $snapname = extract_param($param, 'snapname');
4648
4649         return if !defined($param->{description});
4650
4651         my $updatefn =  sub {
4652
4653             my $conf = PVE::QemuConfig->load_config($vmid);
4654
4655             PVE::QemuConfig->check_lock($conf);
4656
4657             my $snap = $conf->{snapshots}->{$snapname};
4658
4659             die "snapshot '$snapname' does not exist\n" if !defined($snap);
4660
4661             $snap->{description} = $param->{description} if defined($param->{description});
4662
4663              PVE::QemuConfig->write_config($vmid, $conf);
4664         };
4665
4666         PVE::QemuConfig->lock_config($vmid, $updatefn);
4667
4668         return;
4669     }});
4670
4671 __PACKAGE__->register_method({
4672     name => 'get_snapshot_config',
4673     path => '{vmid}/snapshot/{snapname}/config',
4674     method => 'GET',
4675     proxyto => 'node',
4676     description => "Get snapshot configuration",
4677     permissions => {
4678         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback', 'VM.Audit' ], any => 1],
4679     },
4680     parameters => {
4681         additionalProperties => 0,
4682         properties => {
4683             node => get_standard_option('pve-node'),
4684             vmid => get_standard_option('pve-vmid'),
4685             snapname => get_standard_option('pve-snapshot-name'),
4686         },
4687     },
4688     returns => { type => "object" },
4689     code => sub {
4690         my ($param) = @_;
4691
4692         my $rpcenv = PVE::RPCEnvironment::get();
4693
4694         my $authuser = $rpcenv->get_user();
4695
4696         my $vmid = extract_param($param, 'vmid');
4697
4698         my $snapname = extract_param($param, 'snapname');
4699
4700         my $conf = PVE::QemuConfig->load_config($vmid);
4701
4702         my $snap = $conf->{snapshots}->{$snapname};
4703
4704         die "snapshot '$snapname' does not exist\n" if !defined($snap);
4705
4706         return $snap;
4707     }});
4708
4709 __PACKAGE__->register_method({
4710     name => 'rollback',
4711     path => '{vmid}/snapshot/{snapname}/rollback',
4712     method => 'POST',
4713     protected => 1,
4714     proxyto => 'node',
4715     description => "Rollback VM state to specified snapshot.",
4716     permissions => {
4717         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback' ], any => 1],
4718     },
4719     parameters => {
4720         additionalProperties => 0,
4721         properties => {
4722             node => get_standard_option('pve-node'),
4723             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4724             snapname => get_standard_option('pve-snapshot-name'),
4725         },
4726     },
4727     returns => {
4728         type => 'string',
4729         description => "the task ID.",
4730     },
4731     code => sub {
4732         my ($param) = @_;
4733
4734         my $rpcenv = PVE::RPCEnvironment::get();
4735
4736         my $authuser = $rpcenv->get_user();
4737
4738         my $node = extract_param($param, 'node');
4739
4740         my $vmid = extract_param($param, 'vmid');
4741
4742         my $snapname = extract_param($param, 'snapname');
4743
4744         my $realcmd = sub {
4745             PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname");
4746             PVE::QemuConfig->snapshot_rollback($vmid, $snapname);
4747         };
4748
4749         my $worker = sub {
4750             # hold migration lock, this makes sure that nobody create replication snapshots
4751             return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
4752         };
4753
4754         return $rpcenv->fork_worker('qmrollback', $vmid, $authuser, $worker);
4755     }});
4756
4757 __PACKAGE__->register_method({
4758     name => 'delsnapshot',
4759     path => '{vmid}/snapshot/{snapname}',
4760     method => 'DELETE',
4761     protected => 1,
4762     proxyto => 'node',
4763     description => "Delete a VM snapshot.",
4764     permissions => {
4765         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4766     },
4767     parameters => {
4768         additionalProperties => 0,
4769         properties => {
4770             node => get_standard_option('pve-node'),
4771             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4772             snapname => get_standard_option('pve-snapshot-name'),
4773             force => {
4774                 optional => 1,
4775                 type => 'boolean',
4776                 description => "For removal from config file, even if removing disk snapshots fails.",
4777             },
4778         },
4779     },
4780     returns => {
4781         type => 'string',
4782         description => "the task ID.",
4783     },
4784     code => sub {
4785         my ($param) = @_;
4786
4787         my $rpcenv = PVE::RPCEnvironment::get();
4788
4789         my $authuser = $rpcenv->get_user();
4790
4791         my $node = extract_param($param, 'node');
4792
4793         my $vmid = extract_param($param, 'vmid');
4794
4795         my $snapname = extract_param($param, 'snapname');
4796
4797         my $lock_obtained;
4798         my $do_delete = sub {
4799             $lock_obtained = 1;
4800             PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname");
4801             PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force});
4802         };
4803
4804         my $realcmd = sub {
4805             if ($param->{force}) {
4806                 $do_delete->();
4807             } else {
4808                 eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); };
4809                 if (my $err = $@) {
4810                     die $err if $lock_obtained;
4811                     die "Failed to obtain guest migration lock - replication running?\n";
4812                 }
4813             }
4814         };
4815
4816         return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd);
4817     }});
4818
4819 __PACKAGE__->register_method({
4820     name => 'template',
4821     path => '{vmid}/template',
4822     method => 'POST',
4823     protected => 1,
4824     proxyto => 'node',
4825     description => "Create a Template.",
4826     permissions => {
4827         description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
4828         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
4829     },
4830     parameters => {
4831         additionalProperties => 0,
4832         properties => {
4833             node => get_standard_option('pve-node'),
4834             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
4835             disk => {
4836                 optional => 1,
4837                 type => 'string',
4838                 description => "If you want to convert only 1 disk to base image.",
4839                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
4840             },
4841
4842         },
4843     },
4844     returns => {
4845         type => 'string',
4846         description => "the task ID.",
4847     },
4848     code => sub {
4849         my ($param) = @_;
4850
4851         my $rpcenv = PVE::RPCEnvironment::get();
4852
4853         my $authuser = $rpcenv->get_user();
4854
4855         my $node = extract_param($param, 'node');
4856
4857         my $vmid = extract_param($param, 'vmid');
4858
4859         my $disk = extract_param($param, 'disk');
4860
4861         my $load_and_check = sub {
4862             my $conf = PVE::QemuConfig->load_config($vmid);
4863
4864             PVE::QemuConfig->check_lock($conf);
4865
4866             die "unable to create template, because VM contains snapshots\n"
4867                 if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});
4868
4869             die "you can't convert a template to a template\n"
4870                 if PVE::QemuConfig->is_template($conf) && !$disk;
4871
4872             die "you can't convert a VM to template if VM is running\n"
4873                 if PVE::QemuServer::check_running($vmid);
4874
4875             return $conf;
4876         };
4877
4878         $load_and_check->();
4879
4880         my $realcmd = sub {
4881             PVE::QemuConfig->lock_config($vmid, sub {
4882                 my $conf = $load_and_check->();
4883
4884                 $conf->{template} = 1;
4885                 PVE::QemuConfig->write_config($vmid, $conf);
4886
4887                 PVE::QemuServer::template_create($vmid, $conf, $disk);
4888             });
4889         };
4890
4891         return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
4892     }});
4893
4894 __PACKAGE__->register_method({
4895     name => 'cloudinit_generated_config_dump',
4896     path => '{vmid}/cloudinit/dump',
4897     method => 'GET',
4898     proxyto => 'node',
4899     description => "Get automatically generated cloudinit config.",
4900     permissions => {
4901         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
4902     },
4903     parameters => {
4904         additionalProperties => 0,
4905         properties => {
4906             node => get_standard_option('pve-node'),
4907             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4908             type => {
4909                 description => 'Config type.',
4910                 type => 'string',
4911                 enum => ['user', 'network', 'meta'],
4912             },
4913         },
4914     },
4915     returns => {
4916         type => 'string',
4917     },
4918     code => sub {
4919         my ($param) = @_;
4920
4921         my $conf = PVE::QemuConfig->load_config($param->{vmid});
4922
4923         return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
4924     }});
4925
4926 1;