PVE/API2/Qemu.pm

   1 package PVE::API2::Qemu;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd 'abs_path';
   6 use Net::SSLeay;
   7 use IO::Socket::IP;
   8 use IO::Socket::UNIX;
   9 use IPC::Open3;
  10 use JSON;
  11 use URI::Escape;
  12 use Crypt::OpenSSL::Random;
  13 use Socket qw(SOCK_STREAM);
  14
  15 use PVE::APIClient::LWP;
  16 use PVE::CGroup;
  17 use PVE::Cluster qw (cfs_read_file cfs_write_file);;
  18 use PVE::RRD;
  19 use PVE::SafeSyslog;
  20 use PVE::Tools qw(extract_param);
  21 use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
  22 use PVE::Storage;
  23 use PVE::JSONSchema qw(get_standard_option);
  24 use PVE::RESTHandler;
  25 use PVE::ReplicationConfig;
  26 use PVE::GuestHelpers qw(assert_tag_permissions);
  27 use PVE::QemuConfig;
  28 use PVE::QemuServer;
  29 use PVE::QemuServer::Cloudinit;
  30 use PVE::QemuServer::CPUConfig;
  31 use PVE::QemuServer::Drive;
  32 use PVE::QemuServer::ImportDisk;
  33 use PVE::QemuServer::Monitor qw(mon_cmd);
  34 use PVE::QemuServer::Machine;
  35 use PVE::QemuServer::Memory qw(get_current_memory);
  36 use PVE::QemuServer::PCI;
  37 use PVE::QemuServer::USB;
  38 use PVE::QemuMigrate;
  39 use PVE::RPCEnvironment;
  40 use PVE::AccessControl;
  41 use PVE::INotify;
  42 use PVE::Network;
  43 use PVE::Firewall;
  44 use PVE::API2::Firewall::VM;
  45 use PVE::API2::Qemu::Agent;
  46 use PVE::VZDump::Plugin;
  47 use PVE::DataCenterConfig;
  48 use PVE::SSHInfo;
  49 use PVE::Replication;
  50 use PVE::StorageTunnel;
  51 use PVE::RESTEnvironment qw(log_warn);
  52
  53 BEGIN {
  54     if (!$ENV{PVE_GENERATING_DOCS}) {
  55         require PVE::HA::Env::PVE2;
  56         import PVE::HA::Env::PVE2;
  57         require PVE::HA::Config;
  58         import PVE::HA::Config;
  59     }
  60 }
  61
  62 use base qw(PVE::RESTHandler);
  63
  64 my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal.";
  65
  66 my $resolve_cdrom_alias = sub {
  67     my $param = shift;
  68
  69     if (my $value = $param->{cdrom}) {
  70         $value .= ",media=cdrom" if $value !~ m/media=/;
  71         $param->{ide2} = $value;
  72         delete $param->{cdrom};
  73     }
  74 };
  75
  76 # Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema.
  77 my $foreach_volume_with_alloc = sub {
  78     my ($param, $func) = @_;
  79
  80     for my $opt (sort keys $param->%*) {
  81         next if !PVE::QemuServer::is_valid_drivename($opt);
  82
  83         my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1);
  84         next if !$drive;
  85
  86         $func->($opt, $drive);
  87     }
  88 };
  89
  90 my $check_drive_param = sub {
  91     my ($param, $storecfg, $extra_checks) = @_;
  92
  93     for my $opt (sort keys $param->%*) {
  94         next if !PVE::QemuServer::is_valid_drivename($opt);
  95
  96         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
  97         raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
  98
  99         if ($drive->{'import-from'}) {
 100             if ($drive->{file} !~ $PVE::QemuServer::Drive::NEW_DISK_RE || $3 != 0) {
 101                 raise_param_exc({
 102                     $opt => "'import-from' requires special syntax - ".
 103                         "use <storage ID>:0,import-from=<source>",
 104                 });
 105             }
 106
 107             if ($opt eq 'efidisk0') {
 108                 for my $required (qw(efitype pre-enrolled-keys)) {
 109                     if (!defined($drive->{$required})) {
 110                         raise_param_exc({
 111                             $opt => "need to specify '$required' when using 'import-from'",
 112                         });
 113                     }
 114                 }
 115             } elsif ($opt eq 'tpmstate0') {
 116                 raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" })
 117                     if !defined($drive->{version});
 118             }
 119         }
 120
 121         PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 122
 123         $extra_checks->($drive) if $extra_checks;
 124
 125         $param->{$opt} = PVE::QemuServer::print_drive($drive, 1);
 126     }
 127 };
 128
 129 my $check_storage_access = sub {
 130    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 131
 132    $foreach_volume_with_alloc->($settings, sub {
 133         my ($ds, $drive) = @_;
 134
 135         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 136
 137         my $volid = $drive->{file};
 138         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 139
 140         if (!$volid || ($volid eq 'none' || $volid eq 'cloudinit' || (defined($volname) && $volname eq 'cloudinit'))) {
 141             # nothing to check
 142         } elsif ($isCDROM && ($volid eq 'cdrom')) {
 143             $rpcenv->check($authuser, "/", ['Sys.Console']);
 144         } elsif (!$isCDROM && ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE)) {
 145             my ($storeid, $size) = ($2 || $default_storage, $3);
 146             die "no storage ID specified (and no default storage)\n" if !$storeid;
 147             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
 148             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 149             raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
 150                 if !$scfg->{content}->{images};
 151         } else {
 152             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 153             if ($storeid) {
 154                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 155                 raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" })
 156                     if $vtype ne 'images' && $vtype ne 'iso';
 157             }
 158         }
 159
 160         if (my $src_image = $drive->{'import-from'}) {
 161             my $src_vmid;
 162             if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume
 163                 (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image);
 164                 raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" })
 165                     if $vtype ne 'images';
 166             }
 167
 168             if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk()
 169                 $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']);
 170             } else {
 171                 PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image);
 172             }
 173         }
 174     });
 175
 176    $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
 177        if defined($settings->{vmstatestorage});
 178 };
 179
 180 my $check_storage_access_clone = sub {
 181    my ($rpcenv, $authuser, $storecfg, $conf, $storage) = @_;
 182
 183    my $sharedvm = 1;
 184
 185    PVE::QemuConfig->foreach_volume($conf, sub {
 186         my ($ds, $drive) = @_;
 187
 188         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 189
 190         my $volid = $drive->{file};
 191
 192         return if !$volid || $volid eq 'none';
 193
 194         if ($isCDROM) {
 195             if ($volid eq 'cdrom') {
 196                 $rpcenv->check($authuser, "/", ['Sys.Console']);
 197             } else {
 198                 # we simply allow access
 199                 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 200                 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 201                 $sharedvm = 0 if !$scfg->{shared};
 202
 203             }
 204         } else {
 205             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 206             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 207             $sharedvm = 0 if !$scfg->{shared};
 208
 209             $sid = $storage if $storage;
 210             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
 211         }
 212     });
 213
 214    $rpcenv->check($authuser, "/storage/$conf->{vmstatestorage}", ['Datastore.AllocateSpace'])
 215        if defined($conf->{vmstatestorage});
 216
 217    return $sharedvm;
 218 };
 219
 220 my $check_storage_access_migrate = sub {
 221     my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
 222
 223     PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
 224
 225     $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
 226
 227     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
 228     die "storage '$storage' does not support vm images\n"
 229         if !$scfg->{content}->{images};
 230 };
 231
 232 my $import_from_volid = sub {
 233     my ($storecfg, $src_volid, $dest_info, $vollist) = @_;
 234
 235     die "could not get size of $src_volid\n"
 236         if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10);
 237
 238     die "cannot import from cloudinit disk\n"
 239         if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid });
 240
 241     my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2];
 242
 243     my $src_vm_state = sub {
 244         my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0;
 245
 246         my $runs = 0;
 247         if ($exists) {
 248             eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); };
 249             die "owner VM $src_vmid not on local node\n" if $@;
 250             $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0;
 251         }
 252
 253         return ($exists, $runs);
 254     };
 255
 256     my ($src_vm_exists, $running) = $src_vm_state->();
 257
 258     die "cannot import from '$src_volid' - full clone feature is not supported\n"
 259         if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running);
 260
 261     my $clonefn = sub {
 262         my ($src_vm_exists_now, $running_now) = $src_vm_state->();
 263
 264         die "owner VM $src_vmid changed state unexpectedly\n"
 265             if $src_vm_exists_now != $src_vm_exists || $running_now != $running;
 266
 267         my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {};
 268
 269         my $src_drive = { file => $src_volid };
 270         my $src_drivename;
 271         PVE::QemuConfig->foreach_volume($src_conf, sub {
 272             my ($ds, $drive) = @_;
 273
 274             return if $src_drivename;
 275
 276             if ($drive->{file} eq $src_volid) {
 277                 $src_drive = $drive;
 278                 $src_drivename = $ds;
 279             }
 280         });
 281
 282         my $source_info = {
 283             vmid => $src_vmid,
 284             running => $running_now,
 285             drivename => $src_drivename,
 286             drive => $src_drive,
 287             snapname => undef,
 288         };
 289
 290         my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid);
 291
 292         return PVE::QemuServer::clone_disk(
 293             $storecfg,
 294             $source_info,
 295             $dest_info,
 296             1,
 297             $vollist,
 298             undef,
 299             undef,
 300             $src_conf->{agent},
 301             PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]),
 302         );
 303     };
 304
 305     my $cloned;
 306     if ($running) {
 307         $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn);
 308     } elsif ($src_vmid) {
 309         $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn);
 310     } else {
 311         $cloned = $clonefn->();
 312     }
 313
 314     return $cloned->@{qw(file size)};
 315 };
 316
 317 # Note: $pool is only needed when creating a VM, because pool permissions
 318 # are automatically inherited if VM already exists inside a pool.
 319 my sub create_disks : prototype($$$$$$$$$$) {
 320     my (
 321         $rpcenv,
 322         $authuser,
 323         $conf,
 324         $arch,
 325         $storecfg,
 326         $vmid,
 327         $pool,
 328         $settings,
 329         $default_storage,
 330         $is_live_import,
 331     ) = @_;
 332
 333     my $vollist = [];
 334
 335     my $res = {};
 336
 337     my $live_import_mapping = {};
 338
 339     my $code = sub {
 340         my ($ds, $disk) = @_;
 341
 342         my $volid = $disk->{file};
 343         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 344
 345         if (!$volid || $volid eq 'none' || $volid eq 'cdrom') {
 346             delete $disk->{size};
 347             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 348         } elsif (defined($volname) && $volname eq 'cloudinit') {
 349             $storeid = $storeid // $default_storage;
 350             die "no storage ID specified (and no default storage)\n" if !$storeid;
 351
 352             if (
 353                 my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 354                 || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 355                 || PVE::QemuConfig->has_cloudinit($res, $ds)
 356             ) {
 357                 die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 358             }
 359
 360             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 361             my $name = "vm-$vmid-cloudinit";
 362
 363             my $fmt = undef;
 364             if ($scfg->{path}) {
 365                 $fmt = $disk->{format} // "qcow2";
 366                 $name .= ".$fmt";
 367             } else {
 368                 $fmt = $disk->{format} // "raw";
 369             }
 370
 371             # Initial disk created with 4 MB and aligned to 4MB on regeneration
 372             my $ci_size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 373             my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, $name, $ci_size/1024);
 374             $disk->{file} = $volid;
 375             $disk->{media} = 'cdrom';
 376             push @$vollist, $volid;
 377             delete $disk->{format}; # no longer needed
 378             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 379             print "$ds: successfully created disk '$res->{$ds}'\n";
 380         } elsif ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) {
 381             my ($storeid, $size) = ($2 || $default_storage, $3);
 382             die "no storage ID specified (and no default storage)\n" if !$storeid;
 383
 384             $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb
 385
 386             my $live_import = $is_live_import && $ds ne 'efidisk0';
 387             my $needs_creation = 1;
 388
 389             if (my $source = delete $disk->{'import-from'}) {
 390                 my $dst_volid;
 391
 392                 $needs_creation = $live_import;
 393
 394                 if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume
 395                     if ($live_import && $ds ne 'efidisk0') {
 396                         my $path = PVE::Storage::path($storecfg, $source)
 397                             or die "failed to get a path for '$source'\n";
 398                         $source = $path;
 399                         ($size, my $source_format) = PVE::Storage::file_size_info($source);
 400                         die "could not get file size of $source\n" if !$size;
 401                         $live_import_mapping->{$ds} = {
 402                             path => $source,
 403                             format => $source_format,
 404                         };
 405                     } else {
 406                         my $dest_info = {
 407                             vmid => $vmid,
 408                             drivename => $ds,
 409                             storage => $storeid,
 410                             format => $disk->{format},
 411                         };
 412
 413                         $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk)
 414                             if $ds eq 'efidisk0';
 415
 416                         ($dst_volid, $size) = eval {
 417                             $import_from_volid->($storecfg, $source, $dest_info, $vollist);
 418                         };
 419                         die "cannot import from '$source' - $@" if $@;
 420                     }
 421                 } else {
 422                     $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
 423                     ($size, my $source_format) = PVE::Storage::file_size_info($source);
 424                     die "could not get file size of $source\n" if !$size;
 425
 426                     if ($live_import && $ds ne 'efidisk0') {
 427                         $live_import_mapping->{$ds} = {
 428                             path => $source,
 429                             format => $source_format,
 430                         };
 431                     } else {
 432                         (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import(
 433                             $source,
 434                             $vmid,
 435                             $storeid,
 436                             {
 437                                 drive_name => $ds,
 438                                 format => $disk->{format},
 439                                 'skip-config-update' => 1,
 440                             },
 441                         );
 442                         push @$vollist, $dst_volid;
 443                     }
 444                 }
 445
 446                 if ($needs_creation) {
 447                     $size = PVE::Tools::convert_size($size, 'b' => 'kb'); # vdisk_alloc uses kb
 448                 } else {
 449                     $disk->{file} = $dst_volid;
 450                     $disk->{size} = $size;
 451                     delete $disk->{format}; # no longer needed
 452                     $res->{$ds} = PVE::QemuServer::print_drive($disk);
 453                 }
 454             }
 455
 456             if ($needs_creation) {
 457                 my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
 458                 my $fmt = $disk->{format} || $defformat;
 459
 460                 my $volid;
 461                 if ($ds eq 'efidisk0') {
 462                     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 463                     ($volid, $size) = PVE::QemuServer::create_efidisk(
 464                         $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm);
 465                 } elsif ($ds eq 'tpmstate0') {
 466                     # swtpm can only use raw volumes, and uses a fixed size
 467                     $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb');
 468                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size);
 469                 } else {
 470                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
 471                 }
 472                 push @$vollist, $volid;
 473                 $disk->{file} = $volid;
 474                 $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
 475                 delete $disk->{format}; # no longer needed
 476                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 477             }
 478
 479             print "$ds: successfully created disk '$res->{$ds}'\n";
 480         } else {
 481             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 482             if ($storeid) {
 483                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 484                 die "cannot use volume $volid - content type needs to be 'images' or 'iso'"
 485                     if $vtype ne 'images' && $vtype ne 'iso';
 486
 487                 if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) {
 488                     if (
 489                         my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 490                         || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 491                         || PVE::QemuConfig->has_cloudinit($res, $ds)
 492                     ) {
 493                         die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 494                     }
 495                 }
 496             }
 497
 498             PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
 499
 500             my $size = PVE::Storage::volume_size_info($storecfg, $volid);
 501             die "volume $volid does not exist\n" if !$size;
 502             $disk->{size} = $size;
 503
 504             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 505         }
 506     };
 507
 508     eval { $foreach_volume_with_alloc->($settings, $code); };
 509
 510     # free allocated images on error
 511     if (my $err = $@) {
 512         syslog('err', "VM $vmid creating disks failed");
 513         foreach my $volid (@$vollist) {
 514             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 515             warn $@ if $@;
 516         }
 517         die $err;
 518     }
 519
 520     # don't return empty import mappings
 521     $live_import_mapping = undef if !%$live_import_mapping;
 522
 523     return ($vollist, $res, $live_import_mapping);
 524 };
 525
 526 my $check_cpu_model_access = sub {
 527     my ($rpcenv, $authuser, $new, $existing) = @_;
 528
 529     return if !defined($new->{cpu});
 530
 531     my $cpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $new->{cpu});
 532     return if !$cpu || !$cpu->{cputype}; # always allow default
 533     my $cputype = $cpu->{cputype};
 534
 535     if ($existing && $existing->{cpu}) {
 536         # changing only other settings doesn't require permissions for CPU model
 537         my $existingCpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $existing->{cpu});
 538         return if $existingCpu->{cputype} eq $cputype;
 539     }
 540
 541     if (PVE::QemuServer::CPUConfig::is_custom_model($cputype)) {
 542         $rpcenv->check($authuser, "/nodes", ['Sys.Audit']);
 543     }
 544 };
 545
 546 my $cpuoptions = {
 547     'cores' => 1,
 548     'cpu' => 1,
 549     'cpulimit' => 1,
 550     'cpuunits' => 1,
 551     'numa' => 1,
 552     'smp' => 1,
 553     'sockets' => 1,
 554     'vcpus' => 1,
 555 };
 556
 557 my $memoryoptions = {
 558     'memory' => 1,
 559     'balloon' => 1,
 560     'shares' => 1,
 561 };
 562
 563 my $hwtypeoptions = {
 564     'acpi' => 1,
 565     'hotplug' => 1,
 566     'kvm' => 1,
 567     'machine' => 1,
 568     'scsihw' => 1,
 569     'smbios1' => 1,
 570     'tablet' => 1,
 571     'vga' => 1,
 572     'watchdog' => 1,
 573     'audio0' => 1,
 574 };
 575
 576 my $generaloptions = {
 577     'agent' => 1,
 578     'autostart' => 1,
 579     'bios' => 1,
 580     'description' => 1,
 581     'keyboard' => 1,
 582     'localtime' => 1,
 583     'migrate_downtime' => 1,
 584     'migrate_speed' => 1,
 585     'name' => 1,
 586     'onboot' => 1,
 587     'ostype' => 1,
 588     'protection' => 1,
 589     'reboot' => 1,
 590     'startdate' => 1,
 591     'startup' => 1,
 592     'tdf' => 1,
 593     'template' => 1,
 594 };
 595
 596 my $vmpoweroptions = {
 597     'freeze' => 1,
 598 };
 599
 600 my $diskoptions = {
 601     'boot' => 1,
 602     'bootdisk' => 1,
 603     'vmstatestorage' => 1,
 604 };
 605
 606 my $cloudinitoptions = {
 607     cicustom => 1,
 608     cipassword => 1,
 609     citype => 1,
 610     ciuser => 1,
 611     ciupgrade => 1,
 612     nameserver => 1,
 613     searchdomain => 1,
 614     sshkeys => 1,
 615 };
 616
 617 my $check_vm_create_serial_perm = sub {
 618     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 619
 620     return 1 if $authuser eq 'root@pam';
 621
 622     foreach my $opt (keys %{$param}) {
 623         next if $opt !~ m/^serial\d+$/;
 624
 625         if ($param->{$opt} eq 'socket') {
 626             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 627         } else {
 628             die "only root can set '$opt' config for real devices\n";
 629         }
 630     }
 631
 632     return 1;
 633 };
 634
 635 my sub check_usb_perm {
 636     my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_;
 637
 638     return 1 if $authuser eq 'root@pam';
 639
 640     $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 641
 642     my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $value);
 643     if ($device->{host} && $device->{host} !~ m/^spice$/i) {
 644         die "only root can set '$opt' config for real devices\n";
 645     } elsif ($device->{mapping}) {
 646         $rpcenv->check_full($authuser, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
 647     } else {
 648         die "either 'host' or 'mapping' must be set.\n";
 649     }
 650
 651     return 1;
 652 }
 653
 654 my sub check_vm_create_usb_perm {
 655     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 656
 657     return 1 if $authuser eq 'root@pam';
 658
 659     foreach my $opt (keys %{$param}) {
 660         next if $opt !~ m/^usb\d+$/;
 661         check_usb_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt});
 662     }
 663
 664     return 1;
 665 };
 666
 667 my sub check_hostpci_perm {
 668     my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_;
 669
 670     return 1 if $authuser eq 'root@pam';
 671
 672     my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $value);
 673     if ($device->{host}) {
 674         die "only root can set '$opt' config for non-mapped devices\n";
 675     } elsif ($device->{mapping}) {
 676         $rpcenv->check_full($authuser, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
 677         $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 678     } else {
 679         die "either 'host' or 'mapping' must be set.\n";
 680     }
 681
 682     return 1;
 683 }
 684
 685 my sub check_vm_create_hostpci_perm {
 686     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 687
 688     return 1 if $authuser eq 'root@pam';
 689
 690     foreach my $opt (keys %{$param}) {
 691         next if $opt !~ m/^hostpci\d+$/;
 692         check_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt});
 693     }
 694
 695     return 1;
 696 };
 697
 698 my $check_vm_modify_config_perm = sub {
 699     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
 700
 701     return 1 if $authuser eq 'root@pam';
 702
 703     foreach my $opt (@$key_list) {
 704         # some checks (e.g., disk, serial port, usb) need to be done somewhere
 705         # else, as there the permission can be value dependend
 706         next if PVE::QemuServer::is_valid_drivename($opt);
 707         next if $opt eq 'cdrom';
 708         next if $opt =~ m/^(?:unused|serial|usb|hostpci)\d+$/;
 709         next if $opt eq 'tags';
 710
 711
 712         if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) {
 713             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
 714         } elsif ($memoryoptions->{$opt}) {
 715             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
 716         } elsif ($hwtypeoptions->{$opt}) {
 717             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 718         } elsif ($generaloptions->{$opt}) {
 719             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
 720             # special case for startup since it changes host behaviour
 721             if ($opt eq 'startup') {
 722                 $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
 723             }
 724         } elsif ($vmpoweroptions->{$opt}) {
 725             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']);
 726         } elsif ($diskoptions->{$opt}) {
 727             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
 728         } elsif ($opt =~ m/^net\d+$/) {
 729             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
 730         } elsif ($cloudinitoptions->{$opt} || $opt =~ m/^ipconfig\d+$/) {
 731             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1);
 732         } elsif ($opt eq 'vmstate') {
 733             # the user needs Disk and PowerMgmt privileges to change the vmstate
 734             # also needs privileges on the storage, that will be checked later
 735             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]);
 736         } else {
 737             # catches args, lock, etc.
 738             # new options will be checked here
 739             die "only root can set '$opt' config\n";
 740         }
 741     }
 742
 743     return 1;
 744 };
 745
 746 sub assert_scsi_feature_compatibility {
 747     my ($opt, $conf, $storecfg, $drive_attributes) = @_;
 748
 749     my $drive = PVE::QemuServer::Drive::parse_drive($opt, $drive_attributes, 1);
 750
 751     my $machine_type = PVE::QemuServer::get_vm_machine($conf, undef, $conf->{arch});
 752     my $machine_version = PVE::QemuServer::Machine::extract_version(
 753         $machine_type, PVE::QemuServer::kvm_user_version());
 754     my $drivetype = PVE::QemuServer::Drive::get_scsi_device_type(
 755         $drive, $storecfg, $machine_version);
 756
 757     if ($drivetype ne 'hd' && $drivetype ne 'cd') {
 758         if ($drive->{product}) {
 759             raise_param_exc({
 760                 $opt => "Passing of product information is only supported for 'scsi-hd' and "
 761                     ."'scsi-cd' devices (e.g. not pass-through).",
 762             });
 763         }
 764         if ($drive->{vendor}) {
 765             raise_param_exc({
 766                 $opt => "Passing of vendor information is only supported for 'scsi-hd' and "
 767                     ."'scsi-cd' devices (e.g. not pass-through).",
 768             });
 769         }
 770     }
 771 }
 772
 773 __PACKAGE__->register_method({
 774     name => 'vmlist',
 775     path => '',
 776     method => 'GET',
 777     description => "Virtual machine index (per node).",
 778     permissions => {
 779         description => "Only list VMs where you have VM.Audit permissons on /vms/<vmid>.",
 780         user => 'all',
 781     },
 782     proxyto => 'node',
 783     protected => 1, # qemu pid files are only readable by root
 784     parameters => {
 785         additionalProperties => 0,
 786         properties => {
 787             node => get_standard_option('pve-node'),
 788             full => {
 789                 type => 'boolean',
 790                 optional => 1,
 791                 description => "Determine the full status of active VMs.",
 792             },
 793         },
 794     },
 795     returns => {
 796         type => 'array',
 797         items => {
 798             type => "object",
 799             properties => $PVE::QemuServer::vmstatus_return_properties,
 800         },
 801         links => [ { rel => 'child', href => "{vmid}" } ],
 802     },
 803     code => sub {
 804         my ($param) = @_;
 805
 806         my $rpcenv = PVE::RPCEnvironment::get();
 807         my $authuser = $rpcenv->get_user();
 808
 809         my $vmstatus = PVE::QemuServer::vmstatus(undef, $param->{full});
 810
 811         my $res = [];
 812         foreach my $vmid (keys %$vmstatus) {
 813             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
 814
 815             my $data = $vmstatus->{$vmid};
 816             push @$res, $data;
 817         }
 818
 819         return $res;
 820     }});
 821
 822 my $parse_restore_archive = sub {
 823     my ($storecfg, $archive) = @_;
 824
 825     my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1);
 826
 827     my $res = {};
 828
 829     if (defined($archive_storeid)) {
 830         my $scfg =  PVE::Storage::storage_config($storecfg, $archive_storeid);
 831         $res->{volid} = $archive;
 832         if ($scfg->{type} eq 'pbs') {
 833             $res->{type} = 'pbs';
 834             return $res;
 835         }
 836     }
 837     my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive);
 838     $res->{type} = 'file';
 839     $res->{path} = $path;
 840     return $res;
 841 };
 842
 843 __PACKAGE__->register_method({
 844     name => 'create_vm',
 845     path => '',
 846     method => 'POST',
 847     description => "Create or restore a virtual machine.",
 848     permissions => {
 849         description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
 850             "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
 851             "If you create disks you need 'Datastore.AllocateSpace' on any used storage." .
 852             "If you use a bridge/vlan, you need 'SDN.Use' on any used bridge/vlan.",
 853         user => 'all', # check inside
 854     },
 855     protected => 1,
 856     proxyto => 'node',
 857     parameters => {
 858         additionalProperties => 0,
 859         properties => PVE::QemuServer::json_config_properties(
 860             {
 861                 node => get_standard_option('pve-node'),
 862                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
 863                 archive => {
 864                     description => "The backup archive. Either the file system path to a .tar or .vma file (use '-' to pipe data from stdin) or a proxmox storage backup volume identifier.",
 865                     type => 'string',
 866                     optional => 1,
 867                     maxLength => 255,
 868                     completion => \&PVE::QemuServer::complete_backup_archives,
 869                 },
 870                 storage => get_standard_option('pve-storage-id', {
 871                     description => "Default storage.",
 872                     optional => 1,
 873                     completion => \&PVE::QemuServer::complete_storage,
 874                 }),
 875                 force => {
 876                     optional => 1,
 877                     type => 'boolean',
 878                     description => "Allow to overwrite existing VM.",
 879                     requires => 'archive',
 880                 },
 881                 unique => {
 882                     optional => 1,
 883                     type => 'boolean',
 884                     description => "Assign a unique random ethernet address.",
 885                     requires => 'archive',
 886                 },
 887                 'live-restore' => {
 888                     optional => 1,
 889                     type => 'boolean',
 890                     description => "Start the VM immediately while importing or restoring in the background.",
 891                 },
 892                 pool => {
 893                     optional => 1,
 894                     type => 'string', format => 'pve-poolid',
 895                     description => "Add the VM to the specified pool.",
 896                 },
 897                 bwlimit => {
 898                     description => "Override I/O bandwidth limit (in KiB/s).",
 899                     optional => 1,
 900                     type => 'integer',
 901                     minimum => '0',
 902                     default => 'restore limit from datacenter or storage config',
 903                 },
 904                 start => {
 905                     optional => 1,
 906                     type => 'boolean',
 907                     default => 0,
 908                     description => "Start VM after it was created successfully.",
 909                 },
 910             },
 911             1, # with_disk_alloc
 912         ),
 913     },
 914     returns => {
 915         type => 'string',
 916     },
 917     code => sub {
 918         my ($param) = @_;
 919
 920         my $rpcenv = PVE::RPCEnvironment::get();
 921         my $authuser = $rpcenv->get_user();
 922
 923         my $node = extract_param($param, 'node');
 924         my $vmid = extract_param($param, 'vmid');
 925
 926         my $archive = extract_param($param, 'archive');
 927         my $is_restore = !!$archive;
 928
 929         my $bwlimit = extract_param($param, 'bwlimit');
 930         my $force = extract_param($param, 'force');
 931         my $pool = extract_param($param, 'pool');
 932         my $start_after_create = extract_param($param, 'start');
 933         my $storage = extract_param($param, 'storage');
 934         my $unique = extract_param($param, 'unique');
 935         my $live_restore = extract_param($param, 'live-restore');
 936
 937         if (defined(my $ssh_keys = $param->{sshkeys})) {
 938                 $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
 939                 PVE::Tools::validate_ssh_public_keys($ssh_keys);
 940         }
 941
 942         $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
 943             if defined($param->{cpuunits}); # clamp value depending on cgroup version
 944
 945         PVE::Cluster::check_cfs_quorum();
 946
 947         my $filename = PVE::QemuConfig->config_file($vmid);
 948         my $storecfg = PVE::Storage::config();
 949
 950         if (defined($pool)) {
 951             $rpcenv->check_pool_exist($pool);
 952         }
 953
 954         $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace'])
 955             if defined($storage);
 956
 957         if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
 958             # OK
 959         } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
 960             # OK
 961         } elsif ($archive && $force && (-f $filename) &&
 962                  $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
 963             # OK: user has VM.Backup permissions and wants to restore an existing VM
 964         } else {
 965             raise_perm_exc();
 966         }
 967
 968         if ($archive) {
 969             for my $opt (sort keys $param->%*) {
 970                 if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
 971                     raise_param_exc({ $opt => "option conflicts with option 'archive'" });
 972                 }
 973             }
 974
 975             if ($archive eq '-') {
 976                 die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli';
 977                 $archive = { type => 'pipe' };
 978             } else {
 979                 PVE::Storage::check_volume_access(
 980                     $rpcenv,
 981                     $authuser,
 982                     $storecfg,
 983                     $vmid,
 984                     $archive,
 985                     'backup',
 986                 );
 987
 988                 $archive = $parse_restore_archive->($storecfg, $archive);
 989             }
 990         }
 991
 992         if (scalar(keys $param->%*) > 0) {
 993             &$resolve_cdrom_alias($param);
 994
 995             &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage);
 996
 997             &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]);
 998
 999             &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param);
1000             check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param);
1001             check_vm_create_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $param);
1002
1003             PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param);
1004             &$check_cpu_model_access($rpcenv, $authuser, $param);
1005
1006             $check_drive_param->($param, $storecfg);
1007
1008             PVE::QemuServer::add_random_macs($param);
1009         }
1010
1011         my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -";
1012
1013         eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) };
1014         die "$emsg $@" if $@;
1015
1016         my $restored_data = 0;
1017         my $restorefn = sub {
1018             my $conf = PVE::QemuConfig->load_config($vmid);
1019
1020             PVE::QemuConfig->check_protection($conf, $emsg);
1021
1022             die "$emsg vm is running\n" if PVE::QemuServer::check_running($vmid);
1023
1024             my $realcmd = sub {
1025                 my $restore_options = {
1026                     storage => $storage,
1027                     pool => $pool,
1028                     unique => $unique,
1029                     bwlimit => $bwlimit,
1030                     live => $live_restore,
1031                     override_conf => $param,
1032                 };
1033                 if (my $volid = $archive->{volid}) {
1034                     # best effort, real check is after restoring!
1035                     my $merged = eval {
1036                         my $old_conf = PVE::Storage::extract_vzdump_config($storecfg, $volid);
1037                         PVE::QemuServer::restore_merge_config("backup/qemu-server/$vmid.conf", $old_conf, $param);
1038                     };
1039                     if ($@) {
1040                         warn "Could not extract backed up config: $@\n";
1041                         warn "Skipping early checks!\n";
1042                     } else {
1043                         PVE::QemuServer::check_restore_permissions($rpcenv, $authuser, $merged);
1044                     }
1045                 }
1046                 if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') {
1047                     die "live-restore is only compatible with backup images from a Proxmox Backup Server\n"
1048                         if $live_restore;
1049                     PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options);
1050                 } elsif ($archive->{type} eq 'pbs') {
1051                     PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options);
1052                 } else {
1053                     die "unknown backup archive type\n";
1054                 }
1055                 $restored_data = 1;
1056
1057                 my $restored_conf = PVE::QemuConfig->load_config($vmid);
1058                 # Convert restored VM to template if backup was VM template
1059                 if (PVE::QemuConfig->is_template($restored_conf)) {
1060                     warn "Convert to template.\n";
1061                     eval { PVE::QemuServer::template_create($vmid, $restored_conf) };
1062                     warn $@ if $@;
1063                 }
1064
1065                 PVE::QemuServer::create_ifaces_ipams_ips($restored_conf, $vmid) if $unique;
1066             };
1067
1068             # ensure no old replication state are exists
1069             PVE::ReplicationState::delete_guest_states($vmid);
1070
1071             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
1072
1073             if ($start_after_create && !$live_restore) {
1074                 print "Execute autostart\n";
1075                 eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) };
1076                 warn $@ if $@;
1077             }
1078         };
1079
1080         my $createfn = sub {
1081             my $live_import_mapping = {};
1082
1083             # ensure no old replication state are exists
1084             PVE::ReplicationState::delete_guest_states($vmid);
1085
1086             my $realcmd = sub {
1087                 my $conf = $param;
1088                 my $arch = PVE::QemuServer::get_vm_arch($conf);
1089
1090                 for my $opt (sort keys $param->%*) {
1091                     next if $opt !~ m/^scsi\d+$/;
1092                     assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt});
1093                 }
1094
1095                 $conf->{meta} = PVE::QemuServer::new_meta_info_string();
1096
1097                 my $vollist = [];
1098                 eval {
1099                     ($vollist, my $created_opts, $live_import_mapping) = create_disks(
1100                         $rpcenv,
1101                         $authuser,
1102                         $conf,
1103                         $arch,
1104                         $storecfg,
1105                         $vmid,
1106                         $pool,
1107                         $param,
1108                         $storage,
1109                         $live_restore,
1110                     );
1111                     $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*;
1112
1113                     if (!$conf->{boot}) {
1114                         my $devs = PVE::QemuServer::get_default_bootdevices($conf);
1115                         $conf->{boot} = PVE::QemuServer::print_bootorder($devs);
1116                     }
1117
1118                     my $vga = PVE::QemuServer::parse_vga($conf->{vga});
1119                     PVE::QemuServer::assert_clipboard_config($vga);
1120
1121                     # auto generate uuid if user did not specify smbios1 option
1122                     if (!$conf->{smbios1}) {
1123                         $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid();
1124                     }
1125
1126                     if ((!defined($conf->{vmgenid}) || $conf->{vmgenid} eq '1') && $arch ne 'aarch64') {
1127                         $conf->{vmgenid} = PVE::QemuServer::generate_uuid();
1128                     }
1129
1130                     my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
1131                     my $machine = $machine_conf->{type};
1132                     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
1133                         # always pin Windows' machine version on create, they get to easily confused
1134                         if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) {
1135                             $machine_conf->{type} = PVE::QemuServer::windows_get_pinned_machine_version($machine);
1136                             $conf->{machine} = PVE::QemuServer::Machine::print_machine($machine_conf);
1137                         }
1138                     }
1139                     PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
1140
1141                     $conf->{lock} = 'import' if $live_import_mapping;
1142
1143                     PVE::QemuConfig->write_config($vmid, $conf);
1144                 };
1145                 my $err = $@;
1146
1147                 if ($err) {
1148                     foreach my $volid (@$vollist) {
1149                         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
1150                         warn $@ if $@;
1151                     }
1152                     die "$emsg $err";
1153                 }
1154
1155                 PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
1156
1157                 PVE::QemuServer::create_ifaces_ipams_ips($conf, $vmid);
1158             };
1159
1160             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
1161
1162             if ($start_after_create && !$live_restore) {
1163                 print "Execute autostart\n";
1164                 eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) };
1165                 warn $@ if $@;
1166                 return;
1167             } else {
1168                 return $live_import_mapping;
1169             }
1170         };
1171
1172         my ($code, $worker_name);
1173         if ($is_restore) {
1174             $worker_name = 'qmrestore';
1175             $code = sub {
1176                 eval { $restorefn->() };
1177                 if (my $err = $@) {
1178                     eval { PVE::QemuConfig->remove_lock($vmid, 'create') };
1179                     warn $@ if $@;
1180                     if ($restored_data) {
1181                         warn "error after data was restored, VM disks should be OK but config may "
1182                             ."require adaptions. VM $vmid state is NOT cleaned up.\n";
1183                     } else {
1184                         warn "error before or during data restore, some or all disks were not "
1185                             ."completely restored. VM $vmid state is NOT cleaned up.\n";
1186                     }
1187                     die $err;
1188                 }
1189             };
1190         } else {
1191             $worker_name = 'qmcreate';
1192             $code = sub {
1193                 # If a live import was requested the create function returns
1194                 # the mapping for the startup.
1195                 my $live_import_mapping = eval { $createfn->() };
1196                 if (my $err = $@) {
1197                     eval {
1198                         my $conffile = PVE::QemuConfig->config_file($vmid);
1199                         unlink($conffile) or die "failed to remove config file: $!\n";
1200                     };
1201                     warn $@ if $@;
1202                     die $err;
1203                 }
1204
1205                 if ($live_import_mapping) {
1206                     my $import_options = {
1207                         bwlimit => $bwlimit,
1208                         live => 1,
1209                     };
1210
1211                     my $conf = PVE::QemuConfig->load_config($vmid);
1212                     PVE::QemuServer::live_import_from_files(
1213                         $live_import_mapping,
1214                         $vmid,
1215                         $conf,
1216                         $import_options,
1217                     );
1218                 }
1219             };
1220         }
1221
1222         return $rpcenv->fork_worker($worker_name, $vmid, $authuser, $code);
1223     }});
1224
1225 __PACKAGE__->register_method({
1226     name => 'vmdiridx',
1227     path => '{vmid}',
1228     method => 'GET',
1229     proxyto => 'node',
1230     description => "Directory index",
1231     permissions => {
1232         user => 'all',
1233     },
1234     parameters => {
1235         additionalProperties => 0,
1236         properties => {
1237             node => get_standard_option('pve-node'),
1238             vmid => get_standard_option('pve-vmid'),
1239         },
1240     },
1241     returns => {
1242         type => 'array',
1243         items => {
1244             type => "object",
1245             properties => {
1246                 subdir => { type => 'string' },
1247             },
1248         },
1249         links => [ { rel => 'child', href => "{subdir}" } ],
1250     },
1251     code => sub {
1252         my ($param) = @_;
1253
1254         my $res = [
1255             { subdir => 'config' },
1256             { subdir => 'cloudinit' },
1257             { subdir => 'pending' },
1258             { subdir => 'status' },
1259             { subdir => 'unlink' },
1260             { subdir => 'vncproxy' },
1261             { subdir => 'termproxy' },
1262             { subdir => 'migrate' },
1263             { subdir => 'resize' },
1264             { subdir => 'move' },
1265             { subdir => 'rrd' },
1266             { subdir => 'rrddata' },
1267             { subdir => 'monitor' },
1268             { subdir => 'agent' },
1269             { subdir => 'snapshot' },
1270             { subdir => 'spiceproxy' },
1271             { subdir => 'sendkey' },
1272             { subdir => 'firewall' },
1273             { subdir => 'mtunnel' },
1274             { subdir => 'remote_migrate' },
1275         ];
1276
1277         return $res;
1278     }});
1279
1280 __PACKAGE__->register_method ({
1281     subclass => "PVE::API2::Firewall::VM",
1282     path => '{vmid}/firewall',
1283 });
1284
1285 __PACKAGE__->register_method ({
1286     subclass => "PVE::API2::Qemu::Agent",
1287     path => '{vmid}/agent',
1288 });
1289
1290 __PACKAGE__->register_method({
1291     name => 'rrd',
1292     path => '{vmid}/rrd',
1293     method => 'GET',
1294     protected => 1, # fixme: can we avoid that?
1295     permissions => {
1296         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1297     },
1298     description => "Read VM RRD statistics (returns PNG)",
1299     parameters => {
1300         additionalProperties => 0,
1301         properties => {
1302             node => get_standard_option('pve-node'),
1303             vmid => get_standard_option('pve-vmid'),
1304             timeframe => {
1305                 description => "Specify the time frame you are interested in.",
1306                 type => 'string',
1307                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1308             },
1309             ds => {
1310                 description => "The list of datasources you want to display.",
1311                 type => 'string', format => 'pve-configid-list',
1312             },
1313             cf => {
1314                 description => "The RRD consolidation function",
1315                 type => 'string',
1316                 enum => [ 'AVERAGE', 'MAX' ],
1317                 optional => 1,
1318             },
1319         },
1320     },
1321     returns => {
1322         type => "object",
1323         properties => {
1324             filename => { type => 'string' },
1325         },
1326     },
1327     code => sub {
1328         my ($param) = @_;
1329
1330         return PVE::RRD::create_rrd_graph(
1331             "pve2-vm/$param->{vmid}", $param->{timeframe},
1332             $param->{ds}, $param->{cf});
1333
1334     }});
1335
1336 __PACKAGE__->register_method({
1337     name => 'rrddata',
1338     path => '{vmid}/rrddata',
1339     method => 'GET',
1340     protected => 1, # fixme: can we avoid that?
1341     permissions => {
1342         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1343     },
1344     description => "Read VM RRD statistics",
1345     parameters => {
1346         additionalProperties => 0,
1347         properties => {
1348             node => get_standard_option('pve-node'),
1349             vmid => get_standard_option('pve-vmid'),
1350             timeframe => {
1351                 description => "Specify the time frame you are interested in.",
1352                 type => 'string',
1353                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1354             },
1355             cf => {
1356                 description => "The RRD consolidation function",
1357                 type => 'string',
1358                 enum => [ 'AVERAGE', 'MAX' ],
1359                 optional => 1,
1360             },
1361         },
1362     },
1363     returns => {
1364         type => "array",
1365         items => {
1366             type => "object",
1367             properties => {},
1368         },
1369     },
1370     code => sub {
1371         my ($param) = @_;
1372
1373         return PVE::RRD::create_rrd_data(
1374             "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
1375     }});
1376
1377
1378 __PACKAGE__->register_method({
1379     name => 'vm_config',
1380     path => '{vmid}/config',
1381     method => 'GET',
1382     proxyto => 'node',
1383     description => "Get the virtual machine configuration with pending configuration " .
1384         "changes applied. Set the 'current' parameter to get the current configuration instead.",
1385     permissions => {
1386         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1387     },
1388     parameters => {
1389         additionalProperties => 0,
1390         properties => {
1391             node => get_standard_option('pve-node'),
1392             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1393             current => {
1394                 description => "Get current values (instead of pending values).",
1395                 optional => 1,
1396                 default => 0,
1397                 type => 'boolean',
1398             },
1399             snapshot => get_standard_option('pve-snapshot-name', {
1400                 description => "Fetch config values from given snapshot.",
1401                 optional => 1,
1402                 completion => sub {
1403                     my ($cmd, $pname, $cur, $args) = @_;
1404                     PVE::QemuConfig->snapshot_list($args->[0]);
1405                 },
1406             }),
1407         },
1408     },
1409     returns => {
1410         description => "The VM configuration.",
1411         type => "object",
1412         properties => PVE::QemuServer::json_config_properties({
1413             digest => {
1414                 type => 'string',
1415                 description => 'SHA1 digest of configuration file. This can be used to prevent concurrent modifications.',
1416             }
1417         }),
1418     },
1419     code => sub {
1420         my ($param) = @_;
1421
1422         raise_param_exc({ snapshot => "cannot use 'snapshot' parameter with 'current'",
1423                           current => "cannot use 'snapshot' parameter with 'current'"})
1424             if ($param->{snapshot} && $param->{current});
1425
1426         my $conf;
1427         if ($param->{snapshot}) {
1428             $conf = PVE::QemuConfig->load_snapshot_config($param->{vmid}, $param->{snapshot});
1429         } else {
1430             $conf = PVE::QemuConfig->load_current_config($param->{vmid}, $param->{current});
1431         }
1432         $conf->{cipassword} = '**********' if $conf->{cipassword};
1433         return $conf;
1434
1435     }});
1436
1437 __PACKAGE__->register_method({
1438     name => 'vm_pending',
1439     path => '{vmid}/pending',
1440     method => 'GET',
1441     proxyto => 'node',
1442     description => "Get the virtual machine configuration with both current and pending values.",
1443     permissions => {
1444         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1445     },
1446     parameters => {
1447         additionalProperties => 0,
1448         properties => {
1449             node => get_standard_option('pve-node'),
1450             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1451         },
1452     },
1453     returns => {
1454         type => "array",
1455         items => {
1456             type => "object",
1457             properties => {
1458                 key => {
1459                     description => "Configuration option name.",
1460                     type => 'string',
1461                 },
1462                 value => {
1463                     description => "Current value.",
1464                     type => 'string',
1465                     optional => 1,
1466                 },
1467                 pending => {
1468                     description => "Pending value.",
1469                     type => 'string',
1470                     optional => 1,
1471                 },
1472                 delete => {
1473                     description => "Indicates a pending delete request if present and not 0. " .
1474                                    "The value 2 indicates a force-delete request.",
1475                     type => 'integer',
1476                     minimum => 0,
1477                     maximum => 2,
1478                     optional => 1,
1479                 },
1480             },
1481         },
1482     },
1483     code => sub {
1484         my ($param) = @_;
1485
1486         my $conf = PVE::QemuConfig->load_config($param->{vmid});
1487
1488         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
1489
1490         $conf->{cipassword} = '**********' if defined($conf->{cipassword});
1491         $conf->{pending}->{cipassword} = '********** ' if defined($conf->{pending}->{cipassword});
1492
1493         return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
1494    }});
1495
1496 __PACKAGE__->register_method({
1497     name => 'cloudinit_pending',
1498     path => '{vmid}/cloudinit',
1499     method => 'GET',
1500     proxyto => 'node',
1501     description => "Get the cloudinit configuration with both current and pending values.",
1502     permissions => {
1503         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1504     },
1505     parameters => {
1506         additionalProperties => 0,
1507         properties => {
1508             node => get_standard_option('pve-node'),
1509             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1510         },
1511     },
1512     returns => {
1513         type => "array",
1514         items => {
1515             type => "object",
1516             properties => {
1517                 key => {
1518                     description => "Configuration option name.",
1519                     type => 'string',
1520                 },
1521                 value => {
1522                     description => "Value as it was used to generate the current cloudinit image.",
1523                     type => 'string',
1524                     optional => 1,
1525                 },
1526                 pending => {
1527                     description => "The new pending value.",
1528                     type => 'string',
1529                     optional => 1,
1530                 },
1531                 delete => {
1532                     description => "Indicates a pending delete request if present and not 0. ",
1533                     type => 'integer',
1534                     minimum => 0,
1535                     maximum => 1,
1536                     optional => 1,
1537                 },
1538             },
1539         },
1540     },
1541     code => sub {
1542         my ($param) = @_;
1543
1544         my $vmid = $param->{vmid};
1545         my $conf = PVE::QemuConfig->load_config($vmid);
1546
1547         my $ci = $conf->{cloudinit};
1548
1549         $conf->{cipassword} = '**********' if exists $conf->{cipassword};
1550         $ci->{cipassword} = '**********' if exists $ci->{cipassword};
1551
1552         my $res = [];
1553
1554         # All the values that got added
1555         my $added = delete($ci->{added}) // '';
1556         for my $key (PVE::Tools::split_list($added)) {
1557             push @$res, { key => $key, pending => $conf->{$key} };
1558         }
1559
1560         # All already existing values (+ their new value, if it exists)
1561         for my $opt (keys %$cloudinitoptions) {
1562             next if !$conf->{$opt};
1563             next if $added =~ m/$opt/;
1564             my $item = {
1565                 key => $opt,
1566             };
1567
1568             if (my $pending = $ci->{$opt}) {
1569                 $item->{value} = $pending;
1570                 $item->{pending} = $conf->{$opt};
1571             } else {
1572                 $item->{value} = $conf->{$opt},
1573             }
1574
1575             push @$res, $item;
1576         }
1577
1578         # Now, we'll find the deleted ones
1579         for my $opt (keys %$ci) {
1580             next if $conf->{$opt};
1581             push @$res, { key => $opt, delete => 1 };
1582         }
1583
1584         return $res;
1585    }});
1586
1587 __PACKAGE__->register_method({
1588     name => 'cloudinit_update',
1589     path => '{vmid}/cloudinit',
1590     method => 'PUT',
1591     protected => 1,
1592     proxyto => 'node',
1593     description => "Regenerate and change cloudinit config drive.",
1594     permissions => {
1595         check => ['perm', '/vms/{vmid}', ['VM.Config.Cloudinit']],
1596     },
1597     parameters => {
1598         additionalProperties => 0,
1599         properties => {
1600             node => get_standard_option('pve-node'),
1601             vmid => get_standard_option('pve-vmid'),
1602         },
1603     },
1604     returns => { type => 'null' },
1605     code => sub {
1606         my ($param) = @_;
1607
1608         my $rpcenv = PVE::RPCEnvironment::get();
1609         my $authuser = $rpcenv->get_user();
1610
1611         my $vmid = extract_param($param, 'vmid');
1612
1613         PVE::QemuConfig->lock_config($vmid, sub {
1614             my $conf = PVE::QemuConfig->load_config($vmid);
1615             PVE::QemuConfig->check_lock($conf);
1616
1617             my $storecfg = PVE::Storage::config();
1618             PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
1619         });
1620         return;
1621     }});
1622
1623 # POST/PUT {vmid}/config implementation
1624 #
1625 # The original API used PUT (idempotent) an we assumed that all operations
1626 # are fast. But it turned out that almost any configuration change can
1627 # involve hot-plug actions, or disk alloc/free. Such actions can take long
1628 # time to complete and have side effects (not idempotent).
1629 #
1630 # The new implementation uses POST and forks a worker process. We added
1631 # a new option 'background_delay'. If specified we wait up to
1632 # 'background_delay' second for the worker task to complete. It returns null
1633 # if the task is finished within that time, else we return the UPID.
1634
1635 my $update_vm_api  = sub {
1636     my ($param, $sync) = @_;
1637
1638     my $rpcenv = PVE::RPCEnvironment::get();
1639
1640     my $authuser = $rpcenv->get_user();
1641
1642     my $node = extract_param($param, 'node');
1643
1644     my $vmid = extract_param($param, 'vmid');
1645
1646     my $digest = extract_param($param, 'digest');
1647
1648     my $background_delay = extract_param($param, 'background_delay');
1649
1650     my $skip_cloud_init = extract_param($param, 'skip_cloud_init');
1651
1652     if (defined(my $cipassword = $param->{cipassword})) {
1653         # Same logic as in cloud-init (but with the regex fixed...)
1654         $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
1655             if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
1656     }
1657
1658     my @paramarr = (); # used for log message
1659     foreach my $key (sort keys %$param) {
1660         my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
1661         push @paramarr, "-$key", $value;
1662     }
1663
1664     my $skiplock = extract_param($param, 'skiplock');
1665     raise_param_exc({ skiplock => "Only root may use this option." })
1666         if $skiplock && $authuser ne 'root@pam';
1667
1668     my $delete_str = extract_param($param, 'delete');
1669
1670     my $revert_str = extract_param($param, 'revert');
1671
1672     my $force = extract_param($param, 'force');
1673
1674     if (defined(my $ssh_keys = $param->{sshkeys})) {
1675         $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
1676         PVE::Tools::validate_ssh_public_keys($ssh_keys);
1677     }
1678
1679     $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
1680         if defined($param->{cpuunits}); # clamp value depending on cgroup version
1681
1682     die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param);
1683
1684     my $storecfg = PVE::Storage::config();
1685
1686     &$resolve_cdrom_alias($param);
1687
1688     # now try to verify all parameters
1689
1690     my $revert = {};
1691     foreach my $opt (PVE::Tools::split_list($revert_str)) {
1692         if (!PVE::QemuServer::option_exists($opt)) {
1693             raise_param_exc({ revert => "unknown option '$opt'" });
1694         }
1695
1696         raise_param_exc({ delete => "you can't use '-$opt' and " .
1697                               "-revert $opt' at the same time" })
1698             if defined($param->{$opt});
1699
1700         $revert->{$opt} = 1;
1701     }
1702
1703     my @delete = ();
1704     foreach my $opt (PVE::Tools::split_list($delete_str)) {
1705         $opt = 'ide2' if $opt eq 'cdrom';
1706
1707         raise_param_exc({ delete => "you can't use '-$opt' and " .
1708                               "-delete $opt' at the same time" })
1709             if defined($param->{$opt});
1710
1711         raise_param_exc({ revert => "you can't use '-delete $opt' and " .
1712                               "-revert $opt' at the same time" })
1713             if $revert->{$opt};
1714
1715         if (!PVE::QemuServer::option_exists($opt)) {
1716             raise_param_exc({ delete => "unknown option '$opt'" });
1717         }
1718
1719         push @delete, $opt;
1720     }
1721
1722     my $repl_conf = PVE::ReplicationConfig->new();
1723     my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
1724     my $check_replication = sub {
1725         my ($drive) = @_;
1726         return if !$is_replicated;
1727         my $volid = $drive->{file};
1728         return if !$volid || !($drive->{replicate}//1);
1729         return if PVE::QemuServer::drive_is_cdrom($drive);
1730
1731         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1732         die "cannot add non-managed/pass-through volume to a replicated VM\n"
1733             if !defined($storeid);
1734
1735         return if defined($volname) && $volname eq 'cloudinit';
1736
1737         my $format;
1738         if ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) {
1739             $storeid = $2;
1740             $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid);
1741         } else {
1742             $format = (PVE::Storage::parse_volname($storecfg, $volid))[6];
1743         }
1744         return if PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
1745         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1746         return if $scfg->{shared};
1747         die "cannot add non-replicatable volume to a replicated VM\n";
1748     };
1749
1750     $check_drive_param->($param, $storecfg, $check_replication);
1751
1752     foreach my $opt (keys %$param) {
1753         if ($opt =~ m/^net(\d+)$/) {
1754             # add macaddr
1755             my $net = PVE::QemuServer::parse_net($param->{$opt});
1756             $param->{$opt} = PVE::QemuServer::print_net($net);
1757         } elsif ($opt eq 'vmgenid') {
1758             if ($param->{$opt} eq '1') {
1759                 $param->{$opt} = PVE::QemuServer::generate_uuid();
1760             }
1761         } elsif ($opt eq 'hookscript') {
1762             eval { PVE::GuestHelpers::check_hookscript($param->{$opt}, $storecfg); };
1763             raise_param_exc({ $opt => $@ }) if $@;
1764         }
1765     }
1766
1767     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
1768
1769     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
1770
1771     &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
1772
1773     PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param);
1774
1775     my $updatefn =  sub {
1776
1777         my $conf = PVE::QemuConfig->load_config($vmid);
1778
1779         die "checksum missmatch (file change by other user?)\n"
1780             if $digest && $digest ne $conf->{digest};
1781
1782         &$check_cpu_model_access($rpcenv, $authuser, $param, $conf);
1783
1784         # FIXME: 'suspended' lock should probabyl be a state or "weak" lock?!
1785         if (scalar(@delete) && grep { $_ eq 'vmstate'} @delete) {
1786             if (defined($conf->{lock}) && $conf->{lock} eq 'suspended') {
1787                 delete $conf->{lock}; # for check lock check, not written out
1788                 push @delete, 'lock'; # this is the real deal to write it out
1789             }
1790             push @delete, 'runningmachine' if $conf->{runningmachine};
1791             push @delete, 'runningcpu' if $conf->{runningcpu};
1792         }
1793
1794         PVE::QemuConfig->check_lock($conf) if !$skiplock;
1795
1796         foreach my $opt (keys %$revert) {
1797             if (defined($conf->{$opt})) {
1798                 $param->{$opt} = $conf->{$opt};
1799             } elsif (defined($conf->{pending}->{$opt})) {
1800                 push @delete, $opt;
1801             }
1802         }
1803
1804         if ($param->{memory} || defined($param->{balloon})) {
1805
1806             my $memory = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory};
1807             my $maxmem = get_current_memory($memory);
1808             my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon};
1809
1810             die "balloon value too large (must be smaller than assigned memory)\n"
1811                 if $balloon && $balloon > $maxmem;
1812         }
1813
1814         PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: " . join (' ', @paramarr));
1815
1816         my $worker = sub {
1817
1818             print "update VM $vmid: " . join (' ', @paramarr) . "\n";
1819
1820             # write updates to pending section
1821
1822             my $modified = {}; # record what $option we modify
1823
1824             my @bootorder;
1825             if (my $boot = $conf->{boot}) {
1826                 my $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot);
1827                 @bootorder = PVE::Tools::split_list($bootcfg->{order}) if $bootcfg && $bootcfg->{order};
1828             }
1829             my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete;
1830
1831             my $check_drive_perms = sub {
1832                 my ($opt, $val) = @_;
1833                 my $drive = PVE::QemuServer::parse_drive($opt, $val, 1);
1834                 if (PVE::QemuServer::drive_is_cloudinit($drive)) {
1835                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM']);
1836                 } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM
1837                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
1838                 } else {
1839                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1840
1841                 }
1842             };
1843
1844             foreach my $opt (@delete) {
1845                 $modified->{$opt} = 1;
1846                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1847
1848                 # value of what we want to delete, independent if pending or not
1849                 my $val = $conf->{$opt} // $conf->{pending}->{$opt};
1850                 if (!defined($val)) {
1851                     warn "cannot delete '$opt' - not set in current configuration!\n";
1852                     $modified->{$opt} = 0;
1853                     next;
1854                 }
1855                 my $is_pending_val = defined($conf->{pending}->{$opt});
1856                 delete $conf->{pending}->{$opt};
1857
1858                 # remove from bootorder if necessary
1859                 if (!$bootorder_deleted && @bootorder && grep {$_ eq $opt} @bootorder) {
1860                     @bootorder = grep {$_ ne $opt} @bootorder;
1861                     $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1862                     $modified->{boot} = 1;
1863                 }
1864
1865                 if ($opt =~ m/^unused/) {
1866                     my $drive = PVE::QemuServer::parse_drive($opt, $val);
1867                     PVE::QemuConfig->check_protection($conf, "can't remove unused disk '$drive->{file}'");
1868                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1869                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser)) {
1870                         delete $conf->{$opt};
1871                         PVE::QemuConfig->write_config($vmid, $conf);
1872                     }
1873                 } elsif ($opt eq 'vmstate') {
1874                     PVE::QemuConfig->check_protection($conf, "can't remove vmstate '$val'");
1875                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, { file => $val }, $rpcenv, $authuser, 1)) {
1876                         delete $conf->{$opt};
1877                         PVE::QemuConfig->write_config($vmid, $conf);
1878                     }
1879                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
1880                     PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'");
1881                     $check_drive_perms->($opt, $val);
1882                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val))
1883                         if $is_pending_val;
1884                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1885                     PVE::QemuConfig->write_config($vmid, $conf);
1886                 } elsif ($opt =~ m/^serial\d+$/) {
1887                     if ($val eq 'socket') {
1888                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1889                     } elsif ($authuser ne 'root@pam') {
1890                         die "only root can delete '$opt' config for real devices\n";
1891                     }
1892                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1893                     PVE::QemuConfig->write_config($vmid, $conf);
1894                 } elsif ($opt =~ m/^usb\d+$/) {
1895                     check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $val);
1896                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1897                     PVE::QemuConfig->write_config($vmid, $conf);
1898                 } elsif ($opt =~ m/^hostpci\d+$/) {
1899                     check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $val);
1900                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1901                     PVE::QemuConfig->write_config($vmid, $conf);
1902                 } elsif ($opt eq 'tags') {
1903                     assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser);
1904                     delete $conf->{$opt};
1905                     PVE::QemuConfig->write_config($vmid, $conf);
1906                 } elsif ($opt =~ m/^net\d+$/) {
1907                     if ($conf->{$opt}) {
1908                         PVE::QemuServer::check_bridge_access(
1909                             $rpcenv,
1910                             $authuser,
1911                             { $opt => $conf->{$opt} },
1912                         );
1913                     }
1914                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1915                     PVE::QemuConfig->write_config($vmid, $conf);
1916                 } else {
1917                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1918                     PVE::QemuConfig->write_config($vmid, $conf);
1919                 }
1920             }
1921
1922             foreach my $opt (keys %$param) { # add/change
1923                 $modified->{$opt} = 1;
1924                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1925                 next if defined($conf->{pending}->{$opt}) && ($param->{$opt} eq $conf->{pending}->{$opt}); # skip if nothing changed
1926
1927                 my $arch = PVE::QemuServer::get_vm_arch($conf);
1928
1929                 if (PVE::QemuServer::is_valid_drivename($opt)) {
1930                     # old drive
1931                     if ($conf->{$opt}) {
1932                         $check_drive_perms->($opt, $conf->{$opt});
1933                     }
1934
1935                     # new drive
1936                     $check_drive_perms->($opt, $param->{$opt});
1937                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
1938                         if defined($conf->{pending}->{$opt});
1939
1940                     assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt})
1941                         if $opt =~ m/^scsi\d+$/;
1942
1943                     my (undef, $created_opts) = create_disks(
1944                         $rpcenv,
1945                         $authuser,
1946                         $conf,
1947                         $arch,
1948                         $storecfg,
1949                         $vmid,
1950                         undef,
1951                         {$opt => $param->{$opt}},
1952                         undef,
1953                         undef,
1954                     );
1955                     $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*;
1956
1957                     # default legacy boot order implies all cdroms anyway
1958                     if (@bootorder) {
1959                         # append new CD drives to bootorder to mark them bootable
1960                         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
1961                         if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) {
1962                             push @bootorder, $opt;
1963                             $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1964                             $modified->{boot} = 1;
1965                         }
1966                     }
1967                 } elsif ($opt =~ m/^serial\d+/) {
1968                     if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
1969                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1970                     } elsif ($authuser ne 'root@pam') {
1971                         die "only root can modify '$opt' config for real devices\n";
1972                     }
1973                     $conf->{pending}->{$opt} = $param->{$opt};
1974                 } elsif ($opt eq 'vga') {
1975                     my $vga = PVE::QemuServer::parse_vga($param->{$opt});
1976                     PVE::QemuServer::assert_clipboard_config($vga);
1977                     $conf->{pending}->{$opt} = $param->{$opt};
1978                 } elsif ($opt =~ m/^usb\d+/) {
1979                     if (my $olddevice = $conf->{$opt}) {
1980                         check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $conf->{$opt});
1981                     }
1982                     check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt});
1983                     $conf->{pending}->{$opt} = $param->{$opt};
1984                 } elsif ($opt =~ m/^hostpci\d+$/) {
1985                     if (my $oldvalue = $conf->{$opt}) {
1986                         check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $oldvalue);
1987                     }
1988                     check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt});
1989                     $conf->{pending}->{$opt} = $param->{$opt};
1990                 } elsif ($opt eq 'tags') {
1991                     assert_tag_permissions($vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser);
1992                     $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt});
1993                 } elsif ($opt =~ m/^net\d+$/) {
1994                     if ($conf->{$opt}) {
1995                         PVE::QemuServer::check_bridge_access(
1996                             $rpcenv,
1997                             $authuser,
1998                             { $opt => $conf->{$opt} },
1999                         );
2000                     }
2001                     $conf->{pending}->{$opt} = $param->{$opt};
2002                 } elsif ($opt eq 'machine') {
2003                     my $machine_conf = PVE::QemuServer::Machine::parse_machine($param->{$opt});
2004                     PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
2005                     $conf->{pending}->{$opt} = $param->{$opt};
2006                 } else {
2007                     $conf->{pending}->{$opt} = $param->{$opt};
2008
2009                     if ($opt eq 'boot') {
2010                         my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $param->{$opt});
2011                         if ($new_bootcfg->{order}) {
2012                             my @devs = PVE::Tools::split_list($new_bootcfg->{order});
2013                             for my $dev (@devs) {
2014                                 my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev};
2015                                 my $deleted = grep {$_ eq $dev} @delete;
2016                                 die "invalid bootorder: device '$dev' does not exist'\n"
2017                                     if !$exists || $deleted;
2018                             }
2019
2020                             # remove legacy boot order settings if new one set
2021                             $conf->{pending}->{$opt} = PVE::QemuServer::print_bootorder(\@devs);
2022                             PVE::QemuConfig->add_to_pending_delete($conf, "bootdisk")
2023                                 if $conf->{bootdisk};
2024                         }
2025                     }
2026                 }
2027                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
2028                 PVE::QemuConfig->write_config($vmid, $conf);
2029             }
2030
2031             # remove pending changes when nothing changed
2032             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
2033             my $changes = PVE::QemuConfig->cleanup_pending($conf);
2034             PVE::QemuConfig->write_config($vmid, $conf) if $changes;
2035
2036             return if !scalar(keys %{$conf->{pending}});
2037
2038             my $running = PVE::QemuServer::check_running($vmid);
2039
2040             # apply pending changes
2041
2042             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
2043
2044             my $errors = {};
2045             if ($running) {
2046                 PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors);
2047             } else {
2048                 # cloud_init must be skipped if we are in an incoming, remote live migration
2049                 PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors, $skip_cloud_init);
2050             }
2051             raise_param_exc($errors) if scalar(keys %$errors);
2052
2053             return;
2054         };
2055
2056         if ($sync) {
2057             &$worker();
2058             return;
2059         } else {
2060             my $upid = $rpcenv->fork_worker('qmconfig', $vmid, $authuser, $worker);
2061
2062             if ($background_delay) {
2063
2064                 # Note: It would be better to do that in the Event based HTTPServer
2065                 # to avoid blocking call to sleep.
2066
2067                 my $end_time = time() + $background_delay;
2068
2069                 my $task = PVE::Tools::upid_decode($upid);
2070
2071                 my $running = 1;
2072                 while (time() < $end_time) {
2073                     $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
2074                     last if !$running;
2075                     sleep(1); # this gets interrupted when child process ends
2076                 }
2077
2078                 if (!$running) {
2079                     my $status = PVE::Tools::upid_read_status($upid);
2080                     return if !PVE::Tools::upid_status_is_error($status);
2081                     die "failed to update VM $vmid: $status\n";
2082                 }
2083             }
2084
2085             return $upid;
2086         }
2087     };
2088
2089     return PVE::QemuConfig->lock_config($vmid, $updatefn);
2090 };
2091
2092 my $vm_config_perm_list = [
2093             'VM.Config.Disk',
2094             'VM.Config.CDROM',
2095             'VM.Config.CPU',
2096             'VM.Config.Memory',
2097             'VM.Config.Network',
2098             'VM.Config.HWType',
2099             'VM.Config.Options',
2100             'VM.Config.Cloudinit',
2101     ];
2102
2103 __PACKAGE__->register_method({
2104     name => 'update_vm_async',
2105     path => '{vmid}/config',
2106     method => 'POST',
2107     protected => 1,
2108     proxyto => 'node',
2109     description => "Set virtual machine options (asynchrounous API).",
2110     permissions => {
2111         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
2112     },
2113     parameters => {
2114         additionalProperties => 0,
2115         properties => PVE::QemuServer::json_config_properties(
2116             {
2117                 node => get_standard_option('pve-node'),
2118                 vmid => get_standard_option('pve-vmid'),
2119                 skiplock => get_standard_option('skiplock'),
2120                 delete => {
2121                     type => 'string', format => 'pve-configid-list',
2122                     description => "A list of settings you want to delete.",
2123                     optional => 1,
2124                 },
2125                 revert => {
2126                     type => 'string', format => 'pve-configid-list',
2127                     description => "Revert a pending change.",
2128                     optional => 1,
2129                 },
2130                 force => {
2131                     type => 'boolean',
2132                     description => $opt_force_description,
2133                     optional => 1,
2134                     requires => 'delete',
2135                 },
2136                 digest => {
2137                     type => 'string',
2138                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
2139                     maxLength => 40,
2140                     optional => 1,
2141                 },
2142                 background_delay => {
2143                     type => 'integer',
2144                     description => "Time to wait for the task to finish. We return 'null' if the task finish within that time.",
2145                     minimum => 1,
2146                     maximum => 30,
2147                     optional => 1,
2148                 },
2149             },
2150             1, # with_disk_alloc
2151         ),
2152     },
2153     returns => {
2154         type => 'string',
2155         optional => 1,
2156     },
2157     code => $update_vm_api,
2158 });
2159
2160 __PACKAGE__->register_method({
2161     name => 'update_vm',
2162     path => '{vmid}/config',
2163     method => 'PUT',
2164     protected => 1,
2165     proxyto => 'node',
2166     description => "Set virtual machine options (synchrounous API) - You should consider using the POST method instead for any actions involving hotplug or storage allocation.",
2167     permissions => {
2168         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
2169     },
2170     parameters => {
2171         additionalProperties => 0,
2172         properties => PVE::QemuServer::json_config_properties(
2173             {
2174                 node => get_standard_option('pve-node'),
2175                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
2176                 skiplock => get_standard_option('skiplock'),
2177                 delete => {
2178                     type => 'string', format => 'pve-configid-list',
2179                     description => "A list of settings you want to delete.",
2180                     optional => 1,
2181                 },
2182                 revert => {
2183                     type => 'string', format => 'pve-configid-list',
2184                     description => "Revert a pending change.",
2185                     optional => 1,
2186                 },
2187                 force => {
2188                     type => 'boolean',
2189                     description => $opt_force_description,
2190                     optional => 1,
2191                     requires => 'delete',
2192                 },
2193                 digest => {
2194                     type => 'string',
2195                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
2196                     maxLength => 40,
2197                     optional => 1,
2198                 },
2199             },
2200             1, # with_disk_alloc
2201         ),
2202     },
2203     returns => { type => 'null' },
2204     code => sub {
2205         my ($param) = @_;
2206         &$update_vm_api($param, 1);
2207         return;
2208     }
2209 });
2210
2211 __PACKAGE__->register_method({
2212     name => 'destroy_vm',
2213     path => '{vmid}',
2214     method => 'DELETE',
2215     protected => 1,
2216     proxyto => 'node',
2217     description => "Destroy the VM and  all used/owned volumes. Removes any VM specific permissions"
2218         ." and firewall rules",
2219     permissions => {
2220         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
2221     },
2222     parameters => {
2223         additionalProperties => 0,
2224         properties => {
2225             node => get_standard_option('pve-node'),
2226             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2227             skiplock => get_standard_option('skiplock'),
2228             purge => {
2229                 type => 'boolean',
2230                 description => "Remove VMID from configurations, like backup & replication jobs and HA.",
2231                 optional => 1,
2232             },
2233             'destroy-unreferenced-disks' => {
2234                 type => 'boolean',
2235                 description => "If set, destroy additionally all disks not referenced in the config"
2236                  ." but with a matching VMID from all enabled storages.",
2237                 optional => 1,
2238                 default => 0,
2239             },
2240         },
2241     },
2242     returns => {
2243         type => 'string',
2244     },
2245     code => sub {
2246         my ($param) = @_;
2247
2248         my $rpcenv = PVE::RPCEnvironment::get();
2249         my $authuser = $rpcenv->get_user();
2250         my $vmid = $param->{vmid};
2251
2252         my $skiplock = $param->{skiplock};
2253         raise_param_exc({ skiplock => "Only root may use this option." })
2254             if $skiplock && $authuser ne 'root@pam';
2255
2256         my $early_checks = sub {
2257             # test if VM exists
2258             my $conf = PVE::QemuConfig->load_config($vmid);
2259             PVE::QemuConfig->check_protection($conf, "can't remove VM $vmid");
2260
2261             my $ha_managed = PVE::HA::Config::service_is_configured("vm:$vmid");
2262
2263             if (!$param->{purge}) {
2264                 die "unable to remove VM $vmid - used in HA resources and purge parameter not set.\n"
2265                     if $ha_managed;
2266                 # don't allow destroy if with replication jobs but no purge param
2267                 my $repl_conf = PVE::ReplicationConfig->new();
2268                 $repl_conf->check_for_existing_jobs($vmid);
2269             }
2270
2271             die "VM $vmid is running - destroy failed\n"
2272                 if PVE::QemuServer::check_running($vmid);
2273
2274             return $ha_managed;
2275         };
2276
2277         $early_checks->();
2278
2279         my $realcmd = sub {
2280             my $upid = shift;
2281
2282             my $storecfg = PVE::Storage::config();
2283
2284             syslog('info', "destroy VM $vmid: $upid\n");
2285             PVE::QemuConfig->lock_config($vmid, sub {
2286                 # repeat, config might have changed
2287                 my $ha_managed = $early_checks->();
2288
2289                 my $purge_unreferenced = $param->{'destroy-unreferenced-disks'};
2290
2291                 PVE::QemuServer::destroy_vm(
2292                     $storecfg,
2293                     $vmid,
2294                     $skiplock, { lock => 'destroyed' },
2295                     $purge_unreferenced,
2296                 );
2297
2298                 PVE::AccessControl::remove_vm_access($vmid);
2299                 PVE::Firewall::remove_vmfw_conf($vmid);
2300                 if ($param->{purge}) {
2301                     print "purging VM $vmid from related configurations..\n";
2302                     PVE::ReplicationConfig::remove_vmid_jobs($vmid);
2303                     PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);
2304
2305                     if ($ha_managed) {
2306                         PVE::HA::Config::delete_service_from_config("vm:$vmid");
2307                         print "NOTE: removed VM $vmid from HA resource configuration.\n";
2308                     }
2309                 }
2310
2311                 # only now remove the zombie config, else we can have reuse race
2312                 PVE::QemuConfig->destroy_config($vmid);
2313             });
2314         };
2315
2316         return $rpcenv->fork_worker('qmdestroy', $vmid, $authuser, $realcmd);
2317     }});
2318
2319 __PACKAGE__->register_method({
2320     name => 'unlink',
2321     path => '{vmid}/unlink',
2322     method => 'PUT',
2323     protected => 1,
2324     proxyto => 'node',
2325     description => "Unlink/delete disk images.",
2326     permissions => {
2327         check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']],
2328     },
2329     parameters => {
2330         additionalProperties => 0,
2331         properties => {
2332             node => get_standard_option('pve-node'),
2333             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
2334             idlist => {
2335                 type => 'string', format => 'pve-configid-list',
2336                 description => "A list of disk IDs you want to delete.",
2337             },
2338             force => {
2339                 type => 'boolean',
2340                 description => $opt_force_description,
2341                 optional => 1,
2342             },
2343         },
2344     },
2345     returns => { type => 'null'},
2346     code => sub {
2347         my ($param) = @_;
2348
2349         $param->{delete} = extract_param($param, 'idlist');
2350
2351         __PACKAGE__->update_vm($param);
2352
2353         return;
2354     }});
2355
2356 # uses good entropy, each char is limited to 6 bit to get printable chars simply
2357 my $gen_rand_chars = sub {
2358     my ($length) = @_;
2359
2360     die "invalid length $length" if $length < 1;
2361
2362     my $min = ord('!'); # first printable ascii
2363
2364     my $rand_bytes = Crypt::OpenSSL::Random::random_bytes($length);
2365     die "failed to generate random bytes!\n"
2366       if !$rand_bytes;
2367
2368     my $str = join('', map { chr((ord($_) & 0x3F) + $min) } split('', $rand_bytes));
2369
2370     return $str;
2371 };
2372
2373 my $sslcert;
2374
2375 __PACKAGE__->register_method({
2376     name => 'vncproxy',
2377     path => '{vmid}/vncproxy',
2378     method => 'POST',
2379     protected => 1,
2380     permissions => {
2381         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2382     },
2383     description => "Creates a TCP VNC proxy connections.",
2384     parameters => {
2385         additionalProperties => 0,
2386         properties => {
2387             node => get_standard_option('pve-node'),
2388             vmid => get_standard_option('pve-vmid'),
2389             websocket => {
2390                 optional => 1,
2391                 type => 'boolean',
2392                 description => "Prepare for websocket upgrade (only required when using "
2393                     ."serial terminal, otherwise upgrade is always possible).",
2394             },
2395             'generate-password' => {
2396                 optional => 1,
2397                 type => 'boolean',
2398                 default => 0,
2399                 description => "Generates a random password to be used as ticket instead of the API ticket.",
2400             },
2401         },
2402     },
2403     returns => {
2404         additionalProperties => 0,
2405         properties => {
2406             user => { type => 'string' },
2407             ticket => { type => 'string' },
2408             password => {
2409                 optional => 1,
2410                 description => "Returned if requested with 'generate-password' param."
2411                     ." Consists of printable ASCII characters ('!' .. '~').",
2412                 type => 'string',
2413             },
2414             cert => { type => 'string' },
2415             port => { type => 'integer' },
2416             upid => { type => 'string' },
2417         },
2418     },
2419     code => sub {
2420         my ($param) = @_;
2421
2422         my $rpcenv = PVE::RPCEnvironment::get();
2423
2424         my $authuser = $rpcenv->get_user();
2425
2426         my $vmid = $param->{vmid};
2427         my $node = $param->{node};
2428         my $websocket = $param->{websocket};
2429
2430         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2431
2432         my $serial;
2433         if ($conf->{vga}) {
2434             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2435             $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2436         }
2437
2438         my $authpath = "/vms/$vmid";
2439
2440         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2441         my $password = $ticket;
2442         if ($param->{'generate-password'}) {
2443             $password = $gen_rand_chars->(8);
2444         }
2445
2446         $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
2447             if !$sslcert;
2448
2449         my $family;
2450         my $remcmd = [];
2451
2452         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2453             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2454             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2455             # NOTE: kvm VNC traffic is already TLS encrypted or is known unsecure
2456             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, defined($serial) ? '-t' : '-T');
2457         } else {
2458             $family = PVE::Tools::get_host_address_family($node);
2459         }
2460
2461         my $port = PVE::Tools::next_vnc_port($family);
2462
2463         my $timeout = 10;
2464
2465         my $realcmd = sub {
2466             my $upid = shift;
2467
2468             syslog('info', "starting vnc proxy $upid\n");
2469
2470             my $cmd;
2471
2472             if (defined($serial)) {
2473
2474                 my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-iface', $serial, '-escape', '0' ];
2475
2476                 $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
2477                         '-timeout', $timeout, '-authpath', $authpath,
2478                         '-perm', 'Sys.Console'];
2479
2480                 if ($param->{websocket}) {
2481                     $ENV{PVE_VNC_TICKET} = $password; # pass ticket to vncterm
2482                     push @$cmd, '-notls', '-listen', 'localhost';
2483                 }
2484
2485                 push @$cmd, '-c', @$remcmd, @$termcmd;
2486
2487                 PVE::Tools::run_command($cmd);
2488
2489             } else {
2490
2491                 $ENV{LC_PVE_TICKET} = $password; # set ticket with "qm vncproxy"
2492
2493                 $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid];
2494
2495                 my $sock = IO::Socket::IP->new(
2496                     ReuseAddr => 1,
2497                     Listen => 1,
2498                     LocalPort => $port,
2499                     Proto => 'tcp',
2500                     GetAddrInfoFlags => 0,
2501                 ) or die "failed to create socket: $!\n";
2502                 # Inside the worker we shouldn't have any previous alarms
2503                 # running anyway...:
2504                 alarm(0);
2505                 local $SIG{ALRM} = sub { die "connection timed out\n" };
2506                 alarm $timeout;
2507                 accept(my $cli, $sock) or die "connection failed: $!\n";
2508                 alarm(0);
2509                 close($sock);
2510                 if (PVE::Tools::run_command($cmd,
2511                     output => '>&'.fileno($cli),
2512                     input => '<&'.fileno($cli),
2513                     noerr => 1) != 0)
2514                 {
2515                     die "Failed to run vncproxy.\n";
2516                 }
2517             }
2518
2519             return;
2520         };
2521
2522         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2523
2524         PVE::Tools::wait_for_vnc_port($port);
2525
2526         my $res = {
2527             user => $authuser,
2528             ticket => $ticket,
2529             port => $port,
2530             upid => $upid,
2531             cert => $sslcert,
2532         };
2533         $res->{password} = $password if $param->{'generate-password'};
2534
2535         return $res;
2536     }});
2537
2538 __PACKAGE__->register_method({
2539     name => 'termproxy',
2540     path => '{vmid}/termproxy',
2541     method => 'POST',
2542     protected => 1,
2543     permissions => {
2544         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2545     },
2546     description => "Creates a TCP proxy connections.",
2547     parameters => {
2548         additionalProperties => 0,
2549         properties => {
2550             node => get_standard_option('pve-node'),
2551             vmid => get_standard_option('pve-vmid'),
2552             serial=> {
2553                 optional => 1,
2554                 type => 'string',
2555                 enum => [qw(serial0 serial1 serial2 serial3)],
2556                 description => "opens a serial terminal (defaults to display)",
2557             },
2558         },
2559     },
2560     returns => {
2561         additionalProperties => 0,
2562         properties => {
2563             user => { type => 'string' },
2564             ticket => { type => 'string' },
2565             port => { type => 'integer' },
2566             upid => { type => 'string' },
2567         },
2568     },
2569     code => sub {
2570         my ($param) = @_;
2571
2572         my $rpcenv = PVE::RPCEnvironment::get();
2573
2574         my $authuser = $rpcenv->get_user();
2575
2576         my $vmid = $param->{vmid};
2577         my $node = $param->{node};
2578         my $serial = $param->{serial};
2579
2580         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2581
2582         if (!defined($serial)) {
2583             if ($conf->{vga}) {
2584                 my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2585                 $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2586             }
2587         }
2588
2589         my $authpath = "/vms/$vmid";
2590
2591         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2592
2593         my $family;
2594         my $remcmd = [];
2595
2596         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2597             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2598             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2599             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, '-t');
2600             push @$remcmd, '--';
2601         } else {
2602             $family = PVE::Tools::get_host_address_family($node);
2603         }
2604
2605         my $port = PVE::Tools::next_vnc_port($family);
2606
2607         my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-escape', '0'];
2608         push @$termcmd, '-iface', $serial if $serial;
2609
2610         my $realcmd = sub {
2611             my $upid = shift;
2612
2613             syslog('info', "starting qemu termproxy $upid\n");
2614
2615             my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
2616                        '--perm', 'VM.Console', '--'];
2617             push @$cmd, @$remcmd, @$termcmd;
2618
2619             PVE::Tools::run_command($cmd);
2620         };
2621
2622         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2623
2624         PVE::Tools::wait_for_vnc_port($port);
2625
2626         return {
2627             user => $authuser,
2628             ticket => $ticket,
2629             port => $port,
2630             upid => $upid,
2631         };
2632     }});
2633
2634 __PACKAGE__->register_method({
2635     name => 'vncwebsocket',
2636     path => '{vmid}/vncwebsocket',
2637     method => 'GET',
2638     permissions => {
2639         description => "You also need to pass a valid ticket (vncticket).",
2640         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2641     },
2642     description => "Opens a weksocket for VNC traffic.",
2643     parameters => {
2644         additionalProperties => 0,
2645         properties => {
2646             node => get_standard_option('pve-node'),
2647             vmid => get_standard_option('pve-vmid'),
2648             vncticket => {
2649                 description => "Ticket from previous call to vncproxy.",
2650                 type => 'string',
2651                 maxLength => 512,
2652             },
2653             port => {
2654                 description => "Port number returned by previous vncproxy call.",
2655                 type => 'integer',
2656                 minimum => 5900,
2657                 maximum => 5999,
2658             },
2659         },
2660     },
2661     returns => {
2662         type => "object",
2663         properties => {
2664             port => { type => 'string' },
2665         },
2666     },
2667     code => sub {
2668         my ($param) = @_;
2669
2670         my $rpcenv = PVE::RPCEnvironment::get();
2671
2672         my $authuser = $rpcenv->get_user();
2673
2674         my $vmid = $param->{vmid};
2675         my $node = $param->{node};
2676
2677         my $authpath = "/vms/$vmid";
2678
2679         PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);
2680
2681         my $conf = PVE::QemuConfig->load_config($vmid, $node); # VM exists ?
2682
2683         # Note: VNC ports are acessible from outside, so we do not gain any
2684         # security if we verify that $param->{port} belongs to VM $vmid. This
2685         # check is done by verifying the VNC ticket (inside VNC protocol).
2686
2687         my $port = $param->{port};
2688
2689         return { port => $port };
2690     }});
2691
2692 __PACKAGE__->register_method({
2693     name => 'spiceproxy',
2694     path => '{vmid}/spiceproxy',
2695     method => 'POST',
2696     protected => 1,
2697     proxyto => 'node',
2698     permissions => {
2699         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2700     },
2701     description => "Returns a SPICE configuration to connect to the VM.",
2702     parameters => {
2703         additionalProperties => 0,
2704         properties => {
2705             node => get_standard_option('pve-node'),
2706             vmid => get_standard_option('pve-vmid'),
2707             proxy => get_standard_option('spice-proxy', { optional => 1 }),
2708         },
2709     },
2710     returns => get_standard_option('remote-viewer-config'),
2711     code => sub {
2712         my ($param) = @_;
2713
2714         my $rpcenv = PVE::RPCEnvironment::get();
2715
2716         my $authuser = $rpcenv->get_user();
2717
2718         my $vmid = $param->{vmid};
2719         my $node = $param->{node};
2720         my $proxy = $param->{proxy};
2721
2722         my $conf = PVE::QemuConfig->load_config($vmid, $node);
2723         my $title = "VM $vmid";
2724         $title .= " - ". $conf->{name} if $conf->{name};
2725
2726         my $port = PVE::QemuServer::spice_port($vmid);
2727
2728         my ($ticket, undef, $remote_viewer_config) =
2729             PVE::AccessControl::remote_viewer_config($authuser, $vmid, $node, $proxy, $title, $port);
2730
2731         mon_cmd($vmid, "set_password", protocol => 'spice', password => $ticket);
2732         mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
2733
2734         return $remote_viewer_config;
2735     }});
2736
2737 __PACKAGE__->register_method({
2738     name => 'vmcmdidx',
2739     path => '{vmid}/status',
2740     method => 'GET',
2741     proxyto => 'node',
2742     description => "Directory index",
2743     permissions => {
2744         user => 'all',
2745     },
2746     parameters => {
2747         additionalProperties => 0,
2748         properties => {
2749             node => get_standard_option('pve-node'),
2750             vmid => get_standard_option('pve-vmid'),
2751         },
2752     },
2753     returns => {
2754         type => 'array',
2755         items => {
2756             type => "object",
2757             properties => {
2758                 subdir => { type => 'string' },
2759             },
2760         },
2761         links => [ { rel => 'child', href => "{subdir}" } ],
2762     },
2763     code => sub {
2764         my ($param) = @_;
2765
2766         # test if VM exists
2767         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2768
2769         my $res = [
2770             { subdir => 'current' },
2771             { subdir => 'start' },
2772             { subdir => 'stop' },
2773             { subdir => 'reset' },
2774             { subdir => 'shutdown' },
2775             { subdir => 'suspend' },
2776             { subdir => 'reboot' },
2777             ];
2778
2779         return $res;
2780     }});
2781
2782 __PACKAGE__->register_method({
2783     name => 'vm_status',
2784     path => '{vmid}/status/current',
2785     method => 'GET',
2786     proxyto => 'node',
2787     protected => 1, # qemu pid files are only readable by root
2788     description => "Get virtual machine status.",
2789     permissions => {
2790         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
2791     },
2792     parameters => {
2793         additionalProperties => 0,
2794         properties => {
2795             node => get_standard_option('pve-node'),
2796             vmid => get_standard_option('pve-vmid'),
2797         },
2798     },
2799     returns => {
2800         type => 'object',
2801         properties => {
2802             %$PVE::QemuServer::vmstatus_return_properties,
2803             ha => {
2804                 description => "HA manager service status.",
2805                 type => 'object',
2806             },
2807             spice => {
2808                 description => "QEMU VGA configuration supports spice.",
2809                 type => 'boolean',
2810                 optional => 1,
2811             },
2812             agent => {
2813                 description => "QEMU Guest Agent is enabled in config.",
2814                 type => 'boolean',
2815                 optional => 1,
2816             },
2817             clipboard => {
2818                 description => 'Enable a specific clipboard. If not set, depending on'
2819                     .' the display type the SPICE one will be added.',
2820                 type => 'string',
2821                 enum => ['vnc'],
2822                 optional => 1,
2823             },
2824         },
2825     },
2826     code => sub {
2827         my ($param) = @_;
2828
2829         # test if VM exists
2830         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2831
2832         my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1);
2833         my $status = $vmstatus->{$param->{vmid}};
2834
2835         $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}");
2836
2837         if ($conf->{vga}) {
2838             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2839             my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/;
2840             $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga});
2841             $status->{spice} = 1 if $spice;
2842             $status->{clipboard} = $vga->{clipboard};
2843         }
2844         $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled');
2845
2846         return $status;
2847     }});
2848
2849 __PACKAGE__->register_method({
2850     name => 'vm_start',
2851     path => '{vmid}/status/start',
2852     method => 'POST',
2853     protected => 1,
2854     proxyto => 'node',
2855     description => "Start virtual machine.",
2856     permissions => {
2857         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2858     },
2859     parameters => {
2860         additionalProperties => 0,
2861         properties => {
2862             node => get_standard_option('pve-node'),
2863             vmid => get_standard_option('pve-vmid',
2864                                         { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2865             skiplock => get_standard_option('skiplock'),
2866             stateuri => get_standard_option('pve-qm-stateuri'),
2867             migratedfrom => get_standard_option('pve-node',{ optional => 1 }),
2868             migration_type => {
2869                 type => 'string',
2870                 enum => ['secure', 'insecure'],
2871                 description => "Migration traffic is encrypted using an SSH " .
2872                   "tunnel by default. On secure, completely private networks " .
2873                   "this can be disabled to increase performance.",
2874                 optional => 1,
2875             },
2876             migration_network => {
2877                 type => 'string', format => 'CIDR',
2878                 description => "CIDR of the (sub) network that is used for migration.",
2879                 optional => 1,
2880             },
2881             machine => get_standard_option('pve-qemu-machine'),
2882             'force-cpu' => {
2883                 description => "Override QEMU's -cpu argument with the given string.",
2884                 type => 'string',
2885                 optional => 1,
2886             },
2887             targetstorage => get_standard_option('pve-targetstorage'),
2888             timeout => {
2889                 description => "Wait maximal timeout seconds.",
2890                 type => 'integer',
2891                 minimum => 0,
2892                 default => 'max(30, vm memory in GiB)',
2893                 optional => 1,
2894             },
2895         },
2896     },
2897     returns => {
2898         type => 'string',
2899     },
2900     code => sub {
2901         my ($param) = @_;
2902
2903         my $rpcenv = PVE::RPCEnvironment::get();
2904         my $authuser = $rpcenv->get_user();
2905
2906         my $node = extract_param($param, 'node');
2907         my $vmid = extract_param($param, 'vmid');
2908         my $timeout = extract_param($param, 'timeout');
2909         my $machine = extract_param($param, 'machine');
2910
2911         my $get_root_param = sub {
2912             my $value = extract_param($param, $_[0]);
2913             raise_param_exc({ "$_[0]" => "Only root may use this option." })
2914                 if $value && $authuser ne 'root@pam';
2915             return $value;
2916         };
2917
2918         my $stateuri = $get_root_param->('stateuri');
2919         my $skiplock = $get_root_param->('skiplock');
2920         my $migratedfrom = $get_root_param->('migratedfrom');
2921         my $migration_type = $get_root_param->('migration_type');
2922         my $migration_network = $get_root_param->('migration_network');
2923         my $targetstorage = $get_root_param->('targetstorage');
2924         my $force_cpu = $get_root_param->('force-cpu');
2925
2926         my $storagemap;
2927
2928         if ($targetstorage) {
2929             raise_param_exc({ targetstorage => "targetstorage can only by used with migratedfrom." })
2930                 if !$migratedfrom;
2931             $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
2932             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
2933                 if $@;
2934         }
2935
2936         # read spice ticket from STDIN
2937         my $spice_ticket;
2938         my $nbd_protocol_version = 0;
2939         my $replicated_volumes = {};
2940         my $offline_volumes = {};
2941         if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) {
2942             while (defined(my $line = <STDIN>)) {
2943                 chomp $line;
2944                 if ($line =~ m/^spice_ticket: (.+)$/) {
2945                     $spice_ticket = $1;
2946                 } elsif ($line =~ m/^nbd_protocol_version: (\d+)$/) {
2947                     $nbd_protocol_version = $1;
2948                 } elsif ($line =~ m/^replicated_volume: (.*)$/) {
2949                     $replicated_volumes->{$1} = 1;
2950                 } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead
2951                     $offline_volumes->{tpmstate0} = $1;
2952                 } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) {
2953                     $offline_volumes->{$1} = $2;
2954                 } elsif (!$spice_ticket) {
2955                     # fallback for old source node
2956                     $spice_ticket = $line;
2957                 } else {
2958                     warn "unknown 'start' parameter on STDIN: '$line'\n";
2959                 }
2960             }
2961         }
2962
2963         PVE::Cluster::check_cfs_quorum();
2964
2965         my $storecfg = PVE::Storage::config();
2966
2967         if (PVE::HA::Config::vm_is_ha_managed($vmid) && !$stateuri &&  $rpcenv->{type} ne 'ha') {
2968             my $hacmd = sub {
2969                 my $upid = shift;
2970
2971                 print "Requesting HA start for VM $vmid\n";
2972
2973                 my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
2974                 PVE::Tools::run_command($cmd);
2975                 return;
2976             };
2977
2978             return $rpcenv->fork_worker('hastart', $vmid, $authuser, $hacmd);
2979
2980         } else {
2981
2982             my $realcmd = sub {
2983                 my $upid = shift;
2984
2985                 syslog('info', "start VM $vmid: $upid\n");
2986
2987                 my $migrate_opts = {
2988                     migratedfrom => $migratedfrom,
2989                     spice_ticket => $spice_ticket,
2990                     network => $migration_network,
2991                     type => $migration_type,
2992                     storagemap => $storagemap,
2993                     nbd_proto_version => $nbd_protocol_version,
2994                     replicated_volumes => $replicated_volumes,
2995                     offline_volumes => $offline_volumes,
2996                 };
2997
2998                 my $params = {
2999                     statefile => $stateuri,
3000                     skiplock => $skiplock,
3001                     forcemachine => $machine,
3002                     timeout => $timeout,
3003                     forcecpu => $force_cpu,
3004                 };
3005
3006                 PVE::QemuServer::vm_start($storecfg, $vmid, $params, $migrate_opts);
3007                 return;
3008             };
3009
3010             return $rpcenv->fork_worker('qmstart', $vmid, $authuser, $realcmd);
3011         }
3012     }});
3013
3014 __PACKAGE__->register_method({
3015     name => 'vm_stop',
3016     path => '{vmid}/status/stop',
3017     method => 'POST',
3018     protected => 1,
3019     proxyto => 'node',
3020     description => "Stop virtual machine. The qemu process will exit immediately. This"
3021         ." is akin to pulling the power plug of a running computer and may damage the VM data.",
3022     permissions => {
3023         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3024     },
3025     parameters => {
3026         additionalProperties => 0,
3027         properties => {
3028             node => get_standard_option('pve-node'),
3029             vmid => get_standard_option('pve-vmid',
3030                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3031             skiplock => get_standard_option('skiplock'),
3032             migratedfrom => get_standard_option('pve-node', { optional => 1 }),
3033             timeout => {
3034                 description => "Wait maximal timeout seconds.",
3035                 type => 'integer',
3036                 minimum => 0,
3037                 optional => 1,
3038             },
3039             keepActive => {
3040                 description => "Do not deactivate storage volumes.",
3041                 type => 'boolean',
3042                 optional => 1,
3043                 default => 0,
3044             },
3045             'overrule-shutdown' => {
3046                 description => "Try to abort active 'qmshutdown' tasks before stopping.",
3047                 optional => 1,
3048                 type => 'boolean',
3049                 default => 0,
3050             },
3051         },
3052     },
3053     returns => {
3054         type => 'string',
3055     },
3056     code => sub {
3057         my ($param) = @_;
3058
3059         my $rpcenv = PVE::RPCEnvironment::get();
3060         my $authuser = $rpcenv->get_user();
3061
3062         my $node = extract_param($param, 'node');
3063         my $vmid = extract_param($param, 'vmid');
3064
3065         my $skiplock = extract_param($param, 'skiplock');
3066         raise_param_exc({ skiplock => "Only root may use this option." })
3067             if $skiplock && $authuser ne 'root@pam';
3068
3069         my $keepActive = extract_param($param, 'keepActive');
3070         raise_param_exc({ keepActive => "Only root may use this option." })
3071             if $keepActive && $authuser ne 'root@pam';
3072
3073         my $migratedfrom = extract_param($param, 'migratedfrom');
3074         raise_param_exc({ migratedfrom => "Only root may use this option." })
3075             if $migratedfrom && $authuser ne 'root@pam';
3076
3077         my $overrule_shutdown = extract_param($param, 'overrule-shutdown');
3078
3079         my $storecfg = PVE::Storage::config();
3080
3081         if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) {
3082             raise_param_exc({ 'overrule-shutdown' => "Not applicable for HA resources." })
3083                 if $overrule_shutdown;
3084
3085             my $hacmd = sub {
3086                 my $upid = shift;
3087
3088                 print "Requesting HA stop for VM $vmid\n";
3089
3090                 my $cmd = ['ha-manager', 'crm-command', 'stop',  "vm:$vmid", '0'];
3091                 PVE::Tools::run_command($cmd);
3092                 return;
3093             };
3094
3095             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
3096
3097         } else {
3098             my $realcmd = sub {
3099                 my $upid = shift;
3100
3101                 syslog('info', "stop VM $vmid: $upid\n");
3102
3103                 if ($overrule_shutdown) {
3104                     my $overruled_tasks = PVE::GuestHelpers::abort_guest_tasks(
3105                         $rpcenv, 'qmshutdown', $vmid);
3106                     my $overruled_tasks_list = join(", ", $overruled_tasks->@*);
3107                     print "overruled qmshutdown tasks: $overruled_tasks_list\n"
3108                         if @$overruled_tasks;
3109                 };
3110
3111                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0,
3112                                          $param->{timeout}, 0, 1, $keepActive, $migratedfrom);
3113                 return;
3114             };
3115
3116             return $rpcenv->fork_worker('qmstop', $vmid, $authuser, $realcmd);
3117         }
3118     }});
3119
3120 __PACKAGE__->register_method({
3121     name => 'vm_reset',
3122     path => '{vmid}/status/reset',
3123     method => 'POST',
3124     protected => 1,
3125     proxyto => 'node',
3126     description => "Reset virtual machine.",
3127     permissions => {
3128         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3129     },
3130     parameters => {
3131         additionalProperties => 0,
3132         properties => {
3133             node => get_standard_option('pve-node'),
3134             vmid => get_standard_option('pve-vmid',
3135                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3136             skiplock => get_standard_option('skiplock'),
3137         },
3138     },
3139     returns => {
3140         type => 'string',
3141     },
3142     code => sub {
3143         my ($param) = @_;
3144
3145         my $rpcenv = PVE::RPCEnvironment::get();
3146
3147         my $authuser = $rpcenv->get_user();
3148
3149         my $node = extract_param($param, 'node');
3150
3151         my $vmid = extract_param($param, 'vmid');
3152
3153         my $skiplock = extract_param($param, 'skiplock');
3154         raise_param_exc({ skiplock => "Only root may use this option." })
3155             if $skiplock && $authuser ne 'root@pam';
3156
3157         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3158
3159         my $realcmd = sub {
3160             my $upid = shift;
3161
3162             PVE::QemuServer::vm_reset($vmid, $skiplock);
3163
3164             return;
3165         };
3166
3167         return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd);
3168     }});
3169
3170 __PACKAGE__->register_method({
3171     name => 'vm_shutdown',
3172     path => '{vmid}/status/shutdown',
3173     method => 'POST',
3174     protected => 1,
3175     proxyto => 'node',
3176     description => "Shutdown virtual machine. This is similar to pressing the power button on a"
3177         ." physical machine. This will send an ACPI event for the guest OS, which should then"
3178         ." proceed to a clean shutdown.",
3179     permissions => {
3180         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3181     },
3182     parameters => {
3183         additionalProperties => 0,
3184         properties => {
3185             node => get_standard_option('pve-node'),
3186             vmid => get_standard_option('pve-vmid',
3187                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3188             skiplock => get_standard_option('skiplock'),
3189             timeout => {
3190                 description => "Wait maximal timeout seconds.",
3191                 type => 'integer',
3192                 minimum => 0,
3193                 optional => 1,
3194             },
3195             forceStop => {
3196                 description => "Make sure the VM stops.",
3197                 type => 'boolean',
3198                 optional => 1,
3199                 default => 0,
3200             },
3201             keepActive => {
3202                 description => "Do not deactivate storage volumes.",
3203                 type => 'boolean',
3204                 optional => 1,
3205                 default => 0,
3206             }
3207         },
3208     },
3209     returns => {
3210         type => 'string',
3211     },
3212     code => sub {
3213         my ($param) = @_;
3214
3215         my $rpcenv = PVE::RPCEnvironment::get();
3216         my $authuser = $rpcenv->get_user();
3217
3218         my $node = extract_param($param, 'node');
3219         my $vmid = extract_param($param, 'vmid');
3220
3221         my $skiplock = extract_param($param, 'skiplock');
3222         raise_param_exc({ skiplock => "Only root may use this option." })
3223             if $skiplock && $authuser ne 'root@pam';
3224
3225         my $keepActive = extract_param($param, 'keepActive');
3226         raise_param_exc({ keepActive => "Only root may use this option." })
3227             if $keepActive && $authuser ne 'root@pam';
3228
3229         my $storecfg = PVE::Storage::config();
3230
3231         my $shutdown = 1;
3232
3233         # sending a graceful shutdown command to paused VMs runs into timeouts, and even worse, when
3234         # the VM gets resumed later, it still gets the request delivered and powers off
3235         if (PVE::QemuServer::vm_is_paused($vmid, 1)) {
3236             if ($param->{forceStop}) {
3237                 warn "VM is paused - stop instead of shutdown\n";
3238                 $shutdown = 0;
3239             } else {
3240                 die "VM is paused - cannot shutdown\n";
3241             }
3242         }
3243
3244         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
3245
3246             my $timeout = $param->{timeout} // 60;
3247             my $hacmd = sub {
3248                 my $upid = shift;
3249
3250                 print "Requesting HA stop for VM $vmid\n";
3251
3252                 my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", "$timeout"];
3253                 PVE::Tools::run_command($cmd);
3254                 return;
3255             };
3256
3257             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
3258
3259         } else {
3260
3261             my $realcmd = sub {
3262                 my $upid = shift;
3263
3264                 syslog('info', "shutdown VM $vmid: $upid\n");
3265
3266                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0, $param->{timeout},
3267                                          $shutdown, $param->{forceStop}, $keepActive);
3268                 return;
3269             };
3270
3271             return $rpcenv->fork_worker('qmshutdown', $vmid, $authuser, $realcmd);
3272         }
3273     }});
3274
3275 __PACKAGE__->register_method({
3276     name => 'vm_reboot',
3277     path => '{vmid}/status/reboot',
3278     method => 'POST',
3279     protected => 1,
3280     proxyto => 'node',
3281     description => "Reboot the VM by shutting it down, and starting it again. Applies pending changes.",
3282     permissions => {
3283         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3284     },
3285     parameters => {
3286         additionalProperties => 0,
3287         properties => {
3288             node => get_standard_option('pve-node'),
3289             vmid => get_standard_option('pve-vmid',
3290                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3291             timeout => {
3292                 description => "Wait maximal timeout seconds for the shutdown.",
3293                 type => 'integer',
3294                 minimum => 0,
3295                 optional => 1,
3296             },
3297         },
3298     },
3299     returns => {
3300         type => 'string',
3301     },
3302     code => sub {
3303         my ($param) = @_;
3304
3305         my $rpcenv = PVE::RPCEnvironment::get();
3306         my $authuser = $rpcenv->get_user();
3307
3308         my $node = extract_param($param, 'node');
3309         my $vmid = extract_param($param, 'vmid');
3310
3311         die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid, 1);
3312
3313         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3314
3315         my $realcmd = sub {
3316             my $upid = shift;
3317
3318             syslog('info', "requesting reboot of VM $vmid: $upid\n");
3319             PVE::QemuServer::vm_reboot($vmid, $param->{timeout});
3320             return;
3321         };
3322
3323         return $rpcenv->fork_worker('qmreboot', $vmid, $authuser, $realcmd);
3324     }});
3325
3326 __PACKAGE__->register_method({
3327     name => 'vm_suspend',
3328     path => '{vmid}/status/suspend',
3329     method => 'POST',
3330     protected => 1,
3331     proxyto => 'node',
3332     description => "Suspend virtual machine.",
3333     permissions => {
3334         description => "You need 'VM.PowerMgmt' on /vms/{vmid}, and if you have set 'todisk',".
3335             " you need also 'VM.Config.Disk' on /vms/{vmid} and 'Datastore.AllocateSpace'".
3336             " on the storage for the vmstate.",
3337         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3338     },
3339     parameters => {
3340         additionalProperties => 0,
3341         properties => {
3342             node => get_standard_option('pve-node'),
3343             vmid => get_standard_option('pve-vmid',
3344                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3345             skiplock => get_standard_option('skiplock'),
3346             todisk => {
3347                 type => 'boolean',
3348                 default => 0,
3349                 optional => 1,
3350                 description => 'If set, suspends the VM to disk. Will be resumed on next VM start.',
3351             },
3352             statestorage => get_standard_option('pve-storage-id', {
3353                 description => "The storage for the VM state",
3354                 requires => 'todisk',
3355                 optional => 1,
3356                 completion => \&PVE::Storage::complete_storage_enabled,
3357             }),
3358         },
3359     },
3360     returns => {
3361         type => 'string',
3362     },
3363     code => sub {
3364         my ($param) = @_;
3365
3366         my $rpcenv = PVE::RPCEnvironment::get();
3367         my $authuser = $rpcenv->get_user();
3368
3369         my $node = extract_param($param, 'node');
3370         my $vmid = extract_param($param, 'vmid');
3371
3372         my $todisk = extract_param($param, 'todisk') // 0;
3373
3374         my $statestorage = extract_param($param, 'statestorage');
3375
3376         my $skiplock = extract_param($param, 'skiplock');
3377         raise_param_exc({ skiplock => "Only root may use this option." })
3378             if $skiplock && $authuser ne 'root@pam';
3379
3380         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3381
3382         die "Cannot suspend HA managed VM to disk\n"
3383             if $todisk && PVE::HA::Config::vm_is_ha_managed($vmid);
3384
3385         # early check for storage permission, for better user feedback
3386         if ($todisk) {
3387             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
3388             my $conf = PVE::QemuConfig->load_config($vmid);
3389
3390             # cannot save the state of a non-virtualized PCIe device, so resume cannot really work
3391             for my $key (keys %$conf) {
3392                 next if $key !~ /^hostpci\d+/;
3393                 die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the"
3394                     ." possibility to save/restore their internal state\n";
3395             }
3396
3397             if (!$statestorage) {
3398                 # get statestorage from config if none is given
3399                 my $storecfg = PVE::Storage::config();
3400                 $statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
3401             }
3402
3403             $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
3404         }
3405
3406         my $realcmd = sub {
3407             my $upid = shift;
3408
3409             syslog('info', "suspend VM $vmid: $upid\n");
3410
3411             PVE::QemuServer::vm_suspend($vmid, $skiplock, $todisk, $statestorage);
3412
3413             return;
3414         };
3415
3416         my $taskname = $todisk ? 'qmsuspend' : 'qmpause';
3417         return $rpcenv->fork_worker($taskname, $vmid, $authuser, $realcmd);
3418     }});
3419
3420 __PACKAGE__->register_method({
3421     name => 'vm_resume',
3422     path => '{vmid}/status/resume',
3423     method => 'POST',
3424     protected => 1,
3425     proxyto => 'node',
3426     description => "Resume virtual machine.",
3427     permissions => {
3428         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3429     },
3430     parameters => {
3431         additionalProperties => 0,
3432         properties => {
3433             node => get_standard_option('pve-node'),
3434             vmid => get_standard_option('pve-vmid',
3435                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3436             skiplock => get_standard_option('skiplock'),
3437             nocheck => { type => 'boolean', optional => 1 },
3438
3439         },
3440     },
3441     returns => {
3442         type => 'string',
3443     },
3444     code => sub {
3445         my ($param) = @_;
3446
3447         my $rpcenv = PVE::RPCEnvironment::get();
3448
3449         my $authuser = $rpcenv->get_user();
3450
3451         my $node = extract_param($param, 'node');
3452
3453         my $vmid = extract_param($param, 'vmid');
3454
3455         my $skiplock = extract_param($param, 'skiplock');
3456         raise_param_exc({ skiplock => "Only root may use this option." })
3457             if $skiplock && $authuser ne 'root@pam';
3458
3459         # nocheck is used as part of migration when config file might be still
3460         # be on source node
3461         my $nocheck = extract_param($param, 'nocheck');
3462         raise_param_exc({ nocheck => "Only root may use this option." })
3463             if $nocheck && $authuser ne 'root@pam';
3464
3465         my $to_disk_suspended;
3466         eval {
3467             PVE::QemuConfig->lock_config($vmid, sub {
3468                 my $conf = PVE::QemuConfig->load_config($vmid);
3469                 $to_disk_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
3470             });
3471         };
3472
3473         die "VM $vmid not running\n"
3474             if !$to_disk_suspended && !PVE::QemuServer::check_running($vmid, $nocheck);
3475
3476         my $realcmd = sub {
3477             my $upid = shift;
3478
3479             syslog('info', "resume VM $vmid: $upid\n");
3480
3481             if (!$to_disk_suspended) {
3482                 PVE::QemuServer::vm_resume($vmid, $skiplock, $nocheck);
3483             } else {
3484                 my $storecfg = PVE::Storage::config();
3485                 PVE::QemuServer::vm_start($storecfg, $vmid, { skiplock => $skiplock });
3486             }
3487
3488             return;
3489         };
3490
3491         return $rpcenv->fork_worker('qmresume', $vmid, $authuser, $realcmd);
3492     }});
3493
3494 __PACKAGE__->register_method({
3495     name => 'vm_sendkey',
3496     path => '{vmid}/sendkey',
3497     method => 'PUT',
3498     protected => 1,
3499     proxyto => 'node',
3500     description => "Send key event to virtual machine.",
3501     permissions => {
3502         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
3503     },
3504     parameters => {
3505         additionalProperties => 0,
3506         properties => {
3507             node => get_standard_option('pve-node'),
3508             vmid => get_standard_option('pve-vmid',
3509                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3510             skiplock => get_standard_option('skiplock'),
3511             key => {
3512                 description => "The key (qemu monitor encoding).",
3513                 type => 'string'
3514             }
3515         },
3516     },
3517     returns => { type => 'null'},
3518     code => sub {
3519         my ($param) = @_;
3520
3521         my $rpcenv = PVE::RPCEnvironment::get();
3522
3523         my $authuser = $rpcenv->get_user();
3524
3525         my $node = extract_param($param, 'node');
3526
3527         my $vmid = extract_param($param, 'vmid');
3528
3529         my $skiplock = extract_param($param, 'skiplock');
3530         raise_param_exc({ skiplock => "Only root may use this option." })
3531             if $skiplock && $authuser ne 'root@pam';
3532
3533         PVE::QemuServer::vm_sendkey($vmid, $skiplock, $param->{key});
3534
3535         return;
3536     }});
3537
3538 __PACKAGE__->register_method({
3539     name => 'vm_feature',
3540     path => '{vmid}/feature',
3541     method => 'GET',
3542     proxyto => 'node',
3543     protected => 1,
3544     description => "Check if feature for virtual machine is available.",
3545     permissions => {
3546         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
3547     },
3548     parameters => {
3549         additionalProperties => 0,
3550         properties => {
3551             node => get_standard_option('pve-node'),
3552             vmid => get_standard_option('pve-vmid'),
3553             feature => {
3554                 description => "Feature to check.",
3555                 type => 'string',
3556                 enum => [ 'snapshot', 'clone', 'copy' ],
3557             },
3558             snapname => get_standard_option('pve-snapshot-name', {
3559                 optional => 1,
3560             }),
3561         },
3562     },
3563     returns => {
3564         type => "object",
3565         properties => {
3566             hasFeature => { type => 'boolean' },
3567             nodes => {
3568                 type => 'array',
3569                 items => { type => 'string' },
3570             }
3571         },
3572     },
3573     code => sub {
3574         my ($param) = @_;
3575
3576         my $node = extract_param($param, 'node');
3577
3578         my $vmid = extract_param($param, 'vmid');
3579
3580         my $snapname = extract_param($param, 'snapname');
3581
3582         my $feature = extract_param($param, 'feature');
3583
3584         my $running = PVE::QemuServer::check_running($vmid);
3585
3586         my $conf = PVE::QemuConfig->load_config($vmid);
3587
3588         if($snapname){
3589             my $snap = $conf->{snapshots}->{$snapname};
3590             die "snapshot '$snapname' does not exist\n" if !defined($snap);
3591             $conf = $snap;
3592         }
3593         my $storecfg = PVE::Storage::config();
3594
3595         my $nodelist = PVE::QemuServer::shared_nodes($conf, $storecfg);
3596         my $hasFeature = PVE::QemuConfig->has_feature($feature, $conf, $storecfg, $snapname, $running);
3597
3598         return {
3599             hasFeature => $hasFeature,
3600             nodes => [ keys %$nodelist ],
3601         };
3602     }});
3603
3604 __PACKAGE__->register_method({
3605     name => 'clone_vm',
3606     path => '{vmid}/clone',
3607     method => 'POST',
3608     protected => 1,
3609     proxyto => 'node',
3610     description => "Create a copy of virtual machine/template.",
3611     permissions => {
3612         description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " .
3613             "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
3614             "'Datastore.AllocateSpace' on any used storage and 'SDN.Use' on any used bridge/vnet",
3615         check =>
3616         [ 'and',
3617           ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
3618           [ 'or',
3619             [ 'perm', '/vms/{newid}', ['VM.Allocate']],
3620             [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
3621           ],
3622         ]
3623     },
3624     parameters => {
3625         additionalProperties => 0,
3626         properties => {
3627             node => get_standard_option('pve-node'),
3628             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3629             newid => get_standard_option('pve-vmid', {
3630                 completion => \&PVE::Cluster::complete_next_vmid,
3631                 description => 'VMID for the clone.' }),
3632             name => {
3633                 optional => 1,
3634                 type => 'string', format => 'dns-name',
3635                 description => "Set a name for the new VM.",
3636             },
3637             description => {
3638                 optional => 1,
3639                 type => 'string',
3640                 description => "Description for the new VM.",
3641             },
3642             pool => {
3643                 optional => 1,
3644                 type => 'string', format => 'pve-poolid',
3645                 description => "Add the new VM to the specified pool.",
3646             },
3647             snapname => get_standard_option('pve-snapshot-name', {
3648                 optional => 1,
3649             }),
3650             storage => get_standard_option('pve-storage-id', {
3651                 description => "Target storage for full clone.",
3652                 optional => 1,
3653             }),
3654             'format' => {
3655                 description => "Target format for file storage. Only valid for full clone.",
3656                 type => 'string',
3657                 optional => 1,
3658                 enum => [ 'raw', 'qcow2', 'vmdk'],
3659             },
3660             full => {
3661                 optional => 1,
3662                 type => 'boolean',
3663                 description => "Create a full copy of all disks. This is always done when " .
3664                     "you clone a normal VM. For VM templates, we try to create a linked clone by default.",
3665             },
3666             target => get_standard_option('pve-node', {
3667                 description => "Target node. Only allowed if the original VM is on shared storage.",
3668                 optional => 1,
3669             }),
3670             bwlimit => {
3671                 description => "Override I/O bandwidth limit (in KiB/s).",
3672                 optional => 1,
3673                 type => 'integer',
3674                 minimum => '0',
3675                 default => 'clone limit from datacenter or storage config',
3676             },
3677         },
3678     },
3679     returns => {
3680         type => 'string',
3681     },
3682     code => sub {
3683         my ($param) = @_;
3684
3685         my $rpcenv = PVE::RPCEnvironment::get();
3686         my $authuser = $rpcenv->get_user();
3687
3688         my $node = extract_param($param, 'node');
3689         my $vmid = extract_param($param, 'vmid');
3690         my $newid = extract_param($param, 'newid');
3691         my $pool = extract_param($param, 'pool');
3692
3693         my $snapname = extract_param($param, 'snapname');
3694         my $storage = extract_param($param, 'storage');
3695         my $format = extract_param($param, 'format');
3696         my $target = extract_param($param, 'target');
3697
3698         my $localnode = PVE::INotify::nodename();
3699
3700         if ($target && ($target eq $localnode || $target eq 'localhost')) {
3701             undef $target;
3702         }
3703
3704         my $running = PVE::QemuServer::check_running($vmid) || 0;
3705
3706         my $load_and_check = sub {
3707             $rpcenv->check_pool_exist($pool) if defined($pool);
3708             PVE::Cluster::check_node_exists($target) if $target;
3709
3710             my $storecfg = PVE::Storage::config();
3711
3712             if ($storage) {
3713                 # check if storage is enabled on local node
3714                 PVE::Storage::storage_check_enabled($storecfg, $storage);
3715                 if ($target) {
3716                     # check if storage is available on target node
3717                     PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
3718                     # clone only works if target storage is shared
3719                     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
3720                     die "can't clone to non-shared storage '$storage'\n"
3721                         if !$scfg->{shared};
3722                 }
3723             }
3724
3725             PVE::Cluster::check_cfs_quorum();
3726
3727             my $conf = PVE::QemuConfig->load_config($vmid);
3728             PVE::QemuConfig->check_lock($conf);
3729
3730             my $verify_running = PVE::QemuServer::check_running($vmid) || 0;
3731             die "unexpected state change\n" if $verify_running != $running;
3732
3733             die "snapshot '$snapname' does not exist\n"
3734                 if $snapname && !defined( $conf->{snapshots}->{$snapname});
3735
3736             my $full = $param->{full} // !PVE::QemuConfig->is_template($conf);
3737
3738             die "parameter 'storage' not allowed for linked clones\n"
3739                 if defined($storage) && !$full;
3740
3741             die "parameter 'format' not allowed for linked clones\n"
3742                 if defined($format) && !$full;
3743
3744             my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf;
3745
3746             my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage);
3747             PVE::QemuServer::check_mapping_access($rpcenv, $authuser, $oldconf);
3748
3749             PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $oldconf);
3750
3751             die "can't clone VM to node '$target' (VM uses local storage)\n"
3752                 if $target && !$sharedvm;
3753
3754             my $conffile = PVE::QemuConfig->config_file($newid);
3755             die "unable to create VM $newid: config file already exists\n"
3756                 if -f $conffile;
3757
3758             my $newconf = { lock => 'clone' };
3759             my $drives = {};
3760             my $fullclone = {};
3761             my $vollist = [];
3762
3763             foreach my $opt (keys %$oldconf) {
3764                 my $value = $oldconf->{$opt};
3765
3766                 # do not copy snapshot related info
3767                 next if $opt eq 'snapshots' ||  $opt eq 'parent' || $opt eq 'snaptime' ||
3768                     $opt eq 'vmstate' || $opt eq 'snapstate';
3769
3770                 # no need to copy unused images, because VMID(owner) changes anyways
3771                 next if $opt =~ m/^unused\d+$/;
3772
3773                 die "cannot clone TPM state while VM is running\n"
3774                     if $full && $running && !$snapname && $opt eq 'tpmstate0';
3775
3776                 # always change MAC! address
3777                 if ($opt =~ m/^net(\d+)$/) {
3778                     my $net = PVE::QemuServer::parse_net($value);
3779                     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
3780                     $net->{macaddr} =  PVE::Tools::random_ether_addr($dc->{mac_prefix});
3781                     $newconf->{$opt} = PVE::QemuServer::print_net($net);
3782                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
3783                     my $drive = PVE::QemuServer::parse_drive($opt, $value);
3784                     die "unable to parse drive options for '$opt'\n" if !$drive;
3785                     if (PVE::QemuServer::drive_is_cdrom($drive, 1)) {
3786                         $newconf->{$opt} = $value; # simply copy configuration
3787                     } else {
3788                         if ($full || PVE::QemuServer::drive_is_cloudinit($drive)) {
3789                             die "Full clone feature is not supported for drive '$opt'\n"
3790                                 if !PVE::Storage::volume_has_feature($storecfg, 'copy', $drive->{file}, $snapname, $running);
3791                             $fullclone->{$opt} = 1;
3792                         } else {
3793                             # not full means clone instead of copy
3794                             die "Linked clone feature is not supported for drive '$opt'\n"
3795                                 if !PVE::Storage::volume_has_feature($storecfg, 'clone', $drive->{file}, $snapname, $running);
3796                         }
3797                         $drives->{$opt} = $drive;
3798                         next if PVE::QemuServer::drive_is_cloudinit($drive);
3799                         push @$vollist, $drive->{file};
3800                     }
3801                 } else {
3802                     # copy everything else
3803                     $newconf->{$opt} = $value;
3804                 }
3805             }
3806
3807             return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone);
3808         };
3809
3810         my $clonefn = sub {
3811             my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->();
3812             my $storecfg = PVE::Storage::config();
3813
3814             # auto generate a new uuid
3815             my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || '');
3816             $smbios1->{uuid} = PVE::QemuServer::generate_uuid();
3817             $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1);
3818             # auto generate a new vmgenid only if the option was set for template
3819             if ($newconf->{vmgenid}) {
3820                 $newconf->{vmgenid} = PVE::QemuServer::generate_uuid();
3821             }
3822
3823             delete $newconf->{template};
3824
3825             if ($param->{name}) {
3826                 $newconf->{name} = $param->{name};
3827             } else {
3828                 $newconf->{name} = "Copy-of-VM-" . ($oldconf->{name} // $vmid);
3829             }
3830
3831             if ($param->{description}) {
3832                 $newconf->{description} = $param->{description};
3833             }
3834
3835             # create empty/temp config - this fails if VM already exists on other node
3836             # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code
3837             PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n");
3838
3839             PVE::Firewall::clone_vmfw_conf($vmid, $newid);
3840
3841             my $newvollist = [];
3842             my $jobs = {};
3843
3844             eval {
3845                 local $SIG{INT} =
3846                     local $SIG{TERM} =
3847                     local $SIG{QUIT} =
3848                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3849
3850                 PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
3851
3852                 my $bwlimit = extract_param($param, 'bwlimit');
3853
3854                 my $total_jobs = scalar(keys %{$drives});
3855                 my $i = 1;
3856
3857                 foreach my $opt (sort keys %$drives) {
3858                     my $drive = $drives->{$opt};
3859                     my $skipcomplete = ($total_jobs != $i); # finish after last drive
3860                     my $completion = $skipcomplete ? 'skip' : 'complete';
3861
3862                     my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
3863                     my $storage_list = [ $src_sid ];
3864                     push @$storage_list, $storage if defined($storage);
3865                     my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
3866
3867                     my $source_info = {
3868                         vmid => $vmid,
3869                         running => $running,
3870                         drivename => $opt,
3871                         drive => $drive,
3872                         snapname => $snapname,
3873                     };
3874
3875                     my $dest_info = {
3876                         vmid => $newid,
3877                         drivename => $opt,
3878                         storage => $storage,
3879                         format => $format,
3880                     };
3881
3882                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf)
3883                         if $opt eq 'efidisk0';
3884
3885                     my $newdrive = PVE::QemuServer::clone_disk(
3886                         $storecfg,
3887                         $source_info,
3888                         $dest_info,
3889                         $fullclone->{$opt},
3890                         $newvollist,
3891                         $jobs,
3892                         $completion,
3893                         $oldconf->{agent},
3894                         $clonelimit,
3895                     );
3896
3897                     $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
3898
3899                     PVE::QemuConfig->write_config($newid, $newconf);
3900                     $i++;
3901                 }
3902
3903                 delete $newconf->{lock};
3904
3905                 # do not write pending changes
3906                 if (my @changes = keys %{$newconf->{pending}}) {
3907                     my $pending = join(',', @changes);
3908                     warn "found pending changes for '$pending', discarding for clone\n";
3909                     delete $newconf->{pending};
3910                 }
3911
3912                 PVE::QemuConfig->write_config($newid, $newconf);
3913
3914                 PVE::QemuServer::create_ifaces_ipams_ips($newconf, $newid);
3915
3916                 if ($target) {
3917                     if (!$running) {
3918                         # always deactivate volumes – avoids that LVM LVs are active on several nodes
3919                         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) };
3920                         # but only warn when that fails (e.g., parallel clones keeping them active)
3921                         log_warn($@) if $@;
3922                     }
3923
3924                     PVE::Storage::deactivate_volumes($storecfg, $newvollist);
3925
3926                     my $newconffile = PVE::QemuConfig->config_file($newid, $target);
3927                     die "Failed to move config to node '$target' - rename failed: $!\n"
3928                         if !rename($conffile, $newconffile);
3929                 }
3930
3931                 PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
3932             };
3933             if (my $err = $@) {
3934                 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
3935                 sleep 1; # some storage like rbd need to wait before release volume - really?
3936
3937                 foreach my $volid (@$newvollist) {
3938                     eval { PVE::Storage::vdisk_free($storecfg, $volid); };
3939                     warn $@ if $@;
3940                 }
3941
3942                 PVE::Firewall::remove_vmfw_conf($newid);
3943
3944                 unlink $conffile; # avoid races -> last thing before die
3945
3946                 die "clone failed: $err";
3947             }
3948
3949             return;
3950         };
3951
3952         # Aquire exclusive lock lock for $newid
3953         my $lock_target_vm = sub {
3954             return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn);
3955         };
3956
3957         my $lock_source_vm = sub {
3958             # exclusive lock if VM is running - else shared lock is enough;
3959             if ($running) {
3960                 return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
3961             } else {
3962                 return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
3963             }
3964         };
3965
3966         $load_and_check->(); # early checks before forking/locking
3967
3968         return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm);
3969     }});
3970
3971 __PACKAGE__->register_method({
3972     name => 'move_vm_disk',
3973     path => '{vmid}/move_disk',
3974     method => 'POST',
3975     protected => 1,
3976     proxyto => 'node',
3977     description => "Move volume to different storage or to a different VM.",
3978     permissions => {
3979         description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
3980             "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
3981             "a disk to another VM, you need the permissions on the target VM as well.",
3982         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
3983     },
3984     parameters => {
3985         additionalProperties => 0,
3986         properties => {
3987             node => get_standard_option('pve-node'),
3988             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3989             'target-vmid' => get_standard_option('pve-vmid', {
3990                 completion => \&PVE::QemuServer::complete_vmid,
3991                 optional => 1,
3992             }),
3993             disk => {
3994                 type => 'string',
3995                 description => "The disk you want to move.",
3996                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3997             },
3998             storage => get_standard_option('pve-storage-id', {
3999                 description => "Target storage.",
4000                 completion => \&PVE::QemuServer::complete_storage,
4001                 optional => 1,
4002             }),
4003             'format' => {
4004                 type => 'string',
4005                 description => "Target Format.",
4006                 enum => [ 'raw', 'qcow2', 'vmdk' ],
4007                 optional => 1,
4008             },
4009             delete => {
4010                 type => 'boolean',
4011                 description => "Delete the original disk after successful copy. By default the"
4012                     ." original disk is kept as unused disk.",
4013                 optional => 1,
4014                 default => 0,
4015             },
4016             digest => {
4017                 type => 'string',
4018                 description => 'Prevent changes if current configuration file has different SHA1"
4019                     ." digest. This can be used to prevent concurrent modifications.',
4020                 maxLength => 40,
4021                 optional => 1,
4022             },
4023             bwlimit => {
4024                 description => "Override I/O bandwidth limit (in KiB/s).",
4025                 optional => 1,
4026                 type => 'integer',
4027                 minimum => '0',
4028                 default => 'move limit from datacenter or storage config',
4029             },
4030             'target-disk' => {
4031                 type => 'string',
4032                 description => "The config key the disk will be moved to on the target VM"
4033                     ." (for example, ide0 or scsi1). Default is the source disk key.",
4034                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
4035                 optional => 1,
4036             },
4037             'target-digest' => {
4038                 type => 'string',
4039                 description => 'Prevent changes if the current config file of the target VM has a"
4040                     ." different SHA1 digest. This can be used to detect concurrent modifications.',
4041                 maxLength => 40,
4042                 optional => 1,
4043             },
4044         },
4045     },
4046     returns => {
4047         type => 'string',
4048         description => "the task ID.",
4049     },
4050     code => sub {
4051         my ($param) = @_;
4052
4053         my $rpcenv = PVE::RPCEnvironment::get();
4054         my $authuser = $rpcenv->get_user();
4055
4056         my $node = extract_param($param, 'node');
4057         my $vmid = extract_param($param, 'vmid');
4058         my $target_vmid = extract_param($param, 'target-vmid');
4059         my $digest = extract_param($param, 'digest');
4060         my $target_digest = extract_param($param, 'target-digest');
4061         my $disk = extract_param($param, 'disk');
4062         my $target_disk = extract_param($param, 'target-disk') // $disk;
4063         my $storeid = extract_param($param, 'storage');
4064         my $format = extract_param($param, 'format');
4065
4066         my $storecfg = PVE::Storage::config();
4067
4068         my $load_and_check_move = sub {
4069             my $conf = PVE::QemuConfig->load_config($vmid);
4070             PVE::QemuConfig->check_lock($conf);
4071
4072             PVE::Tools::assert_if_modified($digest, $conf->{digest});
4073
4074             die "disk '$disk' does not exist\n" if !$conf->{$disk};
4075
4076             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
4077
4078             die "disk '$disk' has no associated volume\n" if !$drive->{file};
4079             die "you can't move a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive, 1);
4080
4081             my $old_volid = $drive->{file};
4082             my $oldfmt;
4083             my ($oldstoreid, $oldvolname) = PVE::Storage::parse_volume_id($old_volid);
4084             if ($oldvolname =~ m/\.(raw|qcow2|vmdk)$/){
4085                 $oldfmt = $1;
4086             }
4087
4088             die "you can't move to the same storage with same format\n"
4089                 if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format);
4090
4091             # this only checks snapshots because $disk is passed!
4092             my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use(
4093                 $storecfg,
4094                 $conf,
4095                 $disk,
4096                 $old_volid
4097             );
4098             die "you can't move a disk with snapshots and delete the source\n"
4099                 if $snapshotted && $param->{delete};
4100
4101             return ($conf, $drive, $oldstoreid, $snapshotted);
4102         };
4103
4104         my $move_updatefn = sub {
4105             my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->();
4106             my $old_volid = $drive->{file};
4107
4108             PVE::Cluster::log_msg(
4109                 'info',
4110                 $authuser,
4111                 "move disk VM $vmid: move --disk $disk --storage $storeid"
4112             );
4113
4114             my $running = PVE::QemuServer::check_running($vmid);
4115
4116             PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]);
4117
4118             my $newvollist = [];
4119
4120             eval {
4121                 local $SIG{INT} =
4122                     local $SIG{TERM} =
4123                     local $SIG{QUIT} =
4124                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
4125
4126                 warn "moving disk with snapshots, snapshots will not be moved!\n"
4127                     if $snapshotted;
4128
4129                 my $bwlimit = extract_param($param, 'bwlimit');
4130                 my $movelimit = PVE::Storage::get_bandwidth_limit(
4131                     'move',
4132                     [$oldstoreid, $storeid],
4133                     $bwlimit
4134                 );
4135
4136                 my $source_info = {
4137                     vmid => $vmid,
4138                     running => $running,
4139                     drivename => $disk,
4140                     drive => $drive,
4141                     snapname => undef,
4142                 };
4143
4144                 my $dest_info = {
4145                     vmid => $vmid,
4146                     drivename => $disk,
4147                     storage => $storeid,
4148                     format => $format,
4149                 };
4150
4151                 $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf)
4152                     if $disk eq 'efidisk0';
4153
4154                 my $newdrive = PVE::QemuServer::clone_disk(
4155                     $storecfg,
4156                     $source_info,
4157                     $dest_info,
4158                     1,
4159                     $newvollist,
4160                     undef,
4161                     undef,
4162                     undef,
4163                     $movelimit,
4164                 );
4165                 $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
4166
4167                 PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
4168
4169                 # convert moved disk to base if part of template
4170                 PVE::QemuServer::template_create($vmid, $conf, $disk)
4171                     if PVE::QemuConfig->is_template($conf);
4172
4173                 PVE::QemuConfig->write_config($vmid, $conf);
4174
4175                 my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks');
4176                 if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
4177                     eval { mon_cmd($vmid, "guest-fstrim") };
4178                 }
4179
4180                 eval {
4181                     # try to deactivate volumes - avoid lvm LVs to be active on several nodes
4182                     PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
4183                         if !$running;
4184                 };
4185                 warn $@ if $@;
4186             };
4187             if (my $err = $@) {
4188                 foreach my $volid (@$newvollist) {
4189                     eval { PVE::Storage::vdisk_free($storecfg, $volid) };
4190                     warn $@ if $@;
4191                 }
4192                 die "storage migration failed: $err";
4193             }
4194
4195             if ($param->{delete}) {
4196                 eval {
4197                     PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
4198                     PVE::Storage::vdisk_free($storecfg, $old_volid);
4199                 };
4200                 warn $@ if $@;
4201             }
4202         };
4203
4204         my $load_and_check_reassign_configs = sub {
4205             my $vmlist = PVE::Cluster::get_vmlist()->{ids};
4206
4207             die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid});
4208             die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
4209
4210             my $source_node = $vmlist->{$vmid}->{node};
4211             my $target_node = $vmlist->{$target_vmid}->{node};
4212
4213             die "Both VMs need to be on the same node ($source_node != $target_node)\n"
4214                 if $source_node ne $target_node;
4215
4216             my $source_conf = PVE::QemuConfig->load_config($vmid);
4217             PVE::QemuConfig->check_lock($source_conf);
4218             my $target_conf = PVE::QemuConfig->load_config($target_vmid);
4219             PVE::QemuConfig->check_lock($target_conf);
4220
4221             die "Can't move disks from or to template VMs\n"
4222                 if ($source_conf->{template} || $target_conf->{template});
4223
4224             if ($digest) {
4225                 eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
4226                 die "VM ${vmid}: $@" if $@;
4227             }
4228
4229             if ($target_digest) {
4230                 eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
4231                 die "VM ${target_vmid}: $@" if $@;
4232             }
4233
4234             die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk});
4235
4236             die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n"
4237                 if $target_conf->{$target_disk};
4238
4239             my $drive = PVE::QemuServer::parse_drive(
4240                 $disk,
4241                 $source_conf->{$disk},
4242             );
4243             die "failed to parse source disk - $@\n" if !$drive;
4244
4245             my $source_volid = $drive->{file};
4246
4247             die "disk '${disk}' has no associated volume\n" if !$source_volid;
4248             die "CD drive contents can't be moved to another VM\n"
4249                 if PVE::QemuServer::drive_is_cdrom($drive, 1);
4250
4251             my $storeid = PVE::Storage::parse_volume_id($source_volid, 1);
4252             die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid);
4253
4254             die "Can't move disk used by a snapshot to another VM\n"
4255                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid);
4256             die "Storage does not support moving of this disk to another VM\n"
4257                 if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid));
4258             die "Cannot move disk to another VM while the source VM is running - detach first\n"
4259                 if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/;
4260
4261             # now re-parse using target disk slot format
4262             if ($target_disk =~ /^unused\d+$/) {
4263                 $drive = PVE::QemuServer::parse_drive(
4264                     $target_disk,
4265                     $source_volid,
4266                 );
4267             } else {
4268                 $drive = PVE::QemuServer::parse_drive(
4269                     $target_disk,
4270                     $source_conf->{$disk},
4271                 );
4272             }
4273             die "failed to parse source disk for target disk format - $@\n" if !$drive;
4274
4275             my $repl_conf = PVE::ReplicationConfig->new();
4276             if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
4277                 my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4278                 die "Cannot move disk to a replicated VM. Storage does not support replication!\n"
4279                     if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
4280             }
4281
4282             return ($source_conf, $target_conf, $drive);
4283         };
4284
4285         my $logfunc = sub {
4286             my ($msg) = @_;
4287             print STDERR "$msg\n";
4288         };
4289
4290         my $disk_reassignfn = sub {
4291             return PVE::QemuConfig->lock_config($vmid, sub {
4292                 return PVE::QemuConfig->lock_config($target_vmid, sub {
4293                     my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs();
4294
4295                     my $source_volid = $drive->{file};
4296
4297                     print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n";
4298                     my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
4299
4300                     my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4301
4302                     my $new_volid = PVE::Storage::rename_volume(
4303                         $storecfg,
4304                         $source_volid,
4305                         $target_vmid,
4306                     );
4307
4308                     $drive->{file} = $new_volid;
4309
4310                     my $boot_order = PVE::QemuServer::device_bootorder($source_conf);
4311                     if (defined(delete $boot_order->{$disk})) {
4312                         print "removing disk '$disk' from boot order config\n";
4313                         my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ];
4314                         $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs);
4315                     }
4316
4317                     delete $source_conf->{$disk};
4318                     print "removing disk '${disk}' from VM '${vmid}' config\n";
4319                     PVE::QemuConfig->write_config($vmid, $source_conf);
4320
4321                     my $drive_string = PVE::QemuServer::print_drive($drive);
4322
4323                     if ($target_disk =~ /^unused\d+$/) {
4324                         $target_conf->{$target_disk} = $drive_string;
4325                         PVE::QemuConfig->write_config($target_vmid, $target_conf);
4326                     } else {
4327                         &$update_vm_api(
4328                             {
4329                                 node => $node,
4330                                 vmid => $target_vmid,
4331                                 digest => $target_digest,
4332                                 $target_disk => $drive_string,
4333                             },
4334                             1,
4335                         );
4336                     }
4337
4338                     # remove possible replication snapshots
4339                     if (PVE::Storage::volume_has_feature(
4340                             $storecfg,
4341                             'replicate',
4342                             $source_volid),
4343                     ) {
4344                         eval {
4345                             PVE::Replication::prepare(
4346                                 $storecfg,
4347                                 [$new_volid],
4348                                 undef,
4349                                 1,
4350                                 undef,
4351                                 $logfunc,
4352                             )
4353                         };
4354                         if (my $err = $@) {
4355                             print "Failed to remove replication snapshots on moved disk " .
4356                                 "'$target_disk'. Manual cleanup could be necessary.\n";
4357                         }
4358                     }
4359                 });
4360             });
4361         };
4362
4363         if ($target_vmid && $storeid) {
4364             my $msg = "either set 'storage' or 'target-vmid', but not both";
4365             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4366         } elsif ($target_vmid) {
4367             $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
4368                 if $authuser ne 'root@pam';
4369
4370             raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" })
4371                 if $vmid eq $target_vmid;
4372
4373             my (undef, undef, $drive) = &$load_and_check_reassign_configs();
4374             my $storage = PVE::Storage::parse_volume_id($drive->{file});
4375             $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
4376
4377             return $rpcenv->fork_worker(
4378                 'qmmove',
4379                 "${vmid}-${disk}>${target_vmid}-${target_disk}",
4380                 $authuser,
4381                 $disk_reassignfn
4382             );
4383         } elsif ($storeid) {
4384             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4385
4386             $load_and_check_move->(); # early checks before forking/locking
4387
4388             my $realcmd = sub {
4389                 PVE::QemuConfig->lock_config($vmid, $move_updatefn);
4390             };
4391
4392             return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
4393         } else {
4394             my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
4395             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4396         }
4397     }});
4398
4399 my $check_vm_disks_local = sub {
4400     my ($storecfg, $vmconf, $vmid) = @_;
4401
4402     my $local_disks = {};
4403
4404     # add some more information to the disks e.g. cdrom
4405     PVE::QemuServer::foreach_volid($vmconf, sub {
4406         my ($volid, $attr) = @_;
4407
4408         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4409         if ($storeid) {
4410             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4411             return if $scfg->{shared};
4412         }
4413         # The shared attr here is just a special case where the vdisk
4414         # is marked as shared manually
4415         return if $attr->{shared};
4416         return if $attr->{cdrom} and $volid eq "none";
4417
4418         if (exists $local_disks->{$volid}) {
4419             @{$local_disks->{$volid}}{keys %$attr} = values %$attr
4420         } else {
4421             $local_disks->{$volid} = $attr;
4422             # ensure volid is present in case it's needed
4423             $local_disks->{$volid}->{volid} = $volid;
4424         }
4425     });
4426
4427     return $local_disks;
4428 };
4429
4430 __PACKAGE__->register_method({
4431     name => 'migrate_vm_precondition',
4432     path => '{vmid}/migrate',
4433     method => 'GET',
4434     protected => 1,
4435     proxyto => 'node',
4436     description => "Get preconditions for migration.",
4437     permissions => {
4438         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4439     },
4440     parameters => {
4441         additionalProperties => 0,
4442         properties => {
4443             node => get_standard_option('pve-node'),
4444             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4445             target => get_standard_option('pve-node', {
4446                 description => "Target node.",
4447                 completion =>  \&PVE::Cluster::complete_migration_target,
4448                 optional => 1,
4449             }),
4450         },
4451     },
4452     returns => {
4453         type => "object",
4454         properties => {
4455             running => { type => 'boolean' },
4456             allowed_nodes => {
4457                 type => 'array',
4458                 optional => 1,
4459                 description => "List nodes allowed for offline migration, only passed if VM is offline"
4460             },
4461             not_allowed_nodes => {
4462                 type => 'object',
4463                 optional => 1,
4464                 description => "List not allowed nodes with additional informations, only passed if VM is offline"
4465             },
4466             local_disks => {
4467                 type => 'array',
4468                 description => "List local disks including CD-Rom, unsused and not referenced disks"
4469             },
4470             local_resources => {
4471                 type => 'array',
4472                 description => "List local resources e.g. pci, usb"
4473             },
4474             'mapped-resources' => {
4475                 type => 'array',
4476                 description => "List of mapped resources e.g. pci, usb"
4477             },
4478         },
4479     },
4480     code => sub {
4481         my ($param) = @_;
4482
4483         my $rpcenv = PVE::RPCEnvironment::get();
4484
4485         my $authuser = $rpcenv->get_user();
4486
4487         PVE::Cluster::check_cfs_quorum();
4488
4489         my $res = {};
4490
4491         my $vmid = extract_param($param, 'vmid');
4492         my $target = extract_param($param, 'target');
4493         my $localnode = PVE::INotify::nodename();
4494
4495
4496         # test if VM exists
4497         my $vmconf = PVE::QemuConfig->load_config($vmid);
4498         my $storecfg = PVE::Storage::config();
4499
4500
4501         # try to detect errors early
4502         PVE::QemuConfig->check_lock($vmconf);
4503
4504         $res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0;
4505
4506         my ($local_resources, $mapped_resources, $missing_mappings_by_node) =
4507             PVE::QemuServer::check_local_resources($vmconf, 1);
4508         delete $missing_mappings_by_node->{$localnode};
4509
4510         my $vga = PVE::QemuServer::parse_vga($vmconf->{vga});
4511         if ($res->{running} && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
4512             push $local_resources->@*, "clipboard=vnc";
4513         }
4514
4515         # if vm is not running, return target nodes where local storage/mapped devices are available
4516         # for offline migration
4517         if (!$res->{running}) {
4518             $res->{allowed_nodes} = [];
4519             my $checked_nodes = PVE::QemuServer::check_local_storage_availability($vmconf, $storecfg);
4520             delete $checked_nodes->{$localnode};
4521
4522             foreach my $node (keys %$checked_nodes) {
4523                 my $missing_mappings = $missing_mappings_by_node->{$node};
4524                 if (scalar($missing_mappings->@*)) {
4525                     $checked_nodes->{$node}->{'unavailable-resources'} = $missing_mappings;
4526                     next;
4527                 }
4528
4529                 if (!defined($checked_nodes->{$node}->{unavailable_storages})) {
4530                     push @{$res->{allowed_nodes}}, $node;
4531                 }
4532
4533             }
4534             $res->{not_allowed_nodes} = $checked_nodes;
4535         }
4536
4537         my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid);
4538         $res->{local_disks} = [ values %$local_disks ];;
4539
4540         $res->{local_resources} = $local_resources;
4541         $res->{'mapped-resources'} = $mapped_resources;
4542
4543         return $res;
4544
4545
4546     }});
4547
4548 __PACKAGE__->register_method({
4549     name => 'migrate_vm',
4550     path => '{vmid}/migrate',
4551     method => 'POST',
4552     protected => 1,
4553     proxyto => 'node',
4554     description => "Migrate virtual machine. Creates a new migration task.",
4555     permissions => {
4556         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4557     },
4558     parameters => {
4559         additionalProperties => 0,
4560         properties => {
4561             node => get_standard_option('pve-node'),
4562             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4563             target => get_standard_option('pve-node', {
4564                 description => "Target node.",
4565                 completion =>  \&PVE::Cluster::complete_migration_target,
4566             }),
4567             online => {
4568                 type => 'boolean',
4569                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4570                 optional => 1,
4571             },
4572             force => {
4573                 type => 'boolean',
4574                 description => "Allow to migrate VMs which use local devices. Only root may use this option.",
4575                 optional => 1,
4576             },
4577             migration_type => {
4578                 type => 'string',
4579                 enum => ['secure', 'insecure'],
4580                 description => "Migration traffic is encrypted using an SSH tunnel by default. On secure, completely private networks this can be disabled to increase performance.",
4581                 optional => 1,
4582             },
4583             migration_network => {
4584                 type => 'string', format => 'CIDR',
4585                 description => "CIDR of the (sub) network that is used for migration.",
4586                 optional => 1,
4587             },
4588             "with-local-disks" => {
4589                 type => 'boolean',
4590                 description => "Enable live storage migration for local disk",
4591                 optional => 1,
4592             },
4593             targetstorage => get_standard_option('pve-targetstorage', {
4594                 completion => \&PVE::QemuServer::complete_migration_storage,
4595             }),
4596             bwlimit => {
4597                 description => "Override I/O bandwidth limit (in KiB/s).",
4598                 optional => 1,
4599                 type => 'integer',
4600                 minimum => '0',
4601                 default => 'migrate limit from datacenter or storage config',
4602             },
4603         },
4604     },
4605     returns => {
4606         type => 'string',
4607         description => "the task ID.",
4608     },
4609     code => sub {
4610         my ($param) = @_;
4611
4612         my $rpcenv = PVE::RPCEnvironment::get();
4613         my $authuser = $rpcenv->get_user();
4614
4615         my $target = extract_param($param, 'target');
4616
4617         my $localnode = PVE::INotify::nodename();
4618         raise_param_exc({ target => "target is local node."}) if $target eq $localnode;
4619
4620         PVE::Cluster::check_cfs_quorum();
4621
4622         PVE::Cluster::check_node_exists($target);
4623
4624         my $targetip = PVE::Cluster::remote_node_ip($target);
4625
4626         my $vmid = extract_param($param, 'vmid');
4627
4628         raise_param_exc({ force => "Only root may use this option." })
4629             if $param->{force} && $authuser ne 'root@pam';
4630
4631         raise_param_exc({ migration_type => "Only root may use this option." })
4632             if $param->{migration_type} && $authuser ne 'root@pam';
4633
4634         # allow root only until better network permissions are available
4635         raise_param_exc({ migration_network => "Only root may use this option." })
4636             if $param->{migration_network} && $authuser ne 'root@pam';
4637
4638         # test if VM exists
4639         my $conf = PVE::QemuConfig->load_config($vmid);
4640
4641         # try to detect errors early
4642
4643         PVE::QemuConfig->check_lock($conf);
4644
4645         if (PVE::QemuServer::check_running($vmid)) {
4646             die "can't migrate running VM without --online\n" if !$param->{online};
4647
4648             my $repl_conf = PVE::ReplicationConfig->new();
4649             my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
4650             my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target));
4651             if (!$param->{force} && $is_replicated && !$is_replicated_to_target) {
4652                 die "Cannot live-migrate replicated VM to node '$target' - not a replication " .
4653                     "target. Use 'force' to override.\n";
4654             }
4655         } else {
4656             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4657             $param->{online} = 0;
4658         }
4659
4660         my $storecfg = PVE::Storage::config();
4661         if (my $targetstorage = $param->{targetstorage}) {
4662             my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
4663             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
4664                 if $@;
4665
4666             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
4667                 if !defined($storagemap->{identity});
4668
4669             foreach my $target_sid (values %{$storagemap->{entries}}) {
4670                 $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
4671             }
4672
4673             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
4674                 if $storagemap->{default};
4675
4676             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target)
4677                 if $storagemap->{identity};
4678
4679             $param->{storagemap} = $storagemap;
4680         } else {
4681             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target);
4682         }
4683
4684         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
4685
4686             my $hacmd = sub {
4687                 my $upid = shift;
4688
4689                 print "Requesting HA migration for VM $vmid to node $target\n";
4690
4691                 my $cmd = ['ha-manager', 'migrate', "vm:$vmid", $target];
4692                 PVE::Tools::run_command($cmd);
4693                 return;
4694             };
4695
4696             return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);
4697
4698         } else {
4699
4700             my $realcmd = sub {
4701                 PVE::QemuMigrate->migrate($target, $targetip, $vmid, $param);
4702             };
4703
4704             my $worker = sub {
4705                 return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
4706             };
4707
4708             return $rpcenv->fork_worker('qmigrate', $vmid, $authuser, $worker);
4709         }
4710
4711     }});
4712
4713 __PACKAGE__->register_method({
4714     name => 'remote_migrate_vm',
4715     path => '{vmid}/remote_migrate',
4716     method => 'POST',
4717     protected => 1,
4718     proxyto => 'node',
4719     description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!",
4720     permissions => {
4721         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4722     },
4723     parameters => {
4724         additionalProperties => 0,
4725         properties => {
4726             node => get_standard_option('pve-node'),
4727             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4728             'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
4729             'target-endpoint' => get_standard_option('proxmox-remote', {
4730                 description => "Remote target endpoint",
4731             }),
4732             online => {
4733                 type => 'boolean',
4734                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4735                 optional => 1,
4736             },
4737             delete => {
4738                 type => 'boolean',
4739                 description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.",
4740                 optional => 1,
4741                 default => 0,
4742             },
4743             'target-storage' => get_standard_option('pve-targetstorage', {
4744                 completion => \&PVE::QemuServer::complete_migration_storage,
4745                 optional => 0,
4746             }),
4747             'target-bridge' => {
4748                 type => 'string',
4749                 description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
4750                 format => 'bridge-pair-list',
4751             },
4752             bwlimit => {
4753                 description => "Override I/O bandwidth limit (in KiB/s).",
4754                 optional => 1,
4755                 type => 'integer',
4756                 minimum => '0',
4757                 default => 'migrate limit from datacenter or storage config',
4758             },
4759         },
4760     },
4761     returns => {
4762         type => 'string',
4763         description => "the task ID.",
4764     },
4765     code => sub {
4766         my ($param) = @_;
4767
4768         my $rpcenv = PVE::RPCEnvironment::get();
4769         my $authuser = $rpcenv->get_user();
4770
4771         my $source_vmid = extract_param($param, 'vmid');
4772         my $target_endpoint = extract_param($param, 'target-endpoint');
4773         my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
4774
4775         my $delete = extract_param($param, 'delete') // 0;
4776
4777         PVE::Cluster::check_cfs_quorum();
4778
4779         # test if VM exists
4780         my $conf = PVE::QemuConfig->load_config($source_vmid);
4781
4782         PVE::QemuConfig->check_lock($conf);
4783
4784         raise_param_exc({ vmid => "cannot migrate HA-managed VM to remote cluster" })
4785             if PVE::HA::Config::vm_is_ha_managed($source_vmid);
4786
4787         my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);
4788
4789         # TODO: move this as helper somewhere appropriate?
4790         my $conn_args = {
4791             protocol => 'https',
4792             host => $remote->{host},
4793             port => $remote->{port} // 8006,
4794             apitoken => $remote->{apitoken},
4795         };
4796
4797         my $fp;
4798         if ($fp = $remote->{fingerprint}) {
4799             $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
4800         }
4801
4802         print "Establishing API connection with remote at '$remote->{host}'\n";
4803
4804         my $api_client = PVE::APIClient::LWP->new(%$conn_args);
4805
4806         if (!defined($fp)) {
4807             my $cert_info = $api_client->get("/nodes/localhost/certificates/info");
4808             foreach my $cert (@$cert_info) {
4809                 my $filename = $cert->{filename};
4810                 next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem';
4811                 $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem';
4812             }
4813             $conn_args->{cached_fingerprints} = { uc($fp) => 1 }
4814                 if defined($fp);
4815         }
4816
4817         my $repl_conf = PVE::ReplicationConfig->new();
4818         my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1);
4819         die "cannot remote-migrate replicated VM\n" if $is_replicated;
4820
4821         if (PVE::QemuServer::check_running($source_vmid)) {
4822             die "can't migrate running VM without --online\n" if !$param->{online};
4823
4824         } else {
4825             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4826             $param->{online} = 0;
4827         }
4828
4829         my $storecfg = PVE::Storage::config();
4830         my $target_storage = extract_param($param, 'target-storage');
4831         my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
4832         raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
4833             if $@;
4834
4835         my $target_bridge = extract_param($param, 'target-bridge');
4836         my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') };
4837         raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" })
4838             if $@;
4839
4840         die "remote migration requires explicit storage mapping!\n"
4841             if $storagemap->{identity};
4842
4843         $param->{storagemap} = $storagemap;
4844         $param->{bridgemap} = $bridgemap;
4845         $param->{remote} = {
4846             conn => $conn_args, # re-use fingerprint for tunnel
4847             client => $api_client,
4848             vmid => $target_vmid,
4849         };
4850         $param->{migration_type} = 'websocket';
4851         $param->{'with-local-disks'} = 1;
4852         $param->{delete} = $delete if $delete;
4853
4854         my $cluster_status = $api_client->get("/cluster/status");
4855         my $target_node;
4856         foreach my $entry (@$cluster_status) {
4857             next if $entry->{type} ne 'node';
4858             if ($entry->{local}) {
4859                 $target_node = $entry->{name};
4860                 last;
4861             }
4862         }
4863
4864         die "couldn't determine endpoint's node name\n"
4865             if !defined($target_node);
4866
4867         my $realcmd = sub {
4868             PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param);
4869         };
4870
4871         my $worker = sub {
4872             return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd);
4873         };
4874
4875         return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker);
4876     }});
4877
4878 __PACKAGE__->register_method({
4879     name => 'monitor',
4880     path => '{vmid}/monitor',
4881     method => 'POST',
4882     protected => 1,
4883     proxyto => 'node',
4884     description => "Execute QEMU monitor commands.",
4885     permissions => {
4886         description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')",
4887         check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]],
4888     },
4889     parameters => {
4890         additionalProperties => 0,
4891         properties => {
4892             node => get_standard_option('pve-node'),
4893             vmid => get_standard_option('pve-vmid'),
4894             command => {
4895                 type => 'string',
4896                 description => "The monitor command.",
4897             }
4898         },
4899     },
4900     returns => { type => 'string'},
4901     code => sub {
4902         my ($param) = @_;
4903
4904         my $rpcenv = PVE::RPCEnvironment::get();
4905         my $authuser = $rpcenv->get_user();
4906
4907         my $is_ro = sub {
4908             my $command = shift;
4909             return $command =~ m/^\s*info(\s+|$)/
4910                 || $command =~ m/^\s*help\s*$/;
4911         };
4912
4913         $rpcenv->check_full($authuser, "/", ['Sys.Modify'])
4914             if !&$is_ro($param->{command});
4915
4916         my $vmid = $param->{vmid};
4917
4918         my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists
4919
4920         my $res = '';
4921         eval {
4922             $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, $param->{command});
4923         };
4924         $res = "ERROR: $@" if $@;
4925
4926         return $res;
4927     }});
4928
4929 __PACKAGE__->register_method({
4930     name => 'resize_vm',
4931     path => '{vmid}/resize',
4932     method => 'PUT',
4933     protected => 1,
4934     proxyto => 'node',
4935     description => "Extend volume size.",
4936     permissions => {
4937         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
4938     },
4939     parameters => {
4940         additionalProperties => 0,
4941         properties => {
4942             node => get_standard_option('pve-node'),
4943             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4944             skiplock => get_standard_option('skiplock'),
4945             disk => {
4946                 type => 'string',
4947                 description => "The disk you want to resize.",
4948                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
4949             },
4950             size => {
4951                 type => 'string',
4952                 pattern => '\+?\d+(\.\d+)?[KMGT]?',
4953                 description => "The new size. With the `+` sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
4954             },
4955             digest => {
4956                 type => 'string',
4957                 description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
4958                 maxLength => 40,
4959                 optional => 1,
4960             },
4961         },
4962     },
4963     returns => {
4964         type => 'string',
4965         description => "the task ID.",
4966     },
4967     code => sub {
4968         my ($param) = @_;
4969
4970         my $rpcenv = PVE::RPCEnvironment::get();
4971
4972         my $authuser = $rpcenv->get_user();
4973
4974         my $node = extract_param($param, 'node');
4975
4976         my $vmid = extract_param($param, 'vmid');
4977
4978         my $digest = extract_param($param, 'digest');
4979
4980         my $disk = extract_param($param, 'disk');
4981
4982         my $sizestr = extract_param($param, 'size');
4983
4984         my $skiplock = extract_param($param, 'skiplock');
4985         raise_param_exc({ skiplock => "Only root may use this option." })
4986             if $skiplock && $authuser ne 'root@pam';
4987
4988         my $storecfg = PVE::Storage::config();
4989
4990         my $updatefn =  sub {
4991
4992             my $conf = PVE::QemuConfig->load_config($vmid);
4993
4994             die "checksum missmatch (file change by other user?)\n"
4995                 if $digest && $digest ne $conf->{digest};
4996             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4997
4998             die "disk '$disk' does not exist\n" if !$conf->{$disk};
4999
5000             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
5001
5002             my (undef, undef, undef, undef, undef, undef, $format) =
5003                 PVE::Storage::parse_volname($storecfg, $drive->{file});
5004
5005             my $volid = $drive->{file};
5006
5007             die "disk '$disk' has no associated volume\n" if !$volid;
5008
5009             die "you can't resize a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive);
5010
5011             my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5012
5013             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
5014
5015             PVE::Storage::activate_volumes($storecfg, [$volid]);
5016             my $size = PVE::Storage::volume_size_info($storecfg, $volid, 5);
5017
5018             die "Could not determine current size of volume '$volid'\n" if !defined($size);
5019
5020             die "internal error" if $sizestr !~ m/^(\+)?(\d+(\.\d+)?)([KMGT])?$/;
5021             my ($ext, $newsize, $unit) = ($1, $2, $4);
5022             if ($unit) {
5023                 if ($unit eq 'K') {
5024                     $newsize = $newsize * 1024;
5025                 } elsif ($unit eq 'M') {
5026                     $newsize = $newsize * 1024 * 1024;
5027                 } elsif ($unit eq 'G') {
5028                     $newsize = $newsize * 1024 * 1024 * 1024;
5029                 } elsif ($unit eq 'T') {
5030                     $newsize = $newsize * 1024 * 1024 * 1024 * 1024;
5031                 }
5032             }
5033             $newsize += $size if $ext;
5034             $newsize = int($newsize);
5035
5036             die "shrinking disks is not supported\n" if $newsize < $size;
5037
5038             return if $size == $newsize;
5039
5040             PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: resize --disk $disk --size $sizestr");
5041
5042             PVE::QemuServer::qemu_block_resize($vmid, "drive-$disk", $storecfg, $volid, $newsize);
5043
5044             $drive->{size} = $newsize;
5045             $conf->{$disk} = PVE::QemuServer::print_drive($drive);
5046
5047             PVE::QemuConfig->write_config($vmid, $conf);
5048         };
5049
5050         my $worker = sub {
5051             PVE::QemuConfig->lock_config($vmid, $updatefn);
5052         };
5053
5054         return $rpcenv->fork_worker('resize', $vmid, $authuser, $worker);
5055     }});
5056
5057 __PACKAGE__->register_method({
5058     name => 'snapshot_list',
5059     path => '{vmid}/snapshot',
5060     method => 'GET',
5061     description => "List all snapshots.",
5062     permissions => {
5063         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
5064     },
5065     proxyto => 'node',
5066     protected => 1, # qemu pid files are only readable by root
5067     parameters => {
5068         additionalProperties => 0,
5069         properties => {
5070             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5071             node => get_standard_option('pve-node'),
5072         },
5073     },
5074     returns => {
5075         type => 'array',
5076         items => {
5077             type => "object",
5078             properties => {
5079                 name => {
5080                     description => "Snapshot identifier. Value 'current' identifies the current VM.",
5081                     type => 'string',
5082                 },
5083                 vmstate => {
5084                     description => "Snapshot includes RAM.",
5085                     type => 'boolean',
5086                     optional => 1,
5087                 },
5088                 description => {
5089                     description => "Snapshot description.",
5090                     type => 'string',
5091                 },
5092                 snaptime => {
5093                     description => "Snapshot creation time",
5094                     type => 'integer',
5095                     renderer => 'timestamp',
5096                     optional => 1,
5097                 },
5098                 parent => {
5099                     description => "Parent snapshot identifier.",
5100                     type => 'string',
5101                     optional => 1,
5102                 },
5103             },
5104         },
5105         links => [ { rel => 'child', href => "{name}" } ],
5106     },
5107     code => sub {
5108         my ($param) = @_;
5109
5110         my $vmid = $param->{vmid};
5111
5112         my $conf = PVE::QemuConfig->load_config($vmid);
5113         my $snaphash = $conf->{snapshots} || {};
5114
5115         my $res = [];
5116
5117         foreach my $name (keys %$snaphash) {
5118             my $d = $snaphash->{$name};
5119             my $item = {
5120                 name => $name,
5121                 snaptime => $d->{snaptime} || 0,
5122                 vmstate => $d->{vmstate} ? 1 : 0,
5123                 description => $d->{description} || '',
5124             };
5125             $item->{parent} = $d->{parent} if $d->{parent};
5126             $item->{snapstate} = $d->{snapstate} if $d->{snapstate};
5127             push @$res, $item;
5128         }
5129
5130         my $running = PVE::QemuServer::check_running($vmid, 1) ? 1 : 0;
5131         my $current = {
5132             name => 'current',
5133             digest => $conf->{digest},
5134             running => $running,
5135             description => "You are here!",
5136         };
5137         $current->{parent} = $conf->{parent} if $conf->{parent};
5138
5139         push @$res, $current;
5140
5141         return $res;
5142     }});
5143
5144 __PACKAGE__->register_method({
5145     name => 'snapshot',
5146     path => '{vmid}/snapshot',
5147     method => 'POST',
5148     protected => 1,
5149     proxyto => 'node',
5150     description => "Snapshot a VM.",
5151     permissions => {
5152         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
5153     },
5154     parameters => {
5155         additionalProperties => 0,
5156         properties => {
5157             node => get_standard_option('pve-node'),
5158             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5159             snapname => get_standard_option('pve-snapshot-name'),
5160             vmstate => {
5161                 optional => 1,
5162                 type => 'boolean',
5163                 description => "Save the vmstate",
5164             },
5165             description => {
5166                 optional => 1,
5167                 type => 'string',
5168                 description => "A textual description or comment.",
5169             },
5170         },
5171     },
5172     returns => {
5173         type => 'string',
5174         description => "the task ID.",
5175     },
5176     code => sub {
5177         my ($param) = @_;
5178
5179         my $rpcenv = PVE::RPCEnvironment::get();
5180
5181         my $authuser = $rpcenv->get_user();
5182
5183         my $node = extract_param($param, 'node');
5184
5185         my $vmid = extract_param($param, 'vmid');
5186
5187         my $snapname = extract_param($param, 'snapname');
5188
5189         die "unable to use snapshot name 'current' (reserved name)\n"
5190             if $snapname eq 'current';
5191
5192         die "unable to use snapshot name 'pending' (reserved name)\n"
5193             if lc($snapname) eq 'pending';
5194
5195         my $realcmd = sub {
5196             PVE::Cluster::log_msg('info', $authuser, "snapshot VM $vmid: $snapname");
5197             PVE::QemuConfig->snapshot_create($vmid, $snapname, $param->{vmstate},
5198                                              $param->{description});
5199         };
5200
5201         return $rpcenv->fork_worker('qmsnapshot', $vmid, $authuser, $realcmd);
5202     }});
5203
5204 __PACKAGE__->register_method({
5205     name => 'snapshot_cmd_idx',
5206     path => '{vmid}/snapshot/{snapname}',
5207     description => '',
5208     method => 'GET',
5209     permissions => {
5210         user => 'all',
5211     },
5212     parameters => {
5213         additionalProperties => 0,
5214         properties => {
5215             vmid => get_standard_option('pve-vmid'),
5216             node => get_standard_option('pve-node'),
5217             snapname => get_standard_option('pve-snapshot-name'),
5218         },
5219     },
5220     returns => {
5221         type => 'array',
5222         items => {
5223             type => "object",
5224             properties => {},
5225         },
5226         links => [ { rel => 'child', href => "{cmd}" } ],
5227     },
5228     code => sub {
5229         my ($param) = @_;
5230
5231         my $res = [];
5232
5233         push @$res, { cmd => 'rollback' };
5234         push @$res, { cmd => 'config' };
5235
5236         return $res;
5237     }});
5238
5239 __PACKAGE__->register_method({
5240     name => 'update_snapshot_config',
5241     path => '{vmid}/snapshot/{snapname}/config',
5242     method => 'PUT',
5243     protected => 1,
5244     proxyto => 'node',
5245     description => "Update snapshot metadata.",
5246     permissions => {
5247         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
5248     },
5249     parameters => {
5250         additionalProperties => 0,
5251         properties => {
5252             node => get_standard_option('pve-node'),
5253             vmid => get_standard_option('pve-vmid'),
5254             snapname => get_standard_option('pve-snapshot-name'),
5255             description => {
5256                 optional => 1,
5257                 type => 'string',
5258                 description => "A textual description or comment.",
5259             },
5260         },
5261     },
5262     returns => { type => 'null' },
5263     code => sub {
5264         my ($param) = @_;
5265
5266         my $rpcenv = PVE::RPCEnvironment::get();
5267
5268         my $authuser = $rpcenv->get_user();
5269
5270         my $vmid = extract_param($param, 'vmid');
5271
5272         my $snapname = extract_param($param, 'snapname');
5273
5274         return if !defined($param->{description});
5275
5276         my $updatefn =  sub {
5277
5278             my $conf = PVE::QemuConfig->load_config($vmid);
5279
5280             PVE::QemuConfig->check_lock($conf);
5281
5282             my $snap = $conf->{snapshots}->{$snapname};
5283
5284             die "snapshot '$snapname' does not exist\n" if !defined($snap);
5285
5286             $snap->{description} = $param->{description} if defined($param->{description});
5287
5288              PVE::QemuConfig->write_config($vmid, $conf);
5289         };
5290
5291         PVE::QemuConfig->lock_config($vmid, $updatefn);
5292
5293         return;
5294     }});
5295
5296 __PACKAGE__->register_method({
5297     name => 'get_snapshot_config',
5298     path => '{vmid}/snapshot/{snapname}/config',
5299     method => 'GET',
5300     proxyto => 'node',
5301     description => "Get snapshot configuration",
5302     permissions => {
5303         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback', 'VM.Audit' ], any => 1],
5304     },
5305     parameters => {
5306         additionalProperties => 0,
5307         properties => {
5308             node => get_standard_option('pve-node'),
5309             vmid => get_standard_option('pve-vmid'),
5310             snapname => get_standard_option('pve-snapshot-name'),
5311         },
5312     },
5313     returns => { type => "object" },
5314     code => sub {
5315         my ($param) = @_;
5316
5317         my $rpcenv = PVE::RPCEnvironment::get();
5318
5319         my $authuser = $rpcenv->get_user();
5320
5321         my $vmid = extract_param($param, 'vmid');
5322
5323         my $snapname = extract_param($param, 'snapname');
5324
5325         my $conf = PVE::QemuConfig->load_config($vmid);
5326
5327         my $snap = $conf->{snapshots}->{$snapname};
5328
5329         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5330
5331         return $snap;
5332     }});
5333
5334 __PACKAGE__->register_method({
5335     name => 'rollback',
5336     path => '{vmid}/snapshot/{snapname}/rollback',
5337     method => 'POST',
5338     protected => 1,
5339     proxyto => 'node',
5340     description => "Rollback VM state to specified snapshot.",
5341     permissions => {
5342         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback' ], any => 1],
5343     },
5344     parameters => {
5345         additionalProperties => 0,
5346         properties => {
5347             node => get_standard_option('pve-node'),
5348             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5349             snapname => get_standard_option('pve-snapshot-name'),
5350             start => {
5351                 type => 'boolean',
5352                 description => "Whether the VM should get started after rolling back successfully."
5353                     . " (Note: VMs will be automatically started if the snapshot includes RAM.)",
5354                 optional => 1,
5355                 default => 0,
5356             },
5357         },
5358     },
5359     returns => {
5360         type => 'string',
5361         description => "the task ID.",
5362     },
5363     code => sub {
5364         my ($param) = @_;
5365
5366         my $rpcenv = PVE::RPCEnvironment::get();
5367
5368         my $authuser = $rpcenv->get_user();
5369
5370         my $node = extract_param($param, 'node');
5371
5372         my $vmid = extract_param($param, 'vmid');
5373
5374         my $snapname = extract_param($param, 'snapname');
5375
5376         my $realcmd = sub {
5377             PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname");
5378             PVE::QemuConfig->snapshot_rollback($vmid, $snapname);
5379
5380             if ($param->{start} && !PVE::QemuServer::Helpers::vm_running_locally($vmid)) {
5381                 PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node });
5382             }
5383         };
5384
5385         my $worker = sub {
5386             # hold migration lock, this makes sure that nobody create replication snapshots
5387             return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
5388         };
5389
5390         return $rpcenv->fork_worker('qmrollback', $vmid, $authuser, $worker);
5391     }});
5392
5393 __PACKAGE__->register_method({
5394     name => 'delsnapshot',
5395     path => '{vmid}/snapshot/{snapname}',
5396     method => 'DELETE',
5397     protected => 1,
5398     proxyto => 'node',
5399     description => "Delete a VM snapshot.",
5400     permissions => {
5401         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
5402     },
5403     parameters => {
5404         additionalProperties => 0,
5405         properties => {
5406             node => get_standard_option('pve-node'),
5407             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5408             snapname => get_standard_option('pve-snapshot-name'),
5409             force => {
5410                 optional => 1,
5411                 type => 'boolean',
5412                 description => "For removal from config file, even if removing disk snapshots fails.",
5413             },
5414         },
5415     },
5416     returns => {
5417         type => 'string',
5418         description => "the task ID.",
5419     },
5420     code => sub {
5421         my ($param) = @_;
5422
5423         my $rpcenv = PVE::RPCEnvironment::get();
5424
5425         my $authuser = $rpcenv->get_user();
5426
5427         my $node = extract_param($param, 'node');
5428
5429         my $vmid = extract_param($param, 'vmid');
5430
5431         my $snapname = extract_param($param, 'snapname');
5432
5433         my $lock_obtained;
5434         my $do_delete = sub {
5435             $lock_obtained = 1;
5436             PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname");
5437             PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force});
5438         };
5439
5440         my $realcmd = sub {
5441             if ($param->{force}) {
5442                 $do_delete->();
5443             } else {
5444                 eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); };
5445                 if (my $err = $@) {
5446                     die $err if $lock_obtained;
5447                     die "Failed to obtain guest migration lock - replication running?\n";
5448                 }
5449             }
5450         };
5451
5452         return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd);
5453     }});
5454
5455 __PACKAGE__->register_method({
5456     name => 'template',
5457     path => '{vmid}/template',
5458     method => 'POST',
5459     protected => 1,
5460     proxyto => 'node',
5461     description => "Create a Template.",
5462     permissions => {
5463         description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
5464         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
5465     },
5466     parameters => {
5467         additionalProperties => 0,
5468         properties => {
5469             node => get_standard_option('pve-node'),
5470             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
5471             disk => {
5472                 optional => 1,
5473                 type => 'string',
5474                 description => "If you want to convert only 1 disk to base image.",
5475                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
5476             },
5477
5478         },
5479     },
5480     returns => {
5481         type => 'string',
5482         description => "the task ID.",
5483     },
5484     code => sub {
5485         my ($param) = @_;
5486
5487         my $rpcenv = PVE::RPCEnvironment::get();
5488
5489         my $authuser = $rpcenv->get_user();
5490
5491         my $node = extract_param($param, 'node');
5492
5493         my $vmid = extract_param($param, 'vmid');
5494
5495         my $disk = extract_param($param, 'disk');
5496
5497         my $load_and_check = sub {
5498             my $conf = PVE::QemuConfig->load_config($vmid);
5499
5500             PVE::QemuConfig->check_lock($conf);
5501
5502             die "unable to create template, because VM contains snapshots\n"
5503                 if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});
5504
5505             die "you can't convert a template to a template\n"
5506                 if PVE::QemuConfig->is_template($conf) && !$disk;
5507
5508             die "you can't convert a VM to template if VM is running\n"
5509                 if PVE::QemuServer::check_running($vmid);
5510
5511             return $conf;
5512         };
5513
5514         $load_and_check->();
5515
5516         my $realcmd = sub {
5517             PVE::QemuConfig->lock_config($vmid, sub {
5518                 my $conf = $load_and_check->();
5519
5520                 $conf->{template} = 1;
5521                 PVE::QemuConfig->write_config($vmid, $conf);
5522
5523                 PVE::QemuServer::template_create($vmid, $conf, $disk);
5524             });
5525         };
5526
5527         return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
5528     }});
5529
5530 __PACKAGE__->register_method({
5531     name => 'cloudinit_generated_config_dump',
5532     path => '{vmid}/cloudinit/dump',
5533     method => 'GET',
5534     proxyto => 'node',
5535     description => "Get automatically generated cloudinit config.",
5536     permissions => {
5537         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
5538     },
5539     parameters => {
5540         additionalProperties => 0,
5541         properties => {
5542             node => get_standard_option('pve-node'),
5543             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5544             type => {
5545                 description => 'Config type.',
5546                 type => 'string',
5547                 enum => ['user', 'network', 'meta'],
5548             },
5549         },
5550     },
5551     returns => {
5552         type => 'string',
5553     },
5554     code => sub {
5555         my ($param) = @_;
5556
5557         my $conf = PVE::QemuConfig->load_config($param->{vmid});
5558
5559         return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
5560     }});
5561
5562 __PACKAGE__->register_method({
5563     name => 'mtunnel',
5564     path => '{vmid}/mtunnel',
5565     method => 'POST',
5566     protected => 1,
5567     description => 'Migration tunnel endpoint - only for internal use by VM migration.',
5568     permissions => {
5569         check =>
5570         [ 'and',
5571           ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]],
5572           ['perm', '/', [ 'Sys.Incoming' ]],
5573         ],
5574         description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" .
5575                        " on '/'. Further permission checks happen during the actual migration.",
5576     },
5577     parameters => {
5578         additionalProperties => 0,
5579         properties => {
5580             node => get_standard_option('pve-node'),
5581             vmid => get_standard_option('pve-vmid'),
5582             storages => {
5583                 type => 'string',
5584                 format => 'pve-storage-id-list',
5585                 optional => 1,
5586                 description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.',
5587             },
5588             bridges => {
5589                 type => 'string',
5590                 format => 'pve-bridge-id-list',
5591                 optional => 1,
5592                 description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.',
5593             },
5594         },
5595     },
5596     returns => {
5597         additionalProperties => 0,
5598         properties => {
5599             upid => { type => 'string' },
5600             ticket => { type => 'string' },
5601             socket => { type => 'string' },
5602         },
5603     },
5604     code => sub {
5605         my ($param) = @_;
5606
5607         my $rpcenv = PVE::RPCEnvironment::get();
5608         my $authuser = $rpcenv->get_user();
5609
5610         my $node = extract_param($param, 'node');
5611         my $vmid = extract_param($param, 'vmid');
5612
5613         my $storages = extract_param($param, 'storages');
5614         my $bridges = extract_param($param, 'bridges');
5615
5616         my $nodename = PVE::INotify::nodename();
5617
5618         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5619             if $node ne 'localhost' && $node ne $nodename;
5620
5621         $node = $nodename;
5622
5623         my $storecfg = PVE::Storage::config();
5624         foreach my $storeid (PVE::Tools::split_list($storages)) {
5625             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node);
5626         }
5627
5628         foreach my $bridge (PVE::Tools::split_list($bridges)) {
5629             PVE::Network::read_bridge_mtu($bridge);
5630         }
5631
5632         PVE::Cluster::check_cfs_quorum();
5633
5634         my $lock = 'create';
5635         eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); };
5636
5637         raise_param_exc({ vmid => "unable to create empty VM config - $@"})
5638             if $@;
5639
5640         my $realcmd = sub {
5641             my $state = {
5642                 storecfg => PVE::Storage::config(),
5643                 lock => $lock,
5644                 vmid => $vmid,
5645             };
5646
5647             my $run_locked = sub {
5648                 my ($code, $params) = @_;
5649                 return PVE::QemuConfig->lock_config($state->{vmid}, sub {
5650                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5651
5652                     $state->{conf} = $conf;
5653
5654                     die "Encountered wrong lock - aborting mtunnel command handling.\n"
5655                         if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock});
5656
5657                     return $code->($params);
5658                 });
5659             };
5660
5661             my $cmd_desc = {
5662                 config => {
5663                     conf => {
5664                         type => 'string',
5665                         description => 'Full VM config, adapted for target cluster/node',
5666                     },
5667                     'firewall-config' => {
5668                         type => 'string',
5669                         description => 'VM firewall config',
5670                         optional => 1,
5671                     },
5672                 },
5673                 disk => {
5674                     format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'),
5675                     storage => {
5676                         type => 'string',
5677                         format => 'pve-storage-id',
5678                     },
5679                     drive => {
5680                         type => 'object',
5681                         description => 'parsed drive information without volid and format',
5682                     },
5683                 },
5684                 start => {
5685                     start_params => {
5686                         type => 'object',
5687                         description => 'params passed to vm_start_nolock',
5688                     },
5689                     migrate_opts => {
5690                         type => 'object',
5691                         description => 'migrate_opts passed to vm_start_nolock',
5692                     },
5693                 },
5694                 ticket => {
5695                     path => {
5696                         type => 'string',
5697                         description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.',
5698                     },
5699                 },
5700                 quit => {
5701                     cleanup => {
5702                         type => 'boolean',
5703                         description => 'remove VM config and disks, aborting migration',
5704                         default => 0,
5705                     },
5706                 },
5707                 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'},
5708                 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'},
5709                 bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit},
5710             };
5711
5712             my $cmd_handlers = {
5713                 'version' => sub {
5714                     # compared against other end's version
5715                     # bump/reset for breaking changes
5716                     # bump/bump for opt-in changes
5717                     return {
5718                         api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
5719                         age => 0,
5720                     };
5721                 },
5722                 'config' => sub {
5723                     my ($params) = @_;
5724
5725                     # parse and write out VM FW config if given
5726                     if (my $fw_conf = $params->{'firewall-config'}) {
5727                         my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700);
5728
5729                         my $empty_conf = {
5730                             rules => [],
5731                             options => {},
5732                             aliases => {},
5733                             ipset => {} ,
5734                             ipset_comments => {},
5735                         };
5736                         my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf();
5737
5738                         # TODO: add flag for strict parsing?
5739                         # TODO: add import sub that does all this given raw content?
5740                         my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm');
5741                         $vmfw_conf->{vmid} = $state->{vmid};
5742                         PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf);
5743
5744                         $state->{cleanup}->{fw} = 1;
5745                     }
5746
5747                     my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf";
5748                     my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1);
5749                     delete $new_conf->{lock};
5750                     delete $new_conf->{digest};
5751
5752                     # TODO handle properly?
5753                     delete $new_conf->{snapshots};
5754                     delete $new_conf->{parent};
5755                     delete $new_conf->{pending};
5756
5757                     # not handled by update_vm_api
5758                     my $vmgenid = delete $new_conf->{vmgenid};
5759                     my $meta = delete $new_conf->{meta};
5760                     my $cloudinit = delete $new_conf->{cloudinit}; # this is informational only
5761                     $new_conf->{skip_cloud_init} = 1; # re-use image from source side
5762
5763                     $new_conf->{vmid} = $state->{vmid};
5764                     $new_conf->{node} = $node;
5765
5766                     PVE::QemuConfig->remove_lock($state->{vmid}, 'create');
5767
5768                     eval {
5769                         $update_vm_api->($new_conf, 1);
5770                     };
5771                     if (my $err = $@) {
5772                         # revert to locked previous config
5773                         my $conf = PVE::QemuConfig->load_config($state->{vmid});
5774                         $conf->{lock} = 'create';
5775                         PVE::QemuConfig->write_config($state->{vmid}, $conf);
5776
5777                         die $err;
5778                     }
5779
5780                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5781                     $conf->{lock} = 'migrate';
5782                     $conf->{vmgenid} = $vmgenid if defined($vmgenid);
5783                     $conf->{meta} = $meta if defined($meta);
5784                     $conf->{cloudinit} = $cloudinit if defined($cloudinit);
5785                     PVE::QemuConfig->write_config($state->{vmid}, $conf);
5786
5787                     $state->{lock} = 'migrate';
5788
5789                     return;
5790                 },
5791                 'bwlimit' => sub {
5792                     my ($params) = @_;
5793                     return PVE::StorageTunnel::handle_bwlimit($params);
5794                 },
5795                 'disk' => sub {
5796                     my ($params) = @_;
5797
5798                     my $format = $params->{format};
5799                     my $storeid = $params->{storage};
5800                     my $drive = $params->{drive};
5801
5802                     $check_storage_access_migrate->($rpcenv, $authuser, $state->{storecfg}, $storeid, $node);
5803
5804                     my $storagemap = {
5805                         default => $storeid,
5806                     };
5807
5808                     my $source_volumes = {
5809                         'disk' => [
5810                             undef,
5811                             $storeid,
5812                             $drive,
5813                             0,
5814                             $format,
5815                         ],
5816                     };
5817
5818                     my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks($state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap);
5819                     if (defined($res->{disk})) {
5820                         $state->{cleanup}->{volumes}->{$res->{disk}->{volid}} = 1;
5821                         return $res->{disk};
5822                     } else {
5823                         die "failed to allocate NBD disk..\n";
5824                     }
5825                 },
5826                 'disk-import' => sub {
5827                     my ($params) = @_;
5828
5829                     $check_storage_access_migrate->(
5830                         $rpcenv,
5831                         $authuser,
5832                         $state->{storecfg},
5833                         $params->{storage},
5834                         $node
5835                     );
5836
5837                     $params->{unix} = "/run/qemu-server/$state->{vmid}.storage";
5838
5839                     return PVE::StorageTunnel::handle_disk_import($state, $params);
5840                 },
5841                 'query-disk-import' => sub {
5842                     my ($params) = @_;
5843
5844                     return PVE::StorageTunnel::handle_query_disk_import($state, $params);
5845                 },
5846                 'start' => sub {
5847                     my ($params) = @_;
5848
5849                     my $info = PVE::QemuServer::vm_start_nolock(
5850                         $state->{storecfg},
5851                         $state->{vmid},
5852                         $state->{conf},
5853                         $params->{start_params},
5854                         $params->{migrate_opts},
5855                     );
5856
5857
5858                     if ($info->{migrate}->{proto} ne 'unix') {
5859                         PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5860                         die "migration over non-UNIX sockets not possible\n";
5861                     }
5862
5863                     my $socket = $info->{migrate}->{addr};
5864                     chown $state->{socket_uid}, -1, $socket;
5865                     $state->{sockets}->{$socket} = 1;
5866
5867                     my $unix_sockets = $info->{migrate}->{unix_sockets};
5868                     foreach my $socket (@$unix_sockets) {
5869                         chown $state->{socket_uid}, -1, $socket;
5870                         $state->{sockets}->{$socket} = 1;
5871                     }
5872                     return $info;
5873                 },
5874                 'fstrim' => sub {
5875                     if (PVE::QemuServer::qga_check_running($state->{vmid})) {
5876                         eval { mon_cmd($state->{vmid}, "guest-fstrim") };
5877                         warn "fstrim failed: $@\n" if $@;
5878                     }
5879                     return;
5880                 },
5881                 'stop' => sub {
5882                     PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5883                     return;
5884                 },
5885                 'nbdstop' => sub {
5886                     PVE::QemuServer::nbd_stop($state->{vmid});
5887                     return;
5888                 },
5889                 'resume' => sub {
5890                     if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) {
5891                         PVE::QemuServer::vm_resume($state->{vmid}, 1, 1);
5892                     } else {
5893                         die "VM $state->{vmid} not running\n";
5894                     }
5895                     return;
5896                 },
5897                 'unlock' => sub {
5898                     PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock});
5899                     delete $state->{lock};
5900                     return;
5901                 },
5902                 'ticket' => sub {
5903                     my ($params) = @_;
5904
5905                     my $path = $params->{path};
5906
5907                     die "Not allowed to generate ticket for unknown socket '$path'\n"
5908                         if !defined($state->{sockets}->{$path});
5909
5910                     return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") };
5911                 },
5912                 'quit' => sub {
5913                     my ($params) = @_;
5914
5915                     if ($params->{cleanup}) {
5916                         if ($state->{cleanup}->{fw}) {
5917                             PVE::Firewall::remove_vmfw_conf($state->{vmid});
5918                         }
5919
5920                         for my $volid (keys $state->{cleanup}->{volumes}->%*) {
5921                             print "freeing volume '$volid' as part of cleanup\n";
5922                             eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) };
5923                             warn $@ if $@;
5924                         }
5925
5926                         PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5927                     }
5928
5929                     print "switching to exit-mode, waiting for client to disconnect\n";
5930                     $state->{exit} = 1;
5931                     return;
5932                 },
5933             };
5934
5935             $run_locked->(sub {
5936                 my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel";
5937                 unlink $socket_addr;
5938
5939                 $state->{socket} = IO::Socket::UNIX->new(
5940                     Type => SOCK_STREAM(),
5941                     Local => $socket_addr,
5942                     Listen => 1,
5943                 );
5944
5945                 $state->{socket_uid} = getpwnam('www-data')
5946                     or die "Failed to resolve user 'www-data' to numeric UID\n";
5947                 chown $state->{socket_uid}, -1, $socket_addr;
5948             });
5949
5950             print "mtunnel started\n";
5951
5952             my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) };
5953             if ($@) {
5954                 warn "Failed to accept tunnel connection - $@\n";
5955
5956                 warn "Removing tunnel socket..\n";
5957                 unlink $state->{socket};
5958
5959                 warn "Removing temporary VM config..\n";
5960                 $run_locked->(sub {
5961                     PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5962                 });
5963
5964                 die "Exiting mtunnel\n";
5965             }
5966
5967             $state->{conn} = $conn;
5968
5969             my $reply_err = sub {
5970                 my ($msg) = @_;
5971
5972                 my $reply = JSON::encode_json({
5973                     success => JSON::false,
5974                     msg => $msg,
5975                 });
5976                 $conn->print("$reply\n");
5977                 $conn->flush();
5978             };
5979
5980             my $reply_ok = sub {
5981                 my ($res) = @_;
5982
5983                 $res->{success} = JSON::true;
5984                 my $reply = JSON::encode_json($res);
5985                 $conn->print("$reply\n");
5986                 $conn->flush();
5987             };
5988
5989             while (my $line = <$conn>) {
5990                 chomp $line;
5991
5992                 # untaint, we validate below if needed
5993                 ($line) = $line =~ /^(.*)$/;
5994                 my $parsed = eval { JSON::decode_json($line) };
5995                 if ($@) {
5996                     $reply_err->("failed to parse command - $@");
5997                     next;
5998                 }
5999
6000                 my $cmd = delete $parsed->{cmd};
6001                 if (!defined($cmd)) {
6002                     $reply_err->("'cmd' missing");
6003                 } elsif ($state->{exit}) {
6004                     $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible");
6005                     next;
6006                 } elsif (my $handler = $cmd_handlers->{$cmd}) {
6007                     print "received command '$cmd'\n";
6008                     eval {
6009                         if ($cmd_desc->{$cmd}) {
6010                             PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd});
6011                         } else {
6012                             $parsed = {};
6013                         }
6014                         my $res = $run_locked->($handler, $parsed);
6015                         $reply_ok->($res);
6016                     };
6017                     $reply_err->("failed to handle '$cmd' command - $@")
6018                         if $@;
6019                 } else {
6020                     $reply_err->("unknown command '$cmd' given");
6021                 }
6022             }
6023
6024             if ($state->{exit}) {
6025                 print "mtunnel exited\n";
6026             } else {
6027                 die "mtunnel exited unexpectedly\n";
6028             }
6029         };
6030
6031         my $socket_addr = "/run/qemu-server/$vmid.mtunnel";
6032         my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr");
6033         my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd);
6034
6035         return {
6036             ticket => $ticket,
6037             upid => $upid,
6038             socket => $socket_addr,
6039         };
6040     }});
6041
6042 __PACKAGE__->register_method({
6043     name => 'mtunnelwebsocket',
6044     path => '{vmid}/mtunnelwebsocket',
6045     method => 'GET',
6046     permissions => {
6047         description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
6048         user => 'all', # check inside
6049     },
6050     description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
6051     parameters => {
6052         additionalProperties => 0,
6053         properties => {
6054             node => get_standard_option('pve-node'),
6055             vmid => get_standard_option('pve-vmid'),
6056             socket => {
6057                 type => "string",
6058                 description => "unix socket to forward to",
6059             },
6060             ticket => {
6061                 type => "string",
6062                 description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command",
6063             },
6064         },
6065     },
6066     returns => {
6067         type => "object",
6068         properties => {
6069             port => { type => 'string', optional => 1 },
6070             socket => { type => 'string', optional => 1 },
6071         },
6072     },
6073     code => sub {
6074         my ($param) = @_;
6075
6076         my $rpcenv = PVE::RPCEnvironment::get();
6077         my $authuser = $rpcenv->get_user();
6078
6079         my $nodename = PVE::INotify::nodename();
6080         my $node = extract_param($param, 'node');
6081
6082         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
6083             if $node ne 'localhost' && $node ne $nodename;
6084
6085         my $vmid = $param->{vmid};
6086         # check VM exists
6087         PVE::QemuConfig->load_config($vmid);
6088
6089         my $socket = $param->{socket};
6090         PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket");
6091
6092         return { socket => $socket };
6093     }});
6094
6095 1;