PVE/API2/Qemu.pm

   1 package PVE::API2::Qemu;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd 'abs_path';
   6 use Net::SSLeay;
   7 use IO::Socket::IP;
   8 use IO::Socket::UNIX;
   9 use IPC::Open3;
  10 use JSON;
  11 use URI::Escape;
  12 use Crypt::OpenSSL::Random;
  13 use Socket qw(SOCK_STREAM);
  14
  15 use PVE::APIClient::LWP;
  16 use PVE::CGroup;
  17 use PVE::Cluster qw (cfs_read_file cfs_write_file);;
  18 use PVE::RRD;
  19 use PVE::SafeSyslog;
  20 use PVE::Tools qw(extract_param);
  21 use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
  22 use PVE::Storage;
  23 use PVE::JSONSchema qw(get_standard_option);
  24 use PVE::RESTHandler;
  25 use PVE::ReplicationConfig;
  26 use PVE::GuestHelpers qw(assert_tag_permissions);
  27 use PVE::QemuConfig;
  28 use PVE::QemuServer;
  29 use PVE::QemuServer::Cloudinit;
  30 use PVE::QemuServer::CPUConfig;
  31 use PVE::QemuServer::Drive;
  32 use PVE::QemuServer::ImportDisk;
  33 use PVE::QemuServer::Monitor qw(mon_cmd);
  34 use PVE::QemuServer::Machine;
  35 use PVE::QemuMigrate;
  36 use PVE::RPCEnvironment;
  37 use PVE::AccessControl;
  38 use PVE::INotify;
  39 use PVE::Network;
  40 use PVE::Firewall;
  41 use PVE::API2::Firewall::VM;
  42 use PVE::API2::Qemu::Agent;
  43 use PVE::VZDump::Plugin;
  44 use PVE::DataCenterConfig;
  45 use PVE::SSHInfo;
  46 use PVE::Replication;
  47 use PVE::StorageTunnel;
  48
  49 BEGIN {
  50     if (!$ENV{PVE_GENERATING_DOCS}) {
  51         require PVE::HA::Env::PVE2;
  52         import PVE::HA::Env::PVE2;
  53         require PVE::HA::Config;
  54         import PVE::HA::Config;
  55     }
  56 }
  57
  58 use base qw(PVE::RESTHandler);
  59
  60 my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal.";
  61
  62 my $resolve_cdrom_alias = sub {
  63     my $param = shift;
  64
  65     if (my $value = $param->{cdrom}) {
  66         $value .= ",media=cdrom" if $value !~ m/media=/;
  67         $param->{ide2} = $value;
  68         delete $param->{cdrom};
  69     }
  70 };
  71
  72 # Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema.
  73 my $foreach_volume_with_alloc = sub {
  74     my ($param, $func) = @_;
  75
  76     for my $opt (sort keys $param->%*) {
  77         next if !PVE::QemuServer::is_valid_drivename($opt);
  78
  79         my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1);
  80         next if !$drive;
  81
  82         $func->($opt, $drive);
  83     }
  84 };
  85
  86 my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
  87
  88 my $check_drive_param = sub {
  89     my ($param, $storecfg, $extra_checks) = @_;
  90
  91     for my $opt (sort keys $param->%*) {
  92         next if !PVE::QemuServer::is_valid_drivename($opt);
  93
  94         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
  95         raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
  96
  97         if ($drive->{'import-from'}) {
  98             if ($drive->{file} !~ $NEW_DISK_RE || $3 != 0) {
  99                 raise_param_exc({
 100                     $opt => "'import-from' requires special syntax - ".
 101                         "use <storage ID>:0,import-from=<source>",
 102                 });
 103             }
 104
 105             if ($opt eq 'efidisk0') {
 106                 for my $required (qw(efitype pre-enrolled-keys)) {
 107                     if (!defined($drive->{$required})) {
 108                         raise_param_exc({
 109                             $opt => "need to specify '$required' when using 'import-from'",
 110                         });
 111                     }
 112                 }
 113             } elsif ($opt eq 'tpmstate0') {
 114                 raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" })
 115                     if !defined($drive->{version});
 116             }
 117         }
 118
 119         PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 120
 121         $extra_checks->($drive) if $extra_checks;
 122
 123         $param->{$opt} = PVE::QemuServer::print_drive($drive, 1);
 124     }
 125 };
 126
 127 my $check_storage_access = sub {
 128    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 129
 130    $foreach_volume_with_alloc->($settings, sub {
 131         my ($ds, $drive) = @_;
 132
 133         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 134
 135         my $volid = $drive->{file};
 136         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 137
 138         if (!$volid || ($volid eq 'none' || $volid eq 'cloudinit' || (defined($volname) && $volname eq 'cloudinit'))) {
 139             # nothing to check
 140         } elsif ($isCDROM && ($volid eq 'cdrom')) {
 141             $rpcenv->check($authuser, "/", ['Sys.Console']);
 142         } elsif (!$isCDROM && ($volid =~ $NEW_DISK_RE)) {
 143             my ($storeid, $size) = ($2 || $default_storage, $3);
 144             die "no storage ID specified (and no default storage)\n" if !$storeid;
 145             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
 146             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 147             raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
 148                 if !$scfg->{content}->{images};
 149         } else {
 150             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 151             if ($storeid) {
 152                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 153                 raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" })
 154                     if $vtype ne 'images' && $vtype ne 'iso';
 155             }
 156         }
 157
 158         if (my $src_image = $drive->{'import-from'}) {
 159             my $src_vmid;
 160             if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume
 161                 (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image);
 162                 raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" })
 163                     if $vtype ne 'images';
 164             }
 165
 166             if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk()
 167                 $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']);
 168             } else {
 169                 PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image);
 170             }
 171         }
 172     });
 173
 174    $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
 175        if defined($settings->{vmstatestorage});
 176 };
 177
 178 my $check_storage_access_clone = sub {
 179    my ($rpcenv, $authuser, $storecfg, $conf, $storage) = @_;
 180
 181    my $sharedvm = 1;
 182
 183    PVE::QemuConfig->foreach_volume($conf, sub {
 184         my ($ds, $drive) = @_;
 185
 186         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 187
 188         my $volid = $drive->{file};
 189
 190         return if !$volid || $volid eq 'none';
 191
 192         if ($isCDROM) {
 193             if ($volid eq 'cdrom') {
 194                 $rpcenv->check($authuser, "/", ['Sys.Console']);
 195             } else {
 196                 # we simply allow access
 197                 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 198                 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 199                 $sharedvm = 0 if !$scfg->{shared};
 200
 201             }
 202         } else {
 203             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 204             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 205             $sharedvm = 0 if !$scfg->{shared};
 206
 207             $sid = $storage if $storage;
 208             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
 209         }
 210     });
 211
 212    $rpcenv->check($authuser, "/storage/$conf->{vmstatestorage}", ['Datastore.AllocateSpace'])
 213        if defined($conf->{vmstatestorage});
 214
 215    return $sharedvm;
 216 };
 217
 218 my $check_storage_access_migrate = sub {
 219     my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
 220
 221     PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
 222
 223     $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
 224
 225     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
 226     die "storage '$storage' does not support vm images\n"
 227         if !$scfg->{content}->{images};
 228 };
 229
 230 my $import_from_volid = sub {
 231     my ($storecfg, $src_volid, $dest_info, $vollist) = @_;
 232
 233     die "could not get size of $src_volid\n"
 234         if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10);
 235
 236     die "cannot import from cloudinit disk\n"
 237         if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid });
 238
 239     my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2];
 240
 241     my $src_vm_state = sub {
 242         my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0;
 243
 244         my $runs = 0;
 245         if ($exists) {
 246             eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); };
 247             die "owner VM $src_vmid not on local node\n" if $@;
 248             $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0;
 249         }
 250
 251         return ($exists, $runs);
 252     };
 253
 254     my ($src_vm_exists, $running) = $src_vm_state->();
 255
 256     die "cannot import from '$src_volid' - full clone feature is not supported\n"
 257         if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running);
 258
 259     my $clonefn = sub {
 260         my ($src_vm_exists_now, $running_now) = $src_vm_state->();
 261
 262         die "owner VM $src_vmid changed state unexpectedly\n"
 263             if $src_vm_exists_now != $src_vm_exists || $running_now != $running;
 264
 265         my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {};
 266
 267         my $src_drive = { file => $src_volid };
 268         my $src_drivename;
 269         PVE::QemuConfig->foreach_volume($src_conf, sub {
 270             my ($ds, $drive) = @_;
 271
 272             return if $src_drivename;
 273
 274             if ($drive->{file} eq $src_volid) {
 275                 $src_drive = $drive;
 276                 $src_drivename = $ds;
 277             }
 278         });
 279
 280         my $source_info = {
 281             vmid => $src_vmid,
 282             running => $running_now,
 283             drivename => $src_drivename,
 284             drive => $src_drive,
 285             snapname => undef,
 286         };
 287
 288         my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid);
 289
 290         return PVE::QemuServer::clone_disk(
 291             $storecfg,
 292             $source_info,
 293             $dest_info,
 294             1,
 295             $vollist,
 296             undef,
 297             undef,
 298             $src_conf->{agent},
 299             PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]),
 300         );
 301     };
 302
 303     my $cloned;
 304     if ($running) {
 305         $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn);
 306     } elsif ($src_vmid) {
 307         $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn);
 308     } else {
 309         $cloned = $clonefn->();
 310     }
 311
 312     return $cloned->@{qw(file size)};
 313 };
 314
 315 # Note: $pool is only needed when creating a VM, because pool permissions
 316 # are automatically inherited if VM already exists inside a pool.
 317 my $create_disks = sub {
 318     my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
 319
 320     my $vollist = [];
 321
 322     my $res = {};
 323
 324     my $code = sub {
 325         my ($ds, $disk) = @_;
 326
 327         my $volid = $disk->{file};
 328         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 329
 330         if (!$volid || $volid eq 'none' || $volid eq 'cdrom') {
 331             delete $disk->{size};
 332             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 333         } elsif (defined($volname) && $volname eq 'cloudinit') {
 334             $storeid = $storeid // $default_storage;
 335             die "no storage ID specified (and no default storage)\n" if !$storeid;
 336
 337             if (
 338                 my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 339                 || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 340                 || PVE::QemuConfig->has_cloudinit($res, $ds)
 341             ) {
 342                 die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 343             }
 344
 345             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 346             my $name = "vm-$vmid-cloudinit";
 347
 348             my $fmt = undef;
 349             if ($scfg->{path}) {
 350                 $fmt = $disk->{format} // "qcow2";
 351                 $name .= ".$fmt";
 352             } else {
 353                 $fmt = $disk->{format} // "raw";
 354             }
 355
 356             # Initial disk created with 4 MB and aligned to 4MB on regeneration
 357             my $ci_size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 358             my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, $name, $ci_size/1024);
 359             $disk->{file} = $volid;
 360             $disk->{media} = 'cdrom';
 361             push @$vollist, $volid;
 362             delete $disk->{format}; # no longer needed
 363             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 364             print "$ds: successfully created disk '$res->{$ds}'\n";
 365         } elsif ($volid =~ $NEW_DISK_RE) {
 366             my ($storeid, $size) = ($2 || $default_storage, $3);
 367             die "no storage ID specified (and no default storage)\n" if !$storeid;
 368
 369             if (my $source = delete $disk->{'import-from'}) {
 370                 my $dst_volid;
 371
 372                 if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume
 373                     my $dest_info = {
 374                         vmid => $vmid,
 375                         drivename => $ds,
 376                         storage => $storeid,
 377                         format => $disk->{format},
 378                     };
 379
 380                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk)
 381                         if $ds eq 'efidisk0';
 382
 383                     ($dst_volid, $size) = eval {
 384                         $import_from_volid->($storecfg, $source, $dest_info, $vollist);
 385                     };
 386                     die "cannot import from '$source' - $@" if $@;
 387                 } else {
 388                     $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
 389                     $size = PVE::Storage::file_size_info($source);
 390                     die "could not get file size of $source\n" if !$size;
 391
 392                     (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import(
 393                         $source,
 394                         $vmid,
 395                         $storeid,
 396                         {
 397                             drive_name => $ds,
 398                             format => $disk->{format},
 399                             'skip-config-update' => 1,
 400                         },
 401                     );
 402                     push @$vollist, $dst_volid;
 403                 }
 404
 405                 $disk->{file} = $dst_volid;
 406                 $disk->{size} = $size;
 407                 delete $disk->{format}; # no longer needed
 408                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 409             } else {
 410                 my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
 411                 my $fmt = $disk->{format} || $defformat;
 412
 413                 $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb
 414
 415                 my $volid;
 416                 if ($ds eq 'efidisk0') {
 417                     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 418                     ($volid, $size) = PVE::QemuServer::create_efidisk(
 419                         $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm);
 420                 } elsif ($ds eq 'tpmstate0') {
 421                     # swtpm can only use raw volumes, and uses a fixed size
 422                     $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb');
 423                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size);
 424                 } else {
 425                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
 426                 }
 427                 push @$vollist, $volid;
 428                 $disk->{file} = $volid;
 429                 $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
 430                 delete $disk->{format}; # no longer needed
 431                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 432             }
 433
 434             print "$ds: successfully created disk '$res->{$ds}'\n";
 435         } else {
 436             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 437             if ($storeid) {
 438                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 439                 die "cannot use volume $volid - content type needs to be 'images' or 'iso'"
 440                     if $vtype ne 'images' && $vtype ne 'iso';
 441
 442                 if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) {
 443                     if (
 444                         my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 445                         || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 446                         || PVE::QemuConfig->has_cloudinit($res, $ds)
 447                     ) {
 448                         die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 449                     }
 450                 }
 451             }
 452
 453             PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
 454
 455             my $size = PVE::Storage::volume_size_info($storecfg, $volid);
 456             die "volume $volid does not exist\n" if !$size;
 457             $disk->{size} = $size;
 458
 459             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 460         }
 461     };
 462
 463     eval { $foreach_volume_with_alloc->($settings, $code); };
 464
 465     # free allocated images on error
 466     if (my $err = $@) {
 467         syslog('err', "VM $vmid creating disks failed");
 468         foreach my $volid (@$vollist) {
 469             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 470             warn $@ if $@;
 471         }
 472         die $err;
 473     }
 474
 475     return ($vollist, $res);
 476 };
 477
 478 my $check_cpu_model_access = sub {
 479     my ($rpcenv, $authuser, $new, $existing) = @_;
 480
 481     return if !defined($new->{cpu});
 482
 483     my $cpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $new->{cpu});
 484     return if !$cpu || !$cpu->{cputype}; # always allow default
 485     my $cputype = $cpu->{cputype};
 486
 487     if ($existing && $existing->{cpu}) {
 488         # changing only other settings doesn't require permissions for CPU model
 489         my $existingCpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $existing->{cpu});
 490         return if $existingCpu->{cputype} eq $cputype;
 491     }
 492
 493     if (PVE::QemuServer::CPUConfig::is_custom_model($cputype)) {
 494         $rpcenv->check($authuser, "/nodes", ['Sys.Audit']);
 495     }
 496 };
 497
 498 my $cpuoptions = {
 499     'cores' => 1,
 500     'cpu' => 1,
 501     'cpulimit' => 1,
 502     'cpuunits' => 1,
 503     'numa' => 1,
 504     'smp' => 1,
 505     'sockets' => 1,
 506     'vcpus' => 1,
 507 };
 508
 509 my $memoryoptions = {
 510     'memory' => 1,
 511     'balloon' => 1,
 512     'shares' => 1,
 513 };
 514
 515 my $hwtypeoptions = {
 516     'acpi' => 1,
 517     'hotplug' => 1,
 518     'kvm' => 1,
 519     'machine' => 1,
 520     'scsihw' => 1,
 521     'smbios1' => 1,
 522     'tablet' => 1,
 523     'vga' => 1,
 524     'watchdog' => 1,
 525     'audio0' => 1,
 526 };
 527
 528 my $generaloptions = {
 529     'agent' => 1,
 530     'autostart' => 1,
 531     'bios' => 1,
 532     'description' => 1,
 533     'keyboard' => 1,
 534     'localtime' => 1,
 535     'migrate_downtime' => 1,
 536     'migrate_speed' => 1,
 537     'name' => 1,
 538     'onboot' => 1,
 539     'ostype' => 1,
 540     'protection' => 1,
 541     'reboot' => 1,
 542     'startdate' => 1,
 543     'startup' => 1,
 544     'tdf' => 1,
 545     'template' => 1,
 546 };
 547
 548 my $vmpoweroptions = {
 549     'freeze' => 1,
 550 };
 551
 552 my $diskoptions = {
 553     'boot' => 1,
 554     'bootdisk' => 1,
 555     'vmstatestorage' => 1,
 556 };
 557
 558 my $cloudinitoptions = {
 559     cicustom => 1,
 560     cipassword => 1,
 561     citype => 1,
 562     ciuser => 1,
 563     nameserver => 1,
 564     searchdomain => 1,
 565     sshkeys => 1,
 566 };
 567
 568 my $check_vm_create_serial_perm = sub {
 569     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 570
 571     return 1 if $authuser eq 'root@pam';
 572
 573     foreach my $opt (keys %{$param}) {
 574         next if $opt !~ m/^serial\d+$/;
 575
 576         if ($param->{$opt} eq 'socket') {
 577             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 578         } else {
 579             die "only root can set '$opt' config for real devices\n";
 580         }
 581     }
 582
 583     return 1;
 584 };
 585
 586 my $check_vm_create_usb_perm = sub {
 587     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 588
 589     return 1 if $authuser eq 'root@pam';
 590
 591     foreach my $opt (keys %{$param}) {
 592         next if $opt !~ m/^usb\d+$/;
 593
 594         if ($param->{$opt} =~ m/spice/) {
 595             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 596         } else {
 597             die "only root can set '$opt' config for real devices\n";
 598         }
 599     }
 600
 601     return 1;
 602 };
 603
 604 my $check_vm_modify_config_perm = sub {
 605     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
 606
 607     return 1 if $authuser eq 'root@pam';
 608
 609     foreach my $opt (@$key_list) {
 610         # some checks (e.g., disk, serial port, usb) need to be done somewhere
 611         # else, as there the permission can be value dependend
 612         next if PVE::QemuServer::is_valid_drivename($opt);
 613         next if $opt eq 'cdrom';
 614         next if $opt =~ m/^(?:unused|serial|usb)\d+$/;
 615         next if $opt eq 'tags';
 616
 617
 618         if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) {
 619             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
 620         } elsif ($memoryoptions->{$opt}) {
 621             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
 622         } elsif ($hwtypeoptions->{$opt}) {
 623             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 624         } elsif ($generaloptions->{$opt}) {
 625             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
 626             # special case for startup since it changes host behaviour
 627             if ($opt eq 'startup') {
 628                 $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
 629             }
 630         } elsif ($vmpoweroptions->{$opt}) {
 631             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']);
 632         } elsif ($diskoptions->{$opt}) {
 633             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
 634         } elsif ($opt =~ m/^net\d+$/) {
 635             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
 636         } elsif ($cloudinitoptions->{$opt} || $opt =~ m/^ipconfig\d+$/) {
 637             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1);
 638         } elsif ($opt eq 'vmstate') {
 639             # the user needs Disk and PowerMgmt privileges to change the vmstate
 640             # also needs privileges on the storage, that will be checked later
 641             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]);
 642         } else {
 643             # catches hostpci\d+, args, lock, etc.
 644             # new options will be checked here
 645             die "only root can set '$opt' config\n";
 646         }
 647     }
 648
 649     return 1;
 650 };
 651
 652 __PACKAGE__->register_method({
 653     name => 'vmlist',
 654     path => '',
 655     method => 'GET',
 656     description => "Virtual machine index (per node).",
 657     permissions => {
 658         description => "Only list VMs where you have VM.Audit permissons on /vms/<vmid>.",
 659         user => 'all',
 660     },
 661     proxyto => 'node',
 662     protected => 1, # qemu pid files are only readable by root
 663     parameters => {
 664         additionalProperties => 0,
 665         properties => {
 666             node => get_standard_option('pve-node'),
 667             full => {
 668                 type => 'boolean',
 669                 optional => 1,
 670                 description => "Determine the full status of active VMs.",
 671             },
 672         },
 673     },
 674     returns => {
 675         type => 'array',
 676         items => {
 677             type => "object",
 678             properties => $PVE::QemuServer::vmstatus_return_properties,
 679         },
 680         links => [ { rel => 'child', href => "{vmid}" } ],
 681     },
 682     code => sub {
 683         my ($param) = @_;
 684
 685         my $rpcenv = PVE::RPCEnvironment::get();
 686         my $authuser = $rpcenv->get_user();
 687
 688         my $vmstatus = PVE::QemuServer::vmstatus(undef, $param->{full});
 689
 690         my $res = [];
 691         foreach my $vmid (keys %$vmstatus) {
 692             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
 693
 694             my $data = $vmstatus->{$vmid};
 695             push @$res, $data;
 696         }
 697
 698         return $res;
 699     }});
 700
 701 my $parse_restore_archive = sub {
 702     my ($storecfg, $archive) = @_;
 703
 704     my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1);
 705
 706     if (defined($archive_storeid)) {
 707         my $scfg =  PVE::Storage::storage_config($storecfg, $archive_storeid);
 708         if ($scfg->{type} eq 'pbs') {
 709             return {
 710                 type => 'pbs',
 711                 volid => $archive,
 712             };
 713         }
 714     }
 715     my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive);
 716     return {
 717         type => 'file',
 718         path => $path,
 719     };
 720 };
 721
 722
 723 __PACKAGE__->register_method({
 724     name => 'create_vm',
 725     path => '',
 726     method => 'POST',
 727     description => "Create or restore a virtual machine.",
 728     permissions => {
 729         description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
 730             "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
 731             "If you create disks you need 'Datastore.AllocateSpace' on any used storage.",
 732         user => 'all', # check inside
 733     },
 734     protected => 1,
 735     proxyto => 'node',
 736     parameters => {
 737         additionalProperties => 0,
 738         properties => PVE::QemuServer::json_config_properties(
 739             {
 740                 node => get_standard_option('pve-node'),
 741                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
 742                 archive => {
 743                     description => "The backup archive. Either the file system path to a .tar or .vma file (use '-' to pipe data from stdin) or a proxmox storage backup volume identifier.",
 744                     type => 'string',
 745                     optional => 1,
 746                     maxLength => 255,
 747                     completion => \&PVE::QemuServer::complete_backup_archives,
 748                 },
 749                 storage => get_standard_option('pve-storage-id', {
 750                     description => "Default storage.",
 751                     optional => 1,
 752                     completion => \&PVE::QemuServer::complete_storage,
 753                 }),
 754                 force => {
 755                     optional => 1,
 756                     type => 'boolean',
 757                     description => "Allow to overwrite existing VM.",
 758                     requires => 'archive',
 759                 },
 760                 unique => {
 761                     optional => 1,
 762                     type => 'boolean',
 763                     description => "Assign a unique random ethernet address.",
 764                     requires => 'archive',
 765                 },
 766                 'live-restore' => {
 767                     optional => 1,
 768                     type => 'boolean',
 769                     description => "Start the VM immediately from the backup and restore in background. PBS only.",
 770                     requires => 'archive',
 771                 },
 772                 pool => {
 773                     optional => 1,
 774                     type => 'string', format => 'pve-poolid',
 775                     description => "Add the VM to the specified pool.",
 776                 },
 777                 bwlimit => {
 778                     description => "Override I/O bandwidth limit (in KiB/s).",
 779                     optional => 1,
 780                     type => 'integer',
 781                     minimum => '0',
 782                     default => 'restore limit from datacenter or storage config',
 783                 },
 784                 start => {
 785                     optional => 1,
 786                     type => 'boolean',
 787                     default => 0,
 788                     description => "Start VM after it was created successfully.",
 789                 },
 790             },
 791             1, # with_disk_alloc
 792         ),
 793     },
 794     returns => {
 795         type => 'string',
 796     },
 797     code => sub {
 798         my ($param) = @_;
 799
 800         my $rpcenv = PVE::RPCEnvironment::get();
 801         my $authuser = $rpcenv->get_user();
 802
 803         my $node = extract_param($param, 'node');
 804         my $vmid = extract_param($param, 'vmid');
 805
 806         my $archive = extract_param($param, 'archive');
 807         my $is_restore = !!$archive;
 808
 809         my $bwlimit = extract_param($param, 'bwlimit');
 810         my $force = extract_param($param, 'force');
 811         my $pool = extract_param($param, 'pool');
 812         my $start_after_create = extract_param($param, 'start');
 813         my $storage = extract_param($param, 'storage');
 814         my $unique = extract_param($param, 'unique');
 815         my $live_restore = extract_param($param, 'live-restore');
 816
 817         if (defined(my $ssh_keys = $param->{sshkeys})) {
 818                 $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
 819                 PVE::Tools::validate_ssh_public_keys($ssh_keys);
 820         }
 821
 822         $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
 823             if defined($param->{cpuunits}); # clamp value depending on cgroup version
 824
 825         PVE::Cluster::check_cfs_quorum();
 826
 827         my $filename = PVE::QemuConfig->config_file($vmid);
 828         my $storecfg = PVE::Storage::config();
 829
 830         if (defined($pool)) {
 831             $rpcenv->check_pool_exist($pool);
 832         }
 833
 834         $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace'])
 835             if defined($storage);
 836
 837         if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
 838             # OK
 839         } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
 840             # OK
 841         } elsif ($archive && $force && (-f $filename) &&
 842                  $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
 843             # OK: user has VM.Backup permissions and wants to restore an existing VM
 844         } else {
 845             raise_perm_exc();
 846         }
 847
 848         if ($archive) {
 849             for my $opt (sort keys $param->%*) {
 850                 if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
 851                     raise_param_exc({ $opt => "option conflicts with option 'archive'" });
 852                 }
 853             }
 854
 855             if ($archive eq '-') {
 856                 die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli';
 857                 $archive = { type => 'pipe' };
 858             } else {
 859                 PVE::Storage::check_volume_access(
 860                     $rpcenv,
 861                     $authuser,
 862                     $storecfg,
 863                     $vmid,
 864                     $archive,
 865                     'backup',
 866                 );
 867
 868                 $archive = $parse_restore_archive->($storecfg, $archive);
 869             }
 870         }
 871
 872         if (scalar(keys $param->%*) > 0) {
 873             &$resolve_cdrom_alias($param);
 874
 875             &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage);
 876
 877             &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]);
 878
 879             &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param);
 880             &$check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param);
 881
 882             &$check_cpu_model_access($rpcenv, $authuser, $param);
 883
 884             $check_drive_param->($param, $storecfg);
 885
 886             PVE::QemuServer::add_random_macs($param);
 887         }
 888
 889         my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -";
 890
 891         eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) };
 892         die "$emsg $@" if $@;
 893
 894         my $restored_data = 0;
 895         my $restorefn = sub {
 896             my $conf = PVE::QemuConfig->load_config($vmid);
 897
 898             PVE::QemuConfig->check_protection($conf, $emsg);
 899
 900             die "$emsg vm is running\n" if PVE::QemuServer::check_running($vmid);
 901
 902             my $realcmd = sub {
 903                 my $restore_options = {
 904                     storage => $storage,
 905                     pool => $pool,
 906                     unique => $unique,
 907                     bwlimit => $bwlimit,
 908                     live => $live_restore,
 909                     override_conf => $param,
 910                 };
 911                 if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') {
 912                     die "live-restore is only compatible with backup images from a Proxmox Backup Server\n"
 913                         if $live_restore;
 914                     PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options);
 915                 } elsif ($archive->{type} eq 'pbs') {
 916                     PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options);
 917                 } else {
 918                     die "unknown backup archive type\n";
 919                 }
 920                 $restored_data = 1;
 921
 922                 my $restored_conf = PVE::QemuConfig->load_config($vmid);
 923                 # Convert restored VM to template if backup was VM template
 924                 if (PVE::QemuConfig->is_template($restored_conf)) {
 925                     warn "Convert to template.\n";
 926                     eval { PVE::QemuServer::template_create($vmid, $restored_conf) };
 927                     warn $@ if $@;
 928                 }
 929             };
 930
 931             # ensure no old replication state are exists
 932             PVE::ReplicationState::delete_guest_states($vmid);
 933
 934             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
 935
 936             if ($start_after_create && !$live_restore) {
 937                 print "Execute autostart\n";
 938                 eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) };
 939                 warn $@ if $@;
 940             }
 941         };
 942
 943         my $createfn = sub {
 944             # ensure no old replication state are exists
 945             PVE::ReplicationState::delete_guest_states($vmid);
 946
 947             my $realcmd = sub {
 948                 my $conf = $param;
 949                 my $arch = PVE::QemuServer::get_vm_arch($conf);
 950
 951                 $conf->{meta} = PVE::QemuServer::new_meta_info_string();
 952
 953                 my $vollist = [];
 954                 eval {
 955                     ($vollist, my $created_opts) = $create_disks->(
 956                         $rpcenv,
 957                         $authuser,
 958                         $conf,
 959                         $arch,
 960                         $storecfg,
 961                         $vmid,
 962                         $pool,
 963                         $param,
 964                         $storage,
 965                     );
 966                     $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*;
 967
 968                     if (!$conf->{boot}) {
 969                         my $devs = PVE::QemuServer::get_default_bootdevices($conf);
 970                         $conf->{boot} = PVE::QemuServer::print_bootorder($devs);
 971                     }
 972
 973                     # auto generate uuid if user did not specify smbios1 option
 974                     if (!$conf->{smbios1}) {
 975                         $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid();
 976                     }
 977
 978                     if ((!defined($conf->{vmgenid}) || $conf->{vmgenid} eq '1') && $arch ne 'aarch64') {
 979                         $conf->{vmgenid} = PVE::QemuServer::generate_uuid();
 980                     }
 981
 982                     my $machine = $conf->{machine};
 983                     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
 984                         # always pin Windows' machine version on create, they get to easily confused
 985                         if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) {
 986                             $conf->{machine} = PVE::QemuServer::windows_get_pinned_machine_version($machine);
 987                         }
 988                     }
 989
 990                     PVE::QemuConfig->write_config($vmid, $conf);
 991
 992                 };
 993                 my $err = $@;
 994
 995                 if ($err) {
 996                     foreach my $volid (@$vollist) {
 997                         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 998                         warn $@ if $@;
 999                     }
1000                     die "$emsg $err";
1001                 }
1002
1003                 PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
1004             };
1005
1006             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
1007
1008             if ($start_after_create) {
1009                 print "Execute autostart\n";
1010                 eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) };
1011                 warn $@ if $@;
1012             }
1013         };
1014
1015         my ($code, $worker_name);
1016         if ($is_restore) {
1017             $worker_name = 'qmrestore';
1018             $code = sub {
1019                 eval { $restorefn->() };
1020                 if (my $err = $@) {
1021                     eval { PVE::QemuConfig->remove_lock($vmid, 'create') };
1022                     warn $@ if $@;
1023                     if ($restored_data) {
1024                         warn "error after data was restored, VM disks should be OK but config may "
1025                             ."require adaptions. VM $vmid state is NOT cleaned up.\n";
1026                     } else {
1027                         warn "error before or during data restore, some or all disks were not "
1028                             ."completely restored. VM $vmid state is NOT cleaned up.\n";
1029                     }
1030                     die $err;
1031                 }
1032             };
1033         } else {
1034             $worker_name = 'qmcreate';
1035             $code = sub {
1036                 eval { $createfn->() };
1037                 if (my $err = $@) {
1038                     eval {
1039                         my $conffile = PVE::QemuConfig->config_file($vmid);
1040                         unlink($conffile) or die "failed to remove config file: $!\n";
1041                     };
1042                     warn $@ if $@;
1043                     die $err;
1044                 }
1045             };
1046         }
1047
1048         return $rpcenv->fork_worker($worker_name, $vmid, $authuser, $code);
1049     }});
1050
1051 __PACKAGE__->register_method({
1052     name => 'vmdiridx',
1053     path => '{vmid}',
1054     method => 'GET',
1055     proxyto => 'node',
1056     description => "Directory index",
1057     permissions => {
1058         user => 'all',
1059     },
1060     parameters => {
1061         additionalProperties => 0,
1062         properties => {
1063             node => get_standard_option('pve-node'),
1064             vmid => get_standard_option('pve-vmid'),
1065         },
1066     },
1067     returns => {
1068         type => 'array',
1069         items => {
1070             type => "object",
1071             properties => {
1072                 subdir => { type => 'string' },
1073             },
1074         },
1075         links => [ { rel => 'child', href => "{subdir}" } ],
1076     },
1077     code => sub {
1078         my ($param) = @_;
1079
1080         my $res = [
1081             { subdir => 'config' },
1082             { subdir => 'cloudinit' },
1083             { subdir => 'pending' },
1084             { subdir => 'status' },
1085             { subdir => 'unlink' },
1086             { subdir => 'vncproxy' },
1087             { subdir => 'termproxy' },
1088             { subdir => 'migrate' },
1089             { subdir => 'resize' },
1090             { subdir => 'move' },
1091             { subdir => 'rrd' },
1092             { subdir => 'rrddata' },
1093             { subdir => 'monitor' },
1094             { subdir => 'agent' },
1095             { subdir => 'snapshot' },
1096             { subdir => 'spiceproxy' },
1097             { subdir => 'sendkey' },
1098             { subdir => 'firewall' },
1099             { subdir => 'mtunnel' },
1100             { subdir => 'remote_migrate' },
1101         ];
1102
1103         return $res;
1104     }});
1105
1106 __PACKAGE__->register_method ({
1107     subclass => "PVE::API2::Firewall::VM",
1108     path => '{vmid}/firewall',
1109 });
1110
1111 __PACKAGE__->register_method ({
1112     subclass => "PVE::API2::Qemu::Agent",
1113     path => '{vmid}/agent',
1114 });
1115
1116 __PACKAGE__->register_method({
1117     name => 'rrd',
1118     path => '{vmid}/rrd',
1119     method => 'GET',
1120     protected => 1, # fixme: can we avoid that?
1121     permissions => {
1122         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1123     },
1124     description => "Read VM RRD statistics (returns PNG)",
1125     parameters => {
1126         additionalProperties => 0,
1127         properties => {
1128             node => get_standard_option('pve-node'),
1129             vmid => get_standard_option('pve-vmid'),
1130             timeframe => {
1131                 description => "Specify the time frame you are interested in.",
1132                 type => 'string',
1133                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1134             },
1135             ds => {
1136                 description => "The list of datasources you want to display.",
1137                 type => 'string', format => 'pve-configid-list',
1138             },
1139             cf => {
1140                 description => "The RRD consolidation function",
1141                 type => 'string',
1142                 enum => [ 'AVERAGE', 'MAX' ],
1143                 optional => 1,
1144             },
1145         },
1146     },
1147     returns => {
1148         type => "object",
1149         properties => {
1150             filename => { type => 'string' },
1151         },
1152     },
1153     code => sub {
1154         my ($param) = @_;
1155
1156         return PVE::RRD::create_rrd_graph(
1157             "pve2-vm/$param->{vmid}", $param->{timeframe},
1158             $param->{ds}, $param->{cf});
1159
1160     }});
1161
1162 __PACKAGE__->register_method({
1163     name => 'rrddata',
1164     path => '{vmid}/rrddata',
1165     method => 'GET',
1166     protected => 1, # fixme: can we avoid that?
1167     permissions => {
1168         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1169     },
1170     description => "Read VM RRD statistics",
1171     parameters => {
1172         additionalProperties => 0,
1173         properties => {
1174             node => get_standard_option('pve-node'),
1175             vmid => get_standard_option('pve-vmid'),
1176             timeframe => {
1177                 description => "Specify the time frame you are interested in.",
1178                 type => 'string',
1179                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1180             },
1181             cf => {
1182                 description => "The RRD consolidation function",
1183                 type => 'string',
1184                 enum => [ 'AVERAGE', 'MAX' ],
1185                 optional => 1,
1186             },
1187         },
1188     },
1189     returns => {
1190         type => "array",
1191         items => {
1192             type => "object",
1193             properties => {},
1194         },
1195     },
1196     code => sub {
1197         my ($param) = @_;
1198
1199         return PVE::RRD::create_rrd_data(
1200             "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
1201     }});
1202
1203
1204 __PACKAGE__->register_method({
1205     name => 'vm_config',
1206     path => '{vmid}/config',
1207     method => 'GET',
1208     proxyto => 'node',
1209     description => "Get the virtual machine configuration with pending configuration " .
1210         "changes applied. Set the 'current' parameter to get the current configuration instead.",
1211     permissions => {
1212         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1213     },
1214     parameters => {
1215         additionalProperties => 0,
1216         properties => {
1217             node => get_standard_option('pve-node'),
1218             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1219             current => {
1220                 description => "Get current values (instead of pending values).",
1221                 optional => 1,
1222                 default => 0,
1223                 type => 'boolean',
1224             },
1225             snapshot => get_standard_option('pve-snapshot-name', {
1226                 description => "Fetch config values from given snapshot.",
1227                 optional => 1,
1228                 completion => sub {
1229                     my ($cmd, $pname, $cur, $args) = @_;
1230                     PVE::QemuConfig->snapshot_list($args->[0]);
1231                 },
1232             }),
1233         },
1234     },
1235     returns => {
1236         description => "The VM configuration.",
1237         type => "object",
1238         properties => PVE::QemuServer::json_config_properties({
1239             digest => {
1240                 type => 'string',
1241                 description => 'SHA1 digest of configuration file. This can be used to prevent concurrent modifications.',
1242             }
1243         }),
1244     },
1245     code => sub {
1246         my ($param) = @_;
1247
1248         raise_param_exc({ snapshot => "cannot use 'snapshot' parameter with 'current'",
1249                           current => "cannot use 'snapshot' parameter with 'current'"})
1250             if ($param->{snapshot} && $param->{current});
1251
1252         my $conf;
1253         if ($param->{snapshot}) {
1254             $conf = PVE::QemuConfig->load_snapshot_config($param->{vmid}, $param->{snapshot});
1255         } else {
1256             $conf = PVE::QemuConfig->load_current_config($param->{vmid}, $param->{current});
1257         }
1258         $conf->{cipassword} = '**********' if $conf->{cipassword};
1259         return $conf;
1260
1261     }});
1262
1263 __PACKAGE__->register_method({
1264     name => 'vm_pending',
1265     path => '{vmid}/pending',
1266     method => 'GET',
1267     proxyto => 'node',
1268     description => "Get the virtual machine configuration with both current and pending values.",
1269     permissions => {
1270         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1271     },
1272     parameters => {
1273         additionalProperties => 0,
1274         properties => {
1275             node => get_standard_option('pve-node'),
1276             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1277         },
1278     },
1279     returns => {
1280         type => "array",
1281         items => {
1282             type => "object",
1283             properties => {
1284                 key => {
1285                     description => "Configuration option name.",
1286                     type => 'string',
1287                 },
1288                 value => {
1289                     description => "Current value.",
1290                     type => 'string',
1291                     optional => 1,
1292                 },
1293                 pending => {
1294                     description => "Pending value.",
1295                     type => 'string',
1296                     optional => 1,
1297                 },
1298                 delete => {
1299                     description => "Indicates a pending delete request if present and not 0. " .
1300                                    "The value 2 indicates a force-delete request.",
1301                     type => 'integer',
1302                     minimum => 0,
1303                     maximum => 2,
1304                     optional => 1,
1305                 },
1306             },
1307         },
1308     },
1309     code => sub {
1310         my ($param) = @_;
1311
1312         my $conf = PVE::QemuConfig->load_config($param->{vmid});
1313
1314         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
1315
1316         $conf->{cipassword} = '**********' if defined($conf->{cipassword});
1317         $conf->{pending}->{cipassword} = '********** ' if defined($conf->{pending}->{cipassword});
1318
1319         return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
1320    }});
1321
1322 __PACKAGE__->register_method({
1323     name => 'cloudinit_pending',
1324     path => '{vmid}/cloudinit',
1325     method => 'GET',
1326     proxyto => 'node',
1327     description => "Get the cloudinit configuration with both current and pending values.",
1328     permissions => {
1329         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1330     },
1331     parameters => {
1332         additionalProperties => 0,
1333         properties => {
1334             node => get_standard_option('pve-node'),
1335             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1336         },
1337     },
1338     returns => {
1339         type => "array",
1340         items => {
1341             type => "object",
1342             properties => {
1343                 key => {
1344                     description => "Configuration option name.",
1345                     type => 'string',
1346                 },
1347                 value => {
1348                     description => "Value as it was used to generate the current cloudinit image.",
1349                     type => 'string',
1350                     optional => 1,
1351                 },
1352                 pending => {
1353                     description => "The new pending value.",
1354                     type => 'string',
1355                     optional => 1,
1356                 },
1357                 delete => {
1358                     description => "Indicates a pending delete request if present and not 0. ",
1359                     type => 'integer',
1360                     minimum => 0,
1361                     maximum => 1,
1362                     optional => 1,
1363                 },
1364             },
1365         },
1366     },
1367     code => sub {
1368         my ($param) = @_;
1369
1370         my $vmid = $param->{vmid};
1371         my $conf = PVE::QemuConfig->load_config($vmid);
1372
1373         my $ci = $conf->{cloudinit};
1374
1375         $conf->{cipassword} = '**********' if exists $conf->{cipassword};
1376         $ci->{cipassword} = '**********' if exists $ci->{cipassword};
1377
1378         my $res = [];
1379
1380         # All the values that got added
1381         my $added = delete($ci->{added}) // '';
1382         for my $key (PVE::Tools::split_list($added)) {
1383             push @$res, { key => $key, pending => $conf->{$key} };
1384         }
1385
1386         # All already existing values (+ their new value, if it exists)
1387         for my $opt (keys %$cloudinitoptions) {
1388             next if !$conf->{$opt};
1389             next if $added =~ m/$opt/;
1390             my $item = {
1391                 key => $opt,
1392             };
1393
1394             if (my $pending = $ci->{$opt}) {
1395                 $item->{value} = $pending;
1396                 $item->{pending} = $conf->{$opt};
1397             } else {
1398                 $item->{value} = $conf->{$opt},
1399             }
1400
1401             push @$res, $item;
1402         }
1403
1404         # Now, we'll find the deleted ones
1405         for my $opt (keys %$ci) {
1406             next if $conf->{$opt};
1407             push @$res, { key => $opt, delete => 1 };
1408         }
1409
1410         return $res;
1411    }});
1412
1413 __PACKAGE__->register_method({
1414     name => 'cloudinit_update',
1415     path => '{vmid}/cloudinit',
1416     method => 'PUT',
1417     protected => 1,
1418     proxyto => 'node',
1419     description => "Regenerate and change cloudinit config drive.",
1420     permissions => {
1421         check => ['perm', '/vms/{vmid}', 'VM.Config.Cloudinit'],
1422     },
1423     parameters => {
1424         additionalProperties => 0,
1425         properties => {
1426             node => get_standard_option('pve-node'),
1427             vmid => get_standard_option('pve-vmid'),
1428         },
1429     },
1430     returns => { type => 'null' },
1431     code => sub {
1432         my ($param) = @_;
1433
1434         my $rpcenv = PVE::RPCEnvironment::get();
1435         my $authuser = $rpcenv->get_user();
1436
1437         my $vmid = extract_param($param, 'vmid');
1438
1439         PVE::QemuConfig->lock_config($vmid, sub {
1440             my $conf = PVE::QemuConfig->load_config($vmid);
1441             PVE::QemuConfig->check_lock($conf);
1442
1443             my $storecfg = PVE::Storage::config();
1444             PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
1445         });
1446         return;
1447     }});
1448
1449 # POST/PUT {vmid}/config implementation
1450 #
1451 # The original API used PUT (idempotent) an we assumed that all operations
1452 # are fast. But it turned out that almost any configuration change can
1453 # involve hot-plug actions, or disk alloc/free. Such actions can take long
1454 # time to complete and have side effects (not idempotent).
1455 #
1456 # The new implementation uses POST and forks a worker process. We added
1457 # a new option 'background_delay'. If specified we wait up to
1458 # 'background_delay' second for the worker task to complete. It returns null
1459 # if the task is finished within that time, else we return the UPID.
1460
1461 my $update_vm_api  = sub {
1462     my ($param, $sync) = @_;
1463
1464     my $rpcenv = PVE::RPCEnvironment::get();
1465
1466     my $authuser = $rpcenv->get_user();
1467
1468     my $node = extract_param($param, 'node');
1469
1470     my $vmid = extract_param($param, 'vmid');
1471
1472     my $digest = extract_param($param, 'digest');
1473
1474     my $background_delay = extract_param($param, 'background_delay');
1475
1476     my $skip_cloud_init = extract_param($param, 'skip_cloud_init');
1477
1478     if (defined(my $cipassword = $param->{cipassword})) {
1479         # Same logic as in cloud-init (but with the regex fixed...)
1480         $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
1481             if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
1482     }
1483
1484     my @paramarr = (); # used for log message
1485     foreach my $key (sort keys %$param) {
1486         my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
1487         push @paramarr, "-$key", $value;
1488     }
1489
1490     my $skiplock = extract_param($param, 'skiplock');
1491     raise_param_exc({ skiplock => "Only root may use this option." })
1492         if $skiplock && $authuser ne 'root@pam';
1493
1494     my $delete_str = extract_param($param, 'delete');
1495
1496     my $revert_str = extract_param($param, 'revert');
1497
1498     my $force = extract_param($param, 'force');
1499
1500     if (defined(my $ssh_keys = $param->{sshkeys})) {
1501         $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
1502         PVE::Tools::validate_ssh_public_keys($ssh_keys);
1503     }
1504
1505     $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
1506         if defined($param->{cpuunits}); # clamp value depending on cgroup version
1507
1508     die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param);
1509
1510     my $storecfg = PVE::Storage::config();
1511
1512     my $defaults = PVE::QemuServer::load_defaults();
1513
1514     &$resolve_cdrom_alias($param);
1515
1516     # now try to verify all parameters
1517
1518     my $revert = {};
1519     foreach my $opt (PVE::Tools::split_list($revert_str)) {
1520         if (!PVE::QemuServer::option_exists($opt)) {
1521             raise_param_exc({ revert => "unknown option '$opt'" });
1522         }
1523
1524         raise_param_exc({ delete => "you can't use '-$opt' and " .
1525                               "-revert $opt' at the same time" })
1526             if defined($param->{$opt});
1527
1528         $revert->{$opt} = 1;
1529     }
1530
1531     my @delete = ();
1532     foreach my $opt (PVE::Tools::split_list($delete_str)) {
1533         $opt = 'ide2' if $opt eq 'cdrom';
1534
1535         raise_param_exc({ delete => "you can't use '-$opt' and " .
1536                               "-delete $opt' at the same time" })
1537             if defined($param->{$opt});
1538
1539         raise_param_exc({ revert => "you can't use '-delete $opt' and " .
1540                               "-revert $opt' at the same time" })
1541             if $revert->{$opt};
1542
1543         if (!PVE::QemuServer::option_exists($opt)) {
1544             raise_param_exc({ delete => "unknown option '$opt'" });
1545         }
1546
1547         push @delete, $opt;
1548     }
1549
1550     my $repl_conf = PVE::ReplicationConfig->new();
1551     my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
1552     my $check_replication = sub {
1553         my ($drive) = @_;
1554         return if !$is_replicated;
1555         my $volid = $drive->{file};
1556         return if !$volid || !($drive->{replicate}//1);
1557         return if PVE::QemuServer::drive_is_cdrom($drive);
1558
1559         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1560         die "cannot add non-managed/pass-through volume to a replicated VM\n"
1561             if !defined($storeid);
1562
1563         return if defined($volname) && $volname eq 'cloudinit';
1564
1565         my $format;
1566         if ($volid =~ $NEW_DISK_RE) {
1567             $storeid = $2;
1568             $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid);
1569         } else {
1570             $format = (PVE::Storage::parse_volname($storecfg, $volid))[6];
1571         }
1572         return if PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
1573         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1574         return if $scfg->{shared};
1575         die "cannot add non-replicatable volume to a replicated VM\n";
1576     };
1577
1578     $check_drive_param->($param, $storecfg, $check_replication);
1579
1580     foreach my $opt (keys %$param) {
1581         if ($opt =~ m/^net(\d+)$/) {
1582             # add macaddr
1583             my $net = PVE::QemuServer::parse_net($param->{$opt});
1584             $param->{$opt} = PVE::QemuServer::print_net($net);
1585         } elsif ($opt eq 'vmgenid') {
1586             if ($param->{$opt} eq '1') {
1587                 $param->{$opt} = PVE::QemuServer::generate_uuid();
1588             }
1589         } elsif ($opt eq 'hookscript') {
1590             eval { PVE::GuestHelpers::check_hookscript($param->{$opt}, $storecfg); };
1591             raise_param_exc({ $opt => $@ }) if $@;
1592         }
1593     }
1594
1595     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
1596
1597     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
1598
1599     &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
1600
1601     my $updatefn =  sub {
1602
1603         my $conf = PVE::QemuConfig->load_config($vmid);
1604
1605         die "checksum missmatch (file change by other user?)\n"
1606             if $digest && $digest ne $conf->{digest};
1607
1608         &$check_cpu_model_access($rpcenv, $authuser, $param, $conf);
1609
1610         # FIXME: 'suspended' lock should probabyl be a state or "weak" lock?!
1611         if (scalar(@delete) && grep { $_ eq 'vmstate'} @delete) {
1612             if (defined($conf->{lock}) && $conf->{lock} eq 'suspended') {
1613                 delete $conf->{lock}; # for check lock check, not written out
1614                 push @delete, 'lock'; # this is the real deal to write it out
1615             }
1616             push @delete, 'runningmachine' if $conf->{runningmachine};
1617             push @delete, 'runningcpu' if $conf->{runningcpu};
1618         }
1619
1620         PVE::QemuConfig->check_lock($conf) if !$skiplock;
1621
1622         foreach my $opt (keys %$revert) {
1623             if (defined($conf->{$opt})) {
1624                 $param->{$opt} = $conf->{$opt};
1625             } elsif (defined($conf->{pending}->{$opt})) {
1626                 push @delete, $opt;
1627             }
1628         }
1629
1630         if ($param->{memory} || defined($param->{balloon})) {
1631             my $maxmem = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory} || $defaults->{memory};
1632             my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon};
1633
1634             die "balloon value too large (must be smaller than assigned memory)\n"
1635                 if $balloon && $balloon > $maxmem;
1636         }
1637
1638         PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: " . join (' ', @paramarr));
1639
1640         my $worker = sub {
1641
1642             print "update VM $vmid: " . join (' ', @paramarr) . "\n";
1643
1644             # write updates to pending section
1645
1646             my $modified = {}; # record what $option we modify
1647
1648             my @bootorder;
1649             if (my $boot = $conf->{boot}) {
1650                 my $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot);
1651                 @bootorder = PVE::Tools::split_list($bootcfg->{order}) if $bootcfg && $bootcfg->{order};
1652             }
1653             my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete;
1654
1655             my $check_drive_perms = sub {
1656                 my ($opt, $val) = @_;
1657                 my $drive = PVE::QemuServer::parse_drive($opt, $val, 1);
1658                 if (PVE::QemuServer::drive_is_cloudinit($drive)) {
1659                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM']);
1660                 } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM
1661                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
1662                 } else {
1663                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1664
1665                 }
1666             };
1667
1668             foreach my $opt (@delete) {
1669                 $modified->{$opt} = 1;
1670                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1671
1672                 # value of what we want to delete, independent if pending or not
1673                 my $val = $conf->{$opt} // $conf->{pending}->{$opt};
1674                 if (!defined($val)) {
1675                     warn "cannot delete '$opt' - not set in current configuration!\n";
1676                     $modified->{$opt} = 0;
1677                     next;
1678                 }
1679                 my $is_pending_val = defined($conf->{pending}->{$opt});
1680                 delete $conf->{pending}->{$opt};
1681
1682                 # remove from bootorder if necessary
1683                 if (!$bootorder_deleted && @bootorder && grep {$_ eq $opt} @bootorder) {
1684                     @bootorder = grep {$_ ne $opt} @bootorder;
1685                     $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1686                     $modified->{boot} = 1;
1687                 }
1688
1689                 if ($opt =~ m/^unused/) {
1690                     my $drive = PVE::QemuServer::parse_drive($opt, $val);
1691                     PVE::QemuConfig->check_protection($conf, "can't remove unused disk '$drive->{file}'");
1692                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1693                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser)) {
1694                         delete $conf->{$opt};
1695                         PVE::QemuConfig->write_config($vmid, $conf);
1696                     }
1697                 } elsif ($opt eq 'vmstate') {
1698                     PVE::QemuConfig->check_protection($conf, "can't remove vmstate '$val'");
1699                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, { file => $val }, $rpcenv, $authuser, 1)) {
1700                         delete $conf->{$opt};
1701                         PVE::QemuConfig->write_config($vmid, $conf);
1702                     }
1703                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
1704                     PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'");
1705                     $check_drive_perms->($opt, $val);
1706                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val))
1707                         if $is_pending_val;
1708                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1709                     PVE::QemuConfig->write_config($vmid, $conf);
1710                 } elsif ($opt =~ m/^serial\d+$/) {
1711                     if ($val eq 'socket') {
1712                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1713                     } elsif ($authuser ne 'root@pam') {
1714                         die "only root can delete '$opt' config for real devices\n";
1715                     }
1716                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1717                     PVE::QemuConfig->write_config($vmid, $conf);
1718                 } elsif ($opt =~ m/^usb\d+$/) {
1719                     if ($val =~ m/spice/) {
1720                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1721                     } elsif ($authuser ne 'root@pam') {
1722                         die "only root can delete '$opt' config for real devices\n";
1723                     }
1724                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1725                     PVE::QemuConfig->write_config($vmid, $conf);
1726                 } elsif ($opt eq 'tags') {
1727                     assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser);
1728                     delete $conf->{$opt};
1729                     PVE::QemuConfig->write_config($vmid, $conf);
1730                 } else {
1731                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1732                     PVE::QemuConfig->write_config($vmid, $conf);
1733                 }
1734             }
1735
1736             foreach my $opt (keys %$param) { # add/change
1737                 $modified->{$opt} = 1;
1738                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1739                 next if defined($conf->{pending}->{$opt}) && ($param->{$opt} eq $conf->{pending}->{$opt}); # skip if nothing changed
1740
1741                 my $arch = PVE::QemuServer::get_vm_arch($conf);
1742
1743                 if (PVE::QemuServer::is_valid_drivename($opt)) {
1744                     # old drive
1745                     if ($conf->{$opt}) {
1746                         $check_drive_perms->($opt, $conf->{$opt});
1747                     }
1748
1749                     # new drive
1750                     $check_drive_perms->($opt, $param->{$opt});
1751                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
1752                         if defined($conf->{pending}->{$opt});
1753
1754                     my (undef, $created_opts) = $create_disks->(
1755                         $rpcenv,
1756                         $authuser,
1757                         $conf,
1758                         $arch,
1759                         $storecfg,
1760                         $vmid,
1761                         undef,
1762                         {$opt => $param->{$opt}},
1763                     );
1764                     $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*;
1765
1766                     # default legacy boot order implies all cdroms anyway
1767                     if (@bootorder) {
1768                         # append new CD drives to bootorder to mark them bootable
1769                         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
1770                         if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) {
1771                             push @bootorder, $opt;
1772                             $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1773                             $modified->{boot} = 1;
1774                         }
1775                     }
1776                 } elsif ($opt =~ m/^serial\d+/) {
1777                     if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
1778                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1779                     } elsif ($authuser ne 'root@pam') {
1780                         die "only root can modify '$opt' config for real devices\n";
1781                     }
1782                     $conf->{pending}->{$opt} = $param->{$opt};
1783                 } elsif ($opt =~ m/^usb\d+/) {
1784                     if ((!defined($conf->{$opt}) || $conf->{$opt} =~ m/spice/) && $param->{$opt} =~ m/spice/) {
1785                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1786                     } elsif ($authuser ne 'root@pam') {
1787                         die "only root can modify '$opt' config for real devices\n";
1788                     }
1789                     $conf->{pending}->{$opt} = $param->{$opt};
1790                 } elsif ($opt eq 'tags') {
1791                     assert_tag_permissions($vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser);
1792                     $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt});
1793                 } else {
1794                     $conf->{pending}->{$opt} = $param->{$opt};
1795
1796                     if ($opt eq 'boot') {
1797                         my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $param->{$opt});
1798                         if ($new_bootcfg->{order}) {
1799                             my @devs = PVE::Tools::split_list($new_bootcfg->{order});
1800                             for my $dev (@devs) {
1801                                 my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev};
1802                                 my $deleted = grep {$_ eq $dev} @delete;
1803                                 die "invalid bootorder: device '$dev' does not exist'\n"
1804                                     if !$exists || $deleted;
1805                             }
1806
1807                             # remove legacy boot order settings if new one set
1808                             $conf->{pending}->{$opt} = PVE::QemuServer::print_bootorder(\@devs);
1809                             PVE::QemuConfig->add_to_pending_delete($conf, "bootdisk")
1810                                 if $conf->{bootdisk};
1811                         }
1812                     }
1813                 }
1814                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
1815                 PVE::QemuConfig->write_config($vmid, $conf);
1816             }
1817
1818             # remove pending changes when nothing changed
1819             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1820             my $changes = PVE::QemuConfig->cleanup_pending($conf);
1821             PVE::QemuConfig->write_config($vmid, $conf) if $changes;
1822
1823             return if !scalar(keys %{$conf->{pending}});
1824
1825             my $running = PVE::QemuServer::check_running($vmid);
1826
1827             # apply pending changes
1828
1829             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1830
1831             my $errors = {};
1832             if ($running) {
1833                 PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors);
1834             } else {
1835                 # cloud_init must be skipped if we are in an incoming, remote live migration
1836                 PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors, $skip_cloud_init);
1837             }
1838             raise_param_exc($errors) if scalar(keys %$errors);
1839
1840             return;
1841         };
1842
1843         if ($sync) {
1844             &$worker();
1845             return;
1846         } else {
1847             my $upid = $rpcenv->fork_worker('qmconfig', $vmid, $authuser, $worker);
1848
1849             if ($background_delay) {
1850
1851                 # Note: It would be better to do that in the Event based HTTPServer
1852                 # to avoid blocking call to sleep.
1853
1854                 my $end_time = time() + $background_delay;
1855
1856                 my $task = PVE::Tools::upid_decode($upid);
1857
1858                 my $running = 1;
1859                 while (time() < $end_time) {
1860                     $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
1861                     last if !$running;
1862                     sleep(1); # this gets interrupted when child process ends
1863                 }
1864
1865                 if (!$running) {
1866                     my $status = PVE::Tools::upid_read_status($upid);
1867                     return if !PVE::Tools::upid_status_is_error($status);
1868                     die "failed to update VM $vmid: $status\n";
1869                 }
1870             }
1871
1872             return $upid;
1873         }
1874     };
1875
1876     return PVE::QemuConfig->lock_config($vmid, $updatefn);
1877 };
1878
1879 my $vm_config_perm_list = [
1880             'VM.Config.Disk',
1881             'VM.Config.CDROM',
1882             'VM.Config.CPU',
1883             'VM.Config.Memory',
1884             'VM.Config.Network',
1885             'VM.Config.HWType',
1886             'VM.Config.Options',
1887             'VM.Config.Cloudinit',
1888     ];
1889
1890 __PACKAGE__->register_method({
1891     name => 'update_vm_async',
1892     path => '{vmid}/config',
1893     method => 'POST',
1894     protected => 1,
1895     proxyto => 'node',
1896     description => "Set virtual machine options (asynchrounous API).",
1897     permissions => {
1898         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1899     },
1900     parameters => {
1901         additionalProperties => 0,
1902         properties => PVE::QemuServer::json_config_properties(
1903             {
1904                 node => get_standard_option('pve-node'),
1905                 vmid => get_standard_option('pve-vmid'),
1906                 skiplock => get_standard_option('skiplock'),
1907                 delete => {
1908                     type => 'string', format => 'pve-configid-list',
1909                     description => "A list of settings you want to delete.",
1910                     optional => 1,
1911                 },
1912                 revert => {
1913                     type => 'string', format => 'pve-configid-list',
1914                     description => "Revert a pending change.",
1915                     optional => 1,
1916                 },
1917                 force => {
1918                     type => 'boolean',
1919                     description => $opt_force_description,
1920                     optional => 1,
1921                     requires => 'delete',
1922                 },
1923                 digest => {
1924                     type => 'string',
1925                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1926                     maxLength => 40,
1927                     optional => 1,
1928                 },
1929                 background_delay => {
1930                     type => 'integer',
1931                     description => "Time to wait for the task to finish. We return 'null' if the task finish within that time.",
1932                     minimum => 1,
1933                     maximum => 30,
1934                     optional => 1,
1935                 },
1936             },
1937             1, # with_disk_alloc
1938         ),
1939     },
1940     returns => {
1941         type => 'string',
1942         optional => 1,
1943     },
1944     code => $update_vm_api,
1945 });
1946
1947 __PACKAGE__->register_method({
1948     name => 'update_vm',
1949     path => '{vmid}/config',
1950     method => 'PUT',
1951     protected => 1,
1952     proxyto => 'node',
1953     description => "Set virtual machine options (synchrounous API) - You should consider using the POST method instead for any actions involving hotplug or storage allocation.",
1954     permissions => {
1955         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1956     },
1957     parameters => {
1958         additionalProperties => 0,
1959         properties => PVE::QemuServer::json_config_properties(
1960             {
1961                 node => get_standard_option('pve-node'),
1962                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1963                 skiplock => get_standard_option('skiplock'),
1964                 delete => {
1965                     type => 'string', format => 'pve-configid-list',
1966                     description => "A list of settings you want to delete.",
1967                     optional => 1,
1968                 },
1969                 revert => {
1970                     type => 'string', format => 'pve-configid-list',
1971                     description => "Revert a pending change.",
1972                     optional => 1,
1973                 },
1974                 force => {
1975                     type => 'boolean',
1976                     description => $opt_force_description,
1977                     optional => 1,
1978                     requires => 'delete',
1979                 },
1980                 digest => {
1981                     type => 'string',
1982                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1983                     maxLength => 40,
1984                     optional => 1,
1985                 },
1986             },
1987             1, # with_disk_alloc
1988         ),
1989     },
1990     returns => { type => 'null' },
1991     code => sub {
1992         my ($param) = @_;
1993         &$update_vm_api($param, 1);
1994         return;
1995     }
1996 });
1997
1998 __PACKAGE__->register_method({
1999     name => 'destroy_vm',
2000     path => '{vmid}',
2001     method => 'DELETE',
2002     protected => 1,
2003     proxyto => 'node',
2004     description => "Destroy the VM and  all used/owned volumes. Removes any VM specific permissions"
2005         ." and firewall rules",
2006     permissions => {
2007         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
2008     },
2009     parameters => {
2010         additionalProperties => 0,
2011         properties => {
2012             node => get_standard_option('pve-node'),
2013             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2014             skiplock => get_standard_option('skiplock'),
2015             purge => {
2016                 type => 'boolean',
2017                 description => "Remove VMID from configurations, like backup & replication jobs and HA.",
2018                 optional => 1,
2019             },
2020             'destroy-unreferenced-disks' => {
2021                 type => 'boolean',
2022                 description => "If set, destroy additionally all disks not referenced in the config"
2023                  ." but with a matching VMID from all enabled storages.",
2024                 optional => 1,
2025                 default => 0,
2026             },
2027         },
2028     },
2029     returns => {
2030         type => 'string',
2031     },
2032     code => sub {
2033         my ($param) = @_;
2034
2035         my $rpcenv = PVE::RPCEnvironment::get();
2036         my $authuser = $rpcenv->get_user();
2037         my $vmid = $param->{vmid};
2038
2039         my $skiplock = $param->{skiplock};
2040         raise_param_exc({ skiplock => "Only root may use this option." })
2041             if $skiplock && $authuser ne 'root@pam';
2042
2043         my $early_checks = sub {
2044             # test if VM exists
2045             my $conf = PVE::QemuConfig->load_config($vmid);
2046             PVE::QemuConfig->check_protection($conf, "can't remove VM $vmid");
2047
2048             my $ha_managed = PVE::HA::Config::service_is_configured("vm:$vmid");
2049
2050             if (!$param->{purge}) {
2051                 die "unable to remove VM $vmid - used in HA resources and purge parameter not set.\n"
2052                     if $ha_managed;
2053                 # don't allow destroy if with replication jobs but no purge param
2054                 my $repl_conf = PVE::ReplicationConfig->new();
2055                 $repl_conf->check_for_existing_jobs($vmid);
2056             }
2057
2058             die "VM $vmid is running - destroy failed\n"
2059                 if PVE::QemuServer::check_running($vmid);
2060
2061             return $ha_managed;
2062         };
2063
2064         $early_checks->();
2065
2066         my $realcmd = sub {
2067             my $upid = shift;
2068
2069             my $storecfg = PVE::Storage::config();
2070
2071             syslog('info', "destroy VM $vmid: $upid\n");
2072             PVE::QemuConfig->lock_config($vmid, sub {
2073                 # repeat, config might have changed
2074                 my $ha_managed = $early_checks->();
2075
2076                 my $purge_unreferenced = $param->{'destroy-unreferenced-disks'};
2077
2078                 PVE::QemuServer::destroy_vm(
2079                     $storecfg,
2080                     $vmid,
2081                     $skiplock, { lock => 'destroyed' },
2082                     $purge_unreferenced,
2083                 );
2084
2085                 PVE::AccessControl::remove_vm_access($vmid);
2086                 PVE::Firewall::remove_vmfw_conf($vmid);
2087                 if ($param->{purge}) {
2088                     print "purging VM $vmid from related configurations..\n";
2089                     PVE::ReplicationConfig::remove_vmid_jobs($vmid);
2090                     PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);
2091
2092                     if ($ha_managed) {
2093                         PVE::HA::Config::delete_service_from_config("vm:$vmid");
2094                         print "NOTE: removed VM $vmid from HA resource configuration.\n";
2095                     }
2096                 }
2097
2098                 # only now remove the zombie config, else we can have reuse race
2099                 PVE::QemuConfig->destroy_config($vmid);
2100             });
2101         };
2102
2103         return $rpcenv->fork_worker('qmdestroy', $vmid, $authuser, $realcmd);
2104     }});
2105
2106 __PACKAGE__->register_method({
2107     name => 'unlink',
2108     path => '{vmid}/unlink',
2109     method => 'PUT',
2110     protected => 1,
2111     proxyto => 'node',
2112     description => "Unlink/delete disk images.",
2113     permissions => {
2114         check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']],
2115     },
2116     parameters => {
2117         additionalProperties => 0,
2118         properties => {
2119             node => get_standard_option('pve-node'),
2120             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
2121             idlist => {
2122                 type => 'string', format => 'pve-configid-list',
2123                 description => "A list of disk IDs you want to delete.",
2124             },
2125             force => {
2126                 type => 'boolean',
2127                 description => $opt_force_description,
2128                 optional => 1,
2129             },
2130         },
2131     },
2132     returns => { type => 'null'},
2133     code => sub {
2134         my ($param) = @_;
2135
2136         $param->{delete} = extract_param($param, 'idlist');
2137
2138         __PACKAGE__->update_vm($param);
2139
2140         return;
2141     }});
2142
2143 # uses good entropy, each char is limited to 6 bit to get printable chars simply
2144 my $gen_rand_chars = sub {
2145     my ($length) = @_;
2146
2147     die "invalid length $length" if $length < 1;
2148
2149     my $min = ord('!'); # first printable ascii
2150
2151     my $rand_bytes = Crypt::OpenSSL::Random::random_bytes($length);
2152     die "failed to generate random bytes!\n"
2153       if !$rand_bytes;
2154
2155     my $str = join('', map { chr((ord($_) & 0x3F) + $min) } split('', $rand_bytes));
2156
2157     return $str;
2158 };
2159
2160 my $sslcert;
2161
2162 __PACKAGE__->register_method({
2163     name => 'vncproxy',
2164     path => '{vmid}/vncproxy',
2165     method => 'POST',
2166     protected => 1,
2167     permissions => {
2168         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2169     },
2170     description => "Creates a TCP VNC proxy connections.",
2171     parameters => {
2172         additionalProperties => 0,
2173         properties => {
2174             node => get_standard_option('pve-node'),
2175             vmid => get_standard_option('pve-vmid'),
2176             websocket => {
2177                 optional => 1,
2178                 type => 'boolean',
2179                 description => "starts websockify instead of vncproxy",
2180             },
2181             'generate-password' => {
2182                 optional => 1,
2183                 type => 'boolean',
2184                 default => 0,
2185                 description => "Generates a random password to be used as ticket instead of the API ticket.",
2186             },
2187         },
2188     },
2189     returns => {
2190         additionalProperties => 0,
2191         properties => {
2192             user => { type => 'string' },
2193             ticket => { type => 'string' },
2194             password => {
2195                 optional => 1,
2196                 description => "Returned if requested with 'generate-password' param."
2197                     ." Consists of printable ASCII characters ('!' .. '~').",
2198                 type => 'string',
2199             },
2200             cert => { type => 'string' },
2201             port => { type => 'integer' },
2202             upid => { type => 'string' },
2203         },
2204     },
2205     code => sub {
2206         my ($param) = @_;
2207
2208         my $rpcenv = PVE::RPCEnvironment::get();
2209
2210         my $authuser = $rpcenv->get_user();
2211
2212         my $vmid = $param->{vmid};
2213         my $node = $param->{node};
2214         my $websocket = $param->{websocket};
2215
2216         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2217
2218         my $serial;
2219         if ($conf->{vga}) {
2220             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2221             $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2222         }
2223
2224         my $authpath = "/vms/$vmid";
2225
2226         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2227         my $password = $ticket;
2228         if ($param->{'generate-password'}) {
2229             $password = $gen_rand_chars->(8);
2230         }
2231
2232         $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
2233             if !$sslcert;
2234
2235         my $family;
2236         my $remcmd = [];
2237
2238         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2239             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2240             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2241             # NOTE: kvm VNC traffic is already TLS encrypted or is known unsecure
2242             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, defined($serial) ? '-t' : '-T');
2243         } else {
2244             $family = PVE::Tools::get_host_address_family($node);
2245         }
2246
2247         my $port = PVE::Tools::next_vnc_port($family);
2248
2249         my $timeout = 10;
2250
2251         my $realcmd = sub {
2252             my $upid = shift;
2253
2254             syslog('info', "starting vnc proxy $upid\n");
2255
2256             my $cmd;
2257
2258             if (defined($serial)) {
2259
2260                 my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-iface', $serial, '-escape', '0' ];
2261
2262                 $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
2263                         '-timeout', $timeout, '-authpath', $authpath,
2264                         '-perm', 'Sys.Console'];
2265
2266                 if ($param->{websocket}) {
2267                     $ENV{PVE_VNC_TICKET} = $password; # pass ticket to vncterm
2268                     push @$cmd, '-notls', '-listen', 'localhost';
2269                 }
2270
2271                 push @$cmd, '-c', @$remcmd, @$termcmd;
2272
2273                 PVE::Tools::run_command($cmd);
2274
2275             } else {
2276
2277                 $ENV{LC_PVE_TICKET} = $password if $websocket; # set ticket with "qm vncproxy"
2278
2279                 $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid];
2280
2281                 my $sock = IO::Socket::IP->new(
2282                     ReuseAddr => 1,
2283                     Listen => 1,
2284                     LocalPort => $port,
2285                     Proto => 'tcp',
2286                     GetAddrInfoFlags => 0,
2287                 ) or die "failed to create socket: $!\n";
2288                 # Inside the worker we shouldn't have any previous alarms
2289                 # running anyway...:
2290                 alarm(0);
2291                 local $SIG{ALRM} = sub { die "connection timed out\n" };
2292                 alarm $timeout;
2293                 accept(my $cli, $sock) or die "connection failed: $!\n";
2294                 alarm(0);
2295                 close($sock);
2296                 if (PVE::Tools::run_command($cmd,
2297                     output => '>&'.fileno($cli),
2298                     input => '<&'.fileno($cli),
2299                     noerr => 1) != 0)
2300                 {
2301                     die "Failed to run vncproxy.\n";
2302                 }
2303             }
2304
2305             return;
2306         };
2307
2308         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2309
2310         PVE::Tools::wait_for_vnc_port($port);
2311
2312         my $res = {
2313             user => $authuser,
2314             ticket => $ticket,
2315             port => $port,
2316             upid => $upid,
2317             cert => $sslcert,
2318         };
2319         $res->{password} = $password if $param->{'generate-password'};
2320
2321         return $res;
2322     }});
2323
2324 __PACKAGE__->register_method({
2325     name => 'termproxy',
2326     path => '{vmid}/termproxy',
2327     method => 'POST',
2328     protected => 1,
2329     permissions => {
2330         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2331     },
2332     description => "Creates a TCP proxy connections.",
2333     parameters => {
2334         additionalProperties => 0,
2335         properties => {
2336             node => get_standard_option('pve-node'),
2337             vmid => get_standard_option('pve-vmid'),
2338             serial=> {
2339                 optional => 1,
2340                 type => 'string',
2341                 enum => [qw(serial0 serial1 serial2 serial3)],
2342                 description => "opens a serial terminal (defaults to display)",
2343             },
2344         },
2345     },
2346     returns => {
2347         additionalProperties => 0,
2348         properties => {
2349             user => { type => 'string' },
2350             ticket => { type => 'string' },
2351             port => { type => 'integer' },
2352             upid => { type => 'string' },
2353         },
2354     },
2355     code => sub {
2356         my ($param) = @_;
2357
2358         my $rpcenv = PVE::RPCEnvironment::get();
2359
2360         my $authuser = $rpcenv->get_user();
2361
2362         my $vmid = $param->{vmid};
2363         my $node = $param->{node};
2364         my $serial = $param->{serial};
2365
2366         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2367
2368         if (!defined($serial)) {
2369             if ($conf->{vga}) {
2370                 my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2371                 $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2372             }
2373         }
2374
2375         my $authpath = "/vms/$vmid";
2376
2377         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2378
2379         my $family;
2380         my $remcmd = [];
2381
2382         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2383             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2384             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2385             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, '-t');
2386             push @$remcmd, '--';
2387         } else {
2388             $family = PVE::Tools::get_host_address_family($node);
2389         }
2390
2391         my $port = PVE::Tools::next_vnc_port($family);
2392
2393         my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-escape', '0'];
2394         push @$termcmd, '-iface', $serial if $serial;
2395
2396         my $realcmd = sub {
2397             my $upid = shift;
2398
2399             syslog('info', "starting qemu termproxy $upid\n");
2400
2401             my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
2402                        '--perm', 'VM.Console', '--'];
2403             push @$cmd, @$remcmd, @$termcmd;
2404
2405             PVE::Tools::run_command($cmd);
2406         };
2407
2408         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2409
2410         PVE::Tools::wait_for_vnc_port($port);
2411
2412         return {
2413             user => $authuser,
2414             ticket => $ticket,
2415             port => $port,
2416             upid => $upid,
2417         };
2418     }});
2419
2420 __PACKAGE__->register_method({
2421     name => 'vncwebsocket',
2422     path => '{vmid}/vncwebsocket',
2423     method => 'GET',
2424     permissions => {
2425         description => "You also need to pass a valid ticket (vncticket).",
2426         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2427     },
2428     description => "Opens a weksocket for VNC traffic.",
2429     parameters => {
2430         additionalProperties => 0,
2431         properties => {
2432             node => get_standard_option('pve-node'),
2433             vmid => get_standard_option('pve-vmid'),
2434             vncticket => {
2435                 description => "Ticket from previous call to vncproxy.",
2436                 type => 'string',
2437                 maxLength => 512,
2438             },
2439             port => {
2440                 description => "Port number returned by previous vncproxy call.",
2441                 type => 'integer',
2442                 minimum => 5900,
2443                 maximum => 5999,
2444             },
2445         },
2446     },
2447     returns => {
2448         type => "object",
2449         properties => {
2450             port => { type => 'string' },
2451         },
2452     },
2453     code => sub {
2454         my ($param) = @_;
2455
2456         my $rpcenv = PVE::RPCEnvironment::get();
2457
2458         my $authuser = $rpcenv->get_user();
2459
2460         my $vmid = $param->{vmid};
2461         my $node = $param->{node};
2462
2463         my $authpath = "/vms/$vmid";
2464
2465         PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);
2466
2467         my $conf = PVE::QemuConfig->load_config($vmid, $node); # VM exists ?
2468
2469         # Note: VNC ports are acessible from outside, so we do not gain any
2470         # security if we verify that $param->{port} belongs to VM $vmid. This
2471         # check is done by verifying the VNC ticket (inside VNC protocol).
2472
2473         my $port = $param->{port};
2474
2475         return { port => $port };
2476     }});
2477
2478 __PACKAGE__->register_method({
2479     name => 'spiceproxy',
2480     path => '{vmid}/spiceproxy',
2481     method => 'POST',
2482     protected => 1,
2483     proxyto => 'node',
2484     permissions => {
2485         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2486     },
2487     description => "Returns a SPICE configuration to connect to the VM.",
2488     parameters => {
2489         additionalProperties => 0,
2490         properties => {
2491             node => get_standard_option('pve-node'),
2492             vmid => get_standard_option('pve-vmid'),
2493             proxy => get_standard_option('spice-proxy', { optional => 1 }),
2494         },
2495     },
2496     returns => get_standard_option('remote-viewer-config'),
2497     code => sub {
2498         my ($param) = @_;
2499
2500         my $rpcenv = PVE::RPCEnvironment::get();
2501
2502         my $authuser = $rpcenv->get_user();
2503
2504         my $vmid = $param->{vmid};
2505         my $node = $param->{node};
2506         my $proxy = $param->{proxy};
2507
2508         my $conf = PVE::QemuConfig->load_config($vmid, $node);
2509         my $title = "VM $vmid";
2510         $title .= " - ". $conf->{name} if $conf->{name};
2511
2512         my $port = PVE::QemuServer::spice_port($vmid);
2513
2514         my ($ticket, undef, $remote_viewer_config) =
2515             PVE::AccessControl::remote_viewer_config($authuser, $vmid, $node, $proxy, $title, $port);
2516
2517         mon_cmd($vmid, "set_password", protocol => 'spice', password => $ticket);
2518         mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
2519
2520         return $remote_viewer_config;
2521     }});
2522
2523 __PACKAGE__->register_method({
2524     name => 'vmcmdidx',
2525     path => '{vmid}/status',
2526     method => 'GET',
2527     proxyto => 'node',
2528     description => "Directory index",
2529     permissions => {
2530         user => 'all',
2531     },
2532     parameters => {
2533         additionalProperties => 0,
2534         properties => {
2535             node => get_standard_option('pve-node'),
2536             vmid => get_standard_option('pve-vmid'),
2537         },
2538     },
2539     returns => {
2540         type => 'array',
2541         items => {
2542             type => "object",
2543             properties => {
2544                 subdir => { type => 'string' },
2545             },
2546         },
2547         links => [ { rel => 'child', href => "{subdir}" } ],
2548     },
2549     code => sub {
2550         my ($param) = @_;
2551
2552         # test if VM exists
2553         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2554
2555         my $res = [
2556             { subdir => 'current' },
2557             { subdir => 'start' },
2558             { subdir => 'stop' },
2559             { subdir => 'reset' },
2560             { subdir => 'shutdown' },
2561             { subdir => 'suspend' },
2562             { subdir => 'reboot' },
2563             ];
2564
2565         return $res;
2566     }});
2567
2568 __PACKAGE__->register_method({
2569     name => 'vm_status',
2570     path => '{vmid}/status/current',
2571     method => 'GET',
2572     proxyto => 'node',
2573     protected => 1, # qemu pid files are only readable by root
2574     description => "Get virtual machine status.",
2575     permissions => {
2576         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
2577     },
2578     parameters => {
2579         additionalProperties => 0,
2580         properties => {
2581             node => get_standard_option('pve-node'),
2582             vmid => get_standard_option('pve-vmid'),
2583         },
2584     },
2585     returns => {
2586         type => 'object',
2587         properties => {
2588             %$PVE::QemuServer::vmstatus_return_properties,
2589             ha => {
2590                 description => "HA manager service status.",
2591                 type => 'object',
2592             },
2593             spice => {
2594                 description => "QEMU VGA configuration supports spice.",
2595                 type => 'boolean',
2596                 optional => 1,
2597             },
2598             agent => {
2599                 description => "QEMU Guest Agent is enabled in config.",
2600                 type => 'boolean',
2601                 optional => 1,
2602             },
2603         },
2604     },
2605     code => sub {
2606         my ($param) = @_;
2607
2608         # test if VM exists
2609         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2610
2611         my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1);
2612         my $status = $vmstatus->{$param->{vmid}};
2613
2614         $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}");
2615
2616         if ($conf->{vga}) {
2617             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2618             my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/;
2619             $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga});
2620             $status->{spice} = 1 if $spice;
2621         }
2622         $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled');
2623
2624         return $status;
2625     }});
2626
2627 __PACKAGE__->register_method({
2628     name => 'vm_start',
2629     path => '{vmid}/status/start',
2630     method => 'POST',
2631     protected => 1,
2632     proxyto => 'node',
2633     description => "Start virtual machine.",
2634     permissions => {
2635         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2636     },
2637     parameters => {
2638         additionalProperties => 0,
2639         properties => {
2640             node => get_standard_option('pve-node'),
2641             vmid => get_standard_option('pve-vmid',
2642                                         { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2643             skiplock => get_standard_option('skiplock'),
2644             stateuri => get_standard_option('pve-qm-stateuri'),
2645             migratedfrom => get_standard_option('pve-node',{ optional => 1 }),
2646             migration_type => {
2647                 type => 'string',
2648                 enum => ['secure', 'insecure'],
2649                 description => "Migration traffic is encrypted using an SSH " .
2650                   "tunnel by default. On secure, completely private networks " .
2651                   "this can be disabled to increase performance.",
2652                 optional => 1,
2653             },
2654             migration_network => {
2655                 type => 'string', format => 'CIDR',
2656                 description => "CIDR of the (sub) network that is used for migration.",
2657                 optional => 1,
2658             },
2659             machine => get_standard_option('pve-qemu-machine'),
2660             'force-cpu' => {
2661                 description => "Override QEMU's -cpu argument with the given string.",
2662                 type => 'string',
2663                 optional => 1,
2664             },
2665             targetstorage => get_standard_option('pve-targetstorage'),
2666             timeout => {
2667                 description => "Wait maximal timeout seconds.",
2668                 type => 'integer',
2669                 minimum => 0,
2670                 default => 'max(30, vm memory in GiB)',
2671                 optional => 1,
2672             },
2673         },
2674     },
2675     returns => {
2676         type => 'string',
2677     },
2678     code => sub {
2679         my ($param) = @_;
2680
2681         my $rpcenv = PVE::RPCEnvironment::get();
2682         my $authuser = $rpcenv->get_user();
2683
2684         my $node = extract_param($param, 'node');
2685         my $vmid = extract_param($param, 'vmid');
2686         my $timeout = extract_param($param, 'timeout');
2687         my $machine = extract_param($param, 'machine');
2688
2689         my $get_root_param = sub {
2690             my $value = extract_param($param, $_[0]);
2691             raise_param_exc({ "$_[0]" => "Only root may use this option." })
2692                 if $value && $authuser ne 'root@pam';
2693             return $value;
2694         };
2695
2696         my $stateuri = $get_root_param->('stateuri');
2697         my $skiplock = $get_root_param->('skiplock');
2698         my $migratedfrom = $get_root_param->('migratedfrom');
2699         my $migration_type = $get_root_param->('migration_type');
2700         my $migration_network = $get_root_param->('migration_network');
2701         my $targetstorage = $get_root_param->('targetstorage');
2702         my $force_cpu = $get_root_param->('force-cpu');
2703
2704         my $storagemap;
2705
2706         if ($targetstorage) {
2707             raise_param_exc({ targetstorage => "targetstorage can only by used with migratedfrom." })
2708                 if !$migratedfrom;
2709             $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
2710             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
2711                 if $@;
2712         }
2713
2714         # read spice ticket from STDIN
2715         my $spice_ticket;
2716         my $nbd_protocol_version = 0;
2717         my $replicated_volumes = {};
2718         my $offline_volumes = {};
2719         if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) {
2720             while (defined(my $line = <STDIN>)) {
2721                 chomp $line;
2722                 if ($line =~ m/^spice_ticket: (.+)$/) {
2723                     $spice_ticket = $1;
2724                 } elsif ($line =~ m/^nbd_protocol_version: (\d+)$/) {
2725                     $nbd_protocol_version = $1;
2726                 } elsif ($line =~ m/^replicated_volume: (.*)$/) {
2727                     $replicated_volumes->{$1} = 1;
2728                 } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead
2729                     $offline_volumes->{tpmstate0} = $1;
2730                 } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) {
2731                     $offline_volumes->{$1} = $2;
2732                 } elsif (!$spice_ticket) {
2733                     # fallback for old source node
2734                     $spice_ticket = $line;
2735                 } else {
2736                     warn "unknown 'start' parameter on STDIN: '$line'\n";
2737                 }
2738             }
2739         }
2740
2741         PVE::Cluster::check_cfs_quorum();
2742
2743         my $storecfg = PVE::Storage::config();
2744
2745         if (PVE::HA::Config::vm_is_ha_managed($vmid) && !$stateuri &&  $rpcenv->{type} ne 'ha') {
2746             my $hacmd = sub {
2747                 my $upid = shift;
2748
2749                 print "Requesting HA start for VM $vmid\n";
2750
2751                 my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
2752                 PVE::Tools::run_command($cmd);
2753                 return;
2754             };
2755
2756             return $rpcenv->fork_worker('hastart', $vmid, $authuser, $hacmd);
2757
2758         } else {
2759
2760             my $realcmd = sub {
2761                 my $upid = shift;
2762
2763                 syslog('info', "start VM $vmid: $upid\n");
2764
2765                 my $migrate_opts = {
2766                     migratedfrom => $migratedfrom,
2767                     spice_ticket => $spice_ticket,
2768                     network => $migration_network,
2769                     type => $migration_type,
2770                     storagemap => $storagemap,
2771                     nbd_proto_version => $nbd_protocol_version,
2772                     replicated_volumes => $replicated_volumes,
2773                     offline_volumes => $offline_volumes,
2774                 };
2775
2776                 my $params = {
2777                     statefile => $stateuri,
2778                     skiplock => $skiplock,
2779                     forcemachine => $machine,
2780                     timeout => $timeout,
2781                     forcecpu => $force_cpu,
2782                 };
2783
2784                 PVE::QemuServer::vm_start($storecfg, $vmid, $params, $migrate_opts);
2785                 return;
2786             };
2787
2788             return $rpcenv->fork_worker('qmstart', $vmid, $authuser, $realcmd);
2789         }
2790     }});
2791
2792 __PACKAGE__->register_method({
2793     name => 'vm_stop',
2794     path => '{vmid}/status/stop',
2795     method => 'POST',
2796     protected => 1,
2797     proxyto => 'node',
2798     description => "Stop virtual machine. The qemu process will exit immediately. This" .
2799         "is akin to pulling the power plug of a running computer and may damage the VM data",
2800     permissions => {
2801         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2802     },
2803     parameters => {
2804         additionalProperties => 0,
2805         properties => {
2806             node => get_standard_option('pve-node'),
2807             vmid => get_standard_option('pve-vmid',
2808                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2809             skiplock => get_standard_option('skiplock'),
2810             migratedfrom => get_standard_option('pve-node', { optional => 1 }),
2811             timeout => {
2812                 description => "Wait maximal timeout seconds.",
2813                 type => 'integer',
2814                 minimum => 0,
2815                 optional => 1,
2816             },
2817             keepActive => {
2818                 description => "Do not deactivate storage volumes.",
2819                 type => 'boolean',
2820                 optional => 1,
2821                 default => 0,
2822             }
2823         },
2824     },
2825     returns => {
2826         type => 'string',
2827     },
2828     code => sub {
2829         my ($param) = @_;
2830
2831         my $rpcenv = PVE::RPCEnvironment::get();
2832         my $authuser = $rpcenv->get_user();
2833
2834         my $node = extract_param($param, 'node');
2835         my $vmid = extract_param($param, 'vmid');
2836
2837         my $skiplock = extract_param($param, 'skiplock');
2838         raise_param_exc({ skiplock => "Only root may use this option." })
2839             if $skiplock && $authuser ne 'root@pam';
2840
2841         my $keepActive = extract_param($param, 'keepActive');
2842         raise_param_exc({ keepActive => "Only root may use this option." })
2843             if $keepActive && $authuser ne 'root@pam';
2844
2845         my $migratedfrom = extract_param($param, 'migratedfrom');
2846         raise_param_exc({ migratedfrom => "Only root may use this option." })
2847             if $migratedfrom && $authuser ne 'root@pam';
2848
2849
2850         my $storecfg = PVE::Storage::config();
2851
2852         if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) {
2853
2854             my $hacmd = sub {
2855                 my $upid = shift;
2856
2857                 print "Requesting HA stop for VM $vmid\n";
2858
2859                 my $cmd = ['ha-manager', 'crm-command', 'stop',  "vm:$vmid", '0'];
2860                 PVE::Tools::run_command($cmd);
2861                 return;
2862             };
2863
2864             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
2865
2866         } else {
2867             my $realcmd = sub {
2868                 my $upid = shift;
2869
2870                 syslog('info', "stop VM $vmid: $upid\n");
2871
2872                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0,
2873                                          $param->{timeout}, 0, 1, $keepActive, $migratedfrom);
2874                 return;
2875             };
2876
2877             return $rpcenv->fork_worker('qmstop', $vmid, $authuser, $realcmd);
2878         }
2879     }});
2880
2881 __PACKAGE__->register_method({
2882     name => 'vm_reset',
2883     path => '{vmid}/status/reset',
2884     method => 'POST',
2885     protected => 1,
2886     proxyto => 'node',
2887     description => "Reset virtual machine.",
2888     permissions => {
2889         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2890     },
2891     parameters => {
2892         additionalProperties => 0,
2893         properties => {
2894             node => get_standard_option('pve-node'),
2895             vmid => get_standard_option('pve-vmid',
2896                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2897             skiplock => get_standard_option('skiplock'),
2898         },
2899     },
2900     returns => {
2901         type => 'string',
2902     },
2903     code => sub {
2904         my ($param) = @_;
2905
2906         my $rpcenv = PVE::RPCEnvironment::get();
2907
2908         my $authuser = $rpcenv->get_user();
2909
2910         my $node = extract_param($param, 'node');
2911
2912         my $vmid = extract_param($param, 'vmid');
2913
2914         my $skiplock = extract_param($param, 'skiplock');
2915         raise_param_exc({ skiplock => "Only root may use this option." })
2916             if $skiplock && $authuser ne 'root@pam';
2917
2918         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
2919
2920         my $realcmd = sub {
2921             my $upid = shift;
2922
2923             PVE::QemuServer::vm_reset($vmid, $skiplock);
2924
2925             return;
2926         };
2927
2928         return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd);
2929     }});
2930
2931 __PACKAGE__->register_method({
2932     name => 'vm_shutdown',
2933     path => '{vmid}/status/shutdown',
2934     method => 'POST',
2935     protected => 1,
2936     proxyto => 'node',
2937     description => "Shutdown virtual machine. This is similar to pressing the power button on a physical machine." .
2938         "This will send an ACPI event for the guest OS, which should then proceed to a clean shutdown.",
2939     permissions => {
2940         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2941     },
2942     parameters => {
2943         additionalProperties => 0,
2944         properties => {
2945             node => get_standard_option('pve-node'),
2946             vmid => get_standard_option('pve-vmid',
2947                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2948             skiplock => get_standard_option('skiplock'),
2949             timeout => {
2950                 description => "Wait maximal timeout seconds.",
2951                 type => 'integer',
2952                 minimum => 0,
2953                 optional => 1,
2954             },
2955             forceStop => {
2956                 description => "Make sure the VM stops.",
2957                 type => 'boolean',
2958                 optional => 1,
2959                 default => 0,
2960             },
2961             keepActive => {
2962                 description => "Do not deactivate storage volumes.",
2963                 type => 'boolean',
2964                 optional => 1,
2965                 default => 0,
2966             }
2967         },
2968     },
2969     returns => {
2970         type => 'string',
2971     },
2972     code => sub {
2973         my ($param) = @_;
2974
2975         my $rpcenv = PVE::RPCEnvironment::get();
2976         my $authuser = $rpcenv->get_user();
2977
2978         my $node = extract_param($param, 'node');
2979         my $vmid = extract_param($param, 'vmid');
2980
2981         my $skiplock = extract_param($param, 'skiplock');
2982         raise_param_exc({ skiplock => "Only root may use this option." })
2983             if $skiplock && $authuser ne 'root@pam';
2984
2985         my $keepActive = extract_param($param, 'keepActive');
2986         raise_param_exc({ keepActive => "Only root may use this option." })
2987             if $keepActive && $authuser ne 'root@pam';
2988
2989         my $storecfg = PVE::Storage::config();
2990
2991         my $shutdown = 1;
2992
2993         # if vm is paused, do not shutdown (but stop if forceStop = 1)
2994         # otherwise, we will infer a shutdown command, but run into the timeout,
2995         # then when the vm is resumed, it will instantly shutdown
2996         #
2997         # checking the qmp status here to get feedback to the gui/cli/api
2998         # and the status query should not take too long
2999         if (PVE::QemuServer::vm_is_paused($vmid)) {
3000             if ($param->{forceStop}) {
3001                 warn "VM is paused - stop instead of shutdown\n";
3002                 $shutdown = 0;
3003             } else {
3004                 die "VM is paused - cannot shutdown\n";
3005             }
3006         }
3007
3008         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
3009
3010             my $timeout = $param->{timeout} // 60;
3011             my $hacmd = sub {
3012                 my $upid = shift;
3013
3014                 print "Requesting HA stop for VM $vmid\n";
3015
3016                 my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", "$timeout"];
3017                 PVE::Tools::run_command($cmd);
3018                 return;
3019             };
3020
3021             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
3022
3023         } else {
3024
3025             my $realcmd = sub {
3026                 my $upid = shift;
3027
3028                 syslog('info', "shutdown VM $vmid: $upid\n");
3029
3030                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0, $param->{timeout},
3031                                          $shutdown, $param->{forceStop}, $keepActive);
3032                 return;
3033             };
3034
3035             return $rpcenv->fork_worker('qmshutdown', $vmid, $authuser, $realcmd);
3036         }
3037     }});
3038
3039 __PACKAGE__->register_method({
3040     name => 'vm_reboot',
3041     path => '{vmid}/status/reboot',
3042     method => 'POST',
3043     protected => 1,
3044     proxyto => 'node',
3045     description => "Reboot the VM by shutting it down, and starting it again. Applies pending changes.",
3046     permissions => {
3047         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3048     },
3049     parameters => {
3050         additionalProperties => 0,
3051         properties => {
3052             node => get_standard_option('pve-node'),
3053             vmid => get_standard_option('pve-vmid',
3054                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3055             timeout => {
3056                 description => "Wait maximal timeout seconds for the shutdown.",
3057                 type => 'integer',
3058                 minimum => 0,
3059                 optional => 1,
3060             },
3061         },
3062     },
3063     returns => {
3064         type => 'string',
3065     },
3066     code => sub {
3067         my ($param) = @_;
3068
3069         my $rpcenv = PVE::RPCEnvironment::get();
3070         my $authuser = $rpcenv->get_user();
3071
3072         my $node = extract_param($param, 'node');
3073         my $vmid = extract_param($param, 'vmid');
3074
3075         die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid);
3076
3077         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3078
3079         my $realcmd = sub {
3080             my $upid = shift;
3081
3082             syslog('info', "requesting reboot of VM $vmid: $upid\n");
3083             PVE::QemuServer::vm_reboot($vmid, $param->{timeout});
3084             return;
3085         };
3086
3087         return $rpcenv->fork_worker('qmreboot', $vmid, $authuser, $realcmd);
3088     }});
3089
3090 __PACKAGE__->register_method({
3091     name => 'vm_suspend',
3092     path => '{vmid}/status/suspend',
3093     method => 'POST',
3094     protected => 1,
3095     proxyto => 'node',
3096     description => "Suspend virtual machine.",
3097     permissions => {
3098         description => "You need 'VM.PowerMgmt' on /vms/{vmid}, and if you have set 'todisk',".
3099             " you need also 'VM.Config.Disk' on /vms/{vmid} and 'Datastore.AllocateSpace'".
3100             " on the storage for the vmstate.",
3101         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3102     },
3103     parameters => {
3104         additionalProperties => 0,
3105         properties => {
3106             node => get_standard_option('pve-node'),
3107             vmid => get_standard_option('pve-vmid',
3108                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3109             skiplock => get_standard_option('skiplock'),
3110             todisk => {
3111                 type => 'boolean',
3112                 default => 0,
3113                 optional => 1,
3114                 description => 'If set, suspends the VM to disk. Will be resumed on next VM start.',
3115             },
3116             statestorage => get_standard_option('pve-storage-id', {
3117                 description => "The storage for the VM state",
3118                 requires => 'todisk',
3119                 optional => 1,
3120                 completion => \&PVE::Storage::complete_storage_enabled,
3121             }),
3122         },
3123     },
3124     returns => {
3125         type => 'string',
3126     },
3127     code => sub {
3128         my ($param) = @_;
3129
3130         my $rpcenv = PVE::RPCEnvironment::get();
3131         my $authuser = $rpcenv->get_user();
3132
3133         my $node = extract_param($param, 'node');
3134         my $vmid = extract_param($param, 'vmid');
3135
3136         my $todisk = extract_param($param, 'todisk') // 0;
3137
3138         my $statestorage = extract_param($param, 'statestorage');
3139
3140         my $skiplock = extract_param($param, 'skiplock');
3141         raise_param_exc({ skiplock => "Only root may use this option." })
3142             if $skiplock && $authuser ne 'root@pam';
3143
3144         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3145
3146         die "Cannot suspend HA managed VM to disk\n"
3147             if $todisk && PVE::HA::Config::vm_is_ha_managed($vmid);
3148
3149         # early check for storage permission, for better user feedback
3150         if ($todisk) {
3151             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
3152             my $conf = PVE::QemuConfig->load_config($vmid);
3153
3154             # cannot save the state of a non-virtualized PCIe device, so resume cannot really work
3155             for my $key (keys %$conf) {
3156                 next if $key !~ /^hostpci\d+/;
3157                 die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the"
3158                     ." possibility to save/restore their internal state\n";
3159             }
3160
3161             if (!$statestorage) {
3162                 # get statestorage from config if none is given
3163                 my $storecfg = PVE::Storage::config();
3164                 $statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
3165             }
3166
3167             $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
3168         }
3169
3170         my $realcmd = sub {
3171             my $upid = shift;
3172
3173             syslog('info', "suspend VM $vmid: $upid\n");
3174
3175             PVE::QemuServer::vm_suspend($vmid, $skiplock, $todisk, $statestorage);
3176
3177             return;
3178         };
3179
3180         my $taskname = $todisk ? 'qmsuspend' : 'qmpause';
3181         return $rpcenv->fork_worker($taskname, $vmid, $authuser, $realcmd);
3182     }});
3183
3184 __PACKAGE__->register_method({
3185     name => 'vm_resume',
3186     path => '{vmid}/status/resume',
3187     method => 'POST',
3188     protected => 1,
3189     proxyto => 'node',
3190     description => "Resume virtual machine.",
3191     permissions => {
3192         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3193     },
3194     parameters => {
3195         additionalProperties => 0,
3196         properties => {
3197             node => get_standard_option('pve-node'),
3198             vmid => get_standard_option('pve-vmid',
3199                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3200             skiplock => get_standard_option('skiplock'),
3201             nocheck => { type => 'boolean', optional => 1 },
3202
3203         },
3204     },
3205     returns => {
3206         type => 'string',
3207     },
3208     code => sub {
3209         my ($param) = @_;
3210
3211         my $rpcenv = PVE::RPCEnvironment::get();
3212
3213         my $authuser = $rpcenv->get_user();
3214
3215         my $node = extract_param($param, 'node');
3216
3217         my $vmid = extract_param($param, 'vmid');
3218
3219         my $skiplock = extract_param($param, 'skiplock');
3220         raise_param_exc({ skiplock => "Only root may use this option." })
3221             if $skiplock && $authuser ne 'root@pam';
3222
3223         # nocheck is used as part of migration when config file might be still
3224         # be on source node
3225         my $nocheck = extract_param($param, 'nocheck');
3226         raise_param_exc({ nocheck => "Only root may use this option." })
3227             if $nocheck && $authuser ne 'root@pam';
3228
3229         my $to_disk_suspended;
3230         eval {
3231             PVE::QemuConfig->lock_config($vmid, sub {
3232                 my $conf = PVE::QemuConfig->load_config($vmid);
3233                 $to_disk_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
3234             });
3235         };
3236
3237         die "VM $vmid not running\n"
3238             if !$to_disk_suspended && !PVE::QemuServer::check_running($vmid, $nocheck);
3239
3240         my $realcmd = sub {
3241             my $upid = shift;
3242
3243             syslog('info', "resume VM $vmid: $upid\n");
3244
3245             if (!$to_disk_suspended) {
3246                 PVE::QemuServer::vm_resume($vmid, $skiplock, $nocheck);
3247             } else {
3248                 my $storecfg = PVE::Storage::config();
3249                 PVE::QemuServer::vm_start($storecfg, $vmid, { skiplock => $skiplock });
3250             }
3251
3252             return;
3253         };
3254
3255         return $rpcenv->fork_worker('qmresume', $vmid, $authuser, $realcmd);
3256     }});
3257
3258 __PACKAGE__->register_method({
3259     name => 'vm_sendkey',
3260     path => '{vmid}/sendkey',
3261     method => 'PUT',
3262     protected => 1,
3263     proxyto => 'node',
3264     description => "Send key event to virtual machine.",
3265     permissions => {
3266         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
3267     },
3268     parameters => {
3269         additionalProperties => 0,
3270         properties => {
3271             node => get_standard_option('pve-node'),
3272             vmid => get_standard_option('pve-vmid',
3273                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3274             skiplock => get_standard_option('skiplock'),
3275             key => {
3276                 description => "The key (qemu monitor encoding).",
3277                 type => 'string'
3278             }
3279         },
3280     },
3281     returns => { type => 'null'},
3282     code => sub {
3283         my ($param) = @_;
3284
3285         my $rpcenv = PVE::RPCEnvironment::get();
3286
3287         my $authuser = $rpcenv->get_user();
3288
3289         my $node = extract_param($param, 'node');
3290
3291         my $vmid = extract_param($param, 'vmid');
3292
3293         my $skiplock = extract_param($param, 'skiplock');
3294         raise_param_exc({ skiplock => "Only root may use this option." })
3295             if $skiplock && $authuser ne 'root@pam';
3296
3297         PVE::QemuServer::vm_sendkey($vmid, $skiplock, $param->{key});
3298
3299         return;
3300     }});
3301
3302 __PACKAGE__->register_method({
3303     name => 'vm_feature',
3304     path => '{vmid}/feature',
3305     method => 'GET',
3306     proxyto => 'node',
3307     protected => 1,
3308     description => "Check if feature for virtual machine is available.",
3309     permissions => {
3310         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
3311     },
3312     parameters => {
3313         additionalProperties => 0,
3314         properties => {
3315             node => get_standard_option('pve-node'),
3316             vmid => get_standard_option('pve-vmid'),
3317             feature => {
3318                 description => "Feature to check.",
3319                 type => 'string',
3320                 enum => [ 'snapshot', 'clone', 'copy' ],
3321             },
3322             snapname => get_standard_option('pve-snapshot-name', {
3323                 optional => 1,
3324             }),
3325         },
3326     },
3327     returns => {
3328         type => "object",
3329         properties => {
3330             hasFeature => { type => 'boolean' },
3331             nodes => {
3332                 type => 'array',
3333                 items => { type => 'string' },
3334             }
3335         },
3336     },
3337     code => sub {
3338         my ($param) = @_;
3339
3340         my $node = extract_param($param, 'node');
3341
3342         my $vmid = extract_param($param, 'vmid');
3343
3344         my $snapname = extract_param($param, 'snapname');
3345
3346         my $feature = extract_param($param, 'feature');
3347
3348         my $running = PVE::QemuServer::check_running($vmid);
3349
3350         my $conf = PVE::QemuConfig->load_config($vmid);
3351
3352         if($snapname){
3353             my $snap = $conf->{snapshots}->{$snapname};
3354             die "snapshot '$snapname' does not exist\n" if !defined($snap);
3355             $conf = $snap;
3356         }
3357         my $storecfg = PVE::Storage::config();
3358
3359         my $nodelist = PVE::QemuServer::shared_nodes($conf, $storecfg);
3360         my $hasFeature = PVE::QemuConfig->has_feature($feature, $conf, $storecfg, $snapname, $running);
3361
3362         return {
3363             hasFeature => $hasFeature,
3364             nodes => [ keys %$nodelist ],
3365         };
3366     }});
3367
3368 __PACKAGE__->register_method({
3369     name => 'clone_vm',
3370     path => '{vmid}/clone',
3371     method => 'POST',
3372     protected => 1,
3373     proxyto => 'node',
3374     description => "Create a copy of virtual machine/template.",
3375     permissions => {
3376         description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " .
3377             "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
3378             "'Datastore.AllocateSpace' on any used storage.",
3379         check =>
3380         [ 'and',
3381           ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
3382           [ 'or',
3383             [ 'perm', '/vms/{newid}', ['VM.Allocate']],
3384             [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
3385           ],
3386         ]
3387     },
3388     parameters => {
3389         additionalProperties => 0,
3390         properties => {
3391             node => get_standard_option('pve-node'),
3392             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3393             newid => get_standard_option('pve-vmid', {
3394                 completion => \&PVE::Cluster::complete_next_vmid,
3395                 description => 'VMID for the clone.' }),
3396             name => {
3397                 optional => 1,
3398                 type => 'string', format => 'dns-name',
3399                 description => "Set a name for the new VM.",
3400             },
3401             description => {
3402                 optional => 1,
3403                 type => 'string',
3404                 description => "Description for the new VM.",
3405             },
3406             pool => {
3407                 optional => 1,
3408                 type => 'string', format => 'pve-poolid',
3409                 description => "Add the new VM to the specified pool.",
3410             },
3411             snapname => get_standard_option('pve-snapshot-name', {
3412                 optional => 1,
3413             }),
3414             storage => get_standard_option('pve-storage-id', {
3415                 description => "Target storage for full clone.",
3416                 optional => 1,
3417             }),
3418             'format' => {
3419                 description => "Target format for file storage. Only valid for full clone.",
3420                 type => 'string',
3421                 optional => 1,
3422                 enum => [ 'raw', 'qcow2', 'vmdk'],
3423             },
3424             full => {
3425                 optional => 1,
3426                 type => 'boolean',
3427                 description => "Create a full copy of all disks. This is always done when " .
3428                     "you clone a normal VM. For VM templates, we try to create a linked clone by default.",
3429             },
3430             target => get_standard_option('pve-node', {
3431                 description => "Target node. Only allowed if the original VM is on shared storage.",
3432                 optional => 1,
3433             }),
3434             bwlimit => {
3435                 description => "Override I/O bandwidth limit (in KiB/s).",
3436                 optional => 1,
3437                 type => 'integer',
3438                 minimum => '0',
3439                 default => 'clone limit from datacenter or storage config',
3440             },
3441         },
3442     },
3443     returns => {
3444         type => 'string',
3445     },
3446     code => sub {
3447         my ($param) = @_;
3448
3449         my $rpcenv = PVE::RPCEnvironment::get();
3450         my $authuser = $rpcenv->get_user();
3451
3452         my $node = extract_param($param, 'node');
3453         my $vmid = extract_param($param, 'vmid');
3454         my $newid = extract_param($param, 'newid');
3455         my $pool = extract_param($param, 'pool');
3456
3457         my $snapname = extract_param($param, 'snapname');
3458         my $storage = extract_param($param, 'storage');
3459         my $format = extract_param($param, 'format');
3460         my $target = extract_param($param, 'target');
3461
3462         my $localnode = PVE::INotify::nodename();
3463
3464         if ($target && ($target eq $localnode || $target eq 'localhost')) {
3465             undef $target;
3466         }
3467
3468         my $running = PVE::QemuServer::check_running($vmid) || 0;
3469
3470         my $load_and_check = sub {
3471             $rpcenv->check_pool_exist($pool) if defined($pool);
3472             PVE::Cluster::check_node_exists($target) if $target;
3473
3474             my $storecfg = PVE::Storage::config();
3475
3476             if ($storage) {
3477                 # check if storage is enabled on local node
3478                 PVE::Storage::storage_check_enabled($storecfg, $storage);
3479                 if ($target) {
3480                     # check if storage is available on target node
3481                     PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
3482                     # clone only works if target storage is shared
3483                     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
3484                     die "can't clone to non-shared storage '$storage'\n"
3485                         if !$scfg->{shared};
3486                 }
3487             }
3488
3489             PVE::Cluster::check_cfs_quorum();
3490
3491             my $conf = PVE::QemuConfig->load_config($vmid);
3492             PVE::QemuConfig->check_lock($conf);
3493
3494             my $verify_running = PVE::QemuServer::check_running($vmid) || 0;
3495             die "unexpected state change\n" if $verify_running != $running;
3496
3497             die "snapshot '$snapname' does not exist\n"
3498                 if $snapname && !defined( $conf->{snapshots}->{$snapname});
3499
3500             my $full = $param->{full} // !PVE::QemuConfig->is_template($conf);
3501
3502             die "parameter 'storage' not allowed for linked clones\n"
3503                 if defined($storage) && !$full;
3504
3505             die "parameter 'format' not allowed for linked clones\n"
3506                 if defined($format) && !$full;
3507
3508             my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf;
3509
3510             my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage);
3511
3512             die "can't clone VM to node '$target' (VM uses local storage)\n"
3513                 if $target && !$sharedvm;
3514
3515             my $conffile = PVE::QemuConfig->config_file($newid);
3516             die "unable to create VM $newid: config file already exists\n"
3517                 if -f $conffile;
3518
3519             my $newconf = { lock => 'clone' };
3520             my $drives = {};
3521             my $fullclone = {};
3522             my $vollist = [];
3523
3524             foreach my $opt (keys %$oldconf) {
3525                 my $value = $oldconf->{$opt};
3526
3527                 # do not copy snapshot related info
3528                 next if $opt eq 'snapshots' ||  $opt eq 'parent' || $opt eq 'snaptime' ||
3529                     $opt eq 'vmstate' || $opt eq 'snapstate';
3530
3531                 # no need to copy unused images, because VMID(owner) changes anyways
3532                 next if $opt =~ m/^unused\d+$/;
3533
3534                 die "cannot clone TPM state while VM is running\n"
3535                     if $full && $running && !$snapname && $opt eq 'tpmstate0';
3536
3537                 # always change MAC! address
3538                 if ($opt =~ m/^net(\d+)$/) {
3539                     my $net = PVE::QemuServer::parse_net($value);
3540                     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
3541                     $net->{macaddr} =  PVE::Tools::random_ether_addr($dc->{mac_prefix});
3542                     $newconf->{$opt} = PVE::QemuServer::print_net($net);
3543                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
3544                     my $drive = PVE::QemuServer::parse_drive($opt, $value);
3545                     die "unable to parse drive options for '$opt'\n" if !$drive;
3546                     if (PVE::QemuServer::drive_is_cdrom($drive, 1)) {
3547                         $newconf->{$opt} = $value; # simply copy configuration
3548                     } else {
3549                         if ($full || PVE::QemuServer::drive_is_cloudinit($drive)) {
3550                             die "Full clone feature is not supported for drive '$opt'\n"
3551                                 if !PVE::Storage::volume_has_feature($storecfg, 'copy', $drive->{file}, $snapname, $running);
3552                             $fullclone->{$opt} = 1;
3553                         } else {
3554                             # not full means clone instead of copy
3555                             die "Linked clone feature is not supported for drive '$opt'\n"
3556                                 if !PVE::Storage::volume_has_feature($storecfg, 'clone', $drive->{file}, $snapname, $running);
3557                         }
3558                         $drives->{$opt} = $drive;
3559                         next if PVE::QemuServer::drive_is_cloudinit($drive);
3560                         push @$vollist, $drive->{file};
3561                     }
3562                 } else {
3563                     # copy everything else
3564                     $newconf->{$opt} = $value;
3565                 }
3566             }
3567
3568             return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone);
3569         };
3570
3571         my $clonefn = sub {
3572             my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->();
3573             my $storecfg = PVE::Storage::config();
3574
3575             # auto generate a new uuid
3576             my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || '');
3577             $smbios1->{uuid} = PVE::QemuServer::generate_uuid();
3578             $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1);
3579             # auto generate a new vmgenid only if the option was set for template
3580             if ($newconf->{vmgenid}) {
3581                 $newconf->{vmgenid} = PVE::QemuServer::generate_uuid();
3582             }
3583
3584             delete $newconf->{template};
3585
3586             if ($param->{name}) {
3587                 $newconf->{name} = $param->{name};
3588             } else {
3589                 $newconf->{name} = "Copy-of-VM-" . ($oldconf->{name} // $vmid);
3590             }
3591
3592             if ($param->{description}) {
3593                 $newconf->{description} = $param->{description};
3594             }
3595
3596             # create empty/temp config - this fails if VM already exists on other node
3597             # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code
3598             PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n");
3599
3600             PVE::Firewall::clone_vmfw_conf($vmid, $newid);
3601
3602             my $newvollist = [];
3603             my $jobs = {};
3604
3605             eval {
3606                 local $SIG{INT} =
3607                     local $SIG{TERM} =
3608                     local $SIG{QUIT} =
3609                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3610
3611                 PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
3612
3613                 my $bwlimit = extract_param($param, 'bwlimit');
3614
3615                 my $total_jobs = scalar(keys %{$drives});
3616                 my $i = 1;
3617
3618                 foreach my $opt (sort keys %$drives) {
3619                     my $drive = $drives->{$opt};
3620                     my $skipcomplete = ($total_jobs != $i); # finish after last drive
3621                     my $completion = $skipcomplete ? 'skip' : 'complete';
3622
3623                     my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
3624                     my $storage_list = [ $src_sid ];
3625                     push @$storage_list, $storage if defined($storage);
3626                     my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
3627
3628                     my $source_info = {
3629                         vmid => $vmid,
3630                         running => $running,
3631                         drivename => $opt,
3632                         drive => $drive,
3633                         snapname => $snapname,
3634                     };
3635
3636                     my $dest_info = {
3637                         vmid => $newid,
3638                         drivename => $opt,
3639                         storage => $storage,
3640                         format => $format,
3641                     };
3642
3643                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf)
3644                         if $opt eq 'efidisk0';
3645
3646                     my $newdrive = PVE::QemuServer::clone_disk(
3647                         $storecfg,
3648                         $source_info,
3649                         $dest_info,
3650                         $fullclone->{$opt},
3651                         $newvollist,
3652                         $jobs,
3653                         $completion,
3654                         $oldconf->{agent},
3655                         $clonelimit,
3656                     );
3657
3658                     $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
3659
3660                     PVE::QemuConfig->write_config($newid, $newconf);
3661                     $i++;
3662                 }
3663
3664                 delete $newconf->{lock};
3665
3666                 # do not write pending changes
3667                 if (my @changes = keys %{$newconf->{pending}}) {
3668                     my $pending = join(',', @changes);
3669                     warn "found pending changes for '$pending', discarding for clone\n";
3670                     delete $newconf->{pending};
3671                 }
3672
3673                 PVE::QemuConfig->write_config($newid, $newconf);
3674
3675                 if ($target) {
3676                     # always deactivate volumes - avoid lvm LVs to be active on several nodes
3677                     PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
3678                     PVE::Storage::deactivate_volumes($storecfg, $newvollist);
3679
3680                     my $newconffile = PVE::QemuConfig->config_file($newid, $target);
3681                     die "Failed to move config to node '$target' - rename failed: $!\n"
3682                         if !rename($conffile, $newconffile);
3683                 }
3684
3685                 PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
3686             };
3687             if (my $err = $@) {
3688                 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
3689                 sleep 1; # some storage like rbd need to wait before release volume - really?
3690
3691                 foreach my $volid (@$newvollist) {
3692                     eval { PVE::Storage::vdisk_free($storecfg, $volid); };
3693                     warn $@ if $@;
3694                 }
3695
3696                 PVE::Firewall::remove_vmfw_conf($newid);
3697
3698                 unlink $conffile; # avoid races -> last thing before die
3699
3700                 die "clone failed: $err";
3701             }
3702
3703             return;
3704         };
3705
3706         # Aquire exclusive lock lock for $newid
3707         my $lock_target_vm = sub {
3708             return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn);
3709         };
3710
3711         my $lock_source_vm = sub {
3712             # exclusive lock if VM is running - else shared lock is enough;
3713             if ($running) {
3714                 return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
3715             } else {
3716                 return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
3717             }
3718         };
3719
3720         $load_and_check->(); # early checks before forking/locking
3721
3722         return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm);
3723     }});
3724
3725 __PACKAGE__->register_method({
3726     name => 'move_vm_disk',
3727     path => '{vmid}/move_disk',
3728     method => 'POST',
3729     protected => 1,
3730     proxyto => 'node',
3731     description => "Move volume to different storage or to a different VM.",
3732     permissions => {
3733         description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
3734             "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
3735             "a disk to another VM, you need the permissions on the target VM as well.",
3736         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
3737     },
3738     parameters => {
3739         additionalProperties => 0,
3740         properties => {
3741             node => get_standard_option('pve-node'),
3742             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3743             'target-vmid' => get_standard_option('pve-vmid', {
3744                 completion => \&PVE::QemuServer::complete_vmid,
3745                 optional => 1,
3746             }),
3747             disk => {
3748                 type => 'string',
3749                 description => "The disk you want to move.",
3750                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3751             },
3752             storage => get_standard_option('pve-storage-id', {
3753                 description => "Target storage.",
3754                 completion => \&PVE::QemuServer::complete_storage,
3755                 optional => 1,
3756             }),
3757             'format' => {
3758                 type => 'string',
3759                 description => "Target Format.",
3760                 enum => [ 'raw', 'qcow2', 'vmdk' ],
3761                 optional => 1,
3762             },
3763             delete => {
3764                 type => 'boolean',
3765                 description => "Delete the original disk after successful copy. By default the"
3766                     ." original disk is kept as unused disk.",
3767                 optional => 1,
3768                 default => 0,
3769             },
3770             digest => {
3771                 type => 'string',
3772                 description => 'Prevent changes if current configuration file has different SHA1"
3773                     ." digest. This can be used to prevent concurrent modifications.',
3774                 maxLength => 40,
3775                 optional => 1,
3776             },
3777             bwlimit => {
3778                 description => "Override I/O bandwidth limit (in KiB/s).",
3779                 optional => 1,
3780                 type => 'integer',
3781                 minimum => '0',
3782                 default => 'move limit from datacenter or storage config',
3783             },
3784             'target-disk' => {
3785                 type => 'string',
3786                 description => "The config key the disk will be moved to on the target VM"
3787                     ." (for example, ide0 or scsi1). Default is the source disk key.",
3788                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3789                 optional => 1,
3790             },
3791             'target-digest' => {
3792                 type => 'string',
3793                 description => 'Prevent changes if the current config file of the target VM has a"
3794                     ." different SHA1 digest. This can be used to detect concurrent modifications.',
3795                 maxLength => 40,
3796                 optional => 1,
3797             },
3798         },
3799     },
3800     returns => {
3801         type => 'string',
3802         description => "the task ID.",
3803     },
3804     code => sub {
3805         my ($param) = @_;
3806
3807         my $rpcenv = PVE::RPCEnvironment::get();
3808         my $authuser = $rpcenv->get_user();
3809
3810         my $node = extract_param($param, 'node');
3811         my $vmid = extract_param($param, 'vmid');
3812         my $target_vmid = extract_param($param, 'target-vmid');
3813         my $digest = extract_param($param, 'digest');
3814         my $target_digest = extract_param($param, 'target-digest');
3815         my $disk = extract_param($param, 'disk');
3816         my $target_disk = extract_param($param, 'target-disk') // $disk;
3817         my $storeid = extract_param($param, 'storage');
3818         my $format = extract_param($param, 'format');
3819
3820         my $storecfg = PVE::Storage::config();
3821
3822         my $load_and_check_move = sub {
3823             my $conf = PVE::QemuConfig->load_config($vmid);
3824             PVE::QemuConfig->check_lock($conf);
3825
3826             PVE::Tools::assert_if_modified($digest, $conf->{digest});
3827
3828             die "disk '$disk' does not exist\n" if !$conf->{$disk};
3829
3830             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
3831
3832             die "disk '$disk' has no associated volume\n" if !$drive->{file};
3833             die "you can't move a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive, 1);
3834
3835             my $old_volid = $drive->{file};
3836             my $oldfmt;
3837             my ($oldstoreid, $oldvolname) = PVE::Storage::parse_volume_id($old_volid);
3838             if ($oldvolname =~ m/\.(raw|qcow2|vmdk)$/){
3839                 $oldfmt = $1;
3840             }
3841
3842             die "you can't move to the same storage with same format\n"
3843                 if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format);
3844
3845             # this only checks snapshots because $disk is passed!
3846             my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use(
3847                 $storecfg,
3848                 $conf,
3849                 $disk,
3850                 $old_volid
3851             );
3852             die "you can't move a disk with snapshots and delete the source\n"
3853                 if $snapshotted && $param->{delete};
3854
3855             return ($conf, $drive, $oldstoreid, $snapshotted);
3856         };
3857
3858         my $move_updatefn = sub {
3859             my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->();
3860             my $old_volid = $drive->{file};
3861
3862             PVE::Cluster::log_msg(
3863                 'info',
3864                 $authuser,
3865                 "move disk VM $vmid: move --disk $disk --storage $storeid"
3866             );
3867
3868             my $running = PVE::QemuServer::check_running($vmid);
3869
3870             PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]);
3871
3872             my $newvollist = [];
3873
3874             eval {
3875                 local $SIG{INT} =
3876                     local $SIG{TERM} =
3877                     local $SIG{QUIT} =
3878                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3879
3880                 warn "moving disk with snapshots, snapshots will not be moved!\n"
3881                     if $snapshotted;
3882
3883                 my $bwlimit = extract_param($param, 'bwlimit');
3884                 my $movelimit = PVE::Storage::get_bandwidth_limit(
3885                     'move',
3886                     [$oldstoreid, $storeid],
3887                     $bwlimit
3888                 );
3889
3890                 my $source_info = {
3891                     vmid => $vmid,
3892                     running => $running,
3893                     drivename => $disk,
3894                     drive => $drive,
3895                     snapname => undef,
3896                 };
3897
3898                 my $dest_info = {
3899                     vmid => $vmid,
3900                     drivename => $disk,
3901                     storage => $storeid,
3902                     format => $format,
3903                 };
3904
3905                 $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf)
3906                     if $disk eq 'efidisk0';
3907
3908                 my $newdrive = PVE::QemuServer::clone_disk(
3909                     $storecfg,
3910                     $source_info,
3911                     $dest_info,
3912                     1,
3913                     $newvollist,
3914                     undef,
3915                     undef,
3916                     undef,
3917                     $movelimit,
3918                 );
3919                 $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
3920
3921                 PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
3922
3923                 # convert moved disk to base if part of template
3924                 PVE::QemuServer::template_create($vmid, $conf, $disk)
3925                     if PVE::QemuConfig->is_template($conf);
3926
3927                 PVE::QemuConfig->write_config($vmid, $conf);
3928
3929                 my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks');
3930                 if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
3931                     eval { mon_cmd($vmid, "guest-fstrim") };
3932                 }
3933
3934                 eval {
3935                     # try to deactivate volumes - avoid lvm LVs to be active on several nodes
3936                     PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
3937                         if !$running;
3938                 };
3939                 warn $@ if $@;
3940             };
3941             if (my $err = $@) {
3942                 foreach my $volid (@$newvollist) {
3943                     eval { PVE::Storage::vdisk_free($storecfg, $volid) };
3944                     warn $@ if $@;
3945                 }
3946                 die "storage migration failed: $err";
3947             }
3948
3949             if ($param->{delete}) {
3950                 eval {
3951                     PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
3952                     PVE::Storage::vdisk_free($storecfg, $old_volid);
3953                 };
3954                 warn $@ if $@;
3955             }
3956         };
3957
3958         my $load_and_check_reassign_configs = sub {
3959             my $vmlist = PVE::Cluster::get_vmlist()->{ids};
3960
3961             die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid});
3962             die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
3963
3964             my $source_node = $vmlist->{$vmid}->{node};
3965             my $target_node = $vmlist->{$target_vmid}->{node};
3966
3967             die "Both VMs need to be on the same node ($source_node != $target_node)\n"
3968                 if $source_node ne $target_node;
3969
3970             my $source_conf = PVE::QemuConfig->load_config($vmid);
3971             PVE::QemuConfig->check_lock($source_conf);
3972             my $target_conf = PVE::QemuConfig->load_config($target_vmid);
3973             PVE::QemuConfig->check_lock($target_conf);
3974
3975             die "Can't move disks from or to template VMs\n"
3976                 if ($source_conf->{template} || $target_conf->{template});
3977
3978             if ($digest) {
3979                 eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
3980                 die "VM ${vmid}: $@" if $@;
3981             }
3982
3983             if ($target_digest) {
3984                 eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
3985                 die "VM ${target_vmid}: $@" if $@;
3986             }
3987
3988             die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk});
3989
3990             die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n"
3991                 if $target_conf->{$target_disk};
3992
3993             my $drive = PVE::QemuServer::parse_drive(
3994                 $disk,
3995                 $source_conf->{$disk},
3996             );
3997             die "failed to parse source disk - $@\n" if !$drive;
3998
3999             my $source_volid = $drive->{file};
4000
4001             die "disk '${disk}' has no associated volume\n" if !$source_volid;
4002             die "CD drive contents can't be moved to another VM\n"
4003                 if PVE::QemuServer::drive_is_cdrom($drive, 1);
4004
4005             my $storeid = PVE::Storage::parse_volume_id($source_volid, 1);
4006             die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid);
4007
4008             die "Can't move disk used by a snapshot to another VM\n"
4009                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid);
4010             die "Storage does not support moving of this disk to another VM\n"
4011                 if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid));
4012             die "Cannot move disk to another VM while the source VM is running - detach first\n"
4013                 if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/;
4014
4015             # now re-parse using target disk slot format
4016             if ($target_disk =~ /^unused\d+$/) {
4017                 $drive = PVE::QemuServer::parse_drive(
4018                     $target_disk,
4019                     $source_volid,
4020                 );
4021             } else {
4022                 $drive = PVE::QemuServer::parse_drive(
4023                     $target_disk,
4024                     $source_conf->{$disk},
4025                 );
4026             }
4027             die "failed to parse source disk for target disk format - $@\n" if !$drive;
4028
4029             my $repl_conf = PVE::ReplicationConfig->new();
4030             if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
4031                 my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4032                 die "Cannot move disk to a replicated VM. Storage does not support replication!\n"
4033                     if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
4034             }
4035
4036             return ($source_conf, $target_conf, $drive);
4037         };
4038
4039         my $logfunc = sub {
4040             my ($msg) = @_;
4041             print STDERR "$msg\n";
4042         };
4043
4044         my $disk_reassignfn = sub {
4045             return PVE::QemuConfig->lock_config($vmid, sub {
4046                 return PVE::QemuConfig->lock_config($target_vmid, sub {
4047                     my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs();
4048
4049                     my $source_volid = $drive->{file};
4050
4051                     print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n";
4052                     my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
4053
4054                     my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4055
4056                     my $new_volid = PVE::Storage::rename_volume(
4057                         $storecfg,
4058                         $source_volid,
4059                         $target_vmid,
4060                     );
4061
4062                     $drive->{file} = $new_volid;
4063
4064                     my $boot_order = PVE::QemuServer::device_bootorder($source_conf);
4065                     if (defined(delete $boot_order->{$disk})) {
4066                         print "removing disk '$disk' from boot order config\n";
4067                         my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ];
4068                         $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs);
4069                     }
4070
4071                     delete $source_conf->{$disk};
4072                     print "removing disk '${disk}' from VM '${vmid}' config\n";
4073                     PVE::QemuConfig->write_config($vmid, $source_conf);
4074
4075                     my $drive_string = PVE::QemuServer::print_drive($drive);
4076
4077                     if ($target_disk =~ /^unused\d+$/) {
4078                         $target_conf->{$target_disk} = $drive_string;
4079                         PVE::QemuConfig->write_config($target_vmid, $target_conf);
4080                     } else {
4081                         &$update_vm_api(
4082                             {
4083                                 node => $node,
4084                                 vmid => $target_vmid,
4085                                 digest => $target_digest,
4086                                 $target_disk => $drive_string,
4087                             },
4088                             1,
4089                         );
4090                     }
4091
4092                     # remove possible replication snapshots
4093                     if (PVE::Storage::volume_has_feature(
4094                             $storecfg,
4095                             'replicate',
4096                             $source_volid),
4097                     ) {
4098                         eval {
4099                             PVE::Replication::prepare(
4100                                 $storecfg,
4101                                 [$new_volid],
4102                                 undef,
4103                                 1,
4104                                 undef,
4105                                 $logfunc,
4106                             )
4107                         };
4108                         if (my $err = $@) {
4109                             print "Failed to remove replication snapshots on moved disk " .
4110                                 "'$target_disk'. Manual cleanup could be necessary.\n";
4111                         }
4112                     }
4113                 });
4114             });
4115         };
4116
4117         if ($target_vmid && $storeid) {
4118             my $msg = "either set 'storage' or 'target-vmid', but not both";
4119             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4120         } elsif ($target_vmid) {
4121             $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
4122                 if $authuser ne 'root@pam';
4123
4124             raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" })
4125                 if $vmid eq $target_vmid;
4126
4127             my (undef, undef, $drive) = &$load_and_check_reassign_configs();
4128             my $storage = PVE::Storage::parse_volume_id($drive->{file});
4129             $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
4130
4131             return $rpcenv->fork_worker(
4132                 'qmmove',
4133                 "${vmid}-${disk}>${target_vmid}-${target_disk}",
4134                 $authuser,
4135                 $disk_reassignfn
4136             );
4137         } elsif ($storeid) {
4138             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4139
4140             die "cannot move disk '$disk', only configured disks can be moved to another storage\n"
4141                 if $disk =~ m/^unused\d+$/;
4142
4143             $load_and_check_move->(); # early checks before forking/locking
4144
4145             my $realcmd = sub {
4146                 PVE::QemuConfig->lock_config($vmid, $move_updatefn);
4147             };
4148
4149             return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
4150         } else {
4151             my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
4152             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4153         }
4154     }});
4155
4156 my $check_vm_disks_local = sub {
4157     my ($storecfg, $vmconf, $vmid) = @_;
4158
4159     my $local_disks = {};
4160
4161     # add some more information to the disks e.g. cdrom
4162     PVE::QemuServer::foreach_volid($vmconf, sub {
4163         my ($volid, $attr) = @_;
4164
4165         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4166         if ($storeid) {
4167             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4168             return if $scfg->{shared};
4169         }
4170         # The shared attr here is just a special case where the vdisk
4171         # is marked as shared manually
4172         return if $attr->{shared};
4173         return if $attr->{cdrom} and $volid eq "none";
4174
4175         if (exists $local_disks->{$volid}) {
4176             @{$local_disks->{$volid}}{keys %$attr} = values %$attr
4177         } else {
4178             $local_disks->{$volid} = $attr;
4179             # ensure volid is present in case it's needed
4180             $local_disks->{$volid}->{volid} = $volid;
4181         }
4182     });
4183
4184     return $local_disks;
4185 };
4186
4187 __PACKAGE__->register_method({
4188     name => 'migrate_vm_precondition',
4189     path => '{vmid}/migrate',
4190     method => 'GET',
4191     protected => 1,
4192     proxyto => 'node',
4193     description => "Get preconditions for migration.",
4194     permissions => {
4195         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4196     },
4197     parameters => {
4198         additionalProperties => 0,
4199         properties => {
4200             node => get_standard_option('pve-node'),
4201             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4202             target => get_standard_option('pve-node', {
4203                 description => "Target node.",
4204                 completion =>  \&PVE::Cluster::complete_migration_target,
4205                 optional => 1,
4206             }),
4207         },
4208     },
4209     returns => {
4210         type => "object",
4211         properties => {
4212             running => { type => 'boolean' },
4213             allowed_nodes => {
4214                 type => 'array',
4215                 optional => 1,
4216                 description => "List nodes allowed for offline migration, only passed if VM is offline"
4217             },
4218             not_allowed_nodes => {
4219                 type => 'object',
4220                 optional => 1,
4221                 description => "List not allowed nodes with additional informations, only passed if VM is offline"
4222             },
4223             local_disks => {
4224                 type => 'array',
4225                 description => "List local disks including CD-Rom, unsused and not referenced disks"
4226             },
4227             local_resources => {
4228                 type => 'array',
4229                 description => "List local resources e.g. pci, usb"
4230             }
4231         },
4232     },
4233     code => sub {
4234         my ($param) = @_;
4235
4236         my $rpcenv = PVE::RPCEnvironment::get();
4237
4238         my $authuser = $rpcenv->get_user();
4239
4240         PVE::Cluster::check_cfs_quorum();
4241
4242         my $res = {};
4243
4244         my $vmid = extract_param($param, 'vmid');
4245         my $target = extract_param($param, 'target');
4246         my $localnode = PVE::INotify::nodename();
4247
4248
4249         # test if VM exists
4250         my $vmconf = PVE::QemuConfig->load_config($vmid);
4251         my $storecfg = PVE::Storage::config();
4252
4253
4254         # try to detect errors early
4255         PVE::QemuConfig->check_lock($vmconf);
4256
4257         $res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0;
4258
4259         # if vm is not running, return target nodes where local storage is available
4260         # for offline migration
4261         if (!$res->{running}) {
4262             $res->{allowed_nodes} = [];
4263             my $checked_nodes = PVE::QemuServer::check_local_storage_availability($vmconf, $storecfg);
4264             delete $checked_nodes->{$localnode};
4265
4266             foreach my $node (keys %$checked_nodes) {
4267                 if (!defined $checked_nodes->{$node}->{unavailable_storages}) {
4268                     push @{$res->{allowed_nodes}}, $node;
4269                 }
4270
4271             }
4272             $res->{not_allowed_nodes} = $checked_nodes;
4273         }
4274
4275
4276         my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid);
4277         $res->{local_disks} = [ values %$local_disks ];;
4278
4279         my $local_resources =  PVE::QemuServer::check_local_resources($vmconf, 1);
4280
4281         $res->{local_resources} = $local_resources;
4282
4283         return $res;
4284
4285
4286     }});
4287
4288 __PACKAGE__->register_method({
4289     name => 'migrate_vm',
4290     path => '{vmid}/migrate',
4291     method => 'POST',
4292     protected => 1,
4293     proxyto => 'node',
4294     description => "Migrate virtual machine. Creates a new migration task.",
4295     permissions => {
4296         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4297     },
4298     parameters => {
4299         additionalProperties => 0,
4300         properties => {
4301             node => get_standard_option('pve-node'),
4302             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4303             target => get_standard_option('pve-node', {
4304                 description => "Target node.",
4305                 completion =>  \&PVE::Cluster::complete_migration_target,
4306             }),
4307             online => {
4308                 type => 'boolean',
4309                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4310                 optional => 1,
4311             },
4312             force => {
4313                 type => 'boolean',
4314                 description => "Allow to migrate VMs which use local devices. Only root may use this option.",
4315                 optional => 1,
4316             },
4317             migration_type => {
4318                 type => 'string',
4319                 enum => ['secure', 'insecure'],
4320                 description => "Migration traffic is encrypted using an SSH tunnel by default. On secure, completely private networks this can be disabled to increase performance.",
4321                 optional => 1,
4322             },
4323             migration_network => {
4324                 type => 'string', format => 'CIDR',
4325                 description => "CIDR of the (sub) network that is used for migration.",
4326                 optional => 1,
4327             },
4328             "with-local-disks" => {
4329                 type => 'boolean',
4330                 description => "Enable live storage migration for local disk",
4331                 optional => 1,
4332             },
4333             targetstorage => get_standard_option('pve-targetstorage', {
4334                 completion => \&PVE::QemuServer::complete_migration_storage,
4335             }),
4336             bwlimit => {
4337                 description => "Override I/O bandwidth limit (in KiB/s).",
4338                 optional => 1,
4339                 type => 'integer',
4340                 minimum => '0',
4341                 default => 'migrate limit from datacenter or storage config',
4342             },
4343         },
4344     },
4345     returns => {
4346         type => 'string',
4347         description => "the task ID.",
4348     },
4349     code => sub {
4350         my ($param) = @_;
4351
4352         my $rpcenv = PVE::RPCEnvironment::get();
4353         my $authuser = $rpcenv->get_user();
4354
4355         my $target = extract_param($param, 'target');
4356
4357         my $localnode = PVE::INotify::nodename();
4358         raise_param_exc({ target => "target is local node."}) if $target eq $localnode;
4359
4360         PVE::Cluster::check_cfs_quorum();
4361
4362         PVE::Cluster::check_node_exists($target);
4363
4364         my $targetip = PVE::Cluster::remote_node_ip($target);
4365
4366         my $vmid = extract_param($param, 'vmid');
4367
4368         raise_param_exc({ force => "Only root may use this option." })
4369             if $param->{force} && $authuser ne 'root@pam';
4370
4371         raise_param_exc({ migration_type => "Only root may use this option." })
4372             if $param->{migration_type} && $authuser ne 'root@pam';
4373
4374         # allow root only until better network permissions are available
4375         raise_param_exc({ migration_network => "Only root may use this option." })
4376             if $param->{migration_network} && $authuser ne 'root@pam';
4377
4378         # test if VM exists
4379         my $conf = PVE::QemuConfig->load_config($vmid);
4380
4381         # try to detect errors early
4382
4383         PVE::QemuConfig->check_lock($conf);
4384
4385         if (PVE::QemuServer::check_running($vmid)) {
4386             die "can't migrate running VM without --online\n" if !$param->{online};
4387
4388             my $repl_conf = PVE::ReplicationConfig->new();
4389             my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
4390             my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target));
4391             if (!$param->{force} && $is_replicated && !$is_replicated_to_target) {
4392                 die "Cannot live-migrate replicated VM to node '$target' - not a replication " .
4393                     "target. Use 'force' to override.\n";
4394             }
4395         } else {
4396             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4397             $param->{online} = 0;
4398         }
4399
4400         my $storecfg = PVE::Storage::config();
4401         if (my $targetstorage = $param->{targetstorage}) {
4402             my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
4403             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
4404                 if $@;
4405
4406             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
4407                 if !defined($storagemap->{identity});
4408
4409             foreach my $target_sid (values %{$storagemap->{entries}}) {
4410                 $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
4411             }
4412
4413             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
4414                 if $storagemap->{default};
4415
4416             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target)
4417                 if $storagemap->{identity};
4418
4419             $param->{storagemap} = $storagemap;
4420         } else {
4421             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target);
4422         }
4423
4424         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
4425
4426             my $hacmd = sub {
4427                 my $upid = shift;
4428
4429                 print "Requesting HA migration for VM $vmid to node $target\n";
4430
4431                 my $cmd = ['ha-manager', 'migrate', "vm:$vmid", $target];
4432                 PVE::Tools::run_command($cmd);
4433                 return;
4434             };
4435
4436             return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);
4437
4438         } else {
4439
4440             my $realcmd = sub {
4441                 PVE::QemuMigrate->migrate($target, $targetip, $vmid, $param);
4442             };
4443
4444             my $worker = sub {
4445                 return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
4446             };
4447
4448             return $rpcenv->fork_worker('qmigrate', $vmid, $authuser, $worker);
4449         }
4450
4451     }});
4452
4453 __PACKAGE__->register_method({
4454     name => 'remote_migrate_vm',
4455     path => '{vmid}/remote_migrate',
4456     method => 'POST',
4457     protected => 1,
4458     proxyto => 'node',
4459     description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!",
4460     permissions => {
4461         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4462     },
4463     parameters => {
4464         additionalProperties => 0,
4465         properties => {
4466             node => get_standard_option('pve-node'),
4467             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4468             'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
4469             'target-endpoint' => get_standard_option('proxmox-remote', {
4470                 description => "Remote target endpoint",
4471             }),
4472             online => {
4473                 type => 'boolean',
4474                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4475                 optional => 1,
4476             },
4477             delete => {
4478                 type => 'boolean',
4479                 description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.",
4480                 optional => 1,
4481                 default => 0,
4482             },
4483             'target-storage' => get_standard_option('pve-targetstorage', {
4484                 completion => \&PVE::QemuServer::complete_migration_storage,
4485                 optional => 0,
4486             }),
4487             'target-bridge' => {
4488                 type => 'string',
4489                 description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
4490                 format => 'bridge-pair-list',
4491             },
4492             bwlimit => {
4493                 description => "Override I/O bandwidth limit (in KiB/s).",
4494                 optional => 1,
4495                 type => 'integer',
4496                 minimum => '0',
4497                 default => 'migrate limit from datacenter or storage config',
4498             },
4499         },
4500     },
4501     returns => {
4502         type => 'string',
4503         description => "the task ID.",
4504     },
4505     code => sub {
4506         my ($param) = @_;
4507
4508         my $rpcenv = PVE::RPCEnvironment::get();
4509         my $authuser = $rpcenv->get_user();
4510
4511         my $source_vmid = extract_param($param, 'vmid');
4512         my $target_endpoint = extract_param($param, 'target-endpoint');
4513         my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
4514
4515         my $delete = extract_param($param, 'delete') // 0;
4516
4517         PVE::Cluster::check_cfs_quorum();
4518
4519         # test if VM exists
4520         my $conf = PVE::QemuConfig->load_config($source_vmid);
4521
4522         PVE::QemuConfig->check_lock($conf);
4523
4524         raise_param_exc({ vmid => "cannot migrate HA-managed VM to remote cluster" })
4525             if PVE::HA::Config::vm_is_ha_managed($source_vmid);
4526
4527         my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);
4528
4529         # TODO: move this as helper somewhere appropriate?
4530         my $conn_args = {
4531             protocol => 'https',
4532             host => $remote->{host},
4533             port => $remote->{port} // 8006,
4534             apitoken => $remote->{apitoken},
4535         };
4536
4537         my $fp;
4538         if ($fp = $remote->{fingerprint}) {
4539             $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
4540         }
4541
4542         print "Establishing API connection with remote at '$remote->{host}'\n";
4543
4544         my $api_client = PVE::APIClient::LWP->new(%$conn_args);
4545
4546         if (!defined($fp)) {
4547             my $cert_info = $api_client->get("/nodes/localhost/certificates/info");
4548             foreach my $cert (@$cert_info) {
4549                 my $filename = $cert->{filename};
4550                 next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem';
4551                 $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem';
4552             }
4553             $conn_args->{cached_fingerprints} = { uc($fp) => 1 }
4554                 if defined($fp);
4555         }
4556
4557         my $repl_conf = PVE::ReplicationConfig->new();
4558         my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1);
4559         die "cannot remote-migrate replicated VM\n" if $is_replicated;
4560
4561         if (PVE::QemuServer::check_running($source_vmid)) {
4562             die "can't migrate running VM without --online\n" if !$param->{online};
4563
4564         } else {
4565             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4566             $param->{online} = 0;
4567         }
4568
4569         my $storecfg = PVE::Storage::config();
4570         my $target_storage = extract_param($param, 'target-storage');
4571         my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
4572         raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
4573             if $@;
4574
4575         my $target_bridge = extract_param($param, 'target-bridge');
4576         my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') };
4577         raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" })
4578             if $@;
4579
4580         die "remote migration requires explicit storage mapping!\n"
4581             if $storagemap->{identity};
4582
4583         $param->{storagemap} = $storagemap;
4584         $param->{bridgemap} = $bridgemap;
4585         $param->{remote} = {
4586             conn => $conn_args, # re-use fingerprint for tunnel
4587             client => $api_client,
4588             vmid => $target_vmid,
4589         };
4590         $param->{migration_type} = 'websocket';
4591         $param->{'with-local-disks'} = 1;
4592         $param->{delete} = $delete if $delete;
4593
4594         my $cluster_status = $api_client->get("/cluster/status");
4595         my $target_node;
4596         foreach my $entry (@$cluster_status) {
4597             next if $entry->{type} ne 'node';
4598             if ($entry->{local}) {
4599                 $target_node = $entry->{name};
4600                 last;
4601             }
4602         }
4603
4604         die "couldn't determine endpoint's node name\n"
4605             if !defined($target_node);
4606
4607         my $realcmd = sub {
4608             PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param);
4609         };
4610
4611         my $worker = sub {
4612             return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd);
4613         };
4614
4615         return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker);
4616     }});
4617
4618 __PACKAGE__->register_method({
4619     name => 'monitor',
4620     path => '{vmid}/monitor',
4621     method => 'POST',
4622     protected => 1,
4623     proxyto => 'node',
4624     description => "Execute QEMU monitor commands.",
4625     permissions => {
4626         description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')",
4627         check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]],
4628     },
4629     parameters => {
4630         additionalProperties => 0,
4631         properties => {
4632             node => get_standard_option('pve-node'),
4633             vmid => get_standard_option('pve-vmid'),
4634             command => {
4635                 type => 'string',
4636                 description => "The monitor command.",
4637             }
4638         },
4639     },
4640     returns => { type => 'string'},
4641     code => sub {
4642         my ($param) = @_;
4643
4644         my $rpcenv = PVE::RPCEnvironment::get();
4645         my $authuser = $rpcenv->get_user();
4646
4647         my $is_ro = sub {
4648             my $command = shift;
4649             return $command =~ m/^\s*info(\s+|$)/
4650                 || $command =~ m/^\s*help\s*$/;
4651         };
4652
4653         $rpcenv->check_full($authuser, "/", ['Sys.Modify'])
4654             if !&$is_ro($param->{command});
4655
4656         my $vmid = $param->{vmid};
4657
4658         my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists
4659
4660         my $res = '';
4661         eval {
4662             $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, $param->{command});
4663         };
4664         $res = "ERROR: $@" if $@;
4665
4666         return $res;
4667     }});
4668
4669 __PACKAGE__->register_method({
4670     name => 'resize_vm',
4671     path => '{vmid}/resize',
4672     method => 'PUT',
4673     protected => 1,
4674     proxyto => 'node',
4675     description => "Extend volume size.",
4676     permissions => {
4677         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
4678     },
4679     parameters => {
4680         additionalProperties => 0,
4681         properties => {
4682             node => get_standard_option('pve-node'),
4683             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4684             skiplock => get_standard_option('skiplock'),
4685             disk => {
4686                 type => 'string',
4687                 description => "The disk you want to resize.",
4688                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
4689             },
4690             size => {
4691                 type => 'string',
4692                 pattern => '\+?\d+(\.\d+)?[KMGT]?',
4693                 description => "The new size. With the `+` sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
4694             },
4695             digest => {
4696                 type => 'string',
4697                 description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
4698                 maxLength => 40,
4699                 optional => 1,
4700             },
4701         },
4702     },
4703     returns => { type => 'null'},
4704     code => sub {
4705         my ($param) = @_;
4706
4707         my $rpcenv = PVE::RPCEnvironment::get();
4708
4709         my $authuser = $rpcenv->get_user();
4710
4711         my $node = extract_param($param, 'node');
4712
4713         my $vmid = extract_param($param, 'vmid');
4714
4715         my $digest = extract_param($param, 'digest');
4716
4717         my $disk = extract_param($param, 'disk');
4718
4719         my $sizestr = extract_param($param, 'size');
4720
4721         my $skiplock = extract_param($param, 'skiplock');
4722         raise_param_exc({ skiplock => "Only root may use this option." })
4723             if $skiplock && $authuser ne 'root@pam';
4724
4725         my $storecfg = PVE::Storage::config();
4726
4727         my $updatefn =  sub {
4728
4729             my $conf = PVE::QemuConfig->load_config($vmid);
4730
4731             die "checksum missmatch (file change by other user?)\n"
4732                 if $digest && $digest ne $conf->{digest};
4733             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4734
4735             die "disk '$disk' does not exist\n" if !$conf->{$disk};
4736
4737             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
4738
4739             my (undef, undef, undef, undef, undef, undef, $format) =
4740                 PVE::Storage::parse_volname($storecfg, $drive->{file});
4741
4742             die "can't resize volume: $disk if snapshot exists\n"
4743                 if %{$conf->{snapshots}} && $format eq 'qcow2';
4744
4745             my $volid = $drive->{file};
4746
4747             die "disk '$disk' has no associated volume\n" if !$volid;
4748
4749             die "you can't resize a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive);
4750
4751             my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4752
4753             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4754
4755             PVE::Storage::activate_volumes($storecfg, [$volid]);
4756             my $size = PVE::Storage::volume_size_info($storecfg, $volid, 5);
4757
4758             die "Could not determine current size of volume '$volid'\n" if !defined($size);
4759
4760             die "internal error" if $sizestr !~ m/^(\+)?(\d+(\.\d+)?)([KMGT])?$/;
4761             my ($ext, $newsize, $unit) = ($1, $2, $4);
4762             if ($unit) {
4763                 if ($unit eq 'K') {
4764                     $newsize = $newsize * 1024;
4765                 } elsif ($unit eq 'M') {
4766                     $newsize = $newsize * 1024 * 1024;
4767                 } elsif ($unit eq 'G') {
4768                     $newsize = $newsize * 1024 * 1024 * 1024;
4769                 } elsif ($unit eq 'T') {
4770                     $newsize = $newsize * 1024 * 1024 * 1024 * 1024;
4771                 }
4772             }
4773             $newsize += $size if $ext;
4774             $newsize = int($newsize);
4775
4776             die "shrinking disks is not supported\n" if $newsize < $size;
4777
4778             return if $size == $newsize;
4779
4780             PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: resize --disk $disk --size $sizestr");
4781
4782             PVE::QemuServer::qemu_block_resize($vmid, "drive-$disk", $storecfg, $volid, $newsize);
4783
4784             $drive->{size} = $newsize;
4785             $conf->{$disk} = PVE::QemuServer::print_drive($drive);
4786
4787             PVE::QemuConfig->write_config($vmid, $conf);
4788         };
4789
4790         PVE::QemuConfig->lock_config($vmid, $updatefn);
4791         return;
4792     }});
4793
4794 __PACKAGE__->register_method({
4795     name => 'snapshot_list',
4796     path => '{vmid}/snapshot',
4797     method => 'GET',
4798     description => "List all snapshots.",
4799     permissions => {
4800         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
4801     },
4802     proxyto => 'node',
4803     protected => 1, # qemu pid files are only readable by root
4804     parameters => {
4805         additionalProperties => 0,
4806         properties => {
4807             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4808             node => get_standard_option('pve-node'),
4809         },
4810     },
4811     returns => {
4812         type => 'array',
4813         items => {
4814             type => "object",
4815             properties => {
4816                 name => {
4817                     description => "Snapshot identifier. Value 'current' identifies the current VM.",
4818                     type => 'string',
4819                 },
4820                 vmstate => {
4821                     description => "Snapshot includes RAM.",
4822                     type => 'boolean',
4823                     optional => 1,
4824                 },
4825                 description => {
4826                     description => "Snapshot description.",
4827                     type => 'string',
4828                 },
4829                 snaptime => {
4830                     description => "Snapshot creation time",
4831                     type => 'integer',
4832                     renderer => 'timestamp',
4833                     optional => 1,
4834                 },
4835                 parent => {
4836                     description => "Parent snapshot identifier.",
4837                     type => 'string',
4838                     optional => 1,
4839                 },
4840             },
4841         },
4842         links => [ { rel => 'child', href => "{name}" } ],
4843     },
4844     code => sub {
4845         my ($param) = @_;
4846
4847         my $vmid = $param->{vmid};
4848
4849         my $conf = PVE::QemuConfig->load_config($vmid);
4850         my $snaphash = $conf->{snapshots} || {};
4851
4852         my $res = [];
4853
4854         foreach my $name (keys %$snaphash) {
4855             my $d = $snaphash->{$name};
4856             my $item = {
4857                 name => $name,
4858                 snaptime => $d->{snaptime} || 0,
4859                 vmstate => $d->{vmstate} ? 1 : 0,
4860                 description => $d->{description} || '',
4861             };
4862             $item->{parent} = $d->{parent} if $d->{parent};
4863             $item->{snapstate} = $d->{snapstate} if $d->{snapstate};
4864             push @$res, $item;
4865         }
4866
4867         my $running = PVE::QemuServer::check_running($vmid, 1) ? 1 : 0;
4868         my $current = {
4869             name => 'current',
4870             digest => $conf->{digest},
4871             running => $running,
4872             description => "You are here!",
4873         };
4874         $current->{parent} = $conf->{parent} if $conf->{parent};
4875
4876         push @$res, $current;
4877
4878         return $res;
4879     }});
4880
4881 __PACKAGE__->register_method({
4882     name => 'snapshot',
4883     path => '{vmid}/snapshot',
4884     method => 'POST',
4885     protected => 1,
4886     proxyto => 'node',
4887     description => "Snapshot a VM.",
4888     permissions => {
4889         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4890     },
4891     parameters => {
4892         additionalProperties => 0,
4893         properties => {
4894             node => get_standard_option('pve-node'),
4895             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4896             snapname => get_standard_option('pve-snapshot-name'),
4897             vmstate => {
4898                 optional => 1,
4899                 type => 'boolean',
4900                 description => "Save the vmstate",
4901             },
4902             description => {
4903                 optional => 1,
4904                 type => 'string',
4905                 description => "A textual description or comment.",
4906             },
4907         },
4908     },
4909     returns => {
4910         type => 'string',
4911         description => "the task ID.",
4912     },
4913     code => sub {
4914         my ($param) = @_;
4915
4916         my $rpcenv = PVE::RPCEnvironment::get();
4917
4918         my $authuser = $rpcenv->get_user();
4919
4920         my $node = extract_param($param, 'node');
4921
4922         my $vmid = extract_param($param, 'vmid');
4923
4924         my $snapname = extract_param($param, 'snapname');
4925
4926         die "unable to use snapshot name 'current' (reserved name)\n"
4927             if $snapname eq 'current';
4928
4929         die "unable to use snapshot name 'pending' (reserved name)\n"
4930             if lc($snapname) eq 'pending';
4931
4932         my $realcmd = sub {
4933             PVE::Cluster::log_msg('info', $authuser, "snapshot VM $vmid: $snapname");
4934             PVE::QemuConfig->snapshot_create($vmid, $snapname, $param->{vmstate},
4935                                              $param->{description});
4936         };
4937
4938         return $rpcenv->fork_worker('qmsnapshot', $vmid, $authuser, $realcmd);
4939     }});
4940
4941 __PACKAGE__->register_method({
4942     name => 'snapshot_cmd_idx',
4943     path => '{vmid}/snapshot/{snapname}',
4944     description => '',
4945     method => 'GET',
4946     permissions => {
4947         user => 'all',
4948     },
4949     parameters => {
4950         additionalProperties => 0,
4951         properties => {
4952             vmid => get_standard_option('pve-vmid'),
4953             node => get_standard_option('pve-node'),
4954             snapname => get_standard_option('pve-snapshot-name'),
4955         },
4956     },
4957     returns => {
4958         type => 'array',
4959         items => {
4960             type => "object",
4961             properties => {},
4962         },
4963         links => [ { rel => 'child', href => "{cmd}" } ],
4964     },
4965     code => sub {
4966         my ($param) = @_;
4967
4968         my $res = [];
4969
4970         push @$res, { cmd => 'rollback' };
4971         push @$res, { cmd => 'config' };
4972
4973         return $res;
4974     }});
4975
4976 __PACKAGE__->register_method({
4977     name => 'update_snapshot_config',
4978     path => '{vmid}/snapshot/{snapname}/config',
4979     method => 'PUT',
4980     protected => 1,
4981     proxyto => 'node',
4982     description => "Update snapshot metadata.",
4983     permissions => {
4984         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4985     },
4986     parameters => {
4987         additionalProperties => 0,
4988         properties => {
4989             node => get_standard_option('pve-node'),
4990             vmid => get_standard_option('pve-vmid'),
4991             snapname => get_standard_option('pve-snapshot-name'),
4992             description => {
4993                 optional => 1,
4994                 type => 'string',
4995                 description => "A textual description or comment.",
4996             },
4997         },
4998     },
4999     returns => { type => 'null' },
5000     code => sub {
5001         my ($param) = @_;
5002
5003         my $rpcenv = PVE::RPCEnvironment::get();
5004
5005         my $authuser = $rpcenv->get_user();
5006
5007         my $vmid = extract_param($param, 'vmid');
5008
5009         my $snapname = extract_param($param, 'snapname');
5010
5011         return if !defined($param->{description});
5012
5013         my $updatefn =  sub {
5014
5015             my $conf = PVE::QemuConfig->load_config($vmid);
5016
5017             PVE::QemuConfig->check_lock($conf);
5018
5019             my $snap = $conf->{snapshots}->{$snapname};
5020
5021             die "snapshot '$snapname' does not exist\n" if !defined($snap);
5022
5023             $snap->{description} = $param->{description} if defined($param->{description});
5024
5025              PVE::QemuConfig->write_config($vmid, $conf);
5026         };
5027
5028         PVE::QemuConfig->lock_config($vmid, $updatefn);
5029
5030         return;
5031     }});
5032
5033 __PACKAGE__->register_method({
5034     name => 'get_snapshot_config',
5035     path => '{vmid}/snapshot/{snapname}/config',
5036     method => 'GET',
5037     proxyto => 'node',
5038     description => "Get snapshot configuration",
5039     permissions => {
5040         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback', 'VM.Audit' ], any => 1],
5041     },
5042     parameters => {
5043         additionalProperties => 0,
5044         properties => {
5045             node => get_standard_option('pve-node'),
5046             vmid => get_standard_option('pve-vmid'),
5047             snapname => get_standard_option('pve-snapshot-name'),
5048         },
5049     },
5050     returns => { type => "object" },
5051     code => sub {
5052         my ($param) = @_;
5053
5054         my $rpcenv = PVE::RPCEnvironment::get();
5055
5056         my $authuser = $rpcenv->get_user();
5057
5058         my $vmid = extract_param($param, 'vmid');
5059
5060         my $snapname = extract_param($param, 'snapname');
5061
5062         my $conf = PVE::QemuConfig->load_config($vmid);
5063
5064         my $snap = $conf->{snapshots}->{$snapname};
5065
5066         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5067
5068         return $snap;
5069     }});
5070
5071 __PACKAGE__->register_method({
5072     name => 'rollback',
5073     path => '{vmid}/snapshot/{snapname}/rollback',
5074     method => 'POST',
5075     protected => 1,
5076     proxyto => 'node',
5077     description => "Rollback VM state to specified snapshot.",
5078     permissions => {
5079         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback' ], any => 1],
5080     },
5081     parameters => {
5082         additionalProperties => 0,
5083         properties => {
5084             node => get_standard_option('pve-node'),
5085             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5086             snapname => get_standard_option('pve-snapshot-name'),
5087             start => {
5088                 type => 'boolean',
5089                 description => "Whether the VM should get started after rolling back successfully."
5090                     . " (Note: VMs will be automatically started if the snapshot includes RAM.)",
5091                 optional => 1,
5092                 default => 0,
5093             },
5094         },
5095     },
5096     returns => {
5097         type => 'string',
5098         description => "the task ID.",
5099     },
5100     code => sub {
5101         my ($param) = @_;
5102
5103         my $rpcenv = PVE::RPCEnvironment::get();
5104
5105         my $authuser = $rpcenv->get_user();
5106
5107         my $node = extract_param($param, 'node');
5108
5109         my $vmid = extract_param($param, 'vmid');
5110
5111         my $snapname = extract_param($param, 'snapname');
5112
5113         my $realcmd = sub {
5114             PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname");
5115             PVE::QemuConfig->snapshot_rollback($vmid, $snapname);
5116
5117             if ($param->{start} && !PVE::QemuServer::Helpers::vm_running_locally($vmid)) {
5118                 PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node });
5119             }
5120         };
5121
5122         my $worker = sub {
5123             # hold migration lock, this makes sure that nobody create replication snapshots
5124             return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
5125         };
5126
5127         return $rpcenv->fork_worker('qmrollback', $vmid, $authuser, $worker);
5128     }});
5129
5130 __PACKAGE__->register_method({
5131     name => 'delsnapshot',
5132     path => '{vmid}/snapshot/{snapname}',
5133     method => 'DELETE',
5134     protected => 1,
5135     proxyto => 'node',
5136     description => "Delete a VM snapshot.",
5137     permissions => {
5138         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
5139     },
5140     parameters => {
5141         additionalProperties => 0,
5142         properties => {
5143             node => get_standard_option('pve-node'),
5144             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5145             snapname => get_standard_option('pve-snapshot-name'),
5146             force => {
5147                 optional => 1,
5148                 type => 'boolean',
5149                 description => "For removal from config file, even if removing disk snapshots fails.",
5150             },
5151         },
5152     },
5153     returns => {
5154         type => 'string',
5155         description => "the task ID.",
5156     },
5157     code => sub {
5158         my ($param) = @_;
5159
5160         my $rpcenv = PVE::RPCEnvironment::get();
5161
5162         my $authuser = $rpcenv->get_user();
5163
5164         my $node = extract_param($param, 'node');
5165
5166         my $vmid = extract_param($param, 'vmid');
5167
5168         my $snapname = extract_param($param, 'snapname');
5169
5170         my $lock_obtained;
5171         my $do_delete = sub {
5172             $lock_obtained = 1;
5173             PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname");
5174             PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force});
5175         };
5176
5177         my $realcmd = sub {
5178             if ($param->{force}) {
5179                 $do_delete->();
5180             } else {
5181                 eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); };
5182                 if (my $err = $@) {
5183                     die $err if $lock_obtained;
5184                     die "Failed to obtain guest migration lock - replication running?\n";
5185                 }
5186             }
5187         };
5188
5189         return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd);
5190     }});
5191
5192 __PACKAGE__->register_method({
5193     name => 'template',
5194     path => '{vmid}/template',
5195     method => 'POST',
5196     protected => 1,
5197     proxyto => 'node',
5198     description => "Create a Template.",
5199     permissions => {
5200         description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
5201         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
5202     },
5203     parameters => {
5204         additionalProperties => 0,
5205         properties => {
5206             node => get_standard_option('pve-node'),
5207             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
5208             disk => {
5209                 optional => 1,
5210                 type => 'string',
5211                 description => "If you want to convert only 1 disk to base image.",
5212                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
5213             },
5214
5215         },
5216     },
5217     returns => {
5218         type => 'string',
5219         description => "the task ID.",
5220     },
5221     code => sub {
5222         my ($param) = @_;
5223
5224         my $rpcenv = PVE::RPCEnvironment::get();
5225
5226         my $authuser = $rpcenv->get_user();
5227
5228         my $node = extract_param($param, 'node');
5229
5230         my $vmid = extract_param($param, 'vmid');
5231
5232         my $disk = extract_param($param, 'disk');
5233
5234         my $load_and_check = sub {
5235             my $conf = PVE::QemuConfig->load_config($vmid);
5236
5237             PVE::QemuConfig->check_lock($conf);
5238
5239             die "unable to create template, because VM contains snapshots\n"
5240                 if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});
5241
5242             die "you can't convert a template to a template\n"
5243                 if PVE::QemuConfig->is_template($conf) && !$disk;
5244
5245             die "you can't convert a VM to template if VM is running\n"
5246                 if PVE::QemuServer::check_running($vmid);
5247
5248             return $conf;
5249         };
5250
5251         $load_and_check->();
5252
5253         my $realcmd = sub {
5254             PVE::QemuConfig->lock_config($vmid, sub {
5255                 my $conf = $load_and_check->();
5256
5257                 $conf->{template} = 1;
5258                 PVE::QemuConfig->write_config($vmid, $conf);
5259
5260                 PVE::QemuServer::template_create($vmid, $conf, $disk);
5261             });
5262         };
5263
5264         return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
5265     }});
5266
5267 __PACKAGE__->register_method({
5268     name => 'cloudinit_generated_config_dump',
5269     path => '{vmid}/cloudinit/dump',
5270     method => 'GET',
5271     proxyto => 'node',
5272     description => "Get automatically generated cloudinit config.",
5273     permissions => {
5274         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
5275     },
5276     parameters => {
5277         additionalProperties => 0,
5278         properties => {
5279             node => get_standard_option('pve-node'),
5280             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5281             type => {
5282                 description => 'Config type.',
5283                 type => 'string',
5284                 enum => ['user', 'network', 'meta'],
5285             },
5286         },
5287     },
5288     returns => {
5289         type => 'string',
5290     },
5291     code => sub {
5292         my ($param) = @_;
5293
5294         my $conf = PVE::QemuConfig->load_config($param->{vmid});
5295
5296         return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
5297     }});
5298
5299 __PACKAGE__->register_method({
5300     name => 'mtunnel',
5301     path => '{vmid}/mtunnel',
5302     method => 'POST',
5303     protected => 1,
5304     description => 'Migration tunnel endpoint - only for internal use by VM migration.',
5305     permissions => {
5306         check =>
5307         [ 'and',
5308           ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]],
5309           ['perm', '/', [ 'Sys.Incoming' ]],
5310         ],
5311         description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" .
5312                        " on '/'. Further permission checks happen during the actual migration.",
5313     },
5314     parameters => {
5315         additionalProperties => 0,
5316         properties => {
5317             node => get_standard_option('pve-node'),
5318             vmid => get_standard_option('pve-vmid'),
5319             storages => {
5320                 type => 'string',
5321                 format => 'pve-storage-id-list',
5322                 optional => 1,
5323                 description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.',
5324             },
5325             bridges => {
5326                 type => 'string',
5327                 format => 'pve-bridge-id-list',
5328                 optional => 1,
5329                 description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.',
5330             },
5331         },
5332     },
5333     returns => {
5334         additionalProperties => 0,
5335         properties => {
5336             upid => { type => 'string' },
5337             ticket => { type => 'string' },
5338             socket => { type => 'string' },
5339         },
5340     },
5341     code => sub {
5342         my ($param) = @_;
5343
5344         my $rpcenv = PVE::RPCEnvironment::get();
5345         my $authuser = $rpcenv->get_user();
5346
5347         my $node = extract_param($param, 'node');
5348         my $vmid = extract_param($param, 'vmid');
5349
5350         my $storages = extract_param($param, 'storages');
5351         my $bridges = extract_param($param, 'bridges');
5352
5353         my $nodename = PVE::INotify::nodename();
5354
5355         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5356             if $node ne 'localhost' && $node ne $nodename;
5357
5358         $node = $nodename;
5359
5360         my $storecfg = PVE::Storage::config();
5361         foreach my $storeid (PVE::Tools::split_list($storages)) {
5362             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node);
5363         }
5364
5365         foreach my $bridge (PVE::Tools::split_list($bridges)) {
5366             PVE::Network::read_bridge_mtu($bridge);
5367         }
5368
5369         PVE::Cluster::check_cfs_quorum();
5370
5371         my $lock = 'create';
5372         eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); };
5373
5374         raise_param_exc({ vmid => "unable to create empty VM config - $@"})
5375             if $@;
5376
5377         my $realcmd = sub {
5378             my $state = {
5379                 storecfg => PVE::Storage::config(),
5380                 lock => $lock,
5381                 vmid => $vmid,
5382             };
5383
5384             my $run_locked = sub {
5385                 my ($code, $params) = @_;
5386                 return PVE::QemuConfig->lock_config($state->{vmid}, sub {
5387                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5388
5389                     $state->{conf} = $conf;
5390
5391                     die "Encountered wrong lock - aborting mtunnel command handling.\n"
5392                         if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock});
5393
5394                     return $code->($params);
5395                 });
5396             };
5397
5398             my $cmd_desc = {
5399                 config => {
5400                     conf => {
5401                         type => 'string',
5402                         description => 'Full VM config, adapted for target cluster/node',
5403                     },
5404                     'firewall-config' => {
5405                         type => 'string',
5406                         description => 'VM firewall config',
5407                         optional => 1,
5408                     },
5409                 },
5410                 disk => {
5411                     format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'),
5412                     storage => {
5413                         type => 'string',
5414                         format => 'pve-storage-id',
5415                     },
5416                     drive => {
5417                         type => 'object',
5418                         description => 'parsed drive information without volid and format',
5419                     },
5420                 },
5421                 start => {
5422                     start_params => {
5423                         type => 'object',
5424                         description => 'params passed to vm_start_nolock',
5425                     },
5426                     migrate_opts => {
5427                         type => 'object',
5428                         description => 'migrate_opts passed to vm_start_nolock',
5429                     },
5430                 },
5431                 ticket => {
5432                     path => {
5433                         type => 'string',
5434                         description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.',
5435                     },
5436                 },
5437                 quit => {
5438                     cleanup => {
5439                         type => 'boolean',
5440                         description => 'remove VM config and disks, aborting migration',
5441                         default => 0,
5442                     },
5443                 },
5444                 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'},
5445                 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'},
5446                 bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit},
5447             };
5448
5449             my $cmd_handlers = {
5450                 'version' => sub {
5451                     # compared against other end's version
5452                     # bump/reset for breaking changes
5453                     # bump/bump for opt-in changes
5454                     return {
5455                         api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
5456                         age => 0,
5457                     };
5458                 },
5459                 'config' => sub {
5460                     my ($params) = @_;
5461
5462                     # parse and write out VM FW config if given
5463                     if (my $fw_conf = $params->{'firewall-config'}) {
5464                         my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700);
5465
5466                         my $empty_conf = {
5467                             rules => [],
5468                             options => {},
5469                             aliases => {},
5470                             ipset => {} ,
5471                             ipset_comments => {},
5472                         };
5473                         my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf();
5474
5475                         # TODO: add flag for strict parsing?
5476                         # TODO: add import sub that does all this given raw content?
5477                         my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm');
5478                         $vmfw_conf->{vmid} = $state->{vmid};
5479                         PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf);
5480
5481                         $state->{cleanup}->{fw} = 1;
5482                     }
5483
5484                     my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf";
5485                     my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1);
5486                     delete $new_conf->{lock};
5487                     delete $new_conf->{digest};
5488
5489                     # TODO handle properly?
5490                     delete $new_conf->{snapshots};
5491                     delete $new_conf->{parent};
5492                     delete $new_conf->{pending};
5493
5494                     # not handled by update_vm_api
5495                     my $vmgenid = delete $new_conf->{vmgenid};
5496                     my $meta = delete $new_conf->{meta};
5497                     my $cloudinit = delete $new_conf->{cloudinit}; # this is informational only
5498                     $new_conf->{skip_cloud_init} = 1; # re-use image from source side
5499
5500                     $new_conf->{vmid} = $state->{vmid};
5501                     $new_conf->{node} = $node;
5502
5503                     PVE::QemuConfig->remove_lock($state->{vmid}, 'create');
5504
5505                     eval {
5506                         $update_vm_api->($new_conf, 1);
5507                     };
5508                     if (my $err = $@) {
5509                         # revert to locked previous config
5510                         my $conf = PVE::QemuConfig->load_config($state->{vmid});
5511                         $conf->{lock} = 'create';
5512                         PVE::QemuConfig->write_config($state->{vmid}, $conf);
5513
5514                         die $err;
5515                     }
5516
5517                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5518                     $conf->{lock} = 'migrate';
5519                     $conf->{vmgenid} = $vmgenid if defined($vmgenid);
5520                     $conf->{meta} = $meta if defined($meta);
5521                     $conf->{cloudinit} = $cloudinit if defined($cloudinit);
5522                     PVE::QemuConfig->write_config($state->{vmid}, $conf);
5523
5524                     $state->{lock} = 'migrate';
5525
5526                     return;
5527                 },
5528                 'bwlimit' => sub {
5529                     my ($params) = @_;
5530                     return PVE::StorageTunnel::handle_bwlimit($params);
5531                 },
5532                 'disk' => sub {
5533                     my ($params) = @_;
5534
5535                     my $format = $params->{format};
5536                     my $storeid = $params->{storage};
5537                     my $drive = $params->{drive};
5538
5539                     $check_storage_access_migrate->($rpcenv, $authuser, $state->{storecfg}, $storeid, $node);
5540
5541                     my $storagemap = {
5542                         default => $storeid,
5543                     };
5544
5545                     my $source_volumes = {
5546                         'disk' => [
5547                             undef,
5548                             $storeid,
5549                             undef,
5550                             $drive,
5551                             0,
5552                             $format,
5553                         ],
5554                     };
5555
5556                     my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks($state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap);
5557                     if (defined($res->{disk})) {
5558                         $state->{cleanup}->{volumes}->{$res->{disk}->{volid}} = 1;
5559                         return $res->{disk};
5560                     } else {
5561                         die "failed to allocate NBD disk..\n";
5562                     }
5563                 },
5564                 'disk-import' => sub {
5565                     my ($params) = @_;
5566
5567                     $check_storage_access_migrate->(
5568                         $rpcenv,
5569                         $authuser,
5570                         $state->{storecfg},
5571                         $params->{storage},
5572                         $node
5573                     );
5574
5575                     $params->{unix} = "/run/qemu-server/$state->{vmid}.storage";
5576
5577                     return PVE::StorageTunnel::handle_disk_import($state, $params);
5578                 },
5579                 'query-disk-import' => sub {
5580                     my ($params) = @_;
5581
5582                     return PVE::StorageTunnel::handle_query_disk_import($state, $params);
5583                 },
5584                 'start' => sub {
5585                     my ($params) = @_;
5586
5587                     my $info = PVE::QemuServer::vm_start_nolock(
5588                         $state->{storecfg},
5589                         $state->{vmid},
5590                         $state->{conf},
5591                         $params->{start_params},
5592                         $params->{migrate_opts},
5593                     );
5594
5595
5596                     if ($info->{migrate}->{proto} ne 'unix') {
5597                         PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5598                         die "migration over non-UNIX sockets not possible\n";
5599                     }
5600
5601                     my $socket = $info->{migrate}->{addr};
5602                     chown $state->{socket_uid}, -1, $socket;
5603                     $state->{sockets}->{$socket} = 1;
5604
5605                     my $unix_sockets = $info->{migrate}->{unix_sockets};
5606                     foreach my $socket (@$unix_sockets) {
5607                         chown $state->{socket_uid}, -1, $socket;
5608                         $state->{sockets}->{$socket} = 1;
5609                     }
5610                     return $info;
5611                 },
5612                 'fstrim' => sub {
5613                     if (PVE::QemuServer::qga_check_running($state->{vmid})) {
5614                         eval { mon_cmd($state->{vmid}, "guest-fstrim") };
5615                         warn "fstrim failed: $@\n" if $@;
5616                     }
5617                     return;
5618                 },
5619                 'stop' => sub {
5620                     PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5621                     return;
5622                 },
5623                 'nbdstop' => sub {
5624                     PVE::QemuServer::nbd_stop($state->{vmid});
5625                     return;
5626                 },
5627                 'resume' => sub {
5628                     if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) {
5629                         PVE::QemuServer::vm_resume($state->{vmid}, 1, 1);
5630                     } else {
5631                         die "VM $state->{vmid} not running\n";
5632                     }
5633                     return;
5634                 },
5635                 'unlock' => sub {
5636                     PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock});
5637                     delete $state->{lock};
5638                     return;
5639                 },
5640                 'ticket' => sub {
5641                     my ($params) = @_;
5642
5643                     my $path = $params->{path};
5644
5645                     die "Not allowed to generate ticket for unknown socket '$path'\n"
5646                         if !defined($state->{sockets}->{$path});
5647
5648                     return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") };
5649                 },
5650                 'quit' => sub {
5651                     my ($params) = @_;
5652
5653                     if ($params->{cleanup}) {
5654                         if ($state->{cleanup}->{fw}) {
5655                             PVE::Firewall::remove_vmfw_conf($state->{vmid});
5656                         }
5657
5658                         for my $volid (keys $state->{cleanup}->{volumes}->%*) {
5659                             print "freeing volume '$volid' as part of cleanup\n";
5660                             eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) };
5661                             warn $@ if $@;
5662                         }
5663
5664                         PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5665                     }
5666
5667                     print "switching to exit-mode, waiting for client to disconnect\n";
5668                     $state->{exit} = 1;
5669                     return;
5670                 },
5671             };
5672
5673             $run_locked->(sub {
5674                 my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel";
5675                 unlink $socket_addr;
5676
5677                 $state->{socket} = IO::Socket::UNIX->new(
5678                     Type => SOCK_STREAM(),
5679                     Local => $socket_addr,
5680                     Listen => 1,
5681                 );
5682
5683                 $state->{socket_uid} = getpwnam('www-data')
5684                     or die "Failed to resolve user 'www-data' to numeric UID\n";
5685                 chown $state->{socket_uid}, -1, $socket_addr;
5686             });
5687
5688             print "mtunnel started\n";
5689
5690             my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) };
5691             if ($@) {
5692                 warn "Failed to accept tunnel connection - $@\n";
5693
5694                 warn "Removing tunnel socket..\n";
5695                 unlink $state->{socket};
5696
5697                 warn "Removing temporary VM config..\n";
5698                 $run_locked->(sub {
5699                     PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5700                 });
5701
5702                 die "Exiting mtunnel\n";
5703             }
5704
5705             $state->{conn} = $conn;
5706
5707             my $reply_err = sub {
5708                 my ($msg) = @_;
5709
5710                 my $reply = JSON::encode_json({
5711                     success => JSON::false,
5712                     msg => $msg,
5713                 });
5714                 $conn->print("$reply\n");
5715                 $conn->flush();
5716             };
5717
5718             my $reply_ok = sub {
5719                 my ($res) = @_;
5720
5721                 $res->{success} = JSON::true;
5722                 my $reply = JSON::encode_json($res);
5723                 $conn->print("$reply\n");
5724                 $conn->flush();
5725             };
5726
5727             while (my $line = <$conn>) {
5728                 chomp $line;
5729
5730                 # untaint, we validate below if needed
5731                 ($line) = $line =~ /^(.*)$/;
5732                 my $parsed = eval { JSON::decode_json($line) };
5733                 if ($@) {
5734                     $reply_err->("failed to parse command - $@");
5735                     next;
5736                 }
5737
5738                 my $cmd = delete $parsed->{cmd};
5739                 if (!defined($cmd)) {
5740                     $reply_err->("'cmd' missing");
5741                 } elsif ($state->{exit}) {
5742                     $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible");
5743                     next;
5744                 } elsif (my $handler = $cmd_handlers->{$cmd}) {
5745                     print "received command '$cmd'\n";
5746                     eval {
5747                         if ($cmd_desc->{$cmd}) {
5748                             PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd});
5749                         } else {
5750                             $parsed = {};
5751                         }
5752                         my $res = $run_locked->($handler, $parsed);
5753                         $reply_ok->($res);
5754                     };
5755                     $reply_err->("failed to handle '$cmd' command - $@")
5756                         if $@;
5757                 } else {
5758                     $reply_err->("unknown command '$cmd' given");
5759                 }
5760             }
5761
5762             if ($state->{exit}) {
5763                 print "mtunnel exited\n";
5764             } else {
5765                 die "mtunnel exited unexpectedly\n";
5766             }
5767         };
5768
5769         my $socket_addr = "/run/qemu-server/$vmid.mtunnel";
5770         my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr");
5771         my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd);
5772
5773         return {
5774             ticket => $ticket,
5775             upid => $upid,
5776             socket => $socket_addr,
5777         };
5778     }});
5779
5780 __PACKAGE__->register_method({
5781     name => 'mtunnelwebsocket',
5782     path => '{vmid}/mtunnelwebsocket',
5783     method => 'GET',
5784     permissions => {
5785         description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
5786         user => 'all', # check inside
5787     },
5788     description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
5789     parameters => {
5790         additionalProperties => 0,
5791         properties => {
5792             node => get_standard_option('pve-node'),
5793             vmid => get_standard_option('pve-vmid'),
5794             socket => {
5795                 type => "string",
5796                 description => "unix socket to forward to",
5797             },
5798             ticket => {
5799                 type => "string",
5800                 description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command",
5801             },
5802         },
5803     },
5804     returns => {
5805         type => "object",
5806         properties => {
5807             port => { type => 'string', optional => 1 },
5808             socket => { type => 'string', optional => 1 },
5809         },
5810     },
5811     code => sub {
5812         my ($param) = @_;
5813
5814         my $rpcenv = PVE::RPCEnvironment::get();
5815         my $authuser = $rpcenv->get_user();
5816
5817         my $nodename = PVE::INotify::nodename();
5818         my $node = extract_param($param, 'node');
5819
5820         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5821             if $node ne 'localhost' && $node ne $nodename;
5822
5823         my $vmid = $param->{vmid};
5824         # check VM exists
5825         PVE::QemuConfig->load_config($vmid);
5826
5827         my $socket = $param->{socket};
5828         PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket");
5829
5830         return { socket => $socket };
5831     }});
5832
5833 1;