PVE/API2/Qemu.pm

   1 package PVE::API2::Qemu;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd 'abs_path';
   6 use Net::SSLeay;
   7 use IO::Socket::IP;
   8 use IO::Socket::UNIX;
   9 use IPC::Open3;
  10 use JSON;
  11 use URI::Escape;
  12 use Crypt::OpenSSL::Random;
  13 use Socket qw(SOCK_STREAM);
  14
  15 use PVE::APIClient::LWP;
  16 use PVE::CGroup;
  17 use PVE::Cluster qw (cfs_read_file cfs_write_file);;
  18 use PVE::RRD;
  19 use PVE::SafeSyslog;
  20 use PVE::Tools qw(extract_param);
  21 use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
  22 use PVE::Storage;
  23 use PVE::JSONSchema qw(get_standard_option);
  24 use PVE::RESTHandler;
  25 use PVE::ReplicationConfig;
  26 use PVE::GuestHelpers qw(assert_tag_permissions);
  27 use PVE::QemuConfig;
  28 use PVE::QemuServer;
  29 use PVE::QemuServer::Cloudinit;
  30 use PVE::QemuServer::CPUConfig;
  31 use PVE::QemuServer::Drive;
  32 use PVE::QemuServer::ImportDisk;
  33 use PVE::QemuServer::Monitor qw(mon_cmd);
  34 use PVE::QemuServer::Machine;
  35 use PVE::QemuMigrate;
  36 use PVE::RPCEnvironment;
  37 use PVE::AccessControl;
  38 use PVE::INotify;
  39 use PVE::Network;
  40 use PVE::Firewall;
  41 use PVE::API2::Firewall::VM;
  42 use PVE::API2::Qemu::Agent;
  43 use PVE::VZDump::Plugin;
  44 use PVE::DataCenterConfig;
  45 use PVE::SSHInfo;
  46 use PVE::Replication;
  47 use PVE::StorageTunnel;
  48
  49 BEGIN {
  50     if (!$ENV{PVE_GENERATING_DOCS}) {
  51         require PVE::HA::Env::PVE2;
  52         import PVE::HA::Env::PVE2;
  53         require PVE::HA::Config;
  54         import PVE::HA::Config;
  55     }
  56 }
  57
  58 use base qw(PVE::RESTHandler);
  59
  60 my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal.";
  61
  62 my $resolve_cdrom_alias = sub {
  63     my $param = shift;
  64
  65     if (my $value = $param->{cdrom}) {
  66         $value .= ",media=cdrom" if $value !~ m/media=/;
  67         $param->{ide2} = $value;
  68         delete $param->{cdrom};
  69     }
  70 };
  71
  72 # Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema.
  73 my $foreach_volume_with_alloc = sub {
  74     my ($param, $func) = @_;
  75
  76     for my $opt (sort keys $param->%*) {
  77         next if !PVE::QemuServer::is_valid_drivename($opt);
  78
  79         my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1);
  80         next if !$drive;
  81
  82         $func->($opt, $drive);
  83     }
  84 };
  85
  86 my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
  87
  88 my $check_drive_param = sub {
  89     my ($param, $storecfg, $extra_checks) = @_;
  90
  91     for my $opt (sort keys $param->%*) {
  92         next if !PVE::QemuServer::is_valid_drivename($opt);
  93
  94         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
  95         raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
  96
  97         if ($drive->{'import-from'}) {
  98             if ($drive->{file} !~ $NEW_DISK_RE || $3 != 0) {
  99                 raise_param_exc({
 100                     $opt => "'import-from' requires special syntax - ".
 101                         "use <storage ID>:0,import-from=<source>",
 102                 });
 103             }
 104
 105             if ($opt eq 'efidisk0') {
 106                 for my $required (qw(efitype pre-enrolled-keys)) {
 107                     if (!defined($drive->{$required})) {
 108                         raise_param_exc({
 109                             $opt => "need to specify '$required' when using 'import-from'",
 110                         });
 111                     }
 112                 }
 113             } elsif ($opt eq 'tpmstate0') {
 114                 raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" })
 115                     if !defined($drive->{version});
 116             }
 117         }
 118
 119         PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 120
 121         $extra_checks->($drive) if $extra_checks;
 122
 123         $param->{$opt} = PVE::QemuServer::print_drive($drive, 1);
 124     }
 125 };
 126
 127 my $check_storage_access = sub {
 128    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 129
 130    $foreach_volume_with_alloc->($settings, sub {
 131         my ($ds, $drive) = @_;
 132
 133         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 134
 135         my $volid = $drive->{file};
 136         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 137
 138         if (!$volid || ($volid eq 'none' || $volid eq 'cloudinit' || (defined($volname) && $volname eq 'cloudinit'))) {
 139             # nothing to check
 140         } elsif ($isCDROM && ($volid eq 'cdrom')) {
 141             $rpcenv->check($authuser, "/", ['Sys.Console']);
 142         } elsif (!$isCDROM && ($volid =~ $NEW_DISK_RE)) {
 143             my ($storeid, $size) = ($2 || $default_storage, $3);
 144             die "no storage ID specified (and no default storage)\n" if !$storeid;
 145             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
 146             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 147             raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
 148                 if !$scfg->{content}->{images};
 149         } else {
 150             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 151             if ($storeid) {
 152                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 153                 raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" })
 154                     if $vtype ne 'images' && $vtype ne 'iso';
 155             }
 156         }
 157
 158         if (my $src_image = $drive->{'import-from'}) {
 159             my $src_vmid;
 160             if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume
 161                 (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image);
 162                 raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" })
 163                     if $vtype ne 'images';
 164             }
 165
 166             if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk()
 167                 $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']);
 168             } else {
 169                 PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image);
 170             }
 171         }
 172     });
 173
 174    $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
 175        if defined($settings->{vmstatestorage});
 176 };
 177
 178 my $check_storage_access_clone = sub {
 179    my ($rpcenv, $authuser, $storecfg, $conf, $storage) = @_;
 180
 181    my $sharedvm = 1;
 182
 183    PVE::QemuConfig->foreach_volume($conf, sub {
 184         my ($ds, $drive) = @_;
 185
 186         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 187
 188         my $volid = $drive->{file};
 189
 190         return if !$volid || $volid eq 'none';
 191
 192         if ($isCDROM) {
 193             if ($volid eq 'cdrom') {
 194                 $rpcenv->check($authuser, "/", ['Sys.Console']);
 195             } else {
 196                 # we simply allow access
 197                 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 198                 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 199                 $sharedvm = 0 if !$scfg->{shared};
 200
 201             }
 202         } else {
 203             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 204             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 205             $sharedvm = 0 if !$scfg->{shared};
 206
 207             $sid = $storage if $storage;
 208             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
 209         }
 210     });
 211
 212    $rpcenv->check($authuser, "/storage/$conf->{vmstatestorage}", ['Datastore.AllocateSpace'])
 213        if defined($conf->{vmstatestorage});
 214
 215    return $sharedvm;
 216 };
 217
 218 my $check_storage_access_migrate = sub {
 219     my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
 220
 221     PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
 222
 223     $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
 224
 225     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
 226     die "storage '$storage' does not support vm images\n"
 227         if !$scfg->{content}->{images};
 228 };
 229
 230 my $import_from_volid = sub {
 231     my ($storecfg, $src_volid, $dest_info, $vollist) = @_;
 232
 233     die "could not get size of $src_volid\n"
 234         if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10);
 235
 236     die "cannot import from cloudinit disk\n"
 237         if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid });
 238
 239     my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2];
 240
 241     my $src_vm_state = sub {
 242         my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0;
 243
 244         my $runs = 0;
 245         if ($exists) {
 246             eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); };
 247             die "owner VM $src_vmid not on local node\n" if $@;
 248             $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0;
 249         }
 250
 251         return ($exists, $runs);
 252     };
 253
 254     my ($src_vm_exists, $running) = $src_vm_state->();
 255
 256     die "cannot import from '$src_volid' - full clone feature is not supported\n"
 257         if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running);
 258
 259     my $clonefn = sub {
 260         my ($src_vm_exists_now, $running_now) = $src_vm_state->();
 261
 262         die "owner VM $src_vmid changed state unexpectedly\n"
 263             if $src_vm_exists_now != $src_vm_exists || $running_now != $running;
 264
 265         my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {};
 266
 267         my $src_drive = { file => $src_volid };
 268         my $src_drivename;
 269         PVE::QemuConfig->foreach_volume($src_conf, sub {
 270             my ($ds, $drive) = @_;
 271
 272             return if $src_drivename;
 273
 274             if ($drive->{file} eq $src_volid) {
 275                 $src_drive = $drive;
 276                 $src_drivename = $ds;
 277             }
 278         });
 279
 280         my $source_info = {
 281             vmid => $src_vmid,
 282             running => $running_now,
 283             drivename => $src_drivename,
 284             drive => $src_drive,
 285             snapname => undef,
 286         };
 287
 288         my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid);
 289
 290         return PVE::QemuServer::clone_disk(
 291             $storecfg,
 292             $source_info,
 293             $dest_info,
 294             1,
 295             $vollist,
 296             undef,
 297             undef,
 298             $src_conf->{agent},
 299             PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]),
 300         );
 301     };
 302
 303     my $cloned;
 304     if ($running) {
 305         $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn);
 306     } elsif ($src_vmid) {
 307         $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn);
 308     } else {
 309         $cloned = $clonefn->();
 310     }
 311
 312     return $cloned->@{qw(file size)};
 313 };
 314
 315 # Note: $pool is only needed when creating a VM, because pool permissions
 316 # are automatically inherited if VM already exists inside a pool.
 317 my $create_disks = sub {
 318     my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
 319
 320     my $vollist = [];
 321
 322     my $res = {};
 323
 324     my $code = sub {
 325         my ($ds, $disk) = @_;
 326
 327         my $volid = $disk->{file};
 328         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 329
 330         if (!$volid || $volid eq 'none' || $volid eq 'cdrom') {
 331             delete $disk->{size};
 332             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 333         } elsif (defined($volname) && $volname eq 'cloudinit') {
 334             $storeid = $storeid // $default_storage;
 335             die "no storage ID specified (and no default storage)\n" if !$storeid;
 336
 337             if (
 338                 my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 339                 || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 340                 || PVE::QemuConfig->has_cloudinit($res, $ds)
 341             ) {
 342                 die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 343             }
 344
 345             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 346             my $name = "vm-$vmid-cloudinit";
 347
 348             my $fmt = undef;
 349             if ($scfg->{path}) {
 350                 $fmt = $disk->{format} // "qcow2";
 351                 $name .= ".$fmt";
 352             } else {
 353                 $fmt = $disk->{format} // "raw";
 354             }
 355
 356             # Initial disk created with 4 MB and aligned to 4MB on regeneration
 357             my $ci_size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 358             my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, $name, $ci_size/1024);
 359             $disk->{file} = $volid;
 360             $disk->{media} = 'cdrom';
 361             push @$vollist, $volid;
 362             delete $disk->{format}; # no longer needed
 363             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 364             print "$ds: successfully created disk '$res->{$ds}'\n";
 365         } elsif ($volid =~ $NEW_DISK_RE) {
 366             my ($storeid, $size) = ($2 || $default_storage, $3);
 367             die "no storage ID specified (and no default storage)\n" if !$storeid;
 368
 369             if (my $source = delete $disk->{'import-from'}) {
 370                 my $dst_volid;
 371
 372                 if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume
 373                     my $dest_info = {
 374                         vmid => $vmid,
 375                         drivename => $ds,
 376                         storage => $storeid,
 377                         format => $disk->{format},
 378                     };
 379
 380                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk)
 381                         if $ds eq 'efidisk0';
 382
 383                     ($dst_volid, $size) = eval {
 384                         $import_from_volid->($storecfg, $source, $dest_info, $vollist);
 385                     };
 386                     die "cannot import from '$source' - $@" if $@;
 387                 } else {
 388                     $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
 389                     $size = PVE::Storage::file_size_info($source);
 390                     die "could not get file size of $source\n" if !$size;
 391
 392                     (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import(
 393                         $source,
 394                         $vmid,
 395                         $storeid,
 396                         {
 397                             drive_name => $ds,
 398                             format => $disk->{format},
 399                             'skip-config-update' => 1,
 400                         },
 401                     );
 402                     push @$vollist, $dst_volid;
 403                 }
 404
 405                 $disk->{file} = $dst_volid;
 406                 $disk->{size} = $size;
 407                 delete $disk->{format}; # no longer needed
 408                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 409             } else {
 410                 my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
 411                 my $fmt = $disk->{format} || $defformat;
 412
 413                 $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb
 414
 415                 my $volid;
 416                 if ($ds eq 'efidisk0') {
 417                     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 418                     ($volid, $size) = PVE::QemuServer::create_efidisk(
 419                         $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm);
 420                 } elsif ($ds eq 'tpmstate0') {
 421                     # swtpm can only use raw volumes, and uses a fixed size
 422                     $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb');
 423                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size);
 424                 } else {
 425                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
 426                 }
 427                 push @$vollist, $volid;
 428                 $disk->{file} = $volid;
 429                 $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
 430                 delete $disk->{format}; # no longer needed
 431                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 432             }
 433
 434             print "$ds: successfully created disk '$res->{$ds}'\n";
 435         } else {
 436             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 437             if ($storeid) {
 438                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 439                 die "cannot use volume $volid - content type needs to be 'images' or 'iso'"
 440                     if $vtype ne 'images' && $vtype ne 'iso';
 441
 442                 if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) {
 443                     if (
 444                         my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 445                         || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 446                         || PVE::QemuConfig->has_cloudinit($res, $ds)
 447                     ) {
 448                         die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 449                     }
 450                 }
 451             }
 452
 453             PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
 454
 455             my $size = PVE::Storage::volume_size_info($storecfg, $volid);
 456             die "volume $volid does not exist\n" if !$size;
 457             $disk->{size} = $size;
 458
 459             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 460         }
 461     };
 462
 463     eval { $foreach_volume_with_alloc->($settings, $code); };
 464
 465     # free allocated images on error
 466     if (my $err = $@) {
 467         syslog('err', "VM $vmid creating disks failed");
 468         foreach my $volid (@$vollist) {
 469             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 470             warn $@ if $@;
 471         }
 472         die $err;
 473     }
 474
 475     return ($vollist, $res);
 476 };
 477
 478 my $check_cpu_model_access = sub {
 479     my ($rpcenv, $authuser, $new, $existing) = @_;
 480
 481     return if !defined($new->{cpu});
 482
 483     my $cpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $new->{cpu});
 484     return if !$cpu || !$cpu->{cputype}; # always allow default
 485     my $cputype = $cpu->{cputype};
 486
 487     if ($existing && $existing->{cpu}) {
 488         # changing only other settings doesn't require permissions for CPU model
 489         my $existingCpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $existing->{cpu});
 490         return if $existingCpu->{cputype} eq $cputype;
 491     }
 492
 493     if (PVE::QemuServer::CPUConfig::is_custom_model($cputype)) {
 494         $rpcenv->check($authuser, "/nodes", ['Sys.Audit']);
 495     }
 496 };
 497
 498 my $cpuoptions = {
 499     'cores' => 1,
 500     'cpu' => 1,
 501     'cpulimit' => 1,
 502     'cpuunits' => 1,
 503     'numa' => 1,
 504     'smp' => 1,
 505     'sockets' => 1,
 506     'vcpus' => 1,
 507 };
 508
 509 my $memoryoptions = {
 510     'memory' => 1,
 511     'balloon' => 1,
 512     'shares' => 1,
 513 };
 514
 515 my $hwtypeoptions = {
 516     'acpi' => 1,
 517     'hotplug' => 1,
 518     'kvm' => 1,
 519     'machine' => 1,
 520     'scsihw' => 1,
 521     'smbios1' => 1,
 522     'tablet' => 1,
 523     'vga' => 1,
 524     'watchdog' => 1,
 525     'audio0' => 1,
 526 };
 527
 528 my $generaloptions = {
 529     'agent' => 1,
 530     'autostart' => 1,
 531     'bios' => 1,
 532     'description' => 1,
 533     'keyboard' => 1,
 534     'localtime' => 1,
 535     'migrate_downtime' => 1,
 536     'migrate_speed' => 1,
 537     'name' => 1,
 538     'onboot' => 1,
 539     'ostype' => 1,
 540     'protection' => 1,
 541     'reboot' => 1,
 542     'startdate' => 1,
 543     'startup' => 1,
 544     'tdf' => 1,
 545     'template' => 1,
 546 };
 547
 548 my $vmpoweroptions = {
 549     'freeze' => 1,
 550 };
 551
 552 my $diskoptions = {
 553     'boot' => 1,
 554     'bootdisk' => 1,
 555     'vmstatestorage' => 1,
 556 };
 557
 558 my $cloudinitoptions = {
 559     cicustom => 1,
 560     cipassword => 1,
 561     citype => 1,
 562     ciuser => 1,
 563     nameserver => 1,
 564     searchdomain => 1,
 565     sshkeys => 1,
 566 };
 567
 568 my $check_vm_create_serial_perm = sub {
 569     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 570
 571     return 1 if $authuser eq 'root@pam';
 572
 573     foreach my $opt (keys %{$param}) {
 574         next if $opt !~ m/^serial\d+$/;
 575
 576         if ($param->{$opt} eq 'socket') {
 577             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 578         } else {
 579             die "only root can set '$opt' config for real devices\n";
 580         }
 581     }
 582
 583     return 1;
 584 };
 585
 586 my $check_vm_create_usb_perm = sub {
 587     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 588
 589     return 1 if $authuser eq 'root@pam';
 590
 591     foreach my $opt (keys %{$param}) {
 592         next if $opt !~ m/^usb\d+$/;
 593
 594         if ($param->{$opt} =~ m/spice/) {
 595             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 596         } else {
 597             die "only root can set '$opt' config for real devices\n";
 598         }
 599     }
 600
 601     return 1;
 602 };
 603
 604 my $check_vm_modify_config_perm = sub {
 605     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
 606
 607     return 1 if $authuser eq 'root@pam';
 608
 609     foreach my $opt (@$key_list) {
 610         # some checks (e.g., disk, serial port, usb) need to be done somewhere
 611         # else, as there the permission can be value dependend
 612         next if PVE::QemuServer::is_valid_drivename($opt);
 613         next if $opt eq 'cdrom';
 614         next if $opt =~ m/^(?:unused|serial|usb)\d+$/;
 615         next if $opt eq 'tags';
 616
 617
 618         if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) {
 619             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
 620         } elsif ($memoryoptions->{$opt}) {
 621             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
 622         } elsif ($hwtypeoptions->{$opt}) {
 623             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 624         } elsif ($generaloptions->{$opt}) {
 625             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
 626             # special case for startup since it changes host behaviour
 627             if ($opt eq 'startup') {
 628                 $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
 629             }
 630         } elsif ($vmpoweroptions->{$opt}) {
 631             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']);
 632         } elsif ($diskoptions->{$opt}) {
 633             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
 634         } elsif ($opt =~ m/^(?:net|ipconfig)\d+$/) {
 635             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
 636         } elsif ($cloudinitoptions->{$opt}) {
 637             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1);
 638         } elsif ($opt eq 'vmstate') {
 639             # the user needs Disk and PowerMgmt privileges to change the vmstate
 640             # also needs privileges on the storage, that will be checked later
 641             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]);
 642         } else {
 643             # catches hostpci\d+, args, lock, etc.
 644             # new options will be checked here
 645             die "only root can set '$opt' config\n";
 646         }
 647     }
 648
 649     return 1;
 650 };
 651
 652 __PACKAGE__->register_method({
 653     name => 'vmlist',
 654     path => '',
 655     method => 'GET',
 656     description => "Virtual machine index (per node).",
 657     permissions => {
 658         description => "Only list VMs where you have VM.Audit permissons on /vms/<vmid>.",
 659         user => 'all',
 660     },
 661     proxyto => 'node',
 662     protected => 1, # qemu pid files are only readable by root
 663     parameters => {
 664         additionalProperties => 0,
 665         properties => {
 666             node => get_standard_option('pve-node'),
 667             full => {
 668                 type => 'boolean',
 669                 optional => 1,
 670                 description => "Determine the full status of active VMs.",
 671             },
 672         },
 673     },
 674     returns => {
 675         type => 'array',
 676         items => {
 677             type => "object",
 678             properties => $PVE::QemuServer::vmstatus_return_properties,
 679         },
 680         links => [ { rel => 'child', href => "{vmid}" } ],
 681     },
 682     code => sub {
 683         my ($param) = @_;
 684
 685         my $rpcenv = PVE::RPCEnvironment::get();
 686         my $authuser = $rpcenv->get_user();
 687
 688         my $vmstatus = PVE::QemuServer::vmstatus(undef, $param->{full});
 689
 690         my $res = [];
 691         foreach my $vmid (keys %$vmstatus) {
 692             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
 693
 694             my $data = $vmstatus->{$vmid};
 695             push @$res, $data;
 696         }
 697
 698         return $res;
 699     }});
 700
 701 my $parse_restore_archive = sub {
 702     my ($storecfg, $archive) = @_;
 703
 704     my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1);
 705
 706     if (defined($archive_storeid)) {
 707         my $scfg =  PVE::Storage::storage_config($storecfg, $archive_storeid);
 708         if ($scfg->{type} eq 'pbs') {
 709             return {
 710                 type => 'pbs',
 711                 volid => $archive,
 712             };
 713         }
 714     }
 715     my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive);
 716     return {
 717         type => 'file',
 718         path => $path,
 719     };
 720 };
 721
 722
 723 __PACKAGE__->register_method({
 724     name => 'create_vm',
 725     path => '',
 726     method => 'POST',
 727     description => "Create or restore a virtual machine.",
 728     permissions => {
 729         description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
 730             "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
 731             "If you create disks you need 'Datastore.AllocateSpace' on any used storage.",
 732         user => 'all', # check inside
 733     },
 734     protected => 1,
 735     proxyto => 'node',
 736     parameters => {
 737         additionalProperties => 0,
 738         properties => PVE::QemuServer::json_config_properties(
 739             {
 740                 node => get_standard_option('pve-node'),
 741                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
 742                 archive => {
 743                     description => "The backup archive. Either the file system path to a .tar or .vma file (use '-' to pipe data from stdin) or a proxmox storage backup volume identifier.",
 744                     type => 'string',
 745                     optional => 1,
 746                     maxLength => 255,
 747                     completion => \&PVE::QemuServer::complete_backup_archives,
 748                 },
 749                 storage => get_standard_option('pve-storage-id', {
 750                     description => "Default storage.",
 751                     optional => 1,
 752                     completion => \&PVE::QemuServer::complete_storage,
 753                 }),
 754                 force => {
 755                     optional => 1,
 756                     type => 'boolean',
 757                     description => "Allow to overwrite existing VM.",
 758                     requires => 'archive',
 759                 },
 760                 unique => {
 761                     optional => 1,
 762                     type => 'boolean',
 763                     description => "Assign a unique random ethernet address.",
 764                     requires => 'archive',
 765                 },
 766                 'live-restore' => {
 767                     optional => 1,
 768                     type => 'boolean',
 769                     description => "Start the VM immediately from the backup and restore in background. PBS only.",
 770                     requires => 'archive',
 771                 },
 772                 pool => {
 773                     optional => 1,
 774                     type => 'string', format => 'pve-poolid',
 775                     description => "Add the VM to the specified pool.",
 776                 },
 777                 bwlimit => {
 778                     description => "Override I/O bandwidth limit (in KiB/s).",
 779                     optional => 1,
 780                     type => 'integer',
 781                     minimum => '0',
 782                     default => 'restore limit from datacenter or storage config',
 783                 },
 784                 start => {
 785                     optional => 1,
 786                     type => 'boolean',
 787                     default => 0,
 788                     description => "Start VM after it was created successfully.",
 789                 },
 790             },
 791             1, # with_disk_alloc
 792         ),
 793     },
 794     returns => {
 795         type => 'string',
 796     },
 797     code => sub {
 798         my ($param) = @_;
 799
 800         my $rpcenv = PVE::RPCEnvironment::get();
 801         my $authuser = $rpcenv->get_user();
 802
 803         my $node = extract_param($param, 'node');
 804         my $vmid = extract_param($param, 'vmid');
 805
 806         my $archive = extract_param($param, 'archive');
 807         my $is_restore = !!$archive;
 808
 809         my $bwlimit = extract_param($param, 'bwlimit');
 810         my $force = extract_param($param, 'force');
 811         my $pool = extract_param($param, 'pool');
 812         my $start_after_create = extract_param($param, 'start');
 813         my $storage = extract_param($param, 'storage');
 814         my $unique = extract_param($param, 'unique');
 815         my $live_restore = extract_param($param, 'live-restore');
 816
 817         if (defined(my $ssh_keys = $param->{sshkeys})) {
 818                 $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
 819                 PVE::Tools::validate_ssh_public_keys($ssh_keys);
 820         }
 821
 822         $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
 823             if defined($param->{cpuunits}); # clamp value depending on cgroup version
 824
 825         PVE::Cluster::check_cfs_quorum();
 826
 827         my $filename = PVE::QemuConfig->config_file($vmid);
 828         my $storecfg = PVE::Storage::config();
 829
 830         if (defined($pool)) {
 831             $rpcenv->check_pool_exist($pool);
 832         }
 833
 834         $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace'])
 835             if defined($storage);
 836
 837         if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
 838             # OK
 839         } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
 840             # OK
 841         } elsif ($archive && $force && (-f $filename) &&
 842                  $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
 843             # OK: user has VM.Backup permissions, and want to restore an existing VM
 844         } else {
 845             raise_perm_exc();
 846         }
 847
 848         if ($archive) {
 849             for my $opt (sort keys $param->%*) {
 850                 if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
 851                     raise_param_exc({ $opt => "option conflicts with option 'archive'" });
 852                 }
 853             }
 854
 855             if ($archive eq '-') {
 856                 die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli';
 857                 $archive = { type => 'pipe' };
 858             } else {
 859                 PVE::Storage::check_volume_access(
 860                     $rpcenv,
 861                     $authuser,
 862                     $storecfg,
 863                     $vmid,
 864                     $archive,
 865                     'backup',
 866                 );
 867
 868                 $archive = $parse_restore_archive->($storecfg, $archive);
 869             }
 870         }
 871
 872         if (scalar(keys $param->%*) > 0) {
 873             &$resolve_cdrom_alias($param);
 874
 875             &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage);
 876
 877             &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]);
 878
 879             &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param);
 880             &$check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param);
 881
 882             &$check_cpu_model_access($rpcenv, $authuser, $param);
 883
 884             $check_drive_param->($param, $storecfg);
 885
 886             PVE::QemuServer::add_random_macs($param);
 887         }
 888
 889         my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -";
 890
 891         eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) };
 892         die "$emsg $@" if $@;
 893
 894         my $restored_data = 0;
 895         my $restorefn = sub {
 896             my $conf = PVE::QemuConfig->load_config($vmid);
 897
 898             PVE::QemuConfig->check_protection($conf, $emsg);
 899
 900             die "$emsg vm is running\n" if PVE::QemuServer::check_running($vmid);
 901
 902             my $realcmd = sub {
 903                 my $restore_options = {
 904                     storage => $storage,
 905                     pool => $pool,
 906                     unique => $unique,
 907                     bwlimit => $bwlimit,
 908                     live => $live_restore,
 909                     override_conf => $param,
 910                 };
 911                 if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') {
 912                     die "live-restore is only compatible with backup images from a Proxmox Backup Server\n"
 913                         if $live_restore;
 914                     PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options);
 915                 } elsif ($archive->{type} eq 'pbs') {
 916                     PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options);
 917                 } else {
 918                     die "unknown backup archive type\n";
 919                 }
 920                 $restored_data = 1;
 921
 922                 my $restored_conf = PVE::QemuConfig->load_config($vmid);
 923                 # Convert restored VM to template if backup was VM template
 924                 if (PVE::QemuConfig->is_template($restored_conf)) {
 925                     warn "Convert to template.\n";
 926                     eval { PVE::QemuServer::template_create($vmid, $restored_conf) };
 927                     warn $@ if $@;
 928                 }
 929             };
 930
 931             # ensure no old replication state are exists
 932             PVE::ReplicationState::delete_guest_states($vmid);
 933
 934             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
 935
 936             if ($start_after_create && !$live_restore) {
 937                 print "Execute autostart\n";
 938                 eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) };
 939                 warn $@ if $@;
 940             }
 941         };
 942
 943         my $createfn = sub {
 944             # ensure no old replication state are exists
 945             PVE::ReplicationState::delete_guest_states($vmid);
 946
 947             my $realcmd = sub {
 948                 my $conf = $param;
 949                 my $arch = PVE::QemuServer::get_vm_arch($conf);
 950
 951                 $conf->{meta} = PVE::QemuServer::new_meta_info_string();
 952
 953                 my $vollist = [];
 954                 eval {
 955                     ($vollist, my $created_opts) = $create_disks->(
 956                         $rpcenv,
 957                         $authuser,
 958                         $conf,
 959                         $arch,
 960                         $storecfg,
 961                         $vmid,
 962                         $pool,
 963                         $param,
 964                         $storage,
 965                     );
 966                     $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*;
 967
 968                     if (!$conf->{boot}) {
 969                         my $devs = PVE::QemuServer::get_default_bootdevices($conf);
 970                         $conf->{boot} = PVE::QemuServer::print_bootorder($devs);
 971                     }
 972
 973                     # auto generate uuid if user did not specify smbios1 option
 974                     if (!$conf->{smbios1}) {
 975                         $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid();
 976                     }
 977
 978                     if ((!defined($conf->{vmgenid}) || $conf->{vmgenid} eq '1') && $arch ne 'aarch64') {
 979                         $conf->{vmgenid} = PVE::QemuServer::generate_uuid();
 980                     }
 981
 982                     my $machine = $conf->{machine};
 983                     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
 984                         # always pin Windows' machine version on create, they get to easily confused
 985                         if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) {
 986                             $conf->{machine} = PVE::QemuServer::windows_get_pinned_machine_version($machine);
 987                         }
 988                     }
 989
 990                     PVE::QemuConfig->write_config($vmid, $conf);
 991
 992                 };
 993                 my $err = $@;
 994
 995                 if ($err) {
 996                     foreach my $volid (@$vollist) {
 997                         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 998                         warn $@ if $@;
 999                     }
1000                     die "$emsg $err";
1001                 }
1002
1003                 PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
1004             };
1005
1006             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
1007
1008             if ($start_after_create) {
1009                 print "Execute autostart\n";
1010                 eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) };
1011                 warn $@ if $@;
1012             }
1013         };
1014
1015         my ($code, $worker_name);
1016         if ($is_restore) {
1017             $worker_name = 'qmrestore';
1018             $code = sub {
1019                 eval { $restorefn->() };
1020                 if (my $err = $@) {
1021                     eval { PVE::QemuConfig->remove_lock($vmid, 'create') };
1022                     warn $@ if $@;
1023                     if ($restored_data) {
1024                         warn "error after data was restored, VM disks should be OK but config may "
1025                             ."require adaptions. VM $vmid state is NOT cleaned up.\n";
1026                     } else {
1027                         warn "error before or during data restore, some or all disks were not "
1028                             ."completely restored. VM $vmid state is NOT cleaned up.\n";
1029                     }
1030                     die $err;
1031                 }
1032             };
1033         } else {
1034             $worker_name = 'qmcreate';
1035             $code = sub {
1036                 eval { $createfn->() };
1037                 if (my $err = $@) {
1038                     eval {
1039                         my $conffile = PVE::QemuConfig->config_file($vmid);
1040                         unlink($conffile) or die "failed to remove config file: $!\n";
1041                     };
1042                     warn $@ if $@;
1043                     die $err;
1044                 }
1045             };
1046         }
1047
1048         return $rpcenv->fork_worker($worker_name, $vmid, $authuser, $code);
1049     }});
1050
1051 __PACKAGE__->register_method({
1052     name => 'vmdiridx',
1053     path => '{vmid}',
1054     method => 'GET',
1055     proxyto => 'node',
1056     description => "Directory index",
1057     permissions => {
1058         user => 'all',
1059     },
1060     parameters => {
1061         additionalProperties => 0,
1062         properties => {
1063             node => get_standard_option('pve-node'),
1064             vmid => get_standard_option('pve-vmid'),
1065         },
1066     },
1067     returns => {
1068         type => 'array',
1069         items => {
1070             type => "object",
1071             properties => {
1072                 subdir => { type => 'string' },
1073             },
1074         },
1075         links => [ { rel => 'child', href => "{subdir}" } ],
1076     },
1077     code => sub {
1078         my ($param) = @_;
1079
1080         my $res = [
1081             { subdir => 'config' },
1082             { subdir => 'cloudinit' },
1083             { subdir => 'pending' },
1084             { subdir => 'status' },
1085             { subdir => 'unlink' },
1086             { subdir => 'vncproxy' },
1087             { subdir => 'termproxy' },
1088             { subdir => 'migrate' },
1089             { subdir => 'resize' },
1090             { subdir => 'move' },
1091             { subdir => 'rrd' },
1092             { subdir => 'rrddata' },
1093             { subdir => 'monitor' },
1094             { subdir => 'agent' },
1095             { subdir => 'snapshot' },
1096             { subdir => 'spiceproxy' },
1097             { subdir => 'sendkey' },
1098             { subdir => 'firewall' },
1099             { subdir => 'mtunnel' },
1100             { subdir => 'remote_migrate' },
1101         ];
1102
1103         return $res;
1104     }});
1105
1106 __PACKAGE__->register_method ({
1107     subclass => "PVE::API2::Firewall::VM",
1108     path => '{vmid}/firewall',
1109 });
1110
1111 __PACKAGE__->register_method ({
1112     subclass => "PVE::API2::Qemu::Agent",
1113     path => '{vmid}/agent',
1114 });
1115
1116 __PACKAGE__->register_method({
1117     name => 'rrd',
1118     path => '{vmid}/rrd',
1119     method => 'GET',
1120     protected => 1, # fixme: can we avoid that?
1121     permissions => {
1122         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1123     },
1124     description => "Read VM RRD statistics (returns PNG)",
1125     parameters => {
1126         additionalProperties => 0,
1127         properties => {
1128             node => get_standard_option('pve-node'),
1129             vmid => get_standard_option('pve-vmid'),
1130             timeframe => {
1131                 description => "Specify the time frame you are interested in.",
1132                 type => 'string',
1133                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1134             },
1135             ds => {
1136                 description => "The list of datasources you want to display.",
1137                 type => 'string', format => 'pve-configid-list',
1138             },
1139             cf => {
1140                 description => "The RRD consolidation function",
1141                 type => 'string',
1142                 enum => [ 'AVERAGE', 'MAX' ],
1143                 optional => 1,
1144             },
1145         },
1146     },
1147     returns => {
1148         type => "object",
1149         properties => {
1150             filename => { type => 'string' },
1151         },
1152     },
1153     code => sub {
1154         my ($param) = @_;
1155
1156         return PVE::RRD::create_rrd_graph(
1157             "pve2-vm/$param->{vmid}", $param->{timeframe},
1158             $param->{ds}, $param->{cf});
1159
1160     }});
1161
1162 __PACKAGE__->register_method({
1163     name => 'rrddata',
1164     path => '{vmid}/rrddata',
1165     method => 'GET',
1166     protected => 1, # fixme: can we avoid that?
1167     permissions => {
1168         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1169     },
1170     description => "Read VM RRD statistics",
1171     parameters => {
1172         additionalProperties => 0,
1173         properties => {
1174             node => get_standard_option('pve-node'),
1175             vmid => get_standard_option('pve-vmid'),
1176             timeframe => {
1177                 description => "Specify the time frame you are interested in.",
1178                 type => 'string',
1179                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1180             },
1181             cf => {
1182                 description => "The RRD consolidation function",
1183                 type => 'string',
1184                 enum => [ 'AVERAGE', 'MAX' ],
1185                 optional => 1,
1186             },
1187         },
1188     },
1189     returns => {
1190         type => "array",
1191         items => {
1192             type => "object",
1193             properties => {},
1194         },
1195     },
1196     code => sub {
1197         my ($param) = @_;
1198
1199         return PVE::RRD::create_rrd_data(
1200             "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
1201     }});
1202
1203
1204 __PACKAGE__->register_method({
1205     name => 'vm_config',
1206     path => '{vmid}/config',
1207     method => 'GET',
1208     proxyto => 'node',
1209     description => "Get the virtual machine configuration with pending configuration " .
1210         "changes applied. Set the 'current' parameter to get the current configuration instead.",
1211     permissions => {
1212         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1213     },
1214     parameters => {
1215         additionalProperties => 0,
1216         properties => {
1217             node => get_standard_option('pve-node'),
1218             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1219             current => {
1220                 description => "Get current values (instead of pending values).",
1221                 optional => 1,
1222                 default => 0,
1223                 type => 'boolean',
1224             },
1225             snapshot => get_standard_option('pve-snapshot-name', {
1226                 description => "Fetch config values from given snapshot.",
1227                 optional => 1,
1228                 completion => sub {
1229                     my ($cmd, $pname, $cur, $args) = @_;
1230                     PVE::QemuConfig->snapshot_list($args->[0]);
1231                 },
1232             }),
1233         },
1234     },
1235     returns => {
1236         description => "The VM configuration.",
1237         type => "object",
1238         properties => PVE::QemuServer::json_config_properties({
1239             digest => {
1240                 type => 'string',
1241                 description => 'SHA1 digest of configuration file. This can be used to prevent concurrent modifications.',
1242             }
1243         }),
1244     },
1245     code => sub {
1246         my ($param) = @_;
1247
1248         raise_param_exc({ snapshot => "cannot use 'snapshot' parameter with 'current'",
1249                           current => "cannot use 'snapshot' parameter with 'current'"})
1250             if ($param->{snapshot} && $param->{current});
1251
1252         my $conf;
1253         if ($param->{snapshot}) {
1254             $conf = PVE::QemuConfig->load_snapshot_config($param->{vmid}, $param->{snapshot});
1255         } else {
1256             $conf = PVE::QemuConfig->load_current_config($param->{vmid}, $param->{current});
1257         }
1258         $conf->{cipassword} = '**********' if $conf->{cipassword};
1259         return $conf;
1260
1261     }});
1262
1263 __PACKAGE__->register_method({
1264     name => 'vm_pending',
1265     path => '{vmid}/pending',
1266     method => 'GET',
1267     proxyto => 'node',
1268     description => "Get the virtual machine configuration with both current and pending values.",
1269     permissions => {
1270         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1271     },
1272     parameters => {
1273         additionalProperties => 0,
1274         properties => {
1275             node => get_standard_option('pve-node'),
1276             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1277         },
1278     },
1279     returns => {
1280         type => "array",
1281         items => {
1282             type => "object",
1283             properties => {
1284                 key => {
1285                     description => "Configuration option name.",
1286                     type => 'string',
1287                 },
1288                 value => {
1289                     description => "Current value.",
1290                     type => 'string',
1291                     optional => 1,
1292                 },
1293                 pending => {
1294                     description => "Pending value.",
1295                     type => 'string',
1296                     optional => 1,
1297                 },
1298                 delete => {
1299                     description => "Indicates a pending delete request if present and not 0. " .
1300                                    "The value 2 indicates a force-delete request.",
1301                     type => 'integer',
1302                     minimum => 0,
1303                     maximum => 2,
1304                     optional => 1,
1305                 },
1306             },
1307         },
1308     },
1309     code => sub {
1310         my ($param) = @_;
1311
1312         my $conf = PVE::QemuConfig->load_config($param->{vmid});
1313
1314         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
1315
1316         $conf->{cipassword} = '**********' if defined($conf->{cipassword});
1317         $conf->{pending}->{cipassword} = '********** ' if defined($conf->{pending}->{cipassword});
1318
1319         return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
1320    }});
1321
1322 __PACKAGE__->register_method({
1323     name => 'cloudinit_pending',
1324     path => '{vmid}/cloudinit',
1325     method => 'GET',
1326     proxyto => 'node',
1327     description => "Get the cloudinit configuration with both current and pending values.",
1328     permissions => {
1329         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1330     },
1331     parameters => {
1332         additionalProperties => 0,
1333         properties => {
1334             node => get_standard_option('pve-node'),
1335             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1336         },
1337     },
1338     returns => {
1339         type => "array",
1340         items => {
1341             type => "object",
1342             properties => {
1343                 key => {
1344                     description => "Configuration option name.",
1345                     type => 'string',
1346                 },
1347                 old => {
1348                     description => "Value as it was used to generate the current cloudinit image.",
1349                     type => 'string',
1350                     optional => 1,
1351                 },
1352                 new => {
1353                     description => "The new pending value.",
1354                     type => 'string',
1355                     optional => 1,
1356                 },
1357             },
1358         },
1359     },
1360     code => sub {
1361         my ($param) = @_;
1362
1363         my $vmid = $param->{vmid};
1364         my $conf = PVE::QemuConfig->load_config($vmid);
1365
1366         my $ci = $conf->{cloudinit};
1367
1368         my $res = {};
1369         my $added = delete($ci->{added}) // '';
1370         for my $key (PVE::Tools::split_list($added)) {
1371             $res->{$key} = { new => $conf->{$key} };
1372         }
1373
1374         for my $key (keys %$ci) {
1375             if (!exists($conf->{$key})) {
1376                 $res->{$key} = { old => $ci->{$key} };
1377             } else {
1378                 $res->{$key} = {
1379                     old => $ci->{$key},
1380                     new => $conf->{$key},
1381                 };
1382             }
1383         }
1384
1385         if (defined(my $pw = $res->{cipassword})) {
1386             $pw->{old} = '**********' if exists $pw->{old};
1387             $pw->{new} = '**********' if exists $pw->{new};
1388         }
1389
1390         return $res;
1391    }});
1392
1393 __PACKAGE__->register_method({
1394     name => 'cloudinit_update',
1395     path => '{vmid}/cloudinit',
1396     method => 'PUT',
1397     protected => 1,
1398     proxyto => 'node',
1399     description => "Regenerate and change cloudinit config drive.",
1400     permissions => {
1401         check => ['perm', '/vms/{vmid}', 'VM.Config.Cloudinit'],
1402     },
1403     parameters => {
1404         additionalProperties => 0,
1405         properties => {
1406             node => get_standard_option('pve-node'),
1407             vmid => get_standard_option('pve-vmid'),
1408         },
1409     },
1410     returns => { type => 'null' },
1411     code => sub {
1412         my ($param) = @_;
1413
1414         my $rpcenv = PVE::RPCEnvironment::get();
1415         my $authuser = $rpcenv->get_user();
1416
1417         my $vmid = extract_param($param, 'vmid');
1418
1419         PVE::QemuConfig->lock_config($vmid, sub {
1420             my $conf = PVE::QemuConfig->load_config($vmid);
1421             PVE::QemuConfig->check_lock($conf);
1422
1423             my $storecfg = PVE::Storage::config();
1424             PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
1425         });
1426         return;
1427     }});
1428
1429 # POST/PUT {vmid}/config implementation
1430 #
1431 # The original API used PUT (idempotent) an we assumed that all operations
1432 # are fast. But it turned out that almost any configuration change can
1433 # involve hot-plug actions, or disk alloc/free. Such actions can take long
1434 # time to complete and have side effects (not idempotent).
1435 #
1436 # The new implementation uses POST and forks a worker process. We added
1437 # a new option 'background_delay'. If specified we wait up to
1438 # 'background_delay' second for the worker task to complete. It returns null
1439 # if the task is finished within that time, else we return the UPID.
1440
1441 my $update_vm_api  = sub {
1442     my ($param, $sync) = @_;
1443
1444     my $rpcenv = PVE::RPCEnvironment::get();
1445
1446     my $authuser = $rpcenv->get_user();
1447
1448     my $node = extract_param($param, 'node');
1449
1450     my $vmid = extract_param($param, 'vmid');
1451
1452     my $digest = extract_param($param, 'digest');
1453
1454     my $background_delay = extract_param($param, 'background_delay');
1455
1456     my $skip_cloud_init = extract_param($param, 'skip_cloud_init');
1457
1458     if (defined(my $cipassword = $param->{cipassword})) {
1459         # Same logic as in cloud-init (but with the regex fixed...)
1460         $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
1461             if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
1462     }
1463
1464     my @paramarr = (); # used for log message
1465     foreach my $key (sort keys %$param) {
1466         my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
1467         push @paramarr, "-$key", $value;
1468     }
1469
1470     my $skiplock = extract_param($param, 'skiplock');
1471     raise_param_exc({ skiplock => "Only root may use this option." })
1472         if $skiplock && $authuser ne 'root@pam';
1473
1474     my $delete_str = extract_param($param, 'delete');
1475
1476     my $revert_str = extract_param($param, 'revert');
1477
1478     my $force = extract_param($param, 'force');
1479
1480     if (defined(my $ssh_keys = $param->{sshkeys})) {
1481         $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
1482         PVE::Tools::validate_ssh_public_keys($ssh_keys);
1483     }
1484
1485     $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
1486         if defined($param->{cpuunits}); # clamp value depending on cgroup version
1487
1488     die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param);
1489
1490     my $storecfg = PVE::Storage::config();
1491
1492     my $defaults = PVE::QemuServer::load_defaults();
1493
1494     &$resolve_cdrom_alias($param);
1495
1496     # now try to verify all parameters
1497
1498     my $revert = {};
1499     foreach my $opt (PVE::Tools::split_list($revert_str)) {
1500         if (!PVE::QemuServer::option_exists($opt)) {
1501             raise_param_exc({ revert => "unknown option '$opt'" });
1502         }
1503
1504         raise_param_exc({ delete => "you can't use '-$opt' and " .
1505                               "-revert $opt' at the same time" })
1506             if defined($param->{$opt});
1507
1508         $revert->{$opt} = 1;
1509     }
1510
1511     my @delete = ();
1512     foreach my $opt (PVE::Tools::split_list($delete_str)) {
1513         $opt = 'ide2' if $opt eq 'cdrom';
1514
1515         raise_param_exc({ delete => "you can't use '-$opt' and " .
1516                               "-delete $opt' at the same time" })
1517             if defined($param->{$opt});
1518
1519         raise_param_exc({ revert => "you can't use '-delete $opt' and " .
1520                               "-revert $opt' at the same time" })
1521             if $revert->{$opt};
1522
1523         if (!PVE::QemuServer::option_exists($opt)) {
1524             raise_param_exc({ delete => "unknown option '$opt'" });
1525         }
1526
1527         push @delete, $opt;
1528     }
1529
1530     my $repl_conf = PVE::ReplicationConfig->new();
1531     my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
1532     my $check_replication = sub {
1533         my ($drive) = @_;
1534         return if !$is_replicated;
1535         my $volid = $drive->{file};
1536         return if !$volid || !($drive->{replicate}//1);
1537         return if PVE::QemuServer::drive_is_cdrom($drive);
1538
1539         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1540         die "cannot add non-managed/pass-through volume to a replicated VM\n"
1541             if !defined($storeid);
1542
1543         return if defined($volname) && $volname eq 'cloudinit';
1544
1545         my $format;
1546         if ($volid =~ $NEW_DISK_RE) {
1547             $storeid = $2;
1548             $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid);
1549         } else {
1550             $format = (PVE::Storage::parse_volname($storecfg, $volid))[6];
1551         }
1552         return if PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
1553         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1554         return if $scfg->{shared};
1555         die "cannot add non-replicatable volume to a replicated VM\n";
1556     };
1557
1558     $check_drive_param->($param, $storecfg, $check_replication);
1559
1560     foreach my $opt (keys %$param) {
1561         if ($opt =~ m/^net(\d+)$/) {
1562             # add macaddr
1563             my $net = PVE::QemuServer::parse_net($param->{$opt});
1564             $param->{$opt} = PVE::QemuServer::print_net($net);
1565         } elsif ($opt eq 'vmgenid') {
1566             if ($param->{$opt} eq '1') {
1567                 $param->{$opt} = PVE::QemuServer::generate_uuid();
1568             }
1569         } elsif ($opt eq 'hookscript') {
1570             eval { PVE::GuestHelpers::check_hookscript($param->{$opt}, $storecfg); };
1571             raise_param_exc({ $opt => $@ }) if $@;
1572         }
1573     }
1574
1575     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
1576
1577     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
1578
1579     &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
1580
1581     my $updatefn =  sub {
1582
1583         my $conf = PVE::QemuConfig->load_config($vmid);
1584
1585         die "checksum missmatch (file change by other user?)\n"
1586             if $digest && $digest ne $conf->{digest};
1587
1588         &$check_cpu_model_access($rpcenv, $authuser, $param, $conf);
1589
1590         # FIXME: 'suspended' lock should probabyl be a state or "weak" lock?!
1591         if (scalar(@delete) && grep { $_ eq 'vmstate'} @delete) {
1592             if (defined($conf->{lock}) && $conf->{lock} eq 'suspended') {
1593                 delete $conf->{lock}; # for check lock check, not written out
1594                 push @delete, 'lock'; # this is the real deal to write it out
1595             }
1596             push @delete, 'runningmachine' if $conf->{runningmachine};
1597             push @delete, 'runningcpu' if $conf->{runningcpu};
1598         }
1599
1600         PVE::QemuConfig->check_lock($conf) if !$skiplock;
1601
1602         foreach my $opt (keys %$revert) {
1603             if (defined($conf->{$opt})) {
1604                 $param->{$opt} = $conf->{$opt};
1605             } elsif (defined($conf->{pending}->{$opt})) {
1606                 push @delete, $opt;
1607             }
1608         }
1609
1610         if ($param->{memory} || defined($param->{balloon})) {
1611             my $maxmem = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory} || $defaults->{memory};
1612             my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon};
1613
1614             die "balloon value too large (must be smaller than assigned memory)\n"
1615                 if $balloon && $balloon > $maxmem;
1616         }
1617
1618         PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: " . join (' ', @paramarr));
1619
1620         my $worker = sub {
1621
1622             print "update VM $vmid: " . join (' ', @paramarr) . "\n";
1623
1624             # write updates to pending section
1625
1626             my $modified = {}; # record what $option we modify
1627
1628             my @bootorder;
1629             if (my $boot = $conf->{boot}) {
1630                 my $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot);
1631                 @bootorder = PVE::Tools::split_list($bootcfg->{order}) if $bootcfg && $bootcfg->{order};
1632             }
1633             my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete;
1634
1635             my $check_drive_perms = sub {
1636                 my ($opt, $val) = @_;
1637                 my $drive = PVE::QemuServer::parse_drive($opt, $val, 1);
1638                 if (PVE::QemuServer::drive_is_cloudinit($drive)) {
1639                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM']);
1640                 } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM
1641                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
1642                 } else {
1643                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1644
1645                 }
1646             };
1647
1648             foreach my $opt (@delete) {
1649                 $modified->{$opt} = 1;
1650                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1651
1652                 # value of what we want to delete, independent if pending or not
1653                 my $val = $conf->{$opt} // $conf->{pending}->{$opt};
1654                 if (!defined($val)) {
1655                     warn "cannot delete '$opt' - not set in current configuration!\n";
1656                     $modified->{$opt} = 0;
1657                     next;
1658                 }
1659                 my $is_pending_val = defined($conf->{pending}->{$opt});
1660                 delete $conf->{pending}->{$opt};
1661
1662                 # remove from bootorder if necessary
1663                 if (!$bootorder_deleted && @bootorder && grep {$_ eq $opt} @bootorder) {
1664                     @bootorder = grep {$_ ne $opt} @bootorder;
1665                     $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1666                     $modified->{boot} = 1;
1667                 }
1668
1669                 if ($opt =~ m/^unused/) {
1670                     my $drive = PVE::QemuServer::parse_drive($opt, $val);
1671                     PVE::QemuConfig->check_protection($conf, "can't remove unused disk '$drive->{file}'");
1672                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1673                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser)) {
1674                         delete $conf->{$opt};
1675                         PVE::QemuConfig->write_config($vmid, $conf);
1676                     }
1677                 } elsif ($opt eq 'vmstate') {
1678                     PVE::QemuConfig->check_protection($conf, "can't remove vmstate '$val'");
1679                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, { file => $val }, $rpcenv, $authuser, 1)) {
1680                         delete $conf->{$opt};
1681                         PVE::QemuConfig->write_config($vmid, $conf);
1682                     }
1683                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
1684                     PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'");
1685                     $check_drive_perms->($opt, $val);
1686                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val))
1687                         if $is_pending_val;
1688                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1689                     PVE::QemuConfig->write_config($vmid, $conf);
1690                 } elsif ($opt =~ m/^serial\d+$/) {
1691                     if ($val eq 'socket') {
1692                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1693                     } elsif ($authuser ne 'root@pam') {
1694                         die "only root can delete '$opt' config for real devices\n";
1695                     }
1696                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1697                     PVE::QemuConfig->write_config($vmid, $conf);
1698                 } elsif ($opt =~ m/^usb\d+$/) {
1699                     if ($val =~ m/spice/) {
1700                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1701                     } elsif ($authuser ne 'root@pam') {
1702                         die "only root can delete '$opt' config for real devices\n";
1703                     }
1704                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1705                     PVE::QemuConfig->write_config($vmid, $conf);
1706                 } elsif ($opt eq 'tags') {
1707                     assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser);
1708                     delete $conf->{$opt};
1709                     PVE::QemuConfig->write_config($vmid, $conf);
1710                 } else {
1711                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1712                     PVE::QemuConfig->write_config($vmid, $conf);
1713                 }
1714             }
1715
1716             foreach my $opt (keys %$param) { # add/change
1717                 $modified->{$opt} = 1;
1718                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1719                 next if defined($conf->{pending}->{$opt}) && ($param->{$opt} eq $conf->{pending}->{$opt}); # skip if nothing changed
1720
1721                 my $arch = PVE::QemuServer::get_vm_arch($conf);
1722
1723                 if (PVE::QemuServer::is_valid_drivename($opt)) {
1724                     # old drive
1725                     if ($conf->{$opt}) {
1726                         $check_drive_perms->($opt, $conf->{$opt});
1727                     }
1728
1729                     # new drive
1730                     $check_drive_perms->($opt, $param->{$opt});
1731                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
1732                         if defined($conf->{pending}->{$opt});
1733
1734                     my (undef, $created_opts) = $create_disks->(
1735                         $rpcenv,
1736                         $authuser,
1737                         $conf,
1738                         $arch,
1739                         $storecfg,
1740                         $vmid,
1741                         undef,
1742                         {$opt => $param->{$opt}},
1743                     );
1744                     $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*;
1745
1746                     # default legacy boot order implies all cdroms anyway
1747                     if (@bootorder) {
1748                         # append new CD drives to bootorder to mark them bootable
1749                         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
1750                         if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) {
1751                             push @bootorder, $opt;
1752                             $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1753                             $modified->{boot} = 1;
1754                         }
1755                     }
1756                 } elsif ($opt =~ m/^serial\d+/) {
1757                     if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
1758                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1759                     } elsif ($authuser ne 'root@pam') {
1760                         die "only root can modify '$opt' config for real devices\n";
1761                     }
1762                     $conf->{pending}->{$opt} = $param->{$opt};
1763                 } elsif ($opt =~ m/^usb\d+/) {
1764                     if ((!defined($conf->{$opt}) || $conf->{$opt} =~ m/spice/) && $param->{$opt} =~ m/spice/) {
1765                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1766                     } elsif ($authuser ne 'root@pam') {
1767                         die "only root can modify '$opt' config for real devices\n";
1768                     }
1769                     $conf->{pending}->{$opt} = $param->{$opt};
1770                 } elsif ($opt eq 'tags') {
1771                     assert_tag_permissions($vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser);
1772                     $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt});
1773                 } else {
1774                     $conf->{pending}->{$opt} = $param->{$opt};
1775
1776                     if ($opt eq 'boot') {
1777                         my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $param->{$opt});
1778                         if ($new_bootcfg->{order}) {
1779                             my @devs = PVE::Tools::split_list($new_bootcfg->{order});
1780                             for my $dev (@devs) {
1781                                 my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev};
1782                                 my $deleted = grep {$_ eq $dev} @delete;
1783                                 die "invalid bootorder: device '$dev' does not exist'\n"
1784                                     if !$exists || $deleted;
1785                             }
1786
1787                             # remove legacy boot order settings if new one set
1788                             $conf->{pending}->{$opt} = PVE::QemuServer::print_bootorder(\@devs);
1789                             PVE::QemuConfig->add_to_pending_delete($conf, "bootdisk")
1790                                 if $conf->{bootdisk};
1791                         }
1792                     }
1793                 }
1794                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
1795                 PVE::QemuConfig->write_config($vmid, $conf);
1796             }
1797
1798             # remove pending changes when nothing changed
1799             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1800             my $changes = PVE::QemuConfig->cleanup_pending($conf);
1801             PVE::QemuConfig->write_config($vmid, $conf) if $changes;
1802
1803             return if !scalar(keys %{$conf->{pending}});
1804
1805             my $running = PVE::QemuServer::check_running($vmid);
1806
1807             # apply pending changes
1808
1809             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1810
1811             my $errors = {};
1812             if ($running) {
1813                 PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors);
1814             } else {
1815                 # cloud_init must be skipped if we are in an incoming, remote live migration
1816                 PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors, $skip_cloud_init);
1817             }
1818             raise_param_exc($errors) if scalar(keys %$errors);
1819
1820             return;
1821         };
1822
1823         if ($sync) {
1824             &$worker();
1825             return;
1826         } else {
1827             my $upid = $rpcenv->fork_worker('qmconfig', $vmid, $authuser, $worker);
1828
1829             if ($background_delay) {
1830
1831                 # Note: It would be better to do that in the Event based HTTPServer
1832                 # to avoid blocking call to sleep.
1833
1834                 my $end_time = time() + $background_delay;
1835
1836                 my $task = PVE::Tools::upid_decode($upid);
1837
1838                 my $running = 1;
1839                 while (time() < $end_time) {
1840                     $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
1841                     last if !$running;
1842                     sleep(1); # this gets interrupted when child process ends
1843                 }
1844
1845                 if (!$running) {
1846                     my $status = PVE::Tools::upid_read_status($upid);
1847                     return if !PVE::Tools::upid_status_is_error($status);
1848                     die "failed to update VM $vmid: $status\n";
1849                 }
1850             }
1851
1852             return $upid;
1853         }
1854     };
1855
1856     return PVE::QemuConfig->lock_config($vmid, $updatefn);
1857 };
1858
1859 my $vm_config_perm_list = [
1860             'VM.Config.Disk',
1861             'VM.Config.CDROM',
1862             'VM.Config.CPU',
1863             'VM.Config.Memory',
1864             'VM.Config.Network',
1865             'VM.Config.HWType',
1866             'VM.Config.Options',
1867             'VM.Config.Cloudinit',
1868     ];
1869
1870 __PACKAGE__->register_method({
1871     name => 'update_vm_async',
1872     path => '{vmid}/config',
1873     method => 'POST',
1874     protected => 1,
1875     proxyto => 'node',
1876     description => "Set virtual machine options (asynchrounous API).",
1877     permissions => {
1878         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1879     },
1880     parameters => {
1881         additionalProperties => 0,
1882         properties => PVE::QemuServer::json_config_properties(
1883             {
1884                 node => get_standard_option('pve-node'),
1885                 vmid => get_standard_option('pve-vmid'),
1886                 skiplock => get_standard_option('skiplock'),
1887                 delete => {
1888                     type => 'string', format => 'pve-configid-list',
1889                     description => "A list of settings you want to delete.",
1890                     optional => 1,
1891                 },
1892                 revert => {
1893                     type => 'string', format => 'pve-configid-list',
1894                     description => "Revert a pending change.",
1895                     optional => 1,
1896                 },
1897                 force => {
1898                     type => 'boolean',
1899                     description => $opt_force_description,
1900                     optional => 1,
1901                     requires => 'delete',
1902                 },
1903                 digest => {
1904                     type => 'string',
1905                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1906                     maxLength => 40,
1907                     optional => 1,
1908                 },
1909                 background_delay => {
1910                     type => 'integer',
1911                     description => "Time to wait for the task to finish. We return 'null' if the task finish within that time.",
1912                     minimum => 1,
1913                     maximum => 30,
1914                     optional => 1,
1915                 },
1916             },
1917             1, # with_disk_alloc
1918         ),
1919     },
1920     returns => {
1921         type => 'string',
1922         optional => 1,
1923     },
1924     code => $update_vm_api,
1925 });
1926
1927 __PACKAGE__->register_method({
1928     name => 'update_vm',
1929     path => '{vmid}/config',
1930     method => 'PUT',
1931     protected => 1,
1932     proxyto => 'node',
1933     description => "Set virtual machine options (synchrounous API) - You should consider using the POST method instead for any actions involving hotplug or storage allocation.",
1934     permissions => {
1935         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1936     },
1937     parameters => {
1938         additionalProperties => 0,
1939         properties => PVE::QemuServer::json_config_properties(
1940             {
1941                 node => get_standard_option('pve-node'),
1942                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1943                 skiplock => get_standard_option('skiplock'),
1944                 delete => {
1945                     type => 'string', format => 'pve-configid-list',
1946                     description => "A list of settings you want to delete.",
1947                     optional => 1,
1948                 },
1949                 revert => {
1950                     type => 'string', format => 'pve-configid-list',
1951                     description => "Revert a pending change.",
1952                     optional => 1,
1953                 },
1954                 force => {
1955                     type => 'boolean',
1956                     description => $opt_force_description,
1957                     optional => 1,
1958                     requires => 'delete',
1959                 },
1960                 digest => {
1961                     type => 'string',
1962                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1963                     maxLength => 40,
1964                     optional => 1,
1965                 },
1966             },
1967             1, # with_disk_alloc
1968         ),
1969     },
1970     returns => { type => 'null' },
1971     code => sub {
1972         my ($param) = @_;
1973         &$update_vm_api($param, 1);
1974         return;
1975     }
1976 });
1977
1978 __PACKAGE__->register_method({
1979     name => 'destroy_vm',
1980     path => '{vmid}',
1981     method => 'DELETE',
1982     protected => 1,
1983     proxyto => 'node',
1984     description => "Destroy the VM and  all used/owned volumes. Removes any VM specific permissions"
1985         ." and firewall rules",
1986     permissions => {
1987         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
1988     },
1989     parameters => {
1990         additionalProperties => 0,
1991         properties => {
1992             node => get_standard_option('pve-node'),
1993             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
1994             skiplock => get_standard_option('skiplock'),
1995             purge => {
1996                 type => 'boolean',
1997                 description => "Remove VMID from configurations, like backup & replication jobs and HA.",
1998                 optional => 1,
1999             },
2000             'destroy-unreferenced-disks' => {
2001                 type => 'boolean',
2002                 description => "If set, destroy additionally all disks not referenced in the config"
2003                  ." but with a matching VMID from all enabled storages.",
2004                 optional => 1,
2005                 default => 0,
2006             },
2007         },
2008     },
2009     returns => {
2010         type => 'string',
2011     },
2012     code => sub {
2013         my ($param) = @_;
2014
2015         my $rpcenv = PVE::RPCEnvironment::get();
2016         my $authuser = $rpcenv->get_user();
2017         my $vmid = $param->{vmid};
2018
2019         my $skiplock = $param->{skiplock};
2020         raise_param_exc({ skiplock => "Only root may use this option." })
2021             if $skiplock && $authuser ne 'root@pam';
2022
2023         my $early_checks = sub {
2024             # test if VM exists
2025             my $conf = PVE::QemuConfig->load_config($vmid);
2026             PVE::QemuConfig->check_protection($conf, "can't remove VM $vmid");
2027
2028             my $ha_managed = PVE::HA::Config::service_is_configured("vm:$vmid");
2029
2030             if (!$param->{purge}) {
2031                 die "unable to remove VM $vmid - used in HA resources and purge parameter not set.\n"
2032                     if $ha_managed;
2033                 # don't allow destroy if with replication jobs but no purge param
2034                 my $repl_conf = PVE::ReplicationConfig->new();
2035                 $repl_conf->check_for_existing_jobs($vmid);
2036             }
2037
2038             die "VM $vmid is running - destroy failed\n"
2039                 if PVE::QemuServer::check_running($vmid);
2040
2041             return $ha_managed;
2042         };
2043
2044         $early_checks->();
2045
2046         my $realcmd = sub {
2047             my $upid = shift;
2048
2049             my $storecfg = PVE::Storage::config();
2050
2051             syslog('info', "destroy VM $vmid: $upid\n");
2052             PVE::QemuConfig->lock_config($vmid, sub {
2053                 # repeat, config might have changed
2054                 my $ha_managed = $early_checks->();
2055
2056                 my $purge_unreferenced = $param->{'destroy-unreferenced-disks'};
2057
2058                 PVE::QemuServer::destroy_vm(
2059                     $storecfg,
2060                     $vmid,
2061                     $skiplock, { lock => 'destroyed' },
2062                     $purge_unreferenced,
2063                 );
2064
2065                 PVE::AccessControl::remove_vm_access($vmid);
2066                 PVE::Firewall::remove_vmfw_conf($vmid);
2067                 if ($param->{purge}) {
2068                     print "purging VM $vmid from related configurations..\n";
2069                     PVE::ReplicationConfig::remove_vmid_jobs($vmid);
2070                     PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);
2071
2072                     if ($ha_managed) {
2073                         PVE::HA::Config::delete_service_from_config("vm:$vmid");
2074                         print "NOTE: removed VM $vmid from HA resource configuration.\n";
2075                     }
2076                 }
2077
2078                 # only now remove the zombie config, else we can have reuse race
2079                 PVE::QemuConfig->destroy_config($vmid);
2080             });
2081         };
2082
2083         return $rpcenv->fork_worker('qmdestroy', $vmid, $authuser, $realcmd);
2084     }});
2085
2086 __PACKAGE__->register_method({
2087     name => 'unlink',
2088     path => '{vmid}/unlink',
2089     method => 'PUT',
2090     protected => 1,
2091     proxyto => 'node',
2092     description => "Unlink/delete disk images.",
2093     permissions => {
2094         check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']],
2095     },
2096     parameters => {
2097         additionalProperties => 0,
2098         properties => {
2099             node => get_standard_option('pve-node'),
2100             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
2101             idlist => {
2102                 type => 'string', format => 'pve-configid-list',
2103                 description => "A list of disk IDs you want to delete.",
2104             },
2105             force => {
2106                 type => 'boolean',
2107                 description => $opt_force_description,
2108                 optional => 1,
2109             },
2110         },
2111     },
2112     returns => { type => 'null'},
2113     code => sub {
2114         my ($param) = @_;
2115
2116         $param->{delete} = extract_param($param, 'idlist');
2117
2118         __PACKAGE__->update_vm($param);
2119
2120         return;
2121     }});
2122
2123 # uses good entropy, each char is limited to 6 bit to get printable chars simply
2124 my $gen_rand_chars = sub {
2125     my ($length) = @_;
2126
2127     die "invalid length $length" if $length < 1;
2128
2129     my $min = ord('!'); # first printable ascii
2130
2131     my $rand_bytes = Crypt::OpenSSL::Random::random_bytes($length);
2132     die "failed to generate random bytes!\n"
2133       if !$rand_bytes;
2134
2135     my $str = join('', map { chr((ord($_) & 0x3F) + $min) } split('', $rand_bytes));
2136
2137     return $str;
2138 };
2139
2140 my $sslcert;
2141
2142 __PACKAGE__->register_method({
2143     name => 'vncproxy',
2144     path => '{vmid}/vncproxy',
2145     method => 'POST',
2146     protected => 1,
2147     permissions => {
2148         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2149     },
2150     description => "Creates a TCP VNC proxy connections.",
2151     parameters => {
2152         additionalProperties => 0,
2153         properties => {
2154             node => get_standard_option('pve-node'),
2155             vmid => get_standard_option('pve-vmid'),
2156             websocket => {
2157                 optional => 1,
2158                 type => 'boolean',
2159                 description => "starts websockify instead of vncproxy",
2160             },
2161             'generate-password' => {
2162                 optional => 1,
2163                 type => 'boolean',
2164                 default => 0,
2165                 description => "Generates a random password to be used as ticket instead of the API ticket.",
2166             },
2167         },
2168     },
2169     returns => {
2170         additionalProperties => 0,
2171         properties => {
2172             user => { type => 'string' },
2173             ticket => { type => 'string' },
2174             password => {
2175                 optional => 1,
2176                 description => "Returned if requested with 'generate-password' param."
2177                     ." Consists of printable ASCII characters ('!' .. '~').",
2178                 type => 'string',
2179             },
2180             cert => { type => 'string' },
2181             port => { type => 'integer' },
2182             upid => { type => 'string' },
2183         },
2184     },
2185     code => sub {
2186         my ($param) = @_;
2187
2188         my $rpcenv = PVE::RPCEnvironment::get();
2189
2190         my $authuser = $rpcenv->get_user();
2191
2192         my $vmid = $param->{vmid};
2193         my $node = $param->{node};
2194         my $websocket = $param->{websocket};
2195
2196         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2197
2198         my $serial;
2199         if ($conf->{vga}) {
2200             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2201             $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2202         }
2203
2204         my $authpath = "/vms/$vmid";
2205
2206         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2207         my $password = $ticket;
2208         if ($param->{'generate-password'}) {
2209             $password = $gen_rand_chars->(8);
2210         }
2211
2212         $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
2213             if !$sslcert;
2214
2215         my $family;
2216         my $remcmd = [];
2217
2218         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2219             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2220             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2221             # NOTE: kvm VNC traffic is already TLS encrypted or is known unsecure
2222             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, defined($serial) ? '-t' : '-T');
2223         } else {
2224             $family = PVE::Tools::get_host_address_family($node);
2225         }
2226
2227         my $port = PVE::Tools::next_vnc_port($family);
2228
2229         my $timeout = 10;
2230
2231         my $realcmd = sub {
2232             my $upid = shift;
2233
2234             syslog('info', "starting vnc proxy $upid\n");
2235
2236             my $cmd;
2237
2238             if (defined($serial)) {
2239
2240                 my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-iface', $serial, '-escape', '0' ];
2241
2242                 $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
2243                         '-timeout', $timeout, '-authpath', $authpath,
2244                         '-perm', 'Sys.Console'];
2245
2246                 if ($param->{websocket}) {
2247                     $ENV{PVE_VNC_TICKET} = $password; # pass ticket to vncterm
2248                     push @$cmd, '-notls', '-listen', 'localhost';
2249                 }
2250
2251                 push @$cmd, '-c', @$remcmd, @$termcmd;
2252
2253                 PVE::Tools::run_command($cmd);
2254
2255             } else {
2256
2257                 $ENV{LC_PVE_TICKET} = $password if $websocket; # set ticket with "qm vncproxy"
2258
2259                 $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid];
2260
2261                 my $sock = IO::Socket::IP->new(
2262                     ReuseAddr => 1,
2263                     Listen => 1,
2264                     LocalPort => $port,
2265                     Proto => 'tcp',
2266                     GetAddrInfoFlags => 0,
2267                 ) or die "failed to create socket: $!\n";
2268                 # Inside the worker we shouldn't have any previous alarms
2269                 # running anyway...:
2270                 alarm(0);
2271                 local $SIG{ALRM} = sub { die "connection timed out\n" };
2272                 alarm $timeout;
2273                 accept(my $cli, $sock) or die "connection failed: $!\n";
2274                 alarm(0);
2275                 close($sock);
2276                 if (PVE::Tools::run_command($cmd,
2277                     output => '>&'.fileno($cli),
2278                     input => '<&'.fileno($cli),
2279                     noerr => 1) != 0)
2280                 {
2281                     die "Failed to run vncproxy.\n";
2282                 }
2283             }
2284
2285             return;
2286         };
2287
2288         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2289
2290         PVE::Tools::wait_for_vnc_port($port);
2291
2292         my $res = {
2293             user => $authuser,
2294             ticket => $ticket,
2295             port => $port,
2296             upid => $upid,
2297             cert => $sslcert,
2298         };
2299         $res->{password} = $password if $param->{'generate-password'};
2300
2301         return $res;
2302     }});
2303
2304 __PACKAGE__->register_method({
2305     name => 'termproxy',
2306     path => '{vmid}/termproxy',
2307     method => 'POST',
2308     protected => 1,
2309     permissions => {
2310         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2311     },
2312     description => "Creates a TCP proxy connections.",
2313     parameters => {
2314         additionalProperties => 0,
2315         properties => {
2316             node => get_standard_option('pve-node'),
2317             vmid => get_standard_option('pve-vmid'),
2318             serial=> {
2319                 optional => 1,
2320                 type => 'string',
2321                 enum => [qw(serial0 serial1 serial2 serial3)],
2322                 description => "opens a serial terminal (defaults to display)",
2323             },
2324         },
2325     },
2326     returns => {
2327         additionalProperties => 0,
2328         properties => {
2329             user => { type => 'string' },
2330             ticket => { type => 'string' },
2331             port => { type => 'integer' },
2332             upid => { type => 'string' },
2333         },
2334     },
2335     code => sub {
2336         my ($param) = @_;
2337
2338         my $rpcenv = PVE::RPCEnvironment::get();
2339
2340         my $authuser = $rpcenv->get_user();
2341
2342         my $vmid = $param->{vmid};
2343         my $node = $param->{node};
2344         my $serial = $param->{serial};
2345
2346         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2347
2348         if (!defined($serial)) {
2349             if ($conf->{vga}) {
2350                 my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2351                 $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2352             }
2353         }
2354
2355         my $authpath = "/vms/$vmid";
2356
2357         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2358
2359         my $family;
2360         my $remcmd = [];
2361
2362         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2363             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2364             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2365             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, '-t');
2366             push @$remcmd, '--';
2367         } else {
2368             $family = PVE::Tools::get_host_address_family($node);
2369         }
2370
2371         my $port = PVE::Tools::next_vnc_port($family);
2372
2373         my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-escape', '0'];
2374         push @$termcmd, '-iface', $serial if $serial;
2375
2376         my $realcmd = sub {
2377             my $upid = shift;
2378
2379             syslog('info', "starting qemu termproxy $upid\n");
2380
2381             my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
2382                        '--perm', 'VM.Console', '--'];
2383             push @$cmd, @$remcmd, @$termcmd;
2384
2385             PVE::Tools::run_command($cmd);
2386         };
2387
2388         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2389
2390         PVE::Tools::wait_for_vnc_port($port);
2391
2392         return {
2393             user => $authuser,
2394             ticket => $ticket,
2395             port => $port,
2396             upid => $upid,
2397         };
2398     }});
2399
2400 __PACKAGE__->register_method({
2401     name => 'vncwebsocket',
2402     path => '{vmid}/vncwebsocket',
2403     method => 'GET',
2404     permissions => {
2405         description => "You also need to pass a valid ticket (vncticket).",
2406         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2407     },
2408     description => "Opens a weksocket for VNC traffic.",
2409     parameters => {
2410         additionalProperties => 0,
2411         properties => {
2412             node => get_standard_option('pve-node'),
2413             vmid => get_standard_option('pve-vmid'),
2414             vncticket => {
2415                 description => "Ticket from previous call to vncproxy.",
2416                 type => 'string',
2417                 maxLength => 512,
2418             },
2419             port => {
2420                 description => "Port number returned by previous vncproxy call.",
2421                 type => 'integer',
2422                 minimum => 5900,
2423                 maximum => 5999,
2424             },
2425         },
2426     },
2427     returns => {
2428         type => "object",
2429         properties => {
2430             port => { type => 'string' },
2431         },
2432     },
2433     code => sub {
2434         my ($param) = @_;
2435
2436         my $rpcenv = PVE::RPCEnvironment::get();
2437
2438         my $authuser = $rpcenv->get_user();
2439
2440         my $vmid = $param->{vmid};
2441         my $node = $param->{node};
2442
2443         my $authpath = "/vms/$vmid";
2444
2445         PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);
2446
2447         my $conf = PVE::QemuConfig->load_config($vmid, $node); # VM exists ?
2448
2449         # Note: VNC ports are acessible from outside, so we do not gain any
2450         # security if we verify that $param->{port} belongs to VM $vmid. This
2451         # check is done by verifying the VNC ticket (inside VNC protocol).
2452
2453         my $port = $param->{port};
2454
2455         return { port => $port };
2456     }});
2457
2458 __PACKAGE__->register_method({
2459     name => 'spiceproxy',
2460     path => '{vmid}/spiceproxy',
2461     method => 'POST',
2462     protected => 1,
2463     proxyto => 'node',
2464     permissions => {
2465         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2466     },
2467     description => "Returns a SPICE configuration to connect to the VM.",
2468     parameters => {
2469         additionalProperties => 0,
2470         properties => {
2471             node => get_standard_option('pve-node'),
2472             vmid => get_standard_option('pve-vmid'),
2473             proxy => get_standard_option('spice-proxy', { optional => 1 }),
2474         },
2475     },
2476     returns => get_standard_option('remote-viewer-config'),
2477     code => sub {
2478         my ($param) = @_;
2479
2480         my $rpcenv = PVE::RPCEnvironment::get();
2481
2482         my $authuser = $rpcenv->get_user();
2483
2484         my $vmid = $param->{vmid};
2485         my $node = $param->{node};
2486         my $proxy = $param->{proxy};
2487
2488         my $conf = PVE::QemuConfig->load_config($vmid, $node);
2489         my $title = "VM $vmid";
2490         $title .= " - ". $conf->{name} if $conf->{name};
2491
2492         my $port = PVE::QemuServer::spice_port($vmid);
2493
2494         my ($ticket, undef, $remote_viewer_config) =
2495             PVE::AccessControl::remote_viewer_config($authuser, $vmid, $node, $proxy, $title, $port);
2496
2497         mon_cmd($vmid, "set_password", protocol => 'spice', password => $ticket);
2498         mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
2499
2500         return $remote_viewer_config;
2501     }});
2502
2503 __PACKAGE__->register_method({
2504     name => 'vmcmdidx',
2505     path => '{vmid}/status',
2506     method => 'GET',
2507     proxyto => 'node',
2508     description => "Directory index",
2509     permissions => {
2510         user => 'all',
2511     },
2512     parameters => {
2513         additionalProperties => 0,
2514         properties => {
2515             node => get_standard_option('pve-node'),
2516             vmid => get_standard_option('pve-vmid'),
2517         },
2518     },
2519     returns => {
2520         type => 'array',
2521         items => {
2522             type => "object",
2523             properties => {
2524                 subdir => { type => 'string' },
2525             },
2526         },
2527         links => [ { rel => 'child', href => "{subdir}" } ],
2528     },
2529     code => sub {
2530         my ($param) = @_;
2531
2532         # test if VM exists
2533         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2534
2535         my $res = [
2536             { subdir => 'current' },
2537             { subdir => 'start' },
2538             { subdir => 'stop' },
2539             { subdir => 'reset' },
2540             { subdir => 'shutdown' },
2541             { subdir => 'suspend' },
2542             { subdir => 'reboot' },
2543             ];
2544
2545         return $res;
2546     }});
2547
2548 __PACKAGE__->register_method({
2549     name => 'vm_status',
2550     path => '{vmid}/status/current',
2551     method => 'GET',
2552     proxyto => 'node',
2553     protected => 1, # qemu pid files are only readable by root
2554     description => "Get virtual machine status.",
2555     permissions => {
2556         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
2557     },
2558     parameters => {
2559         additionalProperties => 0,
2560         properties => {
2561             node => get_standard_option('pve-node'),
2562             vmid => get_standard_option('pve-vmid'),
2563         },
2564     },
2565     returns => {
2566         type => 'object',
2567         properties => {
2568             %$PVE::QemuServer::vmstatus_return_properties,
2569             ha => {
2570                 description => "HA manager service status.",
2571                 type => 'object',
2572             },
2573             spice => {
2574                 description => "Qemu VGA configuration supports spice.",
2575                 type => 'boolean',
2576                 optional => 1,
2577             },
2578             agent => {
2579                 description => "Qemu GuestAgent enabled in config.",
2580                 type => 'boolean',
2581                 optional => 1,
2582             },
2583         },
2584     },
2585     code => sub {
2586         my ($param) = @_;
2587
2588         # test if VM exists
2589         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2590
2591         my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1);
2592         my $status = $vmstatus->{$param->{vmid}};
2593
2594         $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}");
2595
2596         if ($conf->{vga}) {
2597             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2598             my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/;
2599             $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga});
2600             $status->{spice} = 1 if $spice;
2601         }
2602         $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled');
2603
2604         return $status;
2605     }});
2606
2607 __PACKAGE__->register_method({
2608     name => 'vm_start',
2609     path => '{vmid}/status/start',
2610     method => 'POST',
2611     protected => 1,
2612     proxyto => 'node',
2613     description => "Start virtual machine.",
2614     permissions => {
2615         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2616     },
2617     parameters => {
2618         additionalProperties => 0,
2619         properties => {
2620             node => get_standard_option('pve-node'),
2621             vmid => get_standard_option('pve-vmid',
2622                                         { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2623             skiplock => get_standard_option('skiplock'),
2624             stateuri => get_standard_option('pve-qm-stateuri'),
2625             migratedfrom => get_standard_option('pve-node',{ optional => 1 }),
2626             migration_type => {
2627                 type => 'string',
2628                 enum => ['secure', 'insecure'],
2629                 description => "Migration traffic is encrypted using an SSH " .
2630                   "tunnel by default. On secure, completely private networks " .
2631                   "this can be disabled to increase performance.",
2632                 optional => 1,
2633             },
2634             migration_network => {
2635                 type => 'string', format => 'CIDR',
2636                 description => "CIDR of the (sub) network that is used for migration.",
2637                 optional => 1,
2638             },
2639             machine => get_standard_option('pve-qemu-machine'),
2640             'force-cpu' => {
2641                 description => "Override QEMU's -cpu argument with the given string.",
2642                 type => 'string',
2643                 optional => 1,
2644             },
2645             targetstorage => get_standard_option('pve-targetstorage'),
2646             timeout => {
2647                 description => "Wait maximal timeout seconds.",
2648                 type => 'integer',
2649                 minimum => 0,
2650                 default => 'max(30, vm memory in GiB)',
2651                 optional => 1,
2652             },
2653         },
2654     },
2655     returns => {
2656         type => 'string',
2657     },
2658     code => sub {
2659         my ($param) = @_;
2660
2661         my $rpcenv = PVE::RPCEnvironment::get();
2662         my $authuser = $rpcenv->get_user();
2663
2664         my $node = extract_param($param, 'node');
2665         my $vmid = extract_param($param, 'vmid');
2666         my $timeout = extract_param($param, 'timeout');
2667         my $machine = extract_param($param, 'machine');
2668
2669         my $get_root_param = sub {
2670             my $value = extract_param($param, $_[0]);
2671             raise_param_exc({ "$_[0]" => "Only root may use this option." })
2672                 if $value && $authuser ne 'root@pam';
2673             return $value;
2674         };
2675
2676         my $stateuri = $get_root_param->('stateuri');
2677         my $skiplock = $get_root_param->('skiplock');
2678         my $migratedfrom = $get_root_param->('migratedfrom');
2679         my $migration_type = $get_root_param->('migration_type');
2680         my $migration_network = $get_root_param->('migration_network');
2681         my $targetstorage = $get_root_param->('targetstorage');
2682         my $force_cpu = $get_root_param->('force-cpu');
2683
2684         my $storagemap;
2685
2686         if ($targetstorage) {
2687             raise_param_exc({ targetstorage => "targetstorage can only by used with migratedfrom." })
2688                 if !$migratedfrom;
2689             $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
2690             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
2691                 if $@;
2692         }
2693
2694         # read spice ticket from STDIN
2695         my $spice_ticket;
2696         my $nbd_protocol_version = 0;
2697         my $replicated_volumes = {};
2698         my $offline_volumes = {};
2699         if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) {
2700             while (defined(my $line = <STDIN>)) {
2701                 chomp $line;
2702                 if ($line =~ m/^spice_ticket: (.+)$/) {
2703                     $spice_ticket = $1;
2704                 } elsif ($line =~ m/^nbd_protocol_version: (\d+)$/) {
2705                     $nbd_protocol_version = $1;
2706                 } elsif ($line =~ m/^replicated_volume: (.*)$/) {
2707                     $replicated_volumes->{$1} = 1;
2708                 } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead
2709                     $offline_volumes->{tpmstate0} = $1;
2710                 } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) {
2711                     $offline_volumes->{$1} = $2;
2712                 } elsif (!$spice_ticket) {
2713                     # fallback for old source node
2714                     $spice_ticket = $line;
2715                 } else {
2716                     warn "unknown 'start' parameter on STDIN: '$line'\n";
2717                 }
2718             }
2719         }
2720
2721         PVE::Cluster::check_cfs_quorum();
2722
2723         my $storecfg = PVE::Storage::config();
2724
2725         if (PVE::HA::Config::vm_is_ha_managed($vmid) && !$stateuri &&  $rpcenv->{type} ne 'ha') {
2726             my $hacmd = sub {
2727                 my $upid = shift;
2728
2729                 print "Requesting HA start for VM $vmid\n";
2730
2731                 my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
2732                 PVE::Tools::run_command($cmd);
2733                 return;
2734             };
2735
2736             return $rpcenv->fork_worker('hastart', $vmid, $authuser, $hacmd);
2737
2738         } else {
2739
2740             my $realcmd = sub {
2741                 my $upid = shift;
2742
2743                 syslog('info', "start VM $vmid: $upid\n");
2744
2745                 my $migrate_opts = {
2746                     migratedfrom => $migratedfrom,
2747                     spice_ticket => $spice_ticket,
2748                     network => $migration_network,
2749                     type => $migration_type,
2750                     storagemap => $storagemap,
2751                     nbd_proto_version => $nbd_protocol_version,
2752                     replicated_volumes => $replicated_volumes,
2753                     offline_volumes => $offline_volumes,
2754                 };
2755
2756                 my $params = {
2757                     statefile => $stateuri,
2758                     skiplock => $skiplock,
2759                     forcemachine => $machine,
2760                     timeout => $timeout,
2761                     forcecpu => $force_cpu,
2762                 };
2763
2764                 PVE::QemuServer::vm_start($storecfg, $vmid, $params, $migrate_opts);
2765                 return;
2766             };
2767
2768             return $rpcenv->fork_worker('qmstart', $vmid, $authuser, $realcmd);
2769         }
2770     }});
2771
2772 __PACKAGE__->register_method({
2773     name => 'vm_stop',
2774     path => '{vmid}/status/stop',
2775     method => 'POST',
2776     protected => 1,
2777     proxyto => 'node',
2778     description => "Stop virtual machine. The qemu process will exit immediately. This" .
2779         "is akin to pulling the power plug of a running computer and may damage the VM data",
2780     permissions => {
2781         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2782     },
2783     parameters => {
2784         additionalProperties => 0,
2785         properties => {
2786             node => get_standard_option('pve-node'),
2787             vmid => get_standard_option('pve-vmid',
2788                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2789             skiplock => get_standard_option('skiplock'),
2790             migratedfrom => get_standard_option('pve-node', { optional => 1 }),
2791             timeout => {
2792                 description => "Wait maximal timeout seconds.",
2793                 type => 'integer',
2794                 minimum => 0,
2795                 optional => 1,
2796             },
2797             keepActive => {
2798                 description => "Do not deactivate storage volumes.",
2799                 type => 'boolean',
2800                 optional => 1,
2801                 default => 0,
2802             }
2803         },
2804     },
2805     returns => {
2806         type => 'string',
2807     },
2808     code => sub {
2809         my ($param) = @_;
2810
2811         my $rpcenv = PVE::RPCEnvironment::get();
2812         my $authuser = $rpcenv->get_user();
2813
2814         my $node = extract_param($param, 'node');
2815         my $vmid = extract_param($param, 'vmid');
2816
2817         my $skiplock = extract_param($param, 'skiplock');
2818         raise_param_exc({ skiplock => "Only root may use this option." })
2819             if $skiplock && $authuser ne 'root@pam';
2820
2821         my $keepActive = extract_param($param, 'keepActive');
2822         raise_param_exc({ keepActive => "Only root may use this option." })
2823             if $keepActive && $authuser ne 'root@pam';
2824
2825         my $migratedfrom = extract_param($param, 'migratedfrom');
2826         raise_param_exc({ migratedfrom => "Only root may use this option." })
2827             if $migratedfrom && $authuser ne 'root@pam';
2828
2829
2830         my $storecfg = PVE::Storage::config();
2831
2832         if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) {
2833
2834             my $hacmd = sub {
2835                 my $upid = shift;
2836
2837                 print "Requesting HA stop for VM $vmid\n";
2838
2839                 my $cmd = ['ha-manager', 'crm-command', 'stop',  "vm:$vmid", '0'];
2840                 PVE::Tools::run_command($cmd);
2841                 return;
2842             };
2843
2844             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
2845
2846         } else {
2847             my $realcmd = sub {
2848                 my $upid = shift;
2849
2850                 syslog('info', "stop VM $vmid: $upid\n");
2851
2852                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0,
2853                                          $param->{timeout}, 0, 1, $keepActive, $migratedfrom);
2854                 return;
2855             };
2856
2857             return $rpcenv->fork_worker('qmstop', $vmid, $authuser, $realcmd);
2858         }
2859     }});
2860
2861 __PACKAGE__->register_method({
2862     name => 'vm_reset',
2863     path => '{vmid}/status/reset',
2864     method => 'POST',
2865     protected => 1,
2866     proxyto => 'node',
2867     description => "Reset virtual machine.",
2868     permissions => {
2869         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2870     },
2871     parameters => {
2872         additionalProperties => 0,
2873         properties => {
2874             node => get_standard_option('pve-node'),
2875             vmid => get_standard_option('pve-vmid',
2876                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2877             skiplock => get_standard_option('skiplock'),
2878         },
2879     },
2880     returns => {
2881         type => 'string',
2882     },
2883     code => sub {
2884         my ($param) = @_;
2885
2886         my $rpcenv = PVE::RPCEnvironment::get();
2887
2888         my $authuser = $rpcenv->get_user();
2889
2890         my $node = extract_param($param, 'node');
2891
2892         my $vmid = extract_param($param, 'vmid');
2893
2894         my $skiplock = extract_param($param, 'skiplock');
2895         raise_param_exc({ skiplock => "Only root may use this option." })
2896             if $skiplock && $authuser ne 'root@pam';
2897
2898         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
2899
2900         my $realcmd = sub {
2901             my $upid = shift;
2902
2903             PVE::QemuServer::vm_reset($vmid, $skiplock);
2904
2905             return;
2906         };
2907
2908         return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd);
2909     }});
2910
2911 __PACKAGE__->register_method({
2912     name => 'vm_shutdown',
2913     path => '{vmid}/status/shutdown',
2914     method => 'POST',
2915     protected => 1,
2916     proxyto => 'node',
2917     description => "Shutdown virtual machine. This is similar to pressing the power button on a physical machine." .
2918         "This will send an ACPI event for the guest OS, which should then proceed to a clean shutdown.",
2919     permissions => {
2920         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2921     },
2922     parameters => {
2923         additionalProperties => 0,
2924         properties => {
2925             node => get_standard_option('pve-node'),
2926             vmid => get_standard_option('pve-vmid',
2927                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2928             skiplock => get_standard_option('skiplock'),
2929             timeout => {
2930                 description => "Wait maximal timeout seconds.",
2931                 type => 'integer',
2932                 minimum => 0,
2933                 optional => 1,
2934             },
2935             forceStop => {
2936                 description => "Make sure the VM stops.",
2937                 type => 'boolean',
2938                 optional => 1,
2939                 default => 0,
2940             },
2941             keepActive => {
2942                 description => "Do not deactivate storage volumes.",
2943                 type => 'boolean',
2944                 optional => 1,
2945                 default => 0,
2946             }
2947         },
2948     },
2949     returns => {
2950         type => 'string',
2951     },
2952     code => sub {
2953         my ($param) = @_;
2954
2955         my $rpcenv = PVE::RPCEnvironment::get();
2956         my $authuser = $rpcenv->get_user();
2957
2958         my $node = extract_param($param, 'node');
2959         my $vmid = extract_param($param, 'vmid');
2960
2961         my $skiplock = extract_param($param, 'skiplock');
2962         raise_param_exc({ skiplock => "Only root may use this option." })
2963             if $skiplock && $authuser ne 'root@pam';
2964
2965         my $keepActive = extract_param($param, 'keepActive');
2966         raise_param_exc({ keepActive => "Only root may use this option." })
2967             if $keepActive && $authuser ne 'root@pam';
2968
2969         my $storecfg = PVE::Storage::config();
2970
2971         my $shutdown = 1;
2972
2973         # if vm is paused, do not shutdown (but stop if forceStop = 1)
2974         # otherwise, we will infer a shutdown command, but run into the timeout,
2975         # then when the vm is resumed, it will instantly shutdown
2976         #
2977         # checking the qmp status here to get feedback to the gui/cli/api
2978         # and the status query should not take too long
2979         if (PVE::QemuServer::vm_is_paused($vmid)) {
2980             if ($param->{forceStop}) {
2981                 warn "VM is paused - stop instead of shutdown\n";
2982                 $shutdown = 0;
2983             } else {
2984                 die "VM is paused - cannot shutdown\n";
2985             }
2986         }
2987
2988         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
2989
2990             my $timeout = $param->{timeout} // 60;
2991             my $hacmd = sub {
2992                 my $upid = shift;
2993
2994                 print "Requesting HA stop for VM $vmid\n";
2995
2996                 my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", "$timeout"];
2997                 PVE::Tools::run_command($cmd);
2998                 return;
2999             };
3000
3001             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
3002
3003         } else {
3004
3005             my $realcmd = sub {
3006                 my $upid = shift;
3007
3008                 syslog('info', "shutdown VM $vmid: $upid\n");
3009
3010                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0, $param->{timeout},
3011                                          $shutdown, $param->{forceStop}, $keepActive);
3012                 return;
3013             };
3014
3015             return $rpcenv->fork_worker('qmshutdown', $vmid, $authuser, $realcmd);
3016         }
3017     }});
3018
3019 __PACKAGE__->register_method({
3020     name => 'vm_reboot',
3021     path => '{vmid}/status/reboot',
3022     method => 'POST',
3023     protected => 1,
3024     proxyto => 'node',
3025     description => "Reboot the VM by shutting it down, and starting it again. Applies pending changes.",
3026     permissions => {
3027         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3028     },
3029     parameters => {
3030         additionalProperties => 0,
3031         properties => {
3032             node => get_standard_option('pve-node'),
3033             vmid => get_standard_option('pve-vmid',
3034                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3035             timeout => {
3036                 description => "Wait maximal timeout seconds for the shutdown.",
3037                 type => 'integer',
3038                 minimum => 0,
3039                 optional => 1,
3040             },
3041         },
3042     },
3043     returns => {
3044         type => 'string',
3045     },
3046     code => sub {
3047         my ($param) = @_;
3048
3049         my $rpcenv = PVE::RPCEnvironment::get();
3050         my $authuser = $rpcenv->get_user();
3051
3052         my $node = extract_param($param, 'node');
3053         my $vmid = extract_param($param, 'vmid');
3054
3055         die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid);
3056
3057         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3058
3059         my $realcmd = sub {
3060             my $upid = shift;
3061
3062             syslog('info', "requesting reboot of VM $vmid: $upid\n");
3063             PVE::QemuServer::vm_reboot($vmid, $param->{timeout});
3064             return;
3065         };
3066
3067         return $rpcenv->fork_worker('qmreboot', $vmid, $authuser, $realcmd);
3068     }});
3069
3070 __PACKAGE__->register_method({
3071     name => 'vm_suspend',
3072     path => '{vmid}/status/suspend',
3073     method => 'POST',
3074     protected => 1,
3075     proxyto => 'node',
3076     description => "Suspend virtual machine.",
3077     permissions => {
3078         description => "You need 'VM.PowerMgmt' on /vms/{vmid}, and if you have set 'todisk',".
3079             " you need also 'VM.Config.Disk' on /vms/{vmid} and 'Datastore.AllocateSpace'".
3080             " on the storage for the vmstate.",
3081         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3082     },
3083     parameters => {
3084         additionalProperties => 0,
3085         properties => {
3086             node => get_standard_option('pve-node'),
3087             vmid => get_standard_option('pve-vmid',
3088                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3089             skiplock => get_standard_option('skiplock'),
3090             todisk => {
3091                 type => 'boolean',
3092                 default => 0,
3093                 optional => 1,
3094                 description => 'If set, suspends the VM to disk. Will be resumed on next VM start.',
3095             },
3096             statestorage => get_standard_option('pve-storage-id', {
3097                 description => "The storage for the VM state",
3098                 requires => 'todisk',
3099                 optional => 1,
3100                 completion => \&PVE::Storage::complete_storage_enabled,
3101             }),
3102         },
3103     },
3104     returns => {
3105         type => 'string',
3106     },
3107     code => sub {
3108         my ($param) = @_;
3109
3110         my $rpcenv = PVE::RPCEnvironment::get();
3111         my $authuser = $rpcenv->get_user();
3112
3113         my $node = extract_param($param, 'node');
3114         my $vmid = extract_param($param, 'vmid');
3115
3116         my $todisk = extract_param($param, 'todisk') // 0;
3117
3118         my $statestorage = extract_param($param, 'statestorage');
3119
3120         my $skiplock = extract_param($param, 'skiplock');
3121         raise_param_exc({ skiplock => "Only root may use this option." })
3122             if $skiplock && $authuser ne 'root@pam';
3123
3124         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3125
3126         die "Cannot suspend HA managed VM to disk\n"
3127             if $todisk && PVE::HA::Config::vm_is_ha_managed($vmid);
3128
3129         # early check for storage permission, for better user feedback
3130         if ($todisk) {
3131             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
3132             my $conf = PVE::QemuConfig->load_config($vmid);
3133
3134             # cannot save the state of a non-virtualized PCIe device, so resume cannot really work
3135             for my $key (keys %$conf) {
3136                 next if $key !~ /^hostpci\d+/;
3137                 die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the"
3138                     ." possibility to save/restore their internal state\n";
3139             }
3140
3141             if (!$statestorage) {
3142                 # get statestorage from config if none is given
3143                 my $storecfg = PVE::Storage::config();
3144                 $statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
3145             }
3146
3147             $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
3148         }
3149
3150         my $realcmd = sub {
3151             my $upid = shift;
3152
3153             syslog('info', "suspend VM $vmid: $upid\n");
3154
3155             PVE::QemuServer::vm_suspend($vmid, $skiplock, $todisk, $statestorage);
3156
3157             return;
3158         };
3159
3160         my $taskname = $todisk ? 'qmsuspend' : 'qmpause';
3161         return $rpcenv->fork_worker($taskname, $vmid, $authuser, $realcmd);
3162     }});
3163
3164 __PACKAGE__->register_method({
3165     name => 'vm_resume',
3166     path => '{vmid}/status/resume',
3167     method => 'POST',
3168     protected => 1,
3169     proxyto => 'node',
3170     description => "Resume virtual machine.",
3171     permissions => {
3172         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3173     },
3174     parameters => {
3175         additionalProperties => 0,
3176         properties => {
3177             node => get_standard_option('pve-node'),
3178             vmid => get_standard_option('pve-vmid',
3179                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3180             skiplock => get_standard_option('skiplock'),
3181             nocheck => { type => 'boolean', optional => 1 },
3182
3183         },
3184     },
3185     returns => {
3186         type => 'string',
3187     },
3188     code => sub {
3189         my ($param) = @_;
3190
3191         my $rpcenv = PVE::RPCEnvironment::get();
3192
3193         my $authuser = $rpcenv->get_user();
3194
3195         my $node = extract_param($param, 'node');
3196
3197         my $vmid = extract_param($param, 'vmid');
3198
3199         my $skiplock = extract_param($param, 'skiplock');
3200         raise_param_exc({ skiplock => "Only root may use this option." })
3201             if $skiplock && $authuser ne 'root@pam';
3202
3203         # nocheck is used as part of migration when config file might be still
3204         # be on source node
3205         my $nocheck = extract_param($param, 'nocheck');
3206         raise_param_exc({ nocheck => "Only root may use this option." })
3207             if $nocheck && $authuser ne 'root@pam';
3208
3209         my $to_disk_suspended;
3210         eval {
3211             PVE::QemuConfig->lock_config($vmid, sub {
3212                 my $conf = PVE::QemuConfig->load_config($vmid);
3213                 $to_disk_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
3214             });
3215         };
3216
3217         die "VM $vmid not running\n"
3218             if !$to_disk_suspended && !PVE::QemuServer::check_running($vmid, $nocheck);
3219
3220         my $realcmd = sub {
3221             my $upid = shift;
3222
3223             syslog('info', "resume VM $vmid: $upid\n");
3224
3225             if (!$to_disk_suspended) {
3226                 PVE::QemuServer::vm_resume($vmid, $skiplock, $nocheck);
3227             } else {
3228                 my $storecfg = PVE::Storage::config();
3229                 PVE::QemuServer::vm_start($storecfg, $vmid, { skiplock => $skiplock });
3230             }
3231
3232             return;
3233         };
3234
3235         return $rpcenv->fork_worker('qmresume', $vmid, $authuser, $realcmd);
3236     }});
3237
3238 __PACKAGE__->register_method({
3239     name => 'vm_sendkey',
3240     path => '{vmid}/sendkey',
3241     method => 'PUT',
3242     protected => 1,
3243     proxyto => 'node',
3244     description => "Send key event to virtual machine.",
3245     permissions => {
3246         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
3247     },
3248     parameters => {
3249         additionalProperties => 0,
3250         properties => {
3251             node => get_standard_option('pve-node'),
3252             vmid => get_standard_option('pve-vmid',
3253                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3254             skiplock => get_standard_option('skiplock'),
3255             key => {
3256                 description => "The key (qemu monitor encoding).",
3257                 type => 'string'
3258             }
3259         },
3260     },
3261     returns => { type => 'null'},
3262     code => sub {
3263         my ($param) = @_;
3264
3265         my $rpcenv = PVE::RPCEnvironment::get();
3266
3267         my $authuser = $rpcenv->get_user();
3268
3269         my $node = extract_param($param, 'node');
3270
3271         my $vmid = extract_param($param, 'vmid');
3272
3273         my $skiplock = extract_param($param, 'skiplock');
3274         raise_param_exc({ skiplock => "Only root may use this option." })
3275             if $skiplock && $authuser ne 'root@pam';
3276
3277         PVE::QemuServer::vm_sendkey($vmid, $skiplock, $param->{key});
3278
3279         return;
3280     }});
3281
3282 __PACKAGE__->register_method({
3283     name => 'vm_feature',
3284     path => '{vmid}/feature',
3285     method => 'GET',
3286     proxyto => 'node',
3287     protected => 1,
3288     description => "Check if feature for virtual machine is available.",
3289     permissions => {
3290         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
3291     },
3292     parameters => {
3293         additionalProperties => 0,
3294         properties => {
3295             node => get_standard_option('pve-node'),
3296             vmid => get_standard_option('pve-vmid'),
3297             feature => {
3298                 description => "Feature to check.",
3299                 type => 'string',
3300                 enum => [ 'snapshot', 'clone', 'copy' ],
3301             },
3302             snapname => get_standard_option('pve-snapshot-name', {
3303                 optional => 1,
3304             }),
3305         },
3306     },
3307     returns => {
3308         type => "object",
3309         properties => {
3310             hasFeature => { type => 'boolean' },
3311             nodes => {
3312                 type => 'array',
3313                 items => { type => 'string' },
3314             }
3315         },
3316     },
3317     code => sub {
3318         my ($param) = @_;
3319
3320         my $node = extract_param($param, 'node');
3321
3322         my $vmid = extract_param($param, 'vmid');
3323
3324         my $snapname = extract_param($param, 'snapname');
3325
3326         my $feature = extract_param($param, 'feature');
3327
3328         my $running = PVE::QemuServer::check_running($vmid);
3329
3330         my $conf = PVE::QemuConfig->load_config($vmid);
3331
3332         if($snapname){
3333             my $snap = $conf->{snapshots}->{$snapname};
3334             die "snapshot '$snapname' does not exist\n" if !defined($snap);
3335             $conf = $snap;
3336         }
3337         my $storecfg = PVE::Storage::config();
3338
3339         my $nodelist = PVE::QemuServer::shared_nodes($conf, $storecfg);
3340         my $hasFeature = PVE::QemuConfig->has_feature($feature, $conf, $storecfg, $snapname, $running);
3341
3342         return {
3343             hasFeature => $hasFeature,
3344             nodes => [ keys %$nodelist ],
3345         };
3346     }});
3347
3348 __PACKAGE__->register_method({
3349     name => 'clone_vm',
3350     path => '{vmid}/clone',
3351     method => 'POST',
3352     protected => 1,
3353     proxyto => 'node',
3354     description => "Create a copy of virtual machine/template.",
3355     permissions => {
3356         description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " .
3357             "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
3358             "'Datastore.AllocateSpace' on any used storage.",
3359         check =>
3360         [ 'and',
3361           ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
3362           [ 'or',
3363             [ 'perm', '/vms/{newid}', ['VM.Allocate']],
3364             [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
3365           ],
3366         ]
3367     },
3368     parameters => {
3369         additionalProperties => 0,
3370         properties => {
3371             node => get_standard_option('pve-node'),
3372             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3373             newid => get_standard_option('pve-vmid', {
3374                 completion => \&PVE::Cluster::complete_next_vmid,
3375                 description => 'VMID for the clone.' }),
3376             name => {
3377                 optional => 1,
3378                 type => 'string', format => 'dns-name',
3379                 description => "Set a name for the new VM.",
3380             },
3381             description => {
3382                 optional => 1,
3383                 type => 'string',
3384                 description => "Description for the new VM.",
3385             },
3386             pool => {
3387                 optional => 1,
3388                 type => 'string', format => 'pve-poolid',
3389                 description => "Add the new VM to the specified pool.",
3390             },
3391             snapname => get_standard_option('pve-snapshot-name', {
3392                 optional => 1,
3393             }),
3394             storage => get_standard_option('pve-storage-id', {
3395                 description => "Target storage for full clone.",
3396                 optional => 1,
3397             }),
3398             'format' => {
3399                 description => "Target format for file storage. Only valid for full clone.",
3400                 type => 'string',
3401                 optional => 1,
3402                 enum => [ 'raw', 'qcow2', 'vmdk'],
3403             },
3404             full => {
3405                 optional => 1,
3406                 type => 'boolean',
3407                 description => "Create a full copy of all disks. This is always done when " .
3408                     "you clone a normal VM. For VM templates, we try to create a linked clone by default.",
3409             },
3410             target => get_standard_option('pve-node', {
3411                 description => "Target node. Only allowed if the original VM is on shared storage.",
3412                 optional => 1,
3413             }),
3414             bwlimit => {
3415                 description => "Override I/O bandwidth limit (in KiB/s).",
3416                 optional => 1,
3417                 type => 'integer',
3418                 minimum => '0',
3419                 default => 'clone limit from datacenter or storage config',
3420             },
3421         },
3422     },
3423     returns => {
3424         type => 'string',
3425     },
3426     code => sub {
3427         my ($param) = @_;
3428
3429         my $rpcenv = PVE::RPCEnvironment::get();
3430         my $authuser = $rpcenv->get_user();
3431
3432         my $node = extract_param($param, 'node');
3433         my $vmid = extract_param($param, 'vmid');
3434         my $newid = extract_param($param, 'newid');
3435         my $pool = extract_param($param, 'pool');
3436
3437         my $snapname = extract_param($param, 'snapname');
3438         my $storage = extract_param($param, 'storage');
3439         my $format = extract_param($param, 'format');
3440         my $target = extract_param($param, 'target');
3441
3442         my $localnode = PVE::INotify::nodename();
3443
3444         if ($target && ($target eq $localnode || $target eq 'localhost')) {
3445             undef $target;
3446         }
3447
3448         my $running = PVE::QemuServer::check_running($vmid) || 0;
3449
3450         my $load_and_check = sub {
3451             $rpcenv->check_pool_exist($pool) if defined($pool);
3452             PVE::Cluster::check_node_exists($target) if $target;
3453
3454             my $storecfg = PVE::Storage::config();
3455
3456             if ($storage) {
3457                 # check if storage is enabled on local node
3458                 PVE::Storage::storage_check_enabled($storecfg, $storage);
3459                 if ($target) {
3460                     # check if storage is available on target node
3461                     PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
3462                     # clone only works if target storage is shared
3463                     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
3464                     die "can't clone to non-shared storage '$storage'\n"
3465                         if !$scfg->{shared};
3466                 }
3467             }
3468
3469             PVE::Cluster::check_cfs_quorum();
3470
3471             my $conf = PVE::QemuConfig->load_config($vmid);
3472             PVE::QemuConfig->check_lock($conf);
3473
3474             my $verify_running = PVE::QemuServer::check_running($vmid) || 0;
3475             die "unexpected state change\n" if $verify_running != $running;
3476
3477             die "snapshot '$snapname' does not exist\n"
3478                 if $snapname && !defined( $conf->{snapshots}->{$snapname});
3479
3480             my $full = $param->{full} // !PVE::QemuConfig->is_template($conf);
3481
3482             die "parameter 'storage' not allowed for linked clones\n"
3483                 if defined($storage) && !$full;
3484
3485             die "parameter 'format' not allowed for linked clones\n"
3486                 if defined($format) && !$full;
3487
3488             my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf;
3489
3490             my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage);
3491
3492             die "can't clone VM to node '$target' (VM uses local storage)\n"
3493                 if $target && !$sharedvm;
3494
3495             my $conffile = PVE::QemuConfig->config_file($newid);
3496             die "unable to create VM $newid: config file already exists\n"
3497                 if -f $conffile;
3498
3499             my $newconf = { lock => 'clone' };
3500             my $drives = {};
3501             my $fullclone = {};
3502             my $vollist = [];
3503
3504             foreach my $opt (keys %$oldconf) {
3505                 my $value = $oldconf->{$opt};
3506
3507                 # do not copy snapshot related info
3508                 next if $opt eq 'snapshots' ||  $opt eq 'parent' || $opt eq 'snaptime' ||
3509                     $opt eq 'vmstate' || $opt eq 'snapstate';
3510
3511                 # no need to copy unused images, because VMID(owner) changes anyways
3512                 next if $opt =~ m/^unused\d+$/;
3513
3514                 die "cannot clone TPM state while VM is running\n"
3515                     if $full && $running && !$snapname && $opt eq 'tpmstate0';
3516
3517                 # always change MAC! address
3518                 if ($opt =~ m/^net(\d+)$/) {
3519                     my $net = PVE::QemuServer::parse_net($value);
3520                     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
3521                     $net->{macaddr} =  PVE::Tools::random_ether_addr($dc->{mac_prefix});
3522                     $newconf->{$opt} = PVE::QemuServer::print_net($net);
3523                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
3524                     my $drive = PVE::QemuServer::parse_drive($opt, $value);
3525                     die "unable to parse drive options for '$opt'\n" if !$drive;
3526                     if (PVE::QemuServer::drive_is_cdrom($drive, 1)) {
3527                         $newconf->{$opt} = $value; # simply copy configuration
3528                     } else {
3529                         if ($full || PVE::QemuServer::drive_is_cloudinit($drive)) {
3530                             die "Full clone feature is not supported for drive '$opt'\n"
3531                                 if !PVE::Storage::volume_has_feature($storecfg, 'copy', $drive->{file}, $snapname, $running);
3532                             $fullclone->{$opt} = 1;
3533                         } else {
3534                             # not full means clone instead of copy
3535                             die "Linked clone feature is not supported for drive '$opt'\n"
3536                                 if !PVE::Storage::volume_has_feature($storecfg, 'clone', $drive->{file}, $snapname, $running);
3537                         }
3538                         $drives->{$opt} = $drive;
3539                         next if PVE::QemuServer::drive_is_cloudinit($drive);
3540                         push @$vollist, $drive->{file};
3541                     }
3542                 } else {
3543                     # copy everything else
3544                     $newconf->{$opt} = $value;
3545                 }
3546             }
3547
3548             return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone);
3549         };
3550
3551         my $clonefn = sub {
3552             my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->();
3553             my $storecfg = PVE::Storage::config();
3554
3555             # auto generate a new uuid
3556             my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || '');
3557             $smbios1->{uuid} = PVE::QemuServer::generate_uuid();
3558             $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1);
3559             # auto generate a new vmgenid only if the option was set for template
3560             if ($newconf->{vmgenid}) {
3561                 $newconf->{vmgenid} = PVE::QemuServer::generate_uuid();
3562             }
3563
3564             delete $newconf->{template};
3565
3566             if ($param->{name}) {
3567                 $newconf->{name} = $param->{name};
3568             } else {
3569                 $newconf->{name} = "Copy-of-VM-" . ($oldconf->{name} // $vmid);
3570             }
3571
3572             if ($param->{description}) {
3573                 $newconf->{description} = $param->{description};
3574             }
3575
3576             # create empty/temp config - this fails if VM already exists on other node
3577             # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code
3578             PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n");
3579
3580             PVE::Firewall::clone_vmfw_conf($vmid, $newid);
3581
3582             my $newvollist = [];
3583             my $jobs = {};
3584
3585             eval {
3586                 local $SIG{INT} =
3587                     local $SIG{TERM} =
3588                     local $SIG{QUIT} =
3589                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3590
3591                 PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
3592
3593                 my $bwlimit = extract_param($param, 'bwlimit');
3594
3595                 my $total_jobs = scalar(keys %{$drives});
3596                 my $i = 1;
3597
3598                 foreach my $opt (sort keys %$drives) {
3599                     my $drive = $drives->{$opt};
3600                     my $skipcomplete = ($total_jobs != $i); # finish after last drive
3601                     my $completion = $skipcomplete ? 'skip' : 'complete';
3602
3603                     my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
3604                     my $storage_list = [ $src_sid ];
3605                     push @$storage_list, $storage if defined($storage);
3606                     my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
3607
3608                     my $source_info = {
3609                         vmid => $vmid,
3610                         running => $running,
3611                         drivename => $opt,
3612                         drive => $drive,
3613                         snapname => $snapname,
3614                     };
3615
3616                     my $dest_info = {
3617                         vmid => $newid,
3618                         drivename => $opt,
3619                         storage => $storage,
3620                         format => $format,
3621                     };
3622
3623                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf)
3624                         if $opt eq 'efidisk0';
3625
3626                     my $newdrive = PVE::QemuServer::clone_disk(
3627                         $storecfg,
3628                         $source_info,
3629                         $dest_info,
3630                         $fullclone->{$opt},
3631                         $newvollist,
3632                         $jobs,
3633                         $completion,
3634                         $oldconf->{agent},
3635                         $clonelimit,
3636                     );
3637
3638                     $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
3639
3640                     PVE::QemuConfig->write_config($newid, $newconf);
3641                     $i++;
3642                 }
3643
3644                 delete $newconf->{lock};
3645
3646                 # do not write pending changes
3647                 if (my @changes = keys %{$newconf->{pending}}) {
3648                     my $pending = join(',', @changes);
3649                     warn "found pending changes for '$pending', discarding for clone\n";
3650                     delete $newconf->{pending};
3651                 }
3652
3653                 PVE::QemuConfig->write_config($newid, $newconf);
3654
3655                 if ($target) {
3656                     # always deactivate volumes - avoid lvm LVs to be active on several nodes
3657                     PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
3658                     PVE::Storage::deactivate_volumes($storecfg, $newvollist);
3659
3660                     my $newconffile = PVE::QemuConfig->config_file($newid, $target);
3661                     die "Failed to move config to node '$target' - rename failed: $!\n"
3662                         if !rename($conffile, $newconffile);
3663                 }
3664
3665                 PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
3666             };
3667             if (my $err = $@) {
3668                 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
3669                 sleep 1; # some storage like rbd need to wait before release volume - really?
3670
3671                 foreach my $volid (@$newvollist) {
3672                     eval { PVE::Storage::vdisk_free($storecfg, $volid); };
3673                     warn $@ if $@;
3674                 }
3675
3676                 PVE::Firewall::remove_vmfw_conf($newid);
3677
3678                 unlink $conffile; # avoid races -> last thing before die
3679
3680                 die "clone failed: $err";
3681             }
3682
3683             return;
3684         };
3685
3686         # Aquire exclusive lock lock for $newid
3687         my $lock_target_vm = sub {
3688             return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn);
3689         };
3690
3691         my $lock_source_vm = sub {
3692             # exclusive lock if VM is running - else shared lock is enough;
3693             if ($running) {
3694                 return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
3695             } else {
3696                 return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
3697             }
3698         };
3699
3700         $load_and_check->(); # early checks before forking/locking
3701
3702         return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm);
3703     }});
3704
3705 __PACKAGE__->register_method({
3706     name => 'move_vm_disk',
3707     path => '{vmid}/move_disk',
3708     method => 'POST',
3709     protected => 1,
3710     proxyto => 'node',
3711     description => "Move volume to different storage or to a different VM.",
3712     permissions => {
3713         description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
3714             "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
3715             "a disk to another VM, you need the permissions on the target VM as well.",
3716         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
3717     },
3718     parameters => {
3719         additionalProperties => 0,
3720         properties => {
3721             node => get_standard_option('pve-node'),
3722             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3723             'target-vmid' => get_standard_option('pve-vmid', {
3724                 completion => \&PVE::QemuServer::complete_vmid,
3725                 optional => 1,
3726             }),
3727             disk => {
3728                 type => 'string',
3729                 description => "The disk you want to move.",
3730                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3731             },
3732             storage => get_standard_option('pve-storage-id', {
3733                 description => "Target storage.",
3734                 completion => \&PVE::QemuServer::complete_storage,
3735                 optional => 1,
3736             }),
3737             'format' => {
3738                 type => 'string',
3739                 description => "Target Format.",
3740                 enum => [ 'raw', 'qcow2', 'vmdk' ],
3741                 optional => 1,
3742             },
3743             delete => {
3744                 type => 'boolean',
3745                 description => "Delete the original disk after successful copy. By default the"
3746                     ." original disk is kept as unused disk.",
3747                 optional => 1,
3748                 default => 0,
3749             },
3750             digest => {
3751                 type => 'string',
3752                 description => 'Prevent changes if current configuration file has different SHA1"
3753                     ." digest. This can be used to prevent concurrent modifications.',
3754                 maxLength => 40,
3755                 optional => 1,
3756             },
3757             bwlimit => {
3758                 description => "Override I/O bandwidth limit (in KiB/s).",
3759                 optional => 1,
3760                 type => 'integer',
3761                 minimum => '0',
3762                 default => 'move limit from datacenter or storage config',
3763             },
3764             'target-disk' => {
3765                 type => 'string',
3766                 description => "The config key the disk will be moved to on the target VM"
3767                     ." (for example, ide0 or scsi1). Default is the source disk key.",
3768                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3769                 optional => 1,
3770             },
3771             'target-digest' => {
3772                 type => 'string',
3773                 description => 'Prevent changes if the current config file of the target VM has a"
3774                     ." different SHA1 digest. This can be used to detect concurrent modifications.',
3775                 maxLength => 40,
3776                 optional => 1,
3777             },
3778         },
3779     },
3780     returns => {
3781         type => 'string',
3782         description => "the task ID.",
3783     },
3784     code => sub {
3785         my ($param) = @_;
3786
3787         my $rpcenv = PVE::RPCEnvironment::get();
3788         my $authuser = $rpcenv->get_user();
3789
3790         my $node = extract_param($param, 'node');
3791         my $vmid = extract_param($param, 'vmid');
3792         my $target_vmid = extract_param($param, 'target-vmid');
3793         my $digest = extract_param($param, 'digest');
3794         my $target_digest = extract_param($param, 'target-digest');
3795         my $disk = extract_param($param, 'disk');
3796         my $target_disk = extract_param($param, 'target-disk') // $disk;
3797         my $storeid = extract_param($param, 'storage');
3798         my $format = extract_param($param, 'format');
3799
3800         my $storecfg = PVE::Storage::config();
3801
3802         my $load_and_check_move = sub {
3803             my $conf = PVE::QemuConfig->load_config($vmid);
3804             PVE::QemuConfig->check_lock($conf);
3805
3806             PVE::Tools::assert_if_modified($digest, $conf->{digest});
3807
3808             die "disk '$disk' does not exist\n" if !$conf->{$disk};
3809
3810             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
3811
3812             die "disk '$disk' has no associated volume\n" if !$drive->{file};
3813             die "you can't move a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive, 1);
3814
3815             my $old_volid = $drive->{file};
3816             my $oldfmt;
3817             my ($oldstoreid, $oldvolname) = PVE::Storage::parse_volume_id($old_volid);
3818             if ($oldvolname =~ m/\.(raw|qcow2|vmdk)$/){
3819                 $oldfmt = $1;
3820             }
3821
3822             die "you can't move to the same storage with same format\n"
3823                 if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format);
3824
3825             # this only checks snapshots because $disk is passed!
3826             my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use(
3827                 $storecfg,
3828                 $conf,
3829                 $disk,
3830                 $old_volid
3831             );
3832             die "you can't move a disk with snapshots and delete the source\n"
3833                 if $snapshotted && $param->{delete};
3834
3835             return ($conf, $drive, $oldstoreid, $snapshotted);
3836         };
3837
3838         my $move_updatefn = sub {
3839             my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->();
3840             my $old_volid = $drive->{file};
3841
3842             PVE::Cluster::log_msg(
3843                 'info',
3844                 $authuser,
3845                 "move disk VM $vmid: move --disk $disk --storage $storeid"
3846             );
3847
3848             my $running = PVE::QemuServer::check_running($vmid);
3849
3850             PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]);
3851
3852             my $newvollist = [];
3853
3854             eval {
3855                 local $SIG{INT} =
3856                     local $SIG{TERM} =
3857                     local $SIG{QUIT} =
3858                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3859
3860                 warn "moving disk with snapshots, snapshots will not be moved!\n"
3861                     if $snapshotted;
3862
3863                 my $bwlimit = extract_param($param, 'bwlimit');
3864                 my $movelimit = PVE::Storage::get_bandwidth_limit(
3865                     'move',
3866                     [$oldstoreid, $storeid],
3867                     $bwlimit
3868                 );
3869
3870                 my $source_info = {
3871                     vmid => $vmid,
3872                     running => $running,
3873                     drivename => $disk,
3874                     drive => $drive,
3875                     snapname => undef,
3876                 };
3877
3878                 my $dest_info = {
3879                     vmid => $vmid,
3880                     drivename => $disk,
3881                     storage => $storeid,
3882                     format => $format,
3883                 };
3884
3885                 $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf)
3886                     if $disk eq 'efidisk0';
3887
3888                 my $newdrive = PVE::QemuServer::clone_disk(
3889                     $storecfg,
3890                     $source_info,
3891                     $dest_info,
3892                     1,
3893                     $newvollist,
3894                     undef,
3895                     undef,
3896                     undef,
3897                     $movelimit,
3898                 );
3899                 $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
3900
3901                 PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
3902
3903                 # convert moved disk to base if part of template
3904                 PVE::QemuServer::template_create($vmid, $conf, $disk)
3905                     if PVE::QemuConfig->is_template($conf);
3906
3907                 PVE::QemuConfig->write_config($vmid, $conf);
3908
3909                 my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks');
3910                 if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
3911                     eval { mon_cmd($vmid, "guest-fstrim") };
3912                 }
3913
3914                 eval {
3915                     # try to deactivate volumes - avoid lvm LVs to be active on several nodes
3916                     PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
3917                         if !$running;
3918                 };
3919                 warn $@ if $@;
3920             };
3921             if (my $err = $@) {
3922                 foreach my $volid (@$newvollist) {
3923                     eval { PVE::Storage::vdisk_free($storecfg, $volid) };
3924                     warn $@ if $@;
3925                 }
3926                 die "storage migration failed: $err";
3927             }
3928
3929             if ($param->{delete}) {
3930                 eval {
3931                     PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
3932                     PVE::Storage::vdisk_free($storecfg, $old_volid);
3933                 };
3934                 warn $@ if $@;
3935             }
3936         };
3937
3938         my $load_and_check_reassign_configs = sub {
3939             my $vmlist = PVE::Cluster::get_vmlist()->{ids};
3940
3941             die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid});
3942             die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
3943
3944             my $source_node = $vmlist->{$vmid}->{node};
3945             my $target_node = $vmlist->{$target_vmid}->{node};
3946
3947             die "Both VMs need to be on the same node ($source_node != $target_node)\n"
3948                 if $source_node ne $target_node;
3949
3950             my $source_conf = PVE::QemuConfig->load_config($vmid);
3951             PVE::QemuConfig->check_lock($source_conf);
3952             my $target_conf = PVE::QemuConfig->load_config($target_vmid);
3953             PVE::QemuConfig->check_lock($target_conf);
3954
3955             die "Can't move disks from or to template VMs\n"
3956                 if ($source_conf->{template} || $target_conf->{template});
3957
3958             if ($digest) {
3959                 eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
3960                 die "VM ${vmid}: $@" if $@;
3961             }
3962
3963             if ($target_digest) {
3964                 eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
3965                 die "VM ${target_vmid}: $@" if $@;
3966             }
3967
3968             die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk});
3969
3970             die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n"
3971                 if $target_conf->{$target_disk};
3972
3973             my $drive = PVE::QemuServer::parse_drive(
3974                 $disk,
3975                 $source_conf->{$disk},
3976             );
3977             die "failed to parse source disk - $@\n" if !$drive;
3978
3979             my $source_volid = $drive->{file};
3980
3981             die "disk '${disk}' has no associated volume\n" if !$source_volid;
3982             die "CD drive contents can't be moved to another VM\n"
3983                 if PVE::QemuServer::drive_is_cdrom($drive, 1);
3984
3985             my $storeid = PVE::Storage::parse_volume_id($source_volid, 1);
3986             die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid);
3987
3988             die "Can't move disk used by a snapshot to another VM\n"
3989                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid);
3990             die "Storage does not support moving of this disk to another VM\n"
3991                 if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid));
3992             die "Cannot move disk to another VM while the source VM is running - detach first\n"
3993                 if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/;
3994
3995             # now re-parse using target disk slot format
3996             if ($target_disk =~ /^unused\d+$/) {
3997                 $drive = PVE::QemuServer::parse_drive(
3998                     $target_disk,
3999                     $source_volid,
4000                 );
4001             } else {
4002                 $drive = PVE::QemuServer::parse_drive(
4003                     $target_disk,
4004                     $source_conf->{$disk},
4005                 );
4006             }
4007             die "failed to parse source disk for target disk format - $@\n" if !$drive;
4008
4009             my $repl_conf = PVE::ReplicationConfig->new();
4010             if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
4011                 my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4012                 die "Cannot move disk to a replicated VM. Storage does not support replication!\n"
4013                     if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
4014             }
4015
4016             return ($source_conf, $target_conf, $drive);
4017         };
4018
4019         my $logfunc = sub {
4020             my ($msg) = @_;
4021             print STDERR "$msg\n";
4022         };
4023
4024         my $disk_reassignfn = sub {
4025             return PVE::QemuConfig->lock_config($vmid, sub {
4026                 return PVE::QemuConfig->lock_config($target_vmid, sub {
4027                     my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs();
4028
4029                     my $source_volid = $drive->{file};
4030
4031                     print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n";
4032                     my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
4033
4034                     my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4035
4036                     my $new_volid = PVE::Storage::rename_volume(
4037                         $storecfg,
4038                         $source_volid,
4039                         $target_vmid,
4040                     );
4041
4042                     $drive->{file} = $new_volid;
4043
4044                     my $boot_order = PVE::QemuServer::device_bootorder($source_conf);
4045                     if (defined(delete $boot_order->{$disk})) {
4046                         print "removing disk '$disk' from boot order config\n";
4047                         my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ];
4048                         $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs);
4049                     }
4050
4051                     delete $source_conf->{$disk};
4052                     print "removing disk '${disk}' from VM '${vmid}' config\n";
4053                     PVE::QemuConfig->write_config($vmid, $source_conf);
4054
4055                     my $drive_string = PVE::QemuServer::print_drive($drive);
4056
4057                     if ($target_disk =~ /^unused\d+$/) {
4058                         $target_conf->{$target_disk} = $drive_string;
4059                         PVE::QemuConfig->write_config($target_vmid, $target_conf);
4060                     } else {
4061                         &$update_vm_api(
4062                             {
4063                                 node => $node,
4064                                 vmid => $target_vmid,
4065                                 digest => $target_digest,
4066                                 $target_disk => $drive_string,
4067                             },
4068                             1,
4069                         );
4070                     }
4071
4072                     # remove possible replication snapshots
4073                     if (PVE::Storage::volume_has_feature(
4074                             $storecfg,
4075                             'replicate',
4076                             $source_volid),
4077                     ) {
4078                         eval {
4079                             PVE::Replication::prepare(
4080                                 $storecfg,
4081                                 [$new_volid],
4082                                 undef,
4083                                 1,
4084                                 undef,
4085                                 $logfunc,
4086                             )
4087                         };
4088                         if (my $err = $@) {
4089                             print "Failed to remove replication snapshots on moved disk " .
4090                                 "'$target_disk'. Manual cleanup could be necessary.\n";
4091                         }
4092                     }
4093                 });
4094             });
4095         };
4096
4097         if ($target_vmid && $storeid) {
4098             my $msg = "either set 'storage' or 'target-vmid', but not both";
4099             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4100         } elsif ($target_vmid) {
4101             $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
4102                 if $authuser ne 'root@pam';
4103
4104             raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" })
4105                 if $vmid eq $target_vmid;
4106
4107             my (undef, undef, $drive) = &$load_and_check_reassign_configs();
4108             my $storage = PVE::Storage::parse_volume_id($drive->{file});
4109             $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
4110
4111             return $rpcenv->fork_worker(
4112                 'qmmove',
4113                 "${vmid}-${disk}>${target_vmid}-${target_disk}",
4114                 $authuser,
4115                 $disk_reassignfn
4116             );
4117         } elsif ($storeid) {
4118             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4119
4120             die "cannot move disk '$disk', only configured disks can be moved to another storage\n"
4121                 if $disk =~ m/^unused\d+$/;
4122
4123             $load_and_check_move->(); # early checks before forking/locking
4124
4125             my $realcmd = sub {
4126                 PVE::QemuConfig->lock_config($vmid, $move_updatefn);
4127             };
4128
4129             return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
4130         } else {
4131             my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
4132             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4133         }
4134     }});
4135
4136 my $check_vm_disks_local = sub {
4137     my ($storecfg, $vmconf, $vmid) = @_;
4138
4139     my $local_disks = {};
4140
4141     # add some more information to the disks e.g. cdrom
4142     PVE::QemuServer::foreach_volid($vmconf, sub {
4143         my ($volid, $attr) = @_;
4144
4145         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4146         if ($storeid) {
4147             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4148             return if $scfg->{shared};
4149         }
4150         # The shared attr here is just a special case where the vdisk
4151         # is marked as shared manually
4152         return if $attr->{shared};
4153         return if $attr->{cdrom} and $volid eq "none";
4154
4155         if (exists $local_disks->{$volid}) {
4156             @{$local_disks->{$volid}}{keys %$attr} = values %$attr
4157         } else {
4158             $local_disks->{$volid} = $attr;
4159             # ensure volid is present in case it's needed
4160             $local_disks->{$volid}->{volid} = $volid;
4161         }
4162     });
4163
4164     return $local_disks;
4165 };
4166
4167 __PACKAGE__->register_method({
4168     name => 'migrate_vm_precondition',
4169     path => '{vmid}/migrate',
4170     method => 'GET',
4171     protected => 1,
4172     proxyto => 'node',
4173     description => "Get preconditions for migration.",
4174     permissions => {
4175         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4176     },
4177     parameters => {
4178         additionalProperties => 0,
4179         properties => {
4180             node => get_standard_option('pve-node'),
4181             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4182             target => get_standard_option('pve-node', {
4183                 description => "Target node.",
4184                 completion =>  \&PVE::Cluster::complete_migration_target,
4185                 optional => 1,
4186             }),
4187         },
4188     },
4189     returns => {
4190         type => "object",
4191         properties => {
4192             running => { type => 'boolean' },
4193             allowed_nodes => {
4194                 type => 'array',
4195                 optional => 1,
4196                 description => "List nodes allowed for offline migration, only passed if VM is offline"
4197             },
4198             not_allowed_nodes => {
4199                 type => 'object',
4200                 optional => 1,
4201                 description => "List not allowed nodes with additional informations, only passed if VM is offline"
4202             },
4203             local_disks => {
4204                 type => 'array',
4205                 description => "List local disks including CD-Rom, unsused and not referenced disks"
4206             },
4207             local_resources => {
4208                 type => 'array',
4209                 description => "List local resources e.g. pci, usb"
4210             }
4211         },
4212     },
4213     code => sub {
4214         my ($param) = @_;
4215
4216         my $rpcenv = PVE::RPCEnvironment::get();
4217
4218         my $authuser = $rpcenv->get_user();
4219
4220         PVE::Cluster::check_cfs_quorum();
4221
4222         my $res = {};
4223
4224         my $vmid = extract_param($param, 'vmid');
4225         my $target = extract_param($param, 'target');
4226         my $localnode = PVE::INotify::nodename();
4227
4228
4229         # test if VM exists
4230         my $vmconf = PVE::QemuConfig->load_config($vmid);
4231         my $storecfg = PVE::Storage::config();
4232
4233
4234         # try to detect errors early
4235         PVE::QemuConfig->check_lock($vmconf);
4236
4237         $res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0;
4238
4239         # if vm is not running, return target nodes where local storage is available
4240         # for offline migration
4241         if (!$res->{running}) {
4242             $res->{allowed_nodes} = [];
4243             my $checked_nodes = PVE::QemuServer::check_local_storage_availability($vmconf, $storecfg);
4244             delete $checked_nodes->{$localnode};
4245
4246             foreach my $node (keys %$checked_nodes) {
4247                 if (!defined $checked_nodes->{$node}->{unavailable_storages}) {
4248                     push @{$res->{allowed_nodes}}, $node;
4249                 }
4250
4251             }
4252             $res->{not_allowed_nodes} = $checked_nodes;
4253         }
4254
4255
4256         my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid);
4257         $res->{local_disks} = [ values %$local_disks ];;
4258
4259         my $local_resources =  PVE::QemuServer::check_local_resources($vmconf, 1);
4260
4261         $res->{local_resources} = $local_resources;
4262
4263         return $res;
4264
4265
4266     }});
4267
4268 __PACKAGE__->register_method({
4269     name => 'migrate_vm',
4270     path => '{vmid}/migrate',
4271     method => 'POST',
4272     protected => 1,
4273     proxyto => 'node',
4274     description => "Migrate virtual machine. Creates a new migration task.",
4275     permissions => {
4276         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4277     },
4278     parameters => {
4279         additionalProperties => 0,
4280         properties => {
4281             node => get_standard_option('pve-node'),
4282             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4283             target => get_standard_option('pve-node', {
4284                 description => "Target node.",
4285                 completion =>  \&PVE::Cluster::complete_migration_target,
4286             }),
4287             online => {
4288                 type => 'boolean',
4289                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4290                 optional => 1,
4291             },
4292             force => {
4293                 type => 'boolean',
4294                 description => "Allow to migrate VMs which use local devices. Only root may use this option.",
4295                 optional => 1,
4296             },
4297             migration_type => {
4298                 type => 'string',
4299                 enum => ['secure', 'insecure'],
4300                 description => "Migration traffic is encrypted using an SSH tunnel by default. On secure, completely private networks this can be disabled to increase performance.",
4301                 optional => 1,
4302             },
4303             migration_network => {
4304                 type => 'string', format => 'CIDR',
4305                 description => "CIDR of the (sub) network that is used for migration.",
4306                 optional => 1,
4307             },
4308             "with-local-disks" => {
4309                 type => 'boolean',
4310                 description => "Enable live storage migration for local disk",
4311                 optional => 1,
4312             },
4313             targetstorage => get_standard_option('pve-targetstorage', {
4314                 completion => \&PVE::QemuServer::complete_migration_storage,
4315             }),
4316             bwlimit => {
4317                 description => "Override I/O bandwidth limit (in KiB/s).",
4318                 optional => 1,
4319                 type => 'integer',
4320                 minimum => '0',
4321                 default => 'migrate limit from datacenter or storage config',
4322             },
4323         },
4324     },
4325     returns => {
4326         type => 'string',
4327         description => "the task ID.",
4328     },
4329     code => sub {
4330         my ($param) = @_;
4331
4332         my $rpcenv = PVE::RPCEnvironment::get();
4333         my $authuser = $rpcenv->get_user();
4334
4335         my $target = extract_param($param, 'target');
4336
4337         my $localnode = PVE::INotify::nodename();
4338         raise_param_exc({ target => "target is local node."}) if $target eq $localnode;
4339
4340         PVE::Cluster::check_cfs_quorum();
4341
4342         PVE::Cluster::check_node_exists($target);
4343
4344         my $targetip = PVE::Cluster::remote_node_ip($target);
4345
4346         my $vmid = extract_param($param, 'vmid');
4347
4348         raise_param_exc({ force => "Only root may use this option." })
4349             if $param->{force} && $authuser ne 'root@pam';
4350
4351         raise_param_exc({ migration_type => "Only root may use this option." })
4352             if $param->{migration_type} && $authuser ne 'root@pam';
4353
4354         # allow root only until better network permissions are available
4355         raise_param_exc({ migration_network => "Only root may use this option." })
4356             if $param->{migration_network} && $authuser ne 'root@pam';
4357
4358         # test if VM exists
4359         my $conf = PVE::QemuConfig->load_config($vmid);
4360
4361         # try to detect errors early
4362
4363         PVE::QemuConfig->check_lock($conf);
4364
4365         if (PVE::QemuServer::check_running($vmid)) {
4366             die "can't migrate running VM without --online\n" if !$param->{online};
4367
4368             my $repl_conf = PVE::ReplicationConfig->new();
4369             my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
4370             my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target));
4371             if (!$param->{force} && $is_replicated && !$is_replicated_to_target) {
4372                 die "Cannot live-migrate replicated VM to node '$target' - not a replication " .
4373                     "target. Use 'force' to override.\n";
4374             }
4375         } else {
4376             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4377             $param->{online} = 0;
4378         }
4379
4380         my $storecfg = PVE::Storage::config();
4381         if (my $targetstorage = $param->{targetstorage}) {
4382             my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
4383             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
4384                 if $@;
4385
4386             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
4387                 if !defined($storagemap->{identity});
4388
4389             foreach my $target_sid (values %{$storagemap->{entries}}) {
4390                 $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
4391             }
4392
4393             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
4394                 if $storagemap->{default};
4395
4396             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target)
4397                 if $storagemap->{identity};
4398
4399             $param->{storagemap} = $storagemap;
4400         } else {
4401             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target);
4402         }
4403
4404         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
4405
4406             my $hacmd = sub {
4407                 my $upid = shift;
4408
4409                 print "Requesting HA migration for VM $vmid to node $target\n";
4410
4411                 my $cmd = ['ha-manager', 'migrate', "vm:$vmid", $target];
4412                 PVE::Tools::run_command($cmd);
4413                 return;
4414             };
4415
4416             return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);
4417
4418         } else {
4419
4420             my $realcmd = sub {
4421                 PVE::QemuMigrate->migrate($target, $targetip, $vmid, $param);
4422             };
4423
4424             my $worker = sub {
4425                 return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
4426             };
4427
4428             return $rpcenv->fork_worker('qmigrate', $vmid, $authuser, $worker);
4429         }
4430
4431     }});
4432
4433 __PACKAGE__->register_method({
4434     name => 'remote_migrate_vm',
4435     path => '{vmid}/remote_migrate',
4436     method => 'POST',
4437     protected => 1,
4438     proxyto => 'node',
4439     description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!",
4440     permissions => {
4441         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4442     },
4443     parameters => {
4444         additionalProperties => 0,
4445         properties => {
4446             node => get_standard_option('pve-node'),
4447             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4448             'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
4449             'target-endpoint' => get_standard_option('proxmox-remote', {
4450                 description => "Remote target endpoint",
4451             }),
4452             online => {
4453                 type => 'boolean',
4454                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4455                 optional => 1,
4456             },
4457             delete => {
4458                 type => 'boolean',
4459                 description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.",
4460                 optional => 1,
4461                 default => 0,
4462             },
4463             'target-storage' => get_standard_option('pve-targetstorage', {
4464                 completion => \&PVE::QemuServer::complete_migration_storage,
4465                 optional => 0,
4466             }),
4467             'target-bridge' => {
4468                 type => 'string',
4469                 description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
4470                 format => 'bridge-pair-list',
4471             },
4472             bwlimit => {
4473                 description => "Override I/O bandwidth limit (in KiB/s).",
4474                 optional => 1,
4475                 type => 'integer',
4476                 minimum => '0',
4477                 default => 'migrate limit from datacenter or storage config',
4478             },
4479         },
4480     },
4481     returns => {
4482         type => 'string',
4483         description => "the task ID.",
4484     },
4485     code => sub {
4486         my ($param) = @_;
4487
4488         my $rpcenv = PVE::RPCEnvironment::get();
4489         my $authuser = $rpcenv->get_user();
4490
4491         my $source_vmid = extract_param($param, 'vmid');
4492         my $target_endpoint = extract_param($param, 'target-endpoint');
4493         my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
4494
4495         my $delete = extract_param($param, 'delete') // 0;
4496
4497         PVE::Cluster::check_cfs_quorum();
4498
4499         # test if VM exists
4500         my $conf = PVE::QemuConfig->load_config($source_vmid);
4501
4502         PVE::QemuConfig->check_lock($conf);
4503
4504         raise_param_exc({ vmid => "cannot migrate HA-managed VM to remote cluster" })
4505             if PVE::HA::Config::vm_is_ha_managed($source_vmid);
4506
4507         my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);
4508
4509         # TODO: move this as helper somewhere appropriate?
4510         my $conn_args = {
4511             protocol => 'https',
4512             host => $remote->{host},
4513             port => $remote->{port} // 8006,
4514             apitoken => $remote->{apitoken},
4515         };
4516
4517         my $fp;
4518         if ($fp = $remote->{fingerprint}) {
4519             $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
4520         }
4521
4522         print "Establishing API connection with remote at '$remote->{host}'\n";
4523
4524         my $api_client = PVE::APIClient::LWP->new(%$conn_args);
4525
4526         if (!defined($fp)) {
4527             my $cert_info = $api_client->get("/nodes/localhost/certificates/info");
4528             foreach my $cert (@$cert_info) {
4529                 my $filename = $cert->{filename};
4530                 next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem';
4531                 $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem';
4532             }
4533             $conn_args->{cached_fingerprints} = { uc($fp) => 1 }
4534                 if defined($fp);
4535         }
4536
4537         my $repl_conf = PVE::ReplicationConfig->new();
4538         my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1);
4539         die "cannot remote-migrate replicated VM\n" if $is_replicated;
4540
4541         if (PVE::QemuServer::check_running($source_vmid)) {
4542             die "can't migrate running VM without --online\n" if !$param->{online};
4543
4544         } else {
4545             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4546             $param->{online} = 0;
4547         }
4548
4549         my $storecfg = PVE::Storage::config();
4550         my $target_storage = extract_param($param, 'target-storage');
4551         my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
4552         raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
4553             if $@;
4554
4555         my $target_bridge = extract_param($param, 'target-bridge');
4556         my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') };
4557         raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" })
4558             if $@;
4559
4560         die "remote migration requires explicit storage mapping!\n"
4561             if $storagemap->{identity};
4562
4563         $param->{storagemap} = $storagemap;
4564         $param->{bridgemap} = $bridgemap;
4565         $param->{remote} = {
4566             conn => $conn_args, # re-use fingerprint for tunnel
4567             client => $api_client,
4568             vmid => $target_vmid,
4569         };
4570         $param->{migration_type} = 'websocket';
4571         $param->{'with-local-disks'} = 1;
4572         $param->{delete} = $delete if $delete;
4573
4574         my $cluster_status = $api_client->get("/cluster/status");
4575         my $target_node;
4576         foreach my $entry (@$cluster_status) {
4577             next if $entry->{type} ne 'node';
4578             if ($entry->{local}) {
4579                 $target_node = $entry->{name};
4580                 last;
4581             }
4582         }
4583
4584         die "couldn't determine endpoint's node name\n"
4585             if !defined($target_node);
4586
4587         my $realcmd = sub {
4588             PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param);
4589         };
4590
4591         my $worker = sub {
4592             return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd);
4593         };
4594
4595         return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker);
4596     }});
4597
4598 __PACKAGE__->register_method({
4599     name => 'monitor',
4600     path => '{vmid}/monitor',
4601     method => 'POST',
4602     protected => 1,
4603     proxyto => 'node',
4604     description => "Execute Qemu monitor commands.",
4605     permissions => {
4606         description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')",
4607         check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]],
4608     },
4609     parameters => {
4610         additionalProperties => 0,
4611         properties => {
4612             node => get_standard_option('pve-node'),
4613             vmid => get_standard_option('pve-vmid'),
4614             command => {
4615                 type => 'string',
4616                 description => "The monitor command.",
4617             }
4618         },
4619     },
4620     returns => { type => 'string'},
4621     code => sub {
4622         my ($param) = @_;
4623
4624         my $rpcenv = PVE::RPCEnvironment::get();
4625         my $authuser = $rpcenv->get_user();
4626
4627         my $is_ro = sub {
4628             my $command = shift;
4629             return $command =~ m/^\s*info(\s+|$)/
4630                 || $command =~ m/^\s*help\s*$/;
4631         };
4632
4633         $rpcenv->check_full($authuser, "/", ['Sys.Modify'])
4634             if !&$is_ro($param->{command});
4635
4636         my $vmid = $param->{vmid};
4637
4638         my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists
4639
4640         my $res = '';
4641         eval {
4642             $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, $param->{command});
4643         };
4644         $res = "ERROR: $@" if $@;
4645
4646         return $res;
4647     }});
4648
4649 __PACKAGE__->register_method({
4650     name => 'resize_vm',
4651     path => '{vmid}/resize',
4652     method => 'PUT',
4653     protected => 1,
4654     proxyto => 'node',
4655     description => "Extend volume size.",
4656     permissions => {
4657         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
4658     },
4659     parameters => {
4660         additionalProperties => 0,
4661         properties => {
4662             node => get_standard_option('pve-node'),
4663             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4664             skiplock => get_standard_option('skiplock'),
4665             disk => {
4666                 type => 'string',
4667                 description => "The disk you want to resize.",
4668                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
4669             },
4670             size => {
4671                 type => 'string',
4672                 pattern => '\+?\d+(\.\d+)?[KMGT]?',
4673                 description => "The new size. With the `+` sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
4674             },
4675             digest => {
4676                 type => 'string',
4677                 description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
4678                 maxLength => 40,
4679                 optional => 1,
4680             },
4681         },
4682     },
4683     returns => { type => 'null'},
4684     code => sub {
4685         my ($param) = @_;
4686
4687         my $rpcenv = PVE::RPCEnvironment::get();
4688
4689         my $authuser = $rpcenv->get_user();
4690
4691         my $node = extract_param($param, 'node');
4692
4693         my $vmid = extract_param($param, 'vmid');
4694
4695         my $digest = extract_param($param, 'digest');
4696
4697         my $disk = extract_param($param, 'disk');
4698
4699         my $sizestr = extract_param($param, 'size');
4700
4701         my $skiplock = extract_param($param, 'skiplock');
4702         raise_param_exc({ skiplock => "Only root may use this option." })
4703             if $skiplock && $authuser ne 'root@pam';
4704
4705         my $storecfg = PVE::Storage::config();
4706
4707         my $updatefn =  sub {
4708
4709             my $conf = PVE::QemuConfig->load_config($vmid);
4710
4711             die "checksum missmatch (file change by other user?)\n"
4712                 if $digest && $digest ne $conf->{digest};
4713             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4714
4715             die "disk '$disk' does not exist\n" if !$conf->{$disk};
4716
4717             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
4718
4719             my (undef, undef, undef, undef, undef, undef, $format) =
4720                 PVE::Storage::parse_volname($storecfg, $drive->{file});
4721
4722             die "can't resize volume: $disk if snapshot exists\n"
4723                 if %{$conf->{snapshots}} && $format eq 'qcow2';
4724
4725             my $volid = $drive->{file};
4726
4727             die "disk '$disk' has no associated volume\n" if !$volid;
4728
4729             die "you can't resize a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive);
4730
4731             my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4732
4733             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4734
4735             PVE::Storage::activate_volumes($storecfg, [$volid]);
4736             my $size = PVE::Storage::volume_size_info($storecfg, $volid, 5);
4737
4738             die "Could not determine current size of volume '$volid'\n" if !defined($size);
4739
4740             die "internal error" if $sizestr !~ m/^(\+)?(\d+(\.\d+)?)([KMGT])?$/;
4741             my ($ext, $newsize, $unit) = ($1, $2, $4);
4742             if ($unit) {
4743                 if ($unit eq 'K') {
4744                     $newsize = $newsize * 1024;
4745                 } elsif ($unit eq 'M') {
4746                     $newsize = $newsize * 1024 * 1024;
4747                 } elsif ($unit eq 'G') {
4748                     $newsize = $newsize * 1024 * 1024 * 1024;
4749                 } elsif ($unit eq 'T') {
4750                     $newsize = $newsize * 1024 * 1024 * 1024 * 1024;
4751                 }
4752             }
4753             $newsize += $size if $ext;
4754             $newsize = int($newsize);
4755
4756             die "shrinking disks is not supported\n" if $newsize < $size;
4757
4758             return if $size == $newsize;
4759
4760             PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: resize --disk $disk --size $sizestr");
4761
4762             PVE::QemuServer::qemu_block_resize($vmid, "drive-$disk", $storecfg, $volid, $newsize);
4763
4764             $drive->{size} = $newsize;
4765             $conf->{$disk} = PVE::QemuServer::print_drive($drive);
4766
4767             PVE::QemuConfig->write_config($vmid, $conf);
4768         };
4769
4770         PVE::QemuConfig->lock_config($vmid, $updatefn);
4771         return;
4772     }});
4773
4774 __PACKAGE__->register_method({
4775     name => 'snapshot_list',
4776     path => '{vmid}/snapshot',
4777     method => 'GET',
4778     description => "List all snapshots.",
4779     permissions => {
4780         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
4781     },
4782     proxyto => 'node',
4783     protected => 1, # qemu pid files are only readable by root
4784     parameters => {
4785         additionalProperties => 0,
4786         properties => {
4787             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4788             node => get_standard_option('pve-node'),
4789         },
4790     },
4791     returns => {
4792         type => 'array',
4793         items => {
4794             type => "object",
4795             properties => {
4796                 name => {
4797                     description => "Snapshot identifier. Value 'current' identifies the current VM.",
4798                     type => 'string',
4799                 },
4800                 vmstate => {
4801                     description => "Snapshot includes RAM.",
4802                     type => 'boolean',
4803                     optional => 1,
4804                 },
4805                 description => {
4806                     description => "Snapshot description.",
4807                     type => 'string',
4808                 },
4809                 snaptime => {
4810                     description => "Snapshot creation time",
4811                     type => 'integer',
4812                     renderer => 'timestamp',
4813                     optional => 1,
4814                 },
4815                 parent => {
4816                     description => "Parent snapshot identifier.",
4817                     type => 'string',
4818                     optional => 1,
4819                 },
4820             },
4821         },
4822         links => [ { rel => 'child', href => "{name}" } ],
4823     },
4824     code => sub {
4825         my ($param) = @_;
4826
4827         my $vmid = $param->{vmid};
4828
4829         my $conf = PVE::QemuConfig->load_config($vmid);
4830         my $snaphash = $conf->{snapshots} || {};
4831
4832         my $res = [];
4833
4834         foreach my $name (keys %$snaphash) {
4835             my $d = $snaphash->{$name};
4836             my $item = {
4837                 name => $name,
4838                 snaptime => $d->{snaptime} || 0,
4839                 vmstate => $d->{vmstate} ? 1 : 0,
4840                 description => $d->{description} || '',
4841             };
4842             $item->{parent} = $d->{parent} if $d->{parent};
4843             $item->{snapstate} = $d->{snapstate} if $d->{snapstate};
4844             push @$res, $item;
4845         }
4846
4847         my $running = PVE::QemuServer::check_running($vmid, 1) ? 1 : 0;
4848         my $current = {
4849             name => 'current',
4850             digest => $conf->{digest},
4851             running => $running,
4852             description => "You are here!",
4853         };
4854         $current->{parent} = $conf->{parent} if $conf->{parent};
4855
4856         push @$res, $current;
4857
4858         return $res;
4859     }});
4860
4861 __PACKAGE__->register_method({
4862     name => 'snapshot',
4863     path => '{vmid}/snapshot',
4864     method => 'POST',
4865     protected => 1,
4866     proxyto => 'node',
4867     description => "Snapshot a VM.",
4868     permissions => {
4869         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4870     },
4871     parameters => {
4872         additionalProperties => 0,
4873         properties => {
4874             node => get_standard_option('pve-node'),
4875             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4876             snapname => get_standard_option('pve-snapshot-name'),
4877             vmstate => {
4878                 optional => 1,
4879                 type => 'boolean',
4880                 description => "Save the vmstate",
4881             },
4882             description => {
4883                 optional => 1,
4884                 type => 'string',
4885                 description => "A textual description or comment.",
4886             },
4887         },
4888     },
4889     returns => {
4890         type => 'string',
4891         description => "the task ID.",
4892     },
4893     code => sub {
4894         my ($param) = @_;
4895
4896         my $rpcenv = PVE::RPCEnvironment::get();
4897
4898         my $authuser = $rpcenv->get_user();
4899
4900         my $node = extract_param($param, 'node');
4901
4902         my $vmid = extract_param($param, 'vmid');
4903
4904         my $snapname = extract_param($param, 'snapname');
4905
4906         die "unable to use snapshot name 'current' (reserved name)\n"
4907             if $snapname eq 'current';
4908
4909         die "unable to use snapshot name 'pending' (reserved name)\n"
4910             if lc($snapname) eq 'pending';
4911
4912         my $realcmd = sub {
4913             PVE::Cluster::log_msg('info', $authuser, "snapshot VM $vmid: $snapname");
4914             PVE::QemuConfig->snapshot_create($vmid, $snapname, $param->{vmstate},
4915                                              $param->{description});
4916         };
4917
4918         return $rpcenv->fork_worker('qmsnapshot', $vmid, $authuser, $realcmd);
4919     }});
4920
4921 __PACKAGE__->register_method({
4922     name => 'snapshot_cmd_idx',
4923     path => '{vmid}/snapshot/{snapname}',
4924     description => '',
4925     method => 'GET',
4926     permissions => {
4927         user => 'all',
4928     },
4929     parameters => {
4930         additionalProperties => 0,
4931         properties => {
4932             vmid => get_standard_option('pve-vmid'),
4933             node => get_standard_option('pve-node'),
4934             snapname => get_standard_option('pve-snapshot-name'),
4935         },
4936     },
4937     returns => {
4938         type => 'array',
4939         items => {
4940             type => "object",
4941             properties => {},
4942         },
4943         links => [ { rel => 'child', href => "{cmd}" } ],
4944     },
4945     code => sub {
4946         my ($param) = @_;
4947
4948         my $res = [];
4949
4950         push @$res, { cmd => 'rollback' };
4951         push @$res, { cmd => 'config' };
4952
4953         return $res;
4954     }});
4955
4956 __PACKAGE__->register_method({
4957     name => 'update_snapshot_config',
4958     path => '{vmid}/snapshot/{snapname}/config',
4959     method => 'PUT',
4960     protected => 1,
4961     proxyto => 'node',
4962     description => "Update snapshot metadata.",
4963     permissions => {
4964         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4965     },
4966     parameters => {
4967         additionalProperties => 0,
4968         properties => {
4969             node => get_standard_option('pve-node'),
4970             vmid => get_standard_option('pve-vmid'),
4971             snapname => get_standard_option('pve-snapshot-name'),
4972             description => {
4973                 optional => 1,
4974                 type => 'string',
4975                 description => "A textual description or comment.",
4976             },
4977         },
4978     },
4979     returns => { type => 'null' },
4980     code => sub {
4981         my ($param) = @_;
4982
4983         my $rpcenv = PVE::RPCEnvironment::get();
4984
4985         my $authuser = $rpcenv->get_user();
4986
4987         my $vmid = extract_param($param, 'vmid');
4988
4989         my $snapname = extract_param($param, 'snapname');
4990
4991         return if !defined($param->{description});
4992
4993         my $updatefn =  sub {
4994
4995             my $conf = PVE::QemuConfig->load_config($vmid);
4996
4997             PVE::QemuConfig->check_lock($conf);
4998
4999             my $snap = $conf->{snapshots}->{$snapname};
5000
5001             die "snapshot '$snapname' does not exist\n" if !defined($snap);
5002
5003             $snap->{description} = $param->{description} if defined($param->{description});
5004
5005              PVE::QemuConfig->write_config($vmid, $conf);
5006         };
5007
5008         PVE::QemuConfig->lock_config($vmid, $updatefn);
5009
5010         return;
5011     }});
5012
5013 __PACKAGE__->register_method({
5014     name => 'get_snapshot_config',
5015     path => '{vmid}/snapshot/{snapname}/config',
5016     method => 'GET',
5017     proxyto => 'node',
5018     description => "Get snapshot configuration",
5019     permissions => {
5020         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback', 'VM.Audit' ], any => 1],
5021     },
5022     parameters => {
5023         additionalProperties => 0,
5024         properties => {
5025             node => get_standard_option('pve-node'),
5026             vmid => get_standard_option('pve-vmid'),
5027             snapname => get_standard_option('pve-snapshot-name'),
5028         },
5029     },
5030     returns => { type => "object" },
5031     code => sub {
5032         my ($param) = @_;
5033
5034         my $rpcenv = PVE::RPCEnvironment::get();
5035
5036         my $authuser = $rpcenv->get_user();
5037
5038         my $vmid = extract_param($param, 'vmid');
5039
5040         my $snapname = extract_param($param, 'snapname');
5041
5042         my $conf = PVE::QemuConfig->load_config($vmid);
5043
5044         my $snap = $conf->{snapshots}->{$snapname};
5045
5046         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5047
5048         return $snap;
5049     }});
5050
5051 __PACKAGE__->register_method({
5052     name => 'rollback',
5053     path => '{vmid}/snapshot/{snapname}/rollback',
5054     method => 'POST',
5055     protected => 1,
5056     proxyto => 'node',
5057     description => "Rollback VM state to specified snapshot.",
5058     permissions => {
5059         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback' ], any => 1],
5060     },
5061     parameters => {
5062         additionalProperties => 0,
5063         properties => {
5064             node => get_standard_option('pve-node'),
5065             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5066             snapname => get_standard_option('pve-snapshot-name'),
5067             start => {
5068                 type => 'boolean',
5069                 description => "Whether the VM should get started after rolling back successfully",
5070                 optional => 1,
5071                 default => 0,
5072             },
5073         },
5074     },
5075     returns => {
5076         type => 'string',
5077         description => "the task ID.",
5078     },
5079     code => sub {
5080         my ($param) = @_;
5081
5082         my $rpcenv = PVE::RPCEnvironment::get();
5083
5084         my $authuser = $rpcenv->get_user();
5085
5086         my $node = extract_param($param, 'node');
5087
5088         my $vmid = extract_param($param, 'vmid');
5089
5090         my $snapname = extract_param($param, 'snapname');
5091
5092         my $realcmd = sub {
5093             PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname");
5094             PVE::QemuConfig->snapshot_rollback($vmid, $snapname);
5095
5096             if ($param->{start}) {
5097                 PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node });
5098             }
5099         };
5100
5101         my $worker = sub {
5102             # hold migration lock, this makes sure that nobody create replication snapshots
5103             return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
5104         };
5105
5106         return $rpcenv->fork_worker('qmrollback', $vmid, $authuser, $worker);
5107     }});
5108
5109 __PACKAGE__->register_method({
5110     name => 'delsnapshot',
5111     path => '{vmid}/snapshot/{snapname}',
5112     method => 'DELETE',
5113     protected => 1,
5114     proxyto => 'node',
5115     description => "Delete a VM snapshot.",
5116     permissions => {
5117         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
5118     },
5119     parameters => {
5120         additionalProperties => 0,
5121         properties => {
5122             node => get_standard_option('pve-node'),
5123             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5124             snapname => get_standard_option('pve-snapshot-name'),
5125             force => {
5126                 optional => 1,
5127                 type => 'boolean',
5128                 description => "For removal from config file, even if removing disk snapshots fails.",
5129             },
5130         },
5131     },
5132     returns => {
5133         type => 'string',
5134         description => "the task ID.",
5135     },
5136     code => sub {
5137         my ($param) = @_;
5138
5139         my $rpcenv = PVE::RPCEnvironment::get();
5140
5141         my $authuser = $rpcenv->get_user();
5142
5143         my $node = extract_param($param, 'node');
5144
5145         my $vmid = extract_param($param, 'vmid');
5146
5147         my $snapname = extract_param($param, 'snapname');
5148
5149         my $lock_obtained;
5150         my $do_delete = sub {
5151             $lock_obtained = 1;
5152             PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname");
5153             PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force});
5154         };
5155
5156         my $realcmd = sub {
5157             if ($param->{force}) {
5158                 $do_delete->();
5159             } else {
5160                 eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); };
5161                 if (my $err = $@) {
5162                     die $err if $lock_obtained;
5163                     die "Failed to obtain guest migration lock - replication running?\n";
5164                 }
5165             }
5166         };
5167
5168         return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd);
5169     }});
5170
5171 __PACKAGE__->register_method({
5172     name => 'template',
5173     path => '{vmid}/template',
5174     method => 'POST',
5175     protected => 1,
5176     proxyto => 'node',
5177     description => "Create a Template.",
5178     permissions => {
5179         description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
5180         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
5181     },
5182     parameters => {
5183         additionalProperties => 0,
5184         properties => {
5185             node => get_standard_option('pve-node'),
5186             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
5187             disk => {
5188                 optional => 1,
5189                 type => 'string',
5190                 description => "If you want to convert only 1 disk to base image.",
5191                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
5192             },
5193
5194         },
5195     },
5196     returns => {
5197         type => 'string',
5198         description => "the task ID.",
5199     },
5200     code => sub {
5201         my ($param) = @_;
5202
5203         my $rpcenv = PVE::RPCEnvironment::get();
5204
5205         my $authuser = $rpcenv->get_user();
5206
5207         my $node = extract_param($param, 'node');
5208
5209         my $vmid = extract_param($param, 'vmid');
5210
5211         my $disk = extract_param($param, 'disk');
5212
5213         my $load_and_check = sub {
5214             my $conf = PVE::QemuConfig->load_config($vmid);
5215
5216             PVE::QemuConfig->check_lock($conf);
5217
5218             die "unable to create template, because VM contains snapshots\n"
5219                 if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});
5220
5221             die "you can't convert a template to a template\n"
5222                 if PVE::QemuConfig->is_template($conf) && !$disk;
5223
5224             die "you can't convert a VM to template if VM is running\n"
5225                 if PVE::QemuServer::check_running($vmid);
5226
5227             return $conf;
5228         };
5229
5230         $load_and_check->();
5231
5232         my $realcmd = sub {
5233             PVE::QemuConfig->lock_config($vmid, sub {
5234                 my $conf = $load_and_check->();
5235
5236                 $conf->{template} = 1;
5237                 PVE::QemuConfig->write_config($vmid, $conf);
5238
5239                 PVE::QemuServer::template_create($vmid, $conf, $disk);
5240             });
5241         };
5242
5243         return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
5244     }});
5245
5246 __PACKAGE__->register_method({
5247     name => 'cloudinit_generated_config_dump',
5248     path => '{vmid}/cloudinit/dump',
5249     method => 'GET',
5250     proxyto => 'node',
5251     description => "Get automatically generated cloudinit config.",
5252     permissions => {
5253         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
5254     },
5255     parameters => {
5256         additionalProperties => 0,
5257         properties => {
5258             node => get_standard_option('pve-node'),
5259             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5260             type => {
5261                 description => 'Config type.',
5262                 type => 'string',
5263                 enum => ['user', 'network', 'meta'],
5264             },
5265         },
5266     },
5267     returns => {
5268         type => 'string',
5269     },
5270     code => sub {
5271         my ($param) = @_;
5272
5273         my $conf = PVE::QemuConfig->load_config($param->{vmid});
5274
5275         return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
5276     }});
5277
5278 __PACKAGE__->register_method({
5279     name => 'mtunnel',
5280     path => '{vmid}/mtunnel',
5281     method => 'POST',
5282     protected => 1,
5283     description => 'Migration tunnel endpoint - only for internal use by VM migration.',
5284     permissions => {
5285         check =>
5286         [ 'and',
5287           ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]],
5288           ['perm', '/', [ 'Sys.Incoming' ]],
5289         ],
5290         description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" .
5291                        " on '/'. Further permission checks happen during the actual migration.",
5292     },
5293     parameters => {
5294         additionalProperties => 0,
5295         properties => {
5296             node => get_standard_option('pve-node'),
5297             vmid => get_standard_option('pve-vmid'),
5298             storages => {
5299                 type => 'string',
5300                 format => 'pve-storage-id-list',
5301                 optional => 1,
5302                 description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.',
5303             },
5304             bridges => {
5305                 type => 'string',
5306                 format => 'pve-bridge-id-list',
5307                 optional => 1,
5308                 description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.',
5309             },
5310         },
5311     },
5312     returns => {
5313         additionalProperties => 0,
5314         properties => {
5315             upid => { type => 'string' },
5316             ticket => { type => 'string' },
5317             socket => { type => 'string' },
5318         },
5319     },
5320     code => sub {
5321         my ($param) = @_;
5322
5323         my $rpcenv = PVE::RPCEnvironment::get();
5324         my $authuser = $rpcenv->get_user();
5325
5326         my $node = extract_param($param, 'node');
5327         my $vmid = extract_param($param, 'vmid');
5328
5329         my $storages = extract_param($param, 'storages');
5330         my $bridges = extract_param($param, 'bridges');
5331
5332         my $nodename = PVE::INotify::nodename();
5333
5334         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5335             if $node ne 'localhost' && $node ne $nodename;
5336
5337         $node = $nodename;
5338
5339         my $storecfg = PVE::Storage::config();
5340         foreach my $storeid (PVE::Tools::split_list($storages)) {
5341             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node);
5342         }
5343
5344         foreach my $bridge (PVE::Tools::split_list($bridges)) {
5345             PVE::Network::read_bridge_mtu($bridge);
5346         }
5347
5348         PVE::Cluster::check_cfs_quorum();
5349
5350         my $lock = 'create';
5351         eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); };
5352
5353         raise_param_exc({ vmid => "unable to create empty VM config - $@"})
5354             if $@;
5355
5356         my $realcmd = sub {
5357             my $state = {
5358                 storecfg => PVE::Storage::config(),
5359                 lock => $lock,
5360                 vmid => $vmid,
5361             };
5362
5363             my $run_locked = sub {
5364                 my ($code, $params) = @_;
5365                 return PVE::QemuConfig->lock_config($state->{vmid}, sub {
5366                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5367
5368                     $state->{conf} = $conf;
5369
5370                     die "Encountered wrong lock - aborting mtunnel command handling.\n"
5371                         if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock});
5372
5373                     return $code->($params);
5374                 });
5375             };
5376
5377             my $cmd_desc = {
5378                 config => {
5379                     conf => {
5380                         type => 'string',
5381                         description => 'Full VM config, adapted for target cluster/node',
5382                     },
5383                     'firewall-config' => {
5384                         type => 'string',
5385                         description => 'VM firewall config',
5386                         optional => 1,
5387                     },
5388                 },
5389                 disk => {
5390                     format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'),
5391                     storage => {
5392                         type => 'string',
5393                         format => 'pve-storage-id',
5394                     },
5395                     drive => {
5396                         type => 'object',
5397                         description => 'parsed drive information without volid and format',
5398                     },
5399                 },
5400                 start => {
5401                     start_params => {
5402                         type => 'object',
5403                         description => 'params passed to vm_start_nolock',
5404                     },
5405                     migrate_opts => {
5406                         type => 'object',
5407                         description => 'migrate_opts passed to vm_start_nolock',
5408                     },
5409                 },
5410                 ticket => {
5411                     path => {
5412                         type => 'string',
5413                         description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.',
5414                     },
5415                 },
5416                 quit => {
5417                     cleanup => {
5418                         type => 'boolean',
5419                         description => 'remove VM config and disks, aborting migration',
5420                         default => 0,
5421                     },
5422                 },
5423                 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'},
5424                 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'},
5425                 bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit},
5426             };
5427
5428             my $cmd_handlers = {
5429                 'version' => sub {
5430                     # compared against other end's version
5431                     # bump/reset for breaking changes
5432                     # bump/bump for opt-in changes
5433                     return {
5434                         api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
5435                         age => 0,
5436                     };
5437                 },
5438                 'config' => sub {
5439                     my ($params) = @_;
5440
5441                     # parse and write out VM FW config if given
5442                     if (my $fw_conf = $params->{'firewall-config'}) {
5443                         my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700);
5444
5445                         my $empty_conf = {
5446                             rules => [],
5447                             options => {},
5448                             aliases => {},
5449                             ipset => {} ,
5450                             ipset_comments => {},
5451                         };
5452                         my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf();
5453
5454                         # TODO: add flag for strict parsing?
5455                         # TODO: add import sub that does all this given raw content?
5456                         my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm');
5457                         $vmfw_conf->{vmid} = $state->{vmid};
5458                         PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf);
5459
5460                         $state->{cleanup}->{fw} = 1;
5461                     }
5462
5463                     my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf";
5464                     my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1);
5465                     delete $new_conf->{lock};
5466                     delete $new_conf->{digest};
5467
5468                     # TODO handle properly?
5469                     delete $new_conf->{snapshots};
5470                     delete $new_conf->{parent};
5471                     delete $new_conf->{pending};
5472
5473                     # not handled by update_vm_api
5474                     my $vmgenid = delete $new_conf->{vmgenid};
5475                     my $meta = delete $new_conf->{meta};
5476                     my $cloudinit = delete $new_conf->{cloudinit}; # this is informational only
5477                     $new_conf->{skip_cloud_init} = 1; # re-use image from source side
5478
5479                     $new_conf->{vmid} = $state->{vmid};
5480                     $new_conf->{node} = $node;
5481
5482                     PVE::QemuConfig->remove_lock($state->{vmid}, 'create');
5483
5484                     eval {
5485                         $update_vm_api->($new_conf, 1);
5486                     };
5487                     if (my $err = $@) {
5488                         # revert to locked previous config
5489                         my $conf = PVE::QemuConfig->load_config($state->{vmid});
5490                         $conf->{lock} = 'create';
5491                         PVE::QemuConfig->write_config($state->{vmid}, $conf);
5492
5493                         die $err;
5494                     }
5495
5496                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5497                     $conf->{lock} = 'migrate';
5498                     $conf->{vmgenid} = $vmgenid if defined($vmgenid);
5499                     $conf->{meta} = $meta if defined($meta);
5500                     $conf->{cloudinit} = $cloudinit if defined($cloudinit);
5501                     PVE::QemuConfig->write_config($state->{vmid}, $conf);
5502
5503                     $state->{lock} = 'migrate';
5504
5505                     return;
5506                 },
5507                 'bwlimit' => sub {
5508                     my ($params) = @_;
5509                     return PVE::StorageTunnel::handle_bwlimit($params);
5510                 },
5511                 'disk' => sub {
5512                     my ($params) = @_;
5513
5514                     my $format = $params->{format};
5515                     my $storeid = $params->{storage};
5516                     my $drive = $params->{drive};
5517
5518                     $check_storage_access_migrate->($rpcenv, $authuser, $state->{storecfg}, $storeid, $node);
5519
5520                     my $storagemap = {
5521                         default => $storeid,
5522                     };
5523
5524                     my $source_volumes = {
5525                         'disk' => [
5526                             undef,
5527                             $storeid,
5528                             undef,
5529                             $drive,
5530                             0,
5531                             $format,
5532                         ],
5533                     };
5534
5535                     my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks($state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap);
5536                     if (defined($res->{disk})) {
5537                         $state->{cleanup}->{volumes}->{$res->{disk}->{volid}} = 1;
5538                         return $res->{disk};
5539                     } else {
5540                         die "failed to allocate NBD disk..\n";
5541                     }
5542                 },
5543                 'disk-import' => sub {
5544                     my ($params) = @_;
5545
5546                     $check_storage_access_migrate->(
5547                         $rpcenv,
5548                         $authuser,
5549                         $state->{storecfg},
5550                         $params->{storage},
5551                         $node
5552                     );
5553
5554                     $params->{unix} = "/run/qemu-server/$state->{vmid}.storage";
5555
5556                     return PVE::StorageTunnel::handle_disk_import($state, $params);
5557                 },
5558                 'query-disk-import' => sub {
5559                     my ($params) = @_;
5560
5561                     return PVE::StorageTunnel::handle_query_disk_import($state, $params);
5562                 },
5563                 'start' => sub {
5564                     my ($params) = @_;
5565
5566                     my $info = PVE::QemuServer::vm_start_nolock(
5567                         $state->{storecfg},
5568                         $state->{vmid},
5569                         $state->{conf},
5570                         $params->{start_params},
5571                         $params->{migrate_opts},
5572                     );
5573
5574
5575                     if ($info->{migrate}->{proto} ne 'unix') {
5576                         PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5577                         die "migration over non-UNIX sockets not possible\n";
5578                     }
5579
5580                     my $socket = $info->{migrate}->{addr};
5581                     chown $state->{socket_uid}, -1, $socket;
5582                     $state->{sockets}->{$socket} = 1;
5583
5584                     my $unix_sockets = $info->{migrate}->{unix_sockets};
5585                     foreach my $socket (@$unix_sockets) {
5586                         chown $state->{socket_uid}, -1, $socket;
5587                         $state->{sockets}->{$socket} = 1;
5588                     }
5589                     return $info;
5590                 },
5591                 'fstrim' => sub {
5592                     if (PVE::QemuServer::qga_check_running($state->{vmid})) {
5593                         eval { mon_cmd($state->{vmid}, "guest-fstrim") };
5594                         warn "fstrim failed: $@\n" if $@;
5595                     }
5596                     return;
5597                 },
5598                 'stop' => sub {
5599                     PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5600                     return;
5601                 },
5602                 'nbdstop' => sub {
5603                     PVE::QemuServer::nbd_stop($state->{vmid});
5604                     return;
5605                 },
5606                 'resume' => sub {
5607                     if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) {
5608                         PVE::QemuServer::vm_resume($state->{vmid}, 1, 1);
5609                     } else {
5610                         die "VM $state->{vmid} not running\n";
5611                     }
5612                     return;
5613                 },
5614                 'unlock' => sub {
5615                     PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock});
5616                     delete $state->{lock};
5617                     return;
5618                 },
5619                 'ticket' => sub {
5620                     my ($params) = @_;
5621
5622                     my $path = $params->{path};
5623
5624                     die "Not allowed to generate ticket for unknown socket '$path'\n"
5625                         if !defined($state->{sockets}->{$path});
5626
5627                     return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") };
5628                 },
5629                 'quit' => sub {
5630                     my ($params) = @_;
5631
5632                     if ($params->{cleanup}) {
5633                         if ($state->{cleanup}->{fw}) {
5634                             PVE::Firewall::remove_vmfw_conf($state->{vmid});
5635                         }
5636
5637                         for my $volid (keys $state->{cleanup}->{volumes}->%*) {
5638                             print "freeing volume '$volid' as part of cleanup\n";
5639                             eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) };
5640                             warn $@ if $@;
5641                         }
5642
5643                         PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5644                     }
5645
5646                     print "switching to exit-mode, waiting for client to disconnect\n";
5647                     $state->{exit} = 1;
5648                     return;
5649                 },
5650             };
5651
5652             $run_locked->(sub {
5653                 my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel";
5654                 unlink $socket_addr;
5655
5656                 $state->{socket} = IO::Socket::UNIX->new(
5657                     Type => SOCK_STREAM(),
5658                     Local => $socket_addr,
5659                     Listen => 1,
5660                 );
5661
5662                 $state->{socket_uid} = getpwnam('www-data')
5663                     or die "Failed to resolve user 'www-data' to numeric UID\n";
5664                 chown $state->{socket_uid}, -1, $socket_addr;
5665             });
5666
5667             print "mtunnel started\n";
5668
5669             my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) };
5670             if ($@) {
5671                 warn "Failed to accept tunnel connection - $@\n";
5672
5673                 warn "Removing tunnel socket..\n";
5674                 unlink $state->{socket};
5675
5676                 warn "Removing temporary VM config..\n";
5677                 $run_locked->(sub {
5678                     PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5679                 });
5680
5681                 die "Exiting mtunnel\n";
5682             }
5683
5684             $state->{conn} = $conn;
5685
5686             my $reply_err = sub {
5687                 my ($msg) = @_;
5688
5689                 my $reply = JSON::encode_json({
5690                     success => JSON::false,
5691                     msg => $msg,
5692                 });
5693                 $conn->print("$reply\n");
5694                 $conn->flush();
5695             };
5696
5697             my $reply_ok = sub {
5698                 my ($res) = @_;
5699
5700                 $res->{success} = JSON::true;
5701                 my $reply = JSON::encode_json($res);
5702                 $conn->print("$reply\n");
5703                 $conn->flush();
5704             };
5705
5706             while (my $line = <$conn>) {
5707                 chomp $line;
5708
5709                 # untaint, we validate below if needed
5710                 ($line) = $line =~ /^(.*)$/;
5711                 my $parsed = eval { JSON::decode_json($line) };
5712                 if ($@) {
5713                     $reply_err->("failed to parse command - $@");
5714                     next;
5715                 }
5716
5717                 my $cmd = delete $parsed->{cmd};
5718                 if (!defined($cmd)) {
5719                     $reply_err->("'cmd' missing");
5720                 } elsif ($state->{exit}) {
5721                     $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible");
5722                     next;
5723                 } elsif (my $handler = $cmd_handlers->{$cmd}) {
5724                     print "received command '$cmd'\n";
5725                     eval {
5726                         if ($cmd_desc->{$cmd}) {
5727                             PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd});
5728                         } else {
5729                             $parsed = {};
5730                         }
5731                         my $res = $run_locked->($handler, $parsed);
5732                         $reply_ok->($res);
5733                     };
5734                     $reply_err->("failed to handle '$cmd' command - $@")
5735                         if $@;
5736                 } else {
5737                     $reply_err->("unknown command '$cmd' given");
5738                 }
5739             }
5740
5741             if ($state->{exit}) {
5742                 print "mtunnel exited\n";
5743             } else {
5744                 die "mtunnel exited unexpectedly\n";
5745             }
5746         };
5747
5748         my $socket_addr = "/run/qemu-server/$vmid.mtunnel";
5749         my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr");
5750         my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd);
5751
5752         return {
5753             ticket => $ticket,
5754             upid => $upid,
5755             socket => $socket_addr,
5756         };
5757     }});
5758
5759 __PACKAGE__->register_method({
5760     name => 'mtunnelwebsocket',
5761     path => '{vmid}/mtunnelwebsocket',
5762     method => 'GET',
5763     permissions => {
5764         description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
5765         user => 'all', # check inside
5766     },
5767     description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
5768     parameters => {
5769         additionalProperties => 0,
5770         properties => {
5771             node => get_standard_option('pve-node'),
5772             vmid => get_standard_option('pve-vmid'),
5773             socket => {
5774                 type => "string",
5775                 description => "unix socket to forward to",
5776             },
5777             ticket => {
5778                 type => "string",
5779                 description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command",
5780             },
5781         },
5782     },
5783     returns => {
5784         type => "object",
5785         properties => {
5786             port => { type => 'string', optional => 1 },
5787             socket => { type => 'string', optional => 1 },
5788         },
5789     },
5790     code => sub {
5791         my ($param) = @_;
5792
5793         my $rpcenv = PVE::RPCEnvironment::get();
5794         my $authuser = $rpcenv->get_user();
5795
5796         my $nodename = PVE::INotify::nodename();
5797         my $node = extract_param($param, 'node');
5798
5799         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5800             if $node ne 'localhost' && $node ne $nodename;
5801
5802         my $vmid = $param->{vmid};
5803         # check VM exists
5804         PVE::QemuConfig->load_config($vmid);
5805
5806         my $socket = $param->{socket};
5807         PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket");
5808
5809         return { socket => $socket };
5810     }});
5811
5812 1;