PVE/API2/Qemu.pm

   1 package PVE::API2::Qemu;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd 'abs_path';
   6 use Net::SSLeay;
   7 use IO::Socket::IP;
   8 use IO::Socket::UNIX;
   9 use IPC::Open3;
  10 use JSON;
  11 use URI::Escape;
  12 use Crypt::OpenSSL::Random;
  13 use Socket qw(SOCK_STREAM);
  14
  15 use PVE::APIClient::LWP;
  16 use PVE::CGroup;
  17 use PVE::Cluster qw (cfs_read_file cfs_write_file);;
  18 use PVE::RRD;
  19 use PVE::SafeSyslog;
  20 use PVE::Tools qw(extract_param);
  21 use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
  22 use PVE::Storage;
  23 use PVE::JSONSchema qw(get_standard_option);
  24 use PVE::RESTHandler;
  25 use PVE::ReplicationConfig;
  26 use PVE::GuestHelpers qw(assert_tag_permissions);
  27 use PVE::QemuConfig;
  28 use PVE::QemuServer;
  29 use PVE::QemuServer::Cloudinit;
  30 use PVE::QemuServer::CPUConfig;
  31 use PVE::QemuServer::Drive;
  32 use PVE::QemuServer::ImportDisk;
  33 use PVE::QemuServer::Monitor qw(mon_cmd);
  34 use PVE::QemuServer::Machine;
  35 use PVE::QemuMigrate;
  36 use PVE::RPCEnvironment;
  37 use PVE::AccessControl;
  38 use PVE::INotify;
  39 use PVE::Network;
  40 use PVE::Firewall;
  41 use PVE::API2::Firewall::VM;
  42 use PVE::API2::Qemu::Agent;
  43 use PVE::VZDump::Plugin;
  44 use PVE::DataCenterConfig;
  45 use PVE::SSHInfo;
  46 use PVE::Replication;
  47 use PVE::StorageTunnel;
  48
  49 BEGIN {
  50     if (!$ENV{PVE_GENERATING_DOCS}) {
  51         require PVE::HA::Env::PVE2;
  52         import PVE::HA::Env::PVE2;
  53         require PVE::HA::Config;
  54         import PVE::HA::Config;
  55     }
  56 }
  57
  58 use base qw(PVE::RESTHandler);
  59
  60 my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal.";
  61
  62 my $resolve_cdrom_alias = sub {
  63     my $param = shift;
  64
  65     if (my $value = $param->{cdrom}) {
  66         $value .= ",media=cdrom" if $value !~ m/media=/;
  67         $param->{ide2} = $value;
  68         delete $param->{cdrom};
  69     }
  70 };
  71
  72 # Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema.
  73 my $foreach_volume_with_alloc = sub {
  74     my ($param, $func) = @_;
  75
  76     for my $opt (sort keys $param->%*) {
  77         next if !PVE::QemuServer::is_valid_drivename($opt);
  78
  79         my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1);
  80         next if !$drive;
  81
  82         $func->($opt, $drive);
  83     }
  84 };
  85
  86 my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
  87
  88 my $check_drive_param = sub {
  89     my ($param, $storecfg, $extra_checks) = @_;
  90
  91     for my $opt (sort keys $param->%*) {
  92         next if !PVE::QemuServer::is_valid_drivename($opt);
  93
  94         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
  95         raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
  96
  97         if ($drive->{'import-from'}) {
  98             if ($drive->{file} !~ $NEW_DISK_RE || $3 != 0) {
  99                 raise_param_exc({
 100                     $opt => "'import-from' requires special syntax - ".
 101                         "use <storage ID>:0,import-from=<source>",
 102                 });
 103             }
 104
 105             if ($opt eq 'efidisk0') {
 106                 for my $required (qw(efitype pre-enrolled-keys)) {
 107                     if (!defined($drive->{$required})) {
 108                         raise_param_exc({
 109                             $opt => "need to specify '$required' when using 'import-from'",
 110                         });
 111                     }
 112                 }
 113             } elsif ($opt eq 'tpmstate0') {
 114                 raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" })
 115                     if !defined($drive->{version});
 116             }
 117         }
 118
 119         PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 120
 121         $extra_checks->($drive) if $extra_checks;
 122
 123         $param->{$opt} = PVE::QemuServer::print_drive($drive, 1);
 124     }
 125 };
 126
 127 my $check_storage_access = sub {
 128    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 129
 130    $foreach_volume_with_alloc->($settings, sub {
 131         my ($ds, $drive) = @_;
 132
 133         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 134
 135         my $volid = $drive->{file};
 136         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 137
 138         if (!$volid || ($volid eq 'none' || $volid eq 'cloudinit' || (defined($volname) && $volname eq 'cloudinit'))) {
 139             # nothing to check
 140         } elsif ($isCDROM && ($volid eq 'cdrom')) {
 141             $rpcenv->check($authuser, "/", ['Sys.Console']);
 142         } elsif (!$isCDROM && ($volid =~ $NEW_DISK_RE)) {
 143             my ($storeid, $size) = ($2 || $default_storage, $3);
 144             die "no storage ID specified (and no default storage)\n" if !$storeid;
 145             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
 146             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 147             raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
 148                 if !$scfg->{content}->{images};
 149         } else {
 150             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 151             if ($storeid) {
 152                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 153                 raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" })
 154                     if $vtype ne 'images' && $vtype ne 'iso';
 155             }
 156         }
 157
 158         if (my $src_image = $drive->{'import-from'}) {
 159             my $src_vmid;
 160             if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume
 161                 (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image);
 162                 raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" })
 163                     if $vtype ne 'images';
 164             }
 165
 166             if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk()
 167                 $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']);
 168             } else {
 169                 PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image);
 170             }
 171         }
 172     });
 173
 174    $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
 175        if defined($settings->{vmstatestorage});
 176 };
 177
 178 my $check_storage_access_clone = sub {
 179    my ($rpcenv, $authuser, $storecfg, $conf, $storage) = @_;
 180
 181    my $sharedvm = 1;
 182
 183    PVE::QemuConfig->foreach_volume($conf, sub {
 184         my ($ds, $drive) = @_;
 185
 186         my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
 187
 188         my $volid = $drive->{file};
 189
 190         return if !$volid || $volid eq 'none';
 191
 192         if ($isCDROM) {
 193             if ($volid eq 'cdrom') {
 194                 $rpcenv->check($authuser, "/", ['Sys.Console']);
 195             } else {
 196                 # we simply allow access
 197                 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 198                 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 199                 $sharedvm = 0 if !$scfg->{shared};
 200
 201             }
 202         } else {
 203             my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 204             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
 205             $sharedvm = 0 if !$scfg->{shared};
 206
 207             $sid = $storage if $storage;
 208             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
 209         }
 210     });
 211
 212    $rpcenv->check($authuser, "/storage/$conf->{vmstatestorage}", ['Datastore.AllocateSpace'])
 213        if defined($conf->{vmstatestorage});
 214
 215    return $sharedvm;
 216 };
 217
 218 my $check_storage_access_migrate = sub {
 219     my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
 220
 221     PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
 222
 223     $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
 224
 225     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
 226     die "storage '$storage' does not support vm images\n"
 227         if !$scfg->{content}->{images};
 228 };
 229
 230 my $import_from_volid = sub {
 231     my ($storecfg, $src_volid, $dest_info, $vollist) = @_;
 232
 233     die "could not get size of $src_volid\n"
 234         if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10);
 235
 236     die "cannot import from cloudinit disk\n"
 237         if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid });
 238
 239     my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2];
 240
 241     my $src_vm_state = sub {
 242         my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0;
 243
 244         my $runs = 0;
 245         if ($exists) {
 246             eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); };
 247             die "owner VM $src_vmid not on local node\n" if $@;
 248             $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0;
 249         }
 250
 251         return ($exists, $runs);
 252     };
 253
 254     my ($src_vm_exists, $running) = $src_vm_state->();
 255
 256     die "cannot import from '$src_volid' - full clone feature is not supported\n"
 257         if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running);
 258
 259     my $clonefn = sub {
 260         my ($src_vm_exists_now, $running_now) = $src_vm_state->();
 261
 262         die "owner VM $src_vmid changed state unexpectedly\n"
 263             if $src_vm_exists_now != $src_vm_exists || $running_now != $running;
 264
 265         my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {};
 266
 267         my $src_drive = { file => $src_volid };
 268         my $src_drivename;
 269         PVE::QemuConfig->foreach_volume($src_conf, sub {
 270             my ($ds, $drive) = @_;
 271
 272             return if $src_drivename;
 273
 274             if ($drive->{file} eq $src_volid) {
 275                 $src_drive = $drive;
 276                 $src_drivename = $ds;
 277             }
 278         });
 279
 280         my $source_info = {
 281             vmid => $src_vmid,
 282             running => $running_now,
 283             drivename => $src_drivename,
 284             drive => $src_drive,
 285             snapname => undef,
 286         };
 287
 288         my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid);
 289
 290         return PVE::QemuServer::clone_disk(
 291             $storecfg,
 292             $source_info,
 293             $dest_info,
 294             1,
 295             $vollist,
 296             undef,
 297             undef,
 298             $src_conf->{agent},
 299             PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]),
 300         );
 301     };
 302
 303     my $cloned;
 304     if ($running) {
 305         $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn);
 306     } elsif ($src_vmid) {
 307         $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn);
 308     } else {
 309         $cloned = $clonefn->();
 310     }
 311
 312     return $cloned->@{qw(file size)};
 313 };
 314
 315 # Note: $pool is only needed when creating a VM, because pool permissions
 316 # are automatically inherited if VM already exists inside a pool.
 317 my $create_disks = sub {
 318     my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
 319
 320     my $vollist = [];
 321
 322     my $res = {};
 323
 324     my $code = sub {
 325         my ($ds, $disk) = @_;
 326
 327         my $volid = $disk->{file};
 328         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
 329
 330         if (!$volid || $volid eq 'none' || $volid eq 'cdrom') {
 331             delete $disk->{size};
 332             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 333         } elsif (defined($volname) && $volname eq 'cloudinit') {
 334             $storeid = $storeid // $default_storage;
 335             die "no storage ID specified (and no default storage)\n" if !$storeid;
 336
 337             if (
 338                 my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 339                 || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 340                 || PVE::QemuConfig->has_cloudinit($res, $ds)
 341             ) {
 342                 die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 343             }
 344
 345             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 346             my $name = "vm-$vmid-cloudinit";
 347
 348             my $fmt = undef;
 349             if ($scfg->{path}) {
 350                 $fmt = $disk->{format} // "qcow2";
 351                 $name .= ".$fmt";
 352             } else {
 353                 $fmt = $disk->{format} // "raw";
 354             }
 355
 356             # Initial disk created with 4 MB and aligned to 4MB on regeneration
 357             my $ci_size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 358             my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, $name, $ci_size/1024);
 359             $disk->{file} = $volid;
 360             $disk->{media} = 'cdrom';
 361             push @$vollist, $volid;
 362             delete $disk->{format}; # no longer needed
 363             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 364             print "$ds: successfully created disk '$res->{$ds}'\n";
 365         } elsif ($volid =~ $NEW_DISK_RE) {
 366             my ($storeid, $size) = ($2 || $default_storage, $3);
 367             die "no storage ID specified (and no default storage)\n" if !$storeid;
 368
 369             if (my $source = delete $disk->{'import-from'}) {
 370                 my $dst_volid;
 371
 372                 if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume
 373                     my $dest_info = {
 374                         vmid => $vmid,
 375                         drivename => $ds,
 376                         storage => $storeid,
 377                         format => $disk->{format},
 378                     };
 379
 380                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk)
 381                         if $ds eq 'efidisk0';
 382
 383                     ($dst_volid, $size) = eval {
 384                         $import_from_volid->($storecfg, $source, $dest_info, $vollist);
 385                     };
 386                     die "cannot import from '$source' - $@" if $@;
 387                 } else {
 388                     $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
 389                     $size = PVE::Storage::file_size_info($source);
 390                     die "could not get file size of $source\n" if !$size;
 391
 392                     (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import(
 393                         $source,
 394                         $vmid,
 395                         $storeid,
 396                         {
 397                             drive_name => $ds,
 398                             format => $disk->{format},
 399                             'skip-config-update' => 1,
 400                         },
 401                     );
 402                     push @$vollist, $dst_volid;
 403                 }
 404
 405                 $disk->{file} = $dst_volid;
 406                 $disk->{size} = $size;
 407                 delete $disk->{format}; # no longer needed
 408                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 409             } else {
 410                 my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
 411                 my $fmt = $disk->{format} || $defformat;
 412
 413                 $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb
 414
 415                 my $volid;
 416                 if ($ds eq 'efidisk0') {
 417                     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 418                     ($volid, $size) = PVE::QemuServer::create_efidisk(
 419                         $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm);
 420                 } elsif ($ds eq 'tpmstate0') {
 421                     # swtpm can only use raw volumes, and uses a fixed size
 422                     $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb');
 423                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size);
 424                 } else {
 425                     $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
 426                 }
 427                 push @$vollist, $volid;
 428                 $disk->{file} = $volid;
 429                 $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
 430                 delete $disk->{format}; # no longer needed
 431                 $res->{$ds} = PVE::QemuServer::print_drive($disk);
 432             }
 433
 434             print "$ds: successfully created disk '$res->{$ds}'\n";
 435         } else {
 436             PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 437             if ($storeid) {
 438                 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
 439                 die "cannot use volume $volid - content type needs to be 'images' or 'iso'"
 440                     if $vtype ne 'images' && $vtype ne 'iso';
 441
 442                 if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) {
 443                     if (
 444                         my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
 445                         || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
 446                         || PVE::QemuConfig->has_cloudinit($res, $ds)
 447                     ) {
 448                         die "$ds - cloud-init drive is already attached at '$ci_key'\n";
 449                     }
 450                 }
 451             }
 452
 453             PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
 454
 455             my $size = PVE::Storage::volume_size_info($storecfg, $volid);
 456             die "volume $volid does not exist\n" if !$size;
 457             $disk->{size} = $size;
 458
 459             $res->{$ds} = PVE::QemuServer::print_drive($disk);
 460         }
 461     };
 462
 463     eval { $foreach_volume_with_alloc->($settings, $code); };
 464
 465     # free allocated images on error
 466     if (my $err = $@) {
 467         syslog('err', "VM $vmid creating disks failed");
 468         foreach my $volid (@$vollist) {
 469             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 470             warn $@ if $@;
 471         }
 472         die $err;
 473     }
 474
 475     return ($vollist, $res);
 476 };
 477
 478 my $check_cpu_model_access = sub {
 479     my ($rpcenv, $authuser, $new, $existing) = @_;
 480
 481     return if !defined($new->{cpu});
 482
 483     my $cpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $new->{cpu});
 484     return if !$cpu || !$cpu->{cputype}; # always allow default
 485     my $cputype = $cpu->{cputype};
 486
 487     if ($existing && $existing->{cpu}) {
 488         # changing only other settings doesn't require permissions for CPU model
 489         my $existingCpu = PVE::JSONSchema::check_format('pve-vm-cpu-conf', $existing->{cpu});
 490         return if $existingCpu->{cputype} eq $cputype;
 491     }
 492
 493     if (PVE::QemuServer::CPUConfig::is_custom_model($cputype)) {
 494         $rpcenv->check($authuser, "/nodes", ['Sys.Audit']);
 495     }
 496 };
 497
 498 my $cpuoptions = {
 499     'cores' => 1,
 500     'cpu' => 1,
 501     'cpulimit' => 1,
 502     'cpuunits' => 1,
 503     'numa' => 1,
 504     'smp' => 1,
 505     'sockets' => 1,
 506     'vcpus' => 1,
 507 };
 508
 509 my $memoryoptions = {
 510     'memory' => 1,
 511     'balloon' => 1,
 512     'shares' => 1,
 513 };
 514
 515 my $hwtypeoptions = {
 516     'acpi' => 1,
 517     'hotplug' => 1,
 518     'kvm' => 1,
 519     'machine' => 1,
 520     'scsihw' => 1,
 521     'smbios1' => 1,
 522     'tablet' => 1,
 523     'vga' => 1,
 524     'watchdog' => 1,
 525     'audio0' => 1,
 526 };
 527
 528 my $generaloptions = {
 529     'agent' => 1,
 530     'autostart' => 1,
 531     'bios' => 1,
 532     'description' => 1,
 533     'keyboard' => 1,
 534     'localtime' => 1,
 535     'migrate_downtime' => 1,
 536     'migrate_speed' => 1,
 537     'name' => 1,
 538     'onboot' => 1,
 539     'ostype' => 1,
 540     'protection' => 1,
 541     'reboot' => 1,
 542     'startdate' => 1,
 543     'startup' => 1,
 544     'tdf' => 1,
 545     'template' => 1,
 546 };
 547
 548 my $vmpoweroptions = {
 549     'freeze' => 1,
 550 };
 551
 552 my $diskoptions = {
 553     'boot' => 1,
 554     'bootdisk' => 1,
 555     'vmstatestorage' => 1,
 556 };
 557
 558 my $cloudinitoptions = {
 559     cicustom => 1,
 560     cipassword => 1,
 561     citype => 1,
 562     ciuser => 1,
 563     nameserver => 1,
 564     searchdomain => 1,
 565     sshkeys => 1,
 566 };
 567
 568 my $check_vm_create_serial_perm = sub {
 569     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 570
 571     return 1 if $authuser eq 'root@pam';
 572
 573     foreach my $opt (keys %{$param}) {
 574         next if $opt !~ m/^serial\d+$/;
 575
 576         if ($param->{$opt} eq 'socket') {
 577             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 578         } else {
 579             die "only root can set '$opt' config for real devices\n";
 580         }
 581     }
 582
 583     return 1;
 584 };
 585
 586 my $check_vm_create_usb_perm = sub {
 587     my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
 588
 589     return 1 if $authuser eq 'root@pam';
 590
 591     foreach my $opt (keys %{$param}) {
 592         next if $opt !~ m/^usb\d+$/;
 593
 594         if ($param->{$opt} =~ m/spice/) {
 595             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 596         } else {
 597             die "only root can set '$opt' config for real devices\n";
 598         }
 599     }
 600
 601     return 1;
 602 };
 603
 604 my $check_vm_modify_config_perm = sub {
 605     my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
 606
 607     return 1 if $authuser eq 'root@pam';
 608
 609     foreach my $opt (@$key_list) {
 610         # some checks (e.g., disk, serial port, usb) need to be done somewhere
 611         # else, as there the permission can be value dependend
 612         next if PVE::QemuServer::is_valid_drivename($opt);
 613         next if $opt eq 'cdrom';
 614         next if $opt =~ m/^(?:unused|serial|usb)\d+$/;
 615         next if $opt eq 'tags';
 616
 617
 618         if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) {
 619             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.CPU']);
 620         } elsif ($memoryoptions->{$opt}) {
 621             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Memory']);
 622         } elsif ($hwtypeoptions->{$opt}) {
 623             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
 624         } elsif ($generaloptions->{$opt}) {
 625             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Options']);
 626             # special case for startup since it changes host behaviour
 627             if ($opt eq 'startup') {
 628                 $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
 629             }
 630         } elsif ($vmpoweroptions->{$opt}) {
 631             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']);
 632         } elsif ($diskoptions->{$opt}) {
 633             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
 634         } elsif ($opt =~ m/^(?:net|ipconfig)\d+$/) {
 635             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
 636         } elsif ($cloudinitoptions->{$opt}) {
 637             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1);
 638         } elsif ($opt eq 'vmstate') {
 639             # the user needs Disk and PowerMgmt privileges to change the vmstate
 640             # also needs privileges on the storage, that will be checked later
 641             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]);
 642         } else {
 643             # catches hostpci\d+, args, lock, etc.
 644             # new options will be checked here
 645             die "only root can set '$opt' config\n";
 646         }
 647     }
 648
 649     return 1;
 650 };
 651
 652 __PACKAGE__->register_method({
 653     name => 'vmlist',
 654     path => '',
 655     method => 'GET',
 656     description => "Virtual machine index (per node).",
 657     permissions => {
 658         description => "Only list VMs where you have VM.Audit permissons on /vms/<vmid>.",
 659         user => 'all',
 660     },
 661     proxyto => 'node',
 662     protected => 1, # qemu pid files are only readable by root
 663     parameters => {
 664         additionalProperties => 0,
 665         properties => {
 666             node => get_standard_option('pve-node'),
 667             full => {
 668                 type => 'boolean',
 669                 optional => 1,
 670                 description => "Determine the full status of active VMs.",
 671             },
 672         },
 673     },
 674     returns => {
 675         type => 'array',
 676         items => {
 677             type => "object",
 678             properties => $PVE::QemuServer::vmstatus_return_properties,
 679         },
 680         links => [ { rel => 'child', href => "{vmid}" } ],
 681     },
 682     code => sub {
 683         my ($param) = @_;
 684
 685         my $rpcenv = PVE::RPCEnvironment::get();
 686         my $authuser = $rpcenv->get_user();
 687
 688         my $vmstatus = PVE::QemuServer::vmstatus(undef, $param->{full});
 689
 690         my $res = [];
 691         foreach my $vmid (keys %$vmstatus) {
 692             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
 693
 694             my $data = $vmstatus->{$vmid};
 695             push @$res, $data;
 696         }
 697
 698         return $res;
 699     }});
 700
 701 my $parse_restore_archive = sub {
 702     my ($storecfg, $archive) = @_;
 703
 704     my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1);
 705
 706     if (defined($archive_storeid)) {
 707         my $scfg =  PVE::Storage::storage_config($storecfg, $archive_storeid);
 708         if ($scfg->{type} eq 'pbs') {
 709             return {
 710                 type => 'pbs',
 711                 volid => $archive,
 712             };
 713         }
 714     }
 715     my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive);
 716     return {
 717         type => 'file',
 718         path => $path,
 719     };
 720 };
 721
 722
 723 __PACKAGE__->register_method({
 724     name => 'create_vm',
 725     path => '',
 726     method => 'POST',
 727     description => "Create or restore a virtual machine.",
 728     permissions => {
 729         description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
 730             "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
 731             "If you create disks you need 'Datastore.AllocateSpace' on any used storage.",
 732         user => 'all', # check inside
 733     },
 734     protected => 1,
 735     proxyto => 'node',
 736     parameters => {
 737         additionalProperties => 0,
 738         properties => PVE::QemuServer::json_config_properties(
 739             {
 740                 node => get_standard_option('pve-node'),
 741                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
 742                 archive => {
 743                     description => "The backup archive. Either the file system path to a .tar or .vma file (use '-' to pipe data from stdin) or a proxmox storage backup volume identifier.",
 744                     type => 'string',
 745                     optional => 1,
 746                     maxLength => 255,
 747                     completion => \&PVE::QemuServer::complete_backup_archives,
 748                 },
 749                 storage => get_standard_option('pve-storage-id', {
 750                     description => "Default storage.",
 751                     optional => 1,
 752                     completion => \&PVE::QemuServer::complete_storage,
 753                 }),
 754                 force => {
 755                     optional => 1,
 756                     type => 'boolean',
 757                     description => "Allow to overwrite existing VM.",
 758                     requires => 'archive',
 759                 },
 760                 unique => {
 761                     optional => 1,
 762                     type => 'boolean',
 763                     description => "Assign a unique random ethernet address.",
 764                     requires => 'archive',
 765                 },
 766                 'live-restore' => {
 767                     optional => 1,
 768                     type => 'boolean',
 769                     description => "Start the VM immediately from the backup and restore in background. PBS only.",
 770                     requires => 'archive',
 771                 },
 772                 pool => {
 773                     optional => 1,
 774                     type => 'string', format => 'pve-poolid',
 775                     description => "Add the VM to the specified pool.",
 776                 },
 777                 bwlimit => {
 778                     description => "Override I/O bandwidth limit (in KiB/s).",
 779                     optional => 1,
 780                     type => 'integer',
 781                     minimum => '0',
 782                     default => 'restore limit from datacenter or storage config',
 783                 },
 784                 start => {
 785                     optional => 1,
 786                     type => 'boolean',
 787                     default => 0,
 788                     description => "Start VM after it was created successfully.",
 789                 },
 790             },
 791             1, # with_disk_alloc
 792         ),
 793     },
 794     returns => {
 795         type => 'string',
 796     },
 797     code => sub {
 798         my ($param) = @_;
 799
 800         my $rpcenv = PVE::RPCEnvironment::get();
 801         my $authuser = $rpcenv->get_user();
 802
 803         my $node = extract_param($param, 'node');
 804         my $vmid = extract_param($param, 'vmid');
 805
 806         my $archive = extract_param($param, 'archive');
 807         my $is_restore = !!$archive;
 808
 809         my $bwlimit = extract_param($param, 'bwlimit');
 810         my $force = extract_param($param, 'force');
 811         my $pool = extract_param($param, 'pool');
 812         my $start_after_create = extract_param($param, 'start');
 813         my $storage = extract_param($param, 'storage');
 814         my $unique = extract_param($param, 'unique');
 815         my $live_restore = extract_param($param, 'live-restore');
 816
 817         if (defined(my $ssh_keys = $param->{sshkeys})) {
 818                 $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
 819                 PVE::Tools::validate_ssh_public_keys($ssh_keys);
 820         }
 821
 822         $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
 823             if defined($param->{cpuunits}); # clamp value depending on cgroup version
 824
 825         PVE::Cluster::check_cfs_quorum();
 826
 827         my $filename = PVE::QemuConfig->config_file($vmid);
 828         my $storecfg = PVE::Storage::config();
 829
 830         if (defined($pool)) {
 831             $rpcenv->check_pool_exist($pool);
 832         }
 833
 834         $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace'])
 835             if defined($storage);
 836
 837         if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
 838             # OK
 839         } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
 840             # OK
 841         } elsif ($archive && $force && (-f $filename) &&
 842                  $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
 843             # OK: user has VM.Backup permissions, and want to restore an existing VM
 844         } else {
 845             raise_perm_exc();
 846         }
 847
 848         if ($archive) {
 849             for my $opt (sort keys $param->%*) {
 850                 if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
 851                     raise_param_exc({ $opt => "option conflicts with option 'archive'" });
 852                 }
 853             }
 854
 855             if ($archive eq '-') {
 856                 die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli';
 857                 $archive = { type => 'pipe' };
 858             } else {
 859                 PVE::Storage::check_volume_access(
 860                     $rpcenv,
 861                     $authuser,
 862                     $storecfg,
 863                     $vmid,
 864                     $archive,
 865                     'backup',
 866                 );
 867
 868                 $archive = $parse_restore_archive->($storecfg, $archive);
 869             }
 870         }
 871
 872         if (scalar(keys $param->%*) > 0) {
 873             &$resolve_cdrom_alias($param);
 874
 875             &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage);
 876
 877             &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]);
 878
 879             &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param);
 880             &$check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param);
 881
 882             &$check_cpu_model_access($rpcenv, $authuser, $param);
 883
 884             $check_drive_param->($param, $storecfg);
 885
 886             PVE::QemuServer::add_random_macs($param);
 887         }
 888
 889         my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -";
 890
 891         eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) };
 892         die "$emsg $@" if $@;
 893
 894         my $restored_data = 0;
 895         my $restorefn = sub {
 896             my $conf = PVE::QemuConfig->load_config($vmid);
 897
 898             PVE::QemuConfig->check_protection($conf, $emsg);
 899
 900             die "$emsg vm is running\n" if PVE::QemuServer::check_running($vmid);
 901
 902             my $realcmd = sub {
 903                 my $restore_options = {
 904                     storage => $storage,
 905                     pool => $pool,
 906                     unique => $unique,
 907                     bwlimit => $bwlimit,
 908                     live => $live_restore,
 909                     override_conf => $param,
 910                 };
 911                 if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') {
 912                     die "live-restore is only compatible with backup images from a Proxmox Backup Server\n"
 913                         if $live_restore;
 914                     PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options);
 915                 } elsif ($archive->{type} eq 'pbs') {
 916                     PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options);
 917                 } else {
 918                     die "unknown backup archive type\n";
 919                 }
 920                 $restored_data = 1;
 921
 922                 my $restored_conf = PVE::QemuConfig->load_config($vmid);
 923                 # Convert restored VM to template if backup was VM template
 924                 if (PVE::QemuConfig->is_template($restored_conf)) {
 925                     warn "Convert to template.\n";
 926                     eval { PVE::QemuServer::template_create($vmid, $restored_conf) };
 927                     warn $@ if $@;
 928                 }
 929             };
 930
 931             # ensure no old replication state are exists
 932             PVE::ReplicationState::delete_guest_states($vmid);
 933
 934             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
 935
 936             if ($start_after_create && !$live_restore) {
 937                 print "Execute autostart\n";
 938                 eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) };
 939                 warn $@ if $@;
 940             }
 941         };
 942
 943         my $createfn = sub {
 944             # ensure no old replication state are exists
 945             PVE::ReplicationState::delete_guest_states($vmid);
 946
 947             my $realcmd = sub {
 948                 my $conf = $param;
 949                 my $arch = PVE::QemuServer::get_vm_arch($conf);
 950
 951                 $conf->{meta} = PVE::QemuServer::new_meta_info_string();
 952
 953                 my $vollist = [];
 954                 eval {
 955                     ($vollist, my $created_opts) = $create_disks->(
 956                         $rpcenv,
 957                         $authuser,
 958                         $conf,
 959                         $arch,
 960                         $storecfg,
 961                         $vmid,
 962                         $pool,
 963                         $param,
 964                         $storage,
 965                     );
 966                     $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*;
 967
 968                     if (!$conf->{boot}) {
 969                         my $devs = PVE::QemuServer::get_default_bootdevices($conf);
 970                         $conf->{boot} = PVE::QemuServer::print_bootorder($devs);
 971                     }
 972
 973                     # auto generate uuid if user did not specify smbios1 option
 974                     if (!$conf->{smbios1}) {
 975                         $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid();
 976                     }
 977
 978                     if ((!defined($conf->{vmgenid}) || $conf->{vmgenid} eq '1') && $arch ne 'aarch64') {
 979                         $conf->{vmgenid} = PVE::QemuServer::generate_uuid();
 980                     }
 981
 982                     my $machine = $conf->{machine};
 983                     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
 984                         # always pin Windows' machine version on create, they get to easily confused
 985                         if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) {
 986                             $conf->{machine} = PVE::QemuServer::windows_get_pinned_machine_version($machine);
 987                         }
 988                     }
 989
 990                     PVE::QemuConfig->write_config($vmid, $conf);
 991
 992                 };
 993                 my $err = $@;
 994
 995                 if ($err) {
 996                     foreach my $volid (@$vollist) {
 997                         eval { PVE::Storage::vdisk_free($storecfg, $volid); };
 998                         warn $@ if $@;
 999                     }
1000                     die "$emsg $err";
1001                 }
1002
1003                 PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
1004             };
1005
1006             PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
1007
1008             if ($start_after_create) {
1009                 print "Execute autostart\n";
1010                 eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) };
1011                 warn $@ if $@;
1012             }
1013         };
1014
1015         my ($code, $worker_name);
1016         if ($is_restore) {
1017             $worker_name = 'qmrestore';
1018             $code = sub {
1019                 eval { $restorefn->() };
1020                 if (my $err = $@) {
1021                     eval { PVE::QemuConfig->remove_lock($vmid, 'create') };
1022                     warn $@ if $@;
1023                     if ($restored_data) {
1024                         warn "error after data was restored, VM disks should be OK but config may "
1025                             ."require adaptions. VM $vmid state is NOT cleaned up.\n";
1026                     } else {
1027                         warn "error before or during data restore, some or all disks were not "
1028                             ."completely restored. VM $vmid state is NOT cleaned up.\n";
1029                     }
1030                     die $err;
1031                 }
1032             };
1033         } else {
1034             $worker_name = 'qmcreate';
1035             $code = sub {
1036                 eval { $createfn->() };
1037                 if (my $err = $@) {
1038                     eval {
1039                         my $conffile = PVE::QemuConfig->config_file($vmid);
1040                         unlink($conffile) or die "failed to remove config file: $!\n";
1041                     };
1042                     warn $@ if $@;
1043                     die $err;
1044                 }
1045             };
1046         }
1047
1048         return $rpcenv->fork_worker($worker_name, $vmid, $authuser, $code);
1049     }});
1050
1051 __PACKAGE__->register_method({
1052     name => 'vmdiridx',
1053     path => '{vmid}',
1054     method => 'GET',
1055     proxyto => 'node',
1056     description => "Directory index",
1057     permissions => {
1058         user => 'all',
1059     },
1060     parameters => {
1061         additionalProperties => 0,
1062         properties => {
1063             node => get_standard_option('pve-node'),
1064             vmid => get_standard_option('pve-vmid'),
1065         },
1066     },
1067     returns => {
1068         type => 'array',
1069         items => {
1070             type => "object",
1071             properties => {
1072                 subdir => { type => 'string' },
1073             },
1074         },
1075         links => [ { rel => 'child', href => "{subdir}" } ],
1076     },
1077     code => sub {
1078         my ($param) = @_;
1079
1080         my $res = [
1081             { subdir => 'config' },
1082             { subdir => 'cloudinit' },
1083             { subdir => 'pending' },
1084             { subdir => 'status' },
1085             { subdir => 'unlink' },
1086             { subdir => 'vncproxy' },
1087             { subdir => 'termproxy' },
1088             { subdir => 'migrate' },
1089             { subdir => 'resize' },
1090             { subdir => 'move' },
1091             { subdir => 'rrd' },
1092             { subdir => 'rrddata' },
1093             { subdir => 'monitor' },
1094             { subdir => 'agent' },
1095             { subdir => 'snapshot' },
1096             { subdir => 'spiceproxy' },
1097             { subdir => 'sendkey' },
1098             { subdir => 'firewall' },
1099             { subdir => 'mtunnel' },
1100             { subdir => 'remote_migrate' },
1101         ];
1102
1103         return $res;
1104     }});
1105
1106 __PACKAGE__->register_method ({
1107     subclass => "PVE::API2::Firewall::VM",
1108     path => '{vmid}/firewall',
1109 });
1110
1111 __PACKAGE__->register_method ({
1112     subclass => "PVE::API2::Qemu::Agent",
1113     path => '{vmid}/agent',
1114 });
1115
1116 __PACKAGE__->register_method({
1117     name => 'rrd',
1118     path => '{vmid}/rrd',
1119     method => 'GET',
1120     protected => 1, # fixme: can we avoid that?
1121     permissions => {
1122         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1123     },
1124     description => "Read VM RRD statistics (returns PNG)",
1125     parameters => {
1126         additionalProperties => 0,
1127         properties => {
1128             node => get_standard_option('pve-node'),
1129             vmid => get_standard_option('pve-vmid'),
1130             timeframe => {
1131                 description => "Specify the time frame you are interested in.",
1132                 type => 'string',
1133                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1134             },
1135             ds => {
1136                 description => "The list of datasources you want to display.",
1137                 type => 'string', format => 'pve-configid-list',
1138             },
1139             cf => {
1140                 description => "The RRD consolidation function",
1141                 type => 'string',
1142                 enum => [ 'AVERAGE', 'MAX' ],
1143                 optional => 1,
1144             },
1145         },
1146     },
1147     returns => {
1148         type => "object",
1149         properties => {
1150             filename => { type => 'string' },
1151         },
1152     },
1153     code => sub {
1154         my ($param) = @_;
1155
1156         return PVE::RRD::create_rrd_graph(
1157             "pve2-vm/$param->{vmid}", $param->{timeframe},
1158             $param->{ds}, $param->{cf});
1159
1160     }});
1161
1162 __PACKAGE__->register_method({
1163     name => 'rrddata',
1164     path => '{vmid}/rrddata',
1165     method => 'GET',
1166     protected => 1, # fixme: can we avoid that?
1167     permissions => {
1168         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1169     },
1170     description => "Read VM RRD statistics",
1171     parameters => {
1172         additionalProperties => 0,
1173         properties => {
1174             node => get_standard_option('pve-node'),
1175             vmid => get_standard_option('pve-vmid'),
1176             timeframe => {
1177                 description => "Specify the time frame you are interested in.",
1178                 type => 'string',
1179                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
1180             },
1181             cf => {
1182                 description => "The RRD consolidation function",
1183                 type => 'string',
1184                 enum => [ 'AVERAGE', 'MAX' ],
1185                 optional => 1,
1186             },
1187         },
1188     },
1189     returns => {
1190         type => "array",
1191         items => {
1192             type => "object",
1193             properties => {},
1194         },
1195     },
1196     code => sub {
1197         my ($param) = @_;
1198
1199         return PVE::RRD::create_rrd_data(
1200             "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
1201     }});
1202
1203
1204 __PACKAGE__->register_method({
1205     name => 'vm_config',
1206     path => '{vmid}/config',
1207     method => 'GET',
1208     proxyto => 'node',
1209     description => "Get the virtual machine configuration with pending configuration " .
1210         "changes applied. Set the 'current' parameter to get the current configuration instead.",
1211     permissions => {
1212         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1213     },
1214     parameters => {
1215         additionalProperties => 0,
1216         properties => {
1217             node => get_standard_option('pve-node'),
1218             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1219             current => {
1220                 description => "Get current values (instead of pending values).",
1221                 optional => 1,
1222                 default => 0,
1223                 type => 'boolean',
1224             },
1225             snapshot => get_standard_option('pve-snapshot-name', {
1226                 description => "Fetch config values from given snapshot.",
1227                 optional => 1,
1228                 completion => sub {
1229                     my ($cmd, $pname, $cur, $args) = @_;
1230                     PVE::QemuConfig->snapshot_list($args->[0]);
1231                 },
1232             }),
1233         },
1234     },
1235     returns => {
1236         description => "The VM configuration.",
1237         type => "object",
1238         properties => PVE::QemuServer::json_config_properties({
1239             digest => {
1240                 type => 'string',
1241                 description => 'SHA1 digest of configuration file. This can be used to prevent concurrent modifications.',
1242             }
1243         }),
1244     },
1245     code => sub {
1246         my ($param) = @_;
1247
1248         raise_param_exc({ snapshot => "cannot use 'snapshot' parameter with 'current'",
1249                           current => "cannot use 'snapshot' parameter with 'current'"})
1250             if ($param->{snapshot} && $param->{current});
1251
1252         my $conf;
1253         if ($param->{snapshot}) {
1254             $conf = PVE::QemuConfig->load_snapshot_config($param->{vmid}, $param->{snapshot});
1255         } else {
1256             $conf = PVE::QemuConfig->load_current_config($param->{vmid}, $param->{current});
1257         }
1258         $conf->{cipassword} = '**********' if $conf->{cipassword};
1259         return $conf;
1260
1261     }});
1262
1263 __PACKAGE__->register_method({
1264     name => 'vm_pending',
1265     path => '{vmid}/pending',
1266     method => 'GET',
1267     proxyto => 'node',
1268     description => "Get the virtual machine configuration with both current and pending values.",
1269     permissions => {
1270         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1271     },
1272     parameters => {
1273         additionalProperties => 0,
1274         properties => {
1275             node => get_standard_option('pve-node'),
1276             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1277         },
1278     },
1279     returns => {
1280         type => "array",
1281         items => {
1282             type => "object",
1283             properties => {
1284                 key => {
1285                     description => "Configuration option name.",
1286                     type => 'string',
1287                 },
1288                 value => {
1289                     description => "Current value.",
1290                     type => 'string',
1291                     optional => 1,
1292                 },
1293                 pending => {
1294                     description => "Pending value.",
1295                     type => 'string',
1296                     optional => 1,
1297                 },
1298                 delete => {
1299                     description => "Indicates a pending delete request if present and not 0. " .
1300                                    "The value 2 indicates a force-delete request.",
1301                     type => 'integer',
1302                     minimum => 0,
1303                     maximum => 2,
1304                     optional => 1,
1305                 },
1306             },
1307         },
1308     },
1309     code => sub {
1310         my ($param) = @_;
1311
1312         my $conf = PVE::QemuConfig->load_config($param->{vmid});
1313
1314         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
1315
1316         $conf->{cipassword} = '**********' if defined($conf->{cipassword});
1317         $conf->{pending}->{cipassword} = '********** ' if defined($conf->{pending}->{cipassword});
1318
1319         return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
1320    }});
1321
1322 __PACKAGE__->register_method({
1323     name => 'cloudinit_pending',
1324     path => '{vmid}/cloudinit',
1325     method => 'GET',
1326     proxyto => 'node',
1327     description => "Get the cloudinit configuration with both current and pending values.",
1328     permissions => {
1329         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1330     },
1331     parameters => {
1332         additionalProperties => 0,
1333         properties => {
1334             node => get_standard_option('pve-node'),
1335             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1336         },
1337     },
1338     returns => {
1339         type => "array",
1340         items => {
1341             type => "object",
1342             properties => {
1343                 key => {
1344                     description => "Configuration option name.",
1345                     type => 'string',
1346                 },
1347                 old => {
1348                     description => "Value as it was used to generate the current cloudinit image.",
1349                     type => 'string',
1350                     optional => 1,
1351                 },
1352                 new => {
1353                     description => "The new pending value.",
1354                     type => 'string',
1355                     optional => 1,
1356                 },
1357             },
1358         },
1359     },
1360     code => sub {
1361         my ($param) = @_;
1362
1363         my $vmid = $param->{vmid};
1364         my $conf = PVE::QemuConfig->load_config($vmid);
1365
1366         my $ci = $conf->{cloudinit};
1367
1368         my $res = {};
1369         my $added = delete($ci->{added}) // '';
1370         for my $key (PVE::Tools::split_list($added)) {
1371             $res->{$key} = { new => $conf->{$key} };
1372         }
1373
1374         for my $key (keys %$ci) {
1375             if (!exists($conf->{$key})) {
1376                 $res->{$key} = { old => $ci->{$key} };
1377             } else {
1378                 $res->{$key} = {
1379                     old => $ci->{$key},
1380                     new => $conf->{$key},
1381                 };
1382             }
1383         }
1384
1385         if (defined(my $pw = $res->{cipassword})) {
1386             $pw->{old} = '**********' if exists $pw->{old};
1387             $pw->{new} = '**********' if exists $pw->{new};
1388         }
1389
1390         return $res;
1391    }});
1392
1393 __PACKAGE__->register_method({
1394     name => 'cloudinit_update',
1395     path => '{vmid}/cloudinit',
1396     method => 'PUT',
1397     protected => 1,
1398     proxyto => 'node',
1399     description => "Regenerate and change cloudinit config drive.",
1400     permissions => {
1401         check => ['perm', '/vms/{vmid}', 'VM.Config.Cloudinit'],
1402     },
1403     parameters => {
1404         additionalProperties => 0,
1405         properties => {
1406             node => get_standard_option('pve-node'),
1407             vmid => get_standard_option('pve-vmid'),
1408         },
1409     },
1410     returns => { type => 'null' },
1411     code => sub {
1412         my ($param) = @_;
1413
1414         my $rpcenv = PVE::RPCEnvironment::get();
1415         my $authuser = $rpcenv->get_user();
1416
1417         my $vmid = extract_param($param, 'vmid');
1418
1419         PVE::QemuConfig->lock_config($vmid, sub {
1420             my $conf = PVE::QemuConfig->load_config($vmid);
1421             PVE::QemuConfig->check_lock($conf);
1422
1423             my $storecfg = PVE::Storage::config();
1424             PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
1425         });
1426         return;
1427     }});
1428
1429 # POST/PUT {vmid}/config implementation
1430 #
1431 # The original API used PUT (idempotent) an we assumed that all operations
1432 # are fast. But it turned out that almost any configuration change can
1433 # involve hot-plug actions, or disk alloc/free. Such actions can take long
1434 # time to complete and have side effects (not idempotent).
1435 #
1436 # The new implementation uses POST and forks a worker process. We added
1437 # a new option 'background_delay'. If specified we wait up to
1438 # 'background_delay' second for the worker task to complete. It returns null
1439 # if the task is finished within that time, else we return the UPID.
1440
1441 my $update_vm_api  = sub {
1442     my ($param, $sync) = @_;
1443
1444     my $rpcenv = PVE::RPCEnvironment::get();
1445
1446     my $authuser = $rpcenv->get_user();
1447
1448     my $node = extract_param($param, 'node');
1449
1450     my $vmid = extract_param($param, 'vmid');
1451
1452     my $digest = extract_param($param, 'digest');
1453
1454     my $background_delay = extract_param($param, 'background_delay');
1455
1456     my $skip_cloud_init = extract_param($param, 'skip_cloud_init');
1457
1458     if (defined(my $cipassword = $param->{cipassword})) {
1459         # Same logic as in cloud-init (but with the regex fixed...)
1460         $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
1461             if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
1462     }
1463
1464     my @paramarr = (); # used for log message
1465     foreach my $key (sort keys %$param) {
1466         my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
1467         push @paramarr, "-$key", $value;
1468     }
1469
1470     my $skiplock = extract_param($param, 'skiplock');
1471     raise_param_exc({ skiplock => "Only root may use this option." })
1472         if $skiplock && $authuser ne 'root@pam';
1473
1474     my $delete_str = extract_param($param, 'delete');
1475
1476     my $revert_str = extract_param($param, 'revert');
1477
1478     my $force = extract_param($param, 'force');
1479
1480     if (defined(my $ssh_keys = $param->{sshkeys})) {
1481         $ssh_keys = URI::Escape::uri_unescape($ssh_keys);
1482         PVE::Tools::validate_ssh_public_keys($ssh_keys);
1483     }
1484
1485     $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
1486         if defined($param->{cpuunits}); # clamp value depending on cgroup version
1487
1488     die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param);
1489
1490     my $storecfg = PVE::Storage::config();
1491
1492     my $defaults = PVE::QemuServer::load_defaults();
1493
1494     &$resolve_cdrom_alias($param);
1495
1496     # now try to verify all parameters
1497
1498     my $revert = {};
1499     foreach my $opt (PVE::Tools::split_list($revert_str)) {
1500         if (!PVE::QemuServer::option_exists($opt)) {
1501             raise_param_exc({ revert => "unknown option '$opt'" });
1502         }
1503
1504         raise_param_exc({ delete => "you can't use '-$opt' and " .
1505                               "-revert $opt' at the same time" })
1506             if defined($param->{$opt});
1507
1508         $revert->{$opt} = 1;
1509     }
1510
1511     my @delete = ();
1512     foreach my $opt (PVE::Tools::split_list($delete_str)) {
1513         $opt = 'ide2' if $opt eq 'cdrom';
1514
1515         raise_param_exc({ delete => "you can't use '-$opt' and " .
1516                               "-delete $opt' at the same time" })
1517             if defined($param->{$opt});
1518
1519         raise_param_exc({ revert => "you can't use '-delete $opt' and " .
1520                               "-revert $opt' at the same time" })
1521             if $revert->{$opt};
1522
1523         if (!PVE::QemuServer::option_exists($opt)) {
1524             raise_param_exc({ delete => "unknown option '$opt'" });
1525         }
1526
1527         push @delete, $opt;
1528     }
1529
1530     my $repl_conf = PVE::ReplicationConfig->new();
1531     my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
1532     my $check_replication = sub {
1533         my ($drive) = @_;
1534         return if !$is_replicated;
1535         my $volid = $drive->{file};
1536         return if !$volid || !($drive->{replicate}//1);
1537         return if PVE::QemuServer::drive_is_cdrom($drive);
1538
1539         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1540         die "cannot add non-managed/pass-through volume to a replicated VM\n"
1541             if !defined($storeid);
1542
1543         return if defined($volname) && $volname eq 'cloudinit';
1544
1545         my $format;
1546         if ($volid =~ $NEW_DISK_RE) {
1547             $storeid = $2;
1548             $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid);
1549         } else {
1550             $format = (PVE::Storage::parse_volname($storecfg, $volid))[6];
1551         }
1552         return if PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
1553         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1554         return if $scfg->{shared};
1555         die "cannot add non-replicatable volume to a replicated VM\n";
1556     };
1557
1558     $check_drive_param->($param, $storecfg, $check_replication);
1559
1560     foreach my $opt (keys %$param) {
1561         if ($opt =~ m/^net(\d+)$/) {
1562             # add macaddr
1563             my $net = PVE::QemuServer::parse_net($param->{$opt});
1564             $param->{$opt} = PVE::QemuServer::print_net($net);
1565         } elsif ($opt eq 'vmgenid') {
1566             if ($param->{$opt} eq '1') {
1567                 $param->{$opt} = PVE::QemuServer::generate_uuid();
1568             }
1569         } elsif ($opt eq 'hookscript') {
1570             eval { PVE::GuestHelpers::check_hookscript($param->{$opt}, $storecfg); };
1571             raise_param_exc({ $opt => $@ }) if $@;
1572         }
1573     }
1574
1575     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
1576
1577     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
1578
1579     &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
1580
1581     my $updatefn =  sub {
1582
1583         my $conf = PVE::QemuConfig->load_config($vmid);
1584
1585         die "checksum missmatch (file change by other user?)\n"
1586             if $digest && $digest ne $conf->{digest};
1587
1588         &$check_cpu_model_access($rpcenv, $authuser, $param, $conf);
1589
1590         # FIXME: 'suspended' lock should probabyl be a state or "weak" lock?!
1591         if (scalar(@delete) && grep { $_ eq 'vmstate'} @delete) {
1592             if (defined($conf->{lock}) && $conf->{lock} eq 'suspended') {
1593                 delete $conf->{lock}; # for check lock check, not written out
1594                 push @delete, 'lock'; # this is the real deal to write it out
1595             }
1596             push @delete, 'runningmachine' if $conf->{runningmachine};
1597             push @delete, 'runningcpu' if $conf->{runningcpu};
1598         }
1599
1600         PVE::QemuConfig->check_lock($conf) if !$skiplock;
1601
1602         foreach my $opt (keys %$revert) {
1603             if (defined($conf->{$opt})) {
1604                 $param->{$opt} = $conf->{$opt};
1605             } elsif (defined($conf->{pending}->{$opt})) {
1606                 push @delete, $opt;
1607             }
1608         }
1609
1610         if ($param->{memory} || defined($param->{balloon})) {
1611             my $maxmem = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory} || $defaults->{memory};
1612             my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon};
1613
1614             die "balloon value too large (must be smaller than assigned memory)\n"
1615                 if $balloon && $balloon > $maxmem;
1616         }
1617
1618         PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: " . join (' ', @paramarr));
1619
1620         my $worker = sub {
1621
1622             print "update VM $vmid: " . join (' ', @paramarr) . "\n";
1623
1624             # write updates to pending section
1625
1626             my $modified = {}; # record what $option we modify
1627
1628             my @bootorder;
1629             if (my $boot = $conf->{boot}) {
1630                 my $bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot);
1631                 @bootorder = PVE::Tools::split_list($bootcfg->{order}) if $bootcfg && $bootcfg->{order};
1632             }
1633             my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete;
1634
1635             my $check_drive_perms = sub {
1636                 my ($opt, $val) = @_;
1637                 my $drive = PVE::QemuServer::parse_drive($opt, $val, 1);
1638                 if (PVE::QemuServer::drive_is_cloudinit($drive)) {
1639                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM']);
1640                 } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM
1641                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
1642                 } else {
1643                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1644
1645                 }
1646             };
1647
1648             foreach my $opt (@delete) {
1649                 $modified->{$opt} = 1;
1650                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1651
1652                 # value of what we want to delete, independent if pending or not
1653                 my $val = $conf->{$opt} // $conf->{pending}->{$opt};
1654                 if (!defined($val)) {
1655                     warn "cannot delete '$opt' - not set in current configuration!\n";
1656                     $modified->{$opt} = 0;
1657                     next;
1658                 }
1659                 my $is_pending_val = defined($conf->{pending}->{$opt});
1660                 delete $conf->{pending}->{$opt};
1661
1662                 # remove from bootorder if necessary
1663                 if (!$bootorder_deleted && @bootorder && grep {$_ eq $opt} @bootorder) {
1664                     @bootorder = grep {$_ ne $opt} @bootorder;
1665                     $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1666                     $modified->{boot} = 1;
1667                 }
1668
1669                 if ($opt =~ m/^unused/) {
1670                     my $drive = PVE::QemuServer::parse_drive($opt, $val);
1671                     PVE::QemuConfig->check_protection($conf, "can't remove unused disk '$drive->{file}'");
1672                     $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
1673                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser)) {
1674                         delete $conf->{$opt};
1675                         PVE::QemuConfig->write_config($vmid, $conf);
1676                     }
1677                 } elsif ($opt eq 'vmstate') {
1678                     PVE::QemuConfig->check_protection($conf, "can't remove vmstate '$val'");
1679                     if (PVE::QemuServer::try_deallocate_drive($storecfg, $vmid, $conf, $opt, { file => $val }, $rpcenv, $authuser, 1)) {
1680                         delete $conf->{$opt};
1681                         PVE::QemuConfig->write_config($vmid, $conf);
1682                     }
1683                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
1684                     PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'");
1685                     $check_drive_perms->($opt, $val);
1686                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val))
1687                         if $is_pending_val;
1688                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1689                     PVE::QemuConfig->write_config($vmid, $conf);
1690                 } elsif ($opt =~ m/^serial\d+$/) {
1691                     if ($val eq 'socket') {
1692                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1693                     } elsif ($authuser ne 'root@pam') {
1694                         die "only root can delete '$opt' config for real devices\n";
1695                     }
1696                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1697                     PVE::QemuConfig->write_config($vmid, $conf);
1698                 } elsif ($opt =~ m/^usb\d+$/) {
1699                     if ($val =~ m/spice/) {
1700                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1701                     } elsif ($authuser ne 'root@pam') {
1702                         die "only root can delete '$opt' config for real devices\n";
1703                     }
1704                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1705                     PVE::QemuConfig->write_config($vmid, $conf);
1706                 } elsif ($opt eq 'tags') {
1707                     assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser);
1708                     delete $conf->{$opt};
1709                     PVE::QemuConfig->write_config($vmid, $conf);
1710                 } else {
1711                     PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
1712                     PVE::QemuConfig->write_config($vmid, $conf);
1713                 }
1714             }
1715
1716             foreach my $opt (keys %$param) { # add/change
1717                 $modified->{$opt} = 1;
1718                 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1719                 next if defined($conf->{pending}->{$opt}) && ($param->{$opt} eq $conf->{pending}->{$opt}); # skip if nothing changed
1720
1721                 my $arch = PVE::QemuServer::get_vm_arch($conf);
1722
1723                 if (PVE::QemuServer::is_valid_drivename($opt)) {
1724                     # old drive
1725                     if ($conf->{$opt}) {
1726                         $check_drive_perms->($opt, $conf->{$opt});
1727                     }
1728
1729                     # new drive
1730                     $check_drive_perms->($opt, $param->{$opt});
1731                     PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
1732                         if defined($conf->{pending}->{$opt});
1733
1734                     my (undef, $created_opts) = $create_disks->(
1735                         $rpcenv,
1736                         $authuser,
1737                         $conf,
1738                         $arch,
1739                         $storecfg,
1740                         $vmid,
1741                         undef,
1742                         {$opt => $param->{$opt}},
1743                     );
1744                     $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*;
1745
1746                     # default legacy boot order implies all cdroms anyway
1747                     if (@bootorder) {
1748                         # append new CD drives to bootorder to mark them bootable
1749                         my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
1750                         if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) {
1751                             push @bootorder, $opt;
1752                             $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
1753                             $modified->{boot} = 1;
1754                         }
1755                     }
1756                 } elsif ($opt =~ m/^serial\d+/) {
1757                     if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
1758                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1759                     } elsif ($authuser ne 'root@pam') {
1760                         die "only root can modify '$opt' config for real devices\n";
1761                     }
1762                     $conf->{pending}->{$opt} = $param->{$opt};
1763                 } elsif ($opt =~ m/^usb\d+/) {
1764                     if ((!defined($conf->{$opt}) || $conf->{$opt} =~ m/spice/) && $param->{$opt} =~ m/spice/) {
1765                         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
1766                     } elsif ($authuser ne 'root@pam') {
1767                         die "only root can modify '$opt' config for real devices\n";
1768                     }
1769                     $conf->{pending}->{$opt} = $param->{$opt};
1770                 } elsif ($opt eq 'tags') {
1771                     assert_tag_permissions($vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser);
1772                     $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt});
1773                 } else {
1774                     $conf->{pending}->{$opt} = $param->{$opt};
1775
1776                     if ($opt eq 'boot') {
1777                         my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $param->{$opt});
1778                         if ($new_bootcfg->{order}) {
1779                             my @devs = PVE::Tools::split_list($new_bootcfg->{order});
1780                             for my $dev (@devs) {
1781                                 my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev};
1782                                 my $deleted = grep {$_ eq $dev} @delete;
1783                                 die "invalid bootorder: device '$dev' does not exist'\n"
1784                                     if !$exists || $deleted;
1785                             }
1786
1787                             # remove legacy boot order settings if new one set
1788                             $conf->{pending}->{$opt} = PVE::QemuServer::print_bootorder(\@devs);
1789                             PVE::QemuConfig->add_to_pending_delete($conf, "bootdisk")
1790                                 if $conf->{bootdisk};
1791                         }
1792                     }
1793                 }
1794                 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
1795                 PVE::QemuConfig->write_config($vmid, $conf);
1796             }
1797
1798             # remove pending changes when nothing changed
1799             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1800             my $changes = PVE::QemuConfig->cleanup_pending($conf);
1801             PVE::QemuConfig->write_config($vmid, $conf) if $changes;
1802
1803             return if !scalar(keys %{$conf->{pending}});
1804
1805             my $running = PVE::QemuServer::check_running($vmid);
1806
1807             # apply pending changes
1808
1809             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
1810
1811             my $errors = {};
1812             if ($running) {
1813                 PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors);
1814             } else {
1815                 # cloud_init must be skipped if we are in an incoming, remote live migration
1816                 PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors, $skip_cloud_init);
1817             }
1818             raise_param_exc($errors) if scalar(keys %$errors);
1819
1820             return;
1821         };
1822
1823         if ($sync) {
1824             &$worker();
1825             return;
1826         } else {
1827             my $upid = $rpcenv->fork_worker('qmconfig', $vmid, $authuser, $worker);
1828
1829             if ($background_delay) {
1830
1831                 # Note: It would be better to do that in the Event based HTTPServer
1832                 # to avoid blocking call to sleep.
1833
1834                 my $end_time = time() + $background_delay;
1835
1836                 my $task = PVE::Tools::upid_decode($upid);
1837
1838                 my $running = 1;
1839                 while (time() < $end_time) {
1840                     $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
1841                     last if !$running;
1842                     sleep(1); # this gets interrupted when child process ends
1843                 }
1844
1845                 if (!$running) {
1846                     my $status = PVE::Tools::upid_read_status($upid);
1847                     return if !PVE::Tools::upid_status_is_error($status);
1848                     die "failed to update VM $vmid: $status\n";
1849                 }
1850             }
1851
1852             return $upid;
1853         }
1854     };
1855
1856     return PVE::QemuConfig->lock_config($vmid, $updatefn);
1857 };
1858
1859 my $vm_config_perm_list = [
1860             'VM.Config.Disk',
1861             'VM.Config.CDROM',
1862             'VM.Config.CPU',
1863             'VM.Config.Memory',
1864             'VM.Config.Network',
1865             'VM.Config.HWType',
1866             'VM.Config.Options',
1867             'VM.Config.Cloudinit',
1868     ];
1869
1870 __PACKAGE__->register_method({
1871     name => 'update_vm_async',
1872     path => '{vmid}/config',
1873     method => 'POST',
1874     protected => 1,
1875     proxyto => 'node',
1876     description => "Set virtual machine options (asynchrounous API).",
1877     permissions => {
1878         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1879     },
1880     parameters => {
1881         additionalProperties => 0,
1882         properties => PVE::QemuServer::json_config_properties(
1883             {
1884                 node => get_standard_option('pve-node'),
1885                 vmid => get_standard_option('pve-vmid'),
1886                 skiplock => get_standard_option('skiplock'),
1887                 delete => {
1888                     type => 'string', format => 'pve-configid-list',
1889                     description => "A list of settings you want to delete.",
1890                     optional => 1,
1891                 },
1892                 revert => {
1893                     type => 'string', format => 'pve-configid-list',
1894                     description => "Revert a pending change.",
1895                     optional => 1,
1896                 },
1897                 force => {
1898                     type => 'boolean',
1899                     description => $opt_force_description,
1900                     optional => 1,
1901                     requires => 'delete',
1902                 },
1903                 digest => {
1904                     type => 'string',
1905                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1906                     maxLength => 40,
1907                     optional => 1,
1908                 },
1909                 background_delay => {
1910                     type => 'integer',
1911                     description => "Time to wait for the task to finish. We return 'null' if the task finish within that time.",
1912                     minimum => 1,
1913                     maximum => 30,
1914                     optional => 1,
1915                 },
1916             },
1917             1, # with_disk_alloc
1918         ),
1919     },
1920     returns => {
1921         type => 'string',
1922         optional => 1,
1923     },
1924     code => $update_vm_api,
1925 });
1926
1927 __PACKAGE__->register_method({
1928     name => 'update_vm',
1929     path => '{vmid}/config',
1930     method => 'PUT',
1931     protected => 1,
1932     proxyto => 'node',
1933     description => "Set virtual machine options (synchrounous API) - You should consider using the POST method instead for any actions involving hotplug or storage allocation.",
1934     permissions => {
1935         check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
1936     },
1937     parameters => {
1938         additionalProperties => 0,
1939         properties => PVE::QemuServer::json_config_properties(
1940             {
1941                 node => get_standard_option('pve-node'),
1942                 vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
1943                 skiplock => get_standard_option('skiplock'),
1944                 delete => {
1945                     type => 'string', format => 'pve-configid-list',
1946                     description => "A list of settings you want to delete.",
1947                     optional => 1,
1948                 },
1949                 revert => {
1950                     type => 'string', format => 'pve-configid-list',
1951                     description => "Revert a pending change.",
1952                     optional => 1,
1953                 },
1954                 force => {
1955                     type => 'boolean',
1956                     description => $opt_force_description,
1957                     optional => 1,
1958                     requires => 'delete',
1959                 },
1960                 digest => {
1961                     type => 'string',
1962                     description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1963                     maxLength => 40,
1964                     optional => 1,
1965                 },
1966             },
1967             1, # with_disk_alloc
1968         ),
1969     },
1970     returns => { type => 'null' },
1971     code => sub {
1972         my ($param) = @_;
1973         &$update_vm_api($param, 1);
1974         return;
1975     }
1976 });
1977
1978 __PACKAGE__->register_method({
1979     name => 'destroy_vm',
1980     path => '{vmid}',
1981     method => 'DELETE',
1982     protected => 1,
1983     proxyto => 'node',
1984     description => "Destroy the VM and  all used/owned volumes. Removes any VM specific permissions"
1985         ." and firewall rules",
1986     permissions => {
1987         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
1988     },
1989     parameters => {
1990         additionalProperties => 0,
1991         properties => {
1992             node => get_standard_option('pve-node'),
1993             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
1994             skiplock => get_standard_option('skiplock'),
1995             purge => {
1996                 type => 'boolean',
1997                 description => "Remove VMID from configurations, like backup & replication jobs and HA.",
1998                 optional => 1,
1999             },
2000             'destroy-unreferenced-disks' => {
2001                 type => 'boolean',
2002                 description => "If set, destroy additionally all disks not referenced in the config"
2003                  ." but with a matching VMID from all enabled storages.",
2004                 optional => 1,
2005                 default => 0,
2006             },
2007         },
2008     },
2009     returns => {
2010         type => 'string',
2011     },
2012     code => sub {
2013         my ($param) = @_;
2014
2015         my $rpcenv = PVE::RPCEnvironment::get();
2016         my $authuser = $rpcenv->get_user();
2017         my $vmid = $param->{vmid};
2018
2019         my $skiplock = $param->{skiplock};
2020         raise_param_exc({ skiplock => "Only root may use this option." })
2021             if $skiplock && $authuser ne 'root@pam';
2022
2023         my $early_checks = sub {
2024             # test if VM exists
2025             my $conf = PVE::QemuConfig->load_config($vmid);
2026             PVE::QemuConfig->check_protection($conf, "can't remove VM $vmid");
2027
2028             my $ha_managed = PVE::HA::Config::service_is_configured("vm:$vmid");
2029
2030             if (!$param->{purge}) {
2031                 die "unable to remove VM $vmid - used in HA resources and purge parameter not set.\n"
2032                     if $ha_managed;
2033                 # don't allow destroy if with replication jobs but no purge param
2034                 my $repl_conf = PVE::ReplicationConfig->new();
2035                 $repl_conf->check_for_existing_jobs($vmid);
2036             }
2037
2038             die "VM $vmid is running - destroy failed\n"
2039                 if PVE::QemuServer::check_running($vmid);
2040
2041             return $ha_managed;
2042         };
2043
2044         $early_checks->();
2045
2046         my $realcmd = sub {
2047             my $upid = shift;
2048
2049             my $storecfg = PVE::Storage::config();
2050
2051             syslog('info', "destroy VM $vmid: $upid\n");
2052             PVE::QemuConfig->lock_config($vmid, sub {
2053                 # repeat, config might have changed
2054                 my $ha_managed = $early_checks->();
2055
2056                 my $purge_unreferenced = $param->{'destroy-unreferenced-disks'};
2057
2058                 PVE::QemuServer::destroy_vm(
2059                     $storecfg,
2060                     $vmid,
2061                     $skiplock, { lock => 'destroyed' },
2062                     $purge_unreferenced,
2063                 );
2064
2065                 PVE::AccessControl::remove_vm_access($vmid);
2066                 PVE::Firewall::remove_vmfw_conf($vmid);
2067                 if ($param->{purge}) {
2068                     print "purging VM $vmid from related configurations..\n";
2069                     PVE::ReplicationConfig::remove_vmid_jobs($vmid);
2070                     PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);
2071
2072                     if ($ha_managed) {
2073                         PVE::HA::Config::delete_service_from_config("vm:$vmid");
2074                         print "NOTE: removed VM $vmid from HA resource configuration.\n";
2075                     }
2076                 }
2077
2078                 # only now remove the zombie config, else we can have reuse race
2079                 PVE::QemuConfig->destroy_config($vmid);
2080             });
2081         };
2082
2083         return $rpcenv->fork_worker('qmdestroy', $vmid, $authuser, $realcmd);
2084     }});
2085
2086 __PACKAGE__->register_method({
2087     name => 'unlink',
2088     path => '{vmid}/unlink',
2089     method => 'PUT',
2090     protected => 1,
2091     proxyto => 'node',
2092     description => "Unlink/delete disk images.",
2093     permissions => {
2094         check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']],
2095     },
2096     parameters => {
2097         additionalProperties => 0,
2098         properties => {
2099             node => get_standard_option('pve-node'),
2100             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
2101             idlist => {
2102                 type => 'string', format => 'pve-configid-list',
2103                 description => "A list of disk IDs you want to delete.",
2104             },
2105             force => {
2106                 type => 'boolean',
2107                 description => $opt_force_description,
2108                 optional => 1,
2109             },
2110         },
2111     },
2112     returns => { type => 'null'},
2113     code => sub {
2114         my ($param) = @_;
2115
2116         $param->{delete} = extract_param($param, 'idlist');
2117
2118         __PACKAGE__->update_vm($param);
2119
2120         return;
2121     }});
2122
2123 # uses good entropy, each char is limited to 6 bit to get printable chars simply
2124 my $gen_rand_chars = sub {
2125     my ($length) = @_;
2126
2127     die "invalid length $length" if $length < 1;
2128
2129     my $min = ord('!'); # first printable ascii
2130
2131     my $rand_bytes = Crypt::OpenSSL::Random::random_bytes($length);
2132     die "failed to generate random bytes!\n"
2133       if !$rand_bytes;
2134
2135     my $str = join('', map { chr((ord($_) & 0x3F) + $min) } split('', $rand_bytes));
2136
2137     return $str;
2138 };
2139
2140 my $sslcert;
2141
2142 __PACKAGE__->register_method({
2143     name => 'vncproxy',
2144     path => '{vmid}/vncproxy',
2145     method => 'POST',
2146     protected => 1,
2147     permissions => {
2148         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2149     },
2150     description => "Creates a TCP VNC proxy connections.",
2151     parameters => {
2152         additionalProperties => 0,
2153         properties => {
2154             node => get_standard_option('pve-node'),
2155             vmid => get_standard_option('pve-vmid'),
2156             websocket => {
2157                 optional => 1,
2158                 type => 'boolean',
2159                 description => "starts websockify instead of vncproxy",
2160             },
2161             'generate-password' => {
2162                 optional => 1,
2163                 type => 'boolean',
2164                 default => 0,
2165                 description => "Generates a random password to be used as ticket instead of the API ticket.",
2166             },
2167         },
2168     },
2169     returns => {
2170         additionalProperties => 0,
2171         properties => {
2172             user => { type => 'string' },
2173             ticket => { type => 'string' },
2174             password => {
2175                 optional => 1,
2176                 description => "Returned if requested with 'generate-password' param."
2177                     ." Consists of printable ASCII characters ('!' .. '~').",
2178                 type => 'string',
2179             },
2180             cert => { type => 'string' },
2181             port => { type => 'integer' },
2182             upid => { type => 'string' },
2183         },
2184     },
2185     code => sub {
2186         my ($param) = @_;
2187
2188         my $rpcenv = PVE::RPCEnvironment::get();
2189
2190         my $authuser = $rpcenv->get_user();
2191
2192         my $vmid = $param->{vmid};
2193         my $node = $param->{node};
2194         my $websocket = $param->{websocket};
2195
2196         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2197
2198         my $serial;
2199         if ($conf->{vga}) {
2200             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2201             $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2202         }
2203
2204         my $authpath = "/vms/$vmid";
2205
2206         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2207         my $password = $ticket;
2208         if ($param->{'generate-password'}) {
2209             $password = $gen_rand_chars->(8);
2210         }
2211
2212         $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
2213             if !$sslcert;
2214
2215         my $family;
2216         my $remcmd = [];
2217
2218         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2219             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2220             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2221             # NOTE: kvm VNC traffic is already TLS encrypted or is known unsecure
2222             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, defined($serial) ? '-t' : '-T');
2223         } else {
2224             $family = PVE::Tools::get_host_address_family($node);
2225         }
2226
2227         my $port = PVE::Tools::next_vnc_port($family);
2228
2229         my $timeout = 10;
2230
2231         my $realcmd = sub {
2232             my $upid = shift;
2233
2234             syslog('info', "starting vnc proxy $upid\n");
2235
2236             my $cmd;
2237
2238             if (defined($serial)) {
2239
2240                 my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-iface', $serial, '-escape', '0' ];
2241
2242                 $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
2243                         '-timeout', $timeout, '-authpath', $authpath,
2244                         '-perm', 'Sys.Console'];
2245
2246                 if ($param->{websocket}) {
2247                     $ENV{PVE_VNC_TICKET} = $password; # pass ticket to vncterm
2248                     push @$cmd, '-notls', '-listen', 'localhost';
2249                 }
2250
2251                 push @$cmd, '-c', @$remcmd, @$termcmd;
2252
2253                 PVE::Tools::run_command($cmd);
2254
2255             } else {
2256
2257                 $ENV{LC_PVE_TICKET} = $password if $websocket; # set ticket with "qm vncproxy"
2258
2259                 $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid];
2260
2261                 my $sock = IO::Socket::IP->new(
2262                     ReuseAddr => 1,
2263                     Listen => 1,
2264                     LocalPort => $port,
2265                     Proto => 'tcp',
2266                     GetAddrInfoFlags => 0,
2267                 ) or die "failed to create socket: $!\n";
2268                 # Inside the worker we shouldn't have any previous alarms
2269                 # running anyway...:
2270                 alarm(0);
2271                 local $SIG{ALRM} = sub { die "connection timed out\n" };
2272                 alarm $timeout;
2273                 accept(my $cli, $sock) or die "connection failed: $!\n";
2274                 alarm(0);
2275                 close($sock);
2276                 if (PVE::Tools::run_command($cmd,
2277                     output => '>&'.fileno($cli),
2278                     input => '<&'.fileno($cli),
2279                     noerr => 1) != 0)
2280                 {
2281                     die "Failed to run vncproxy.\n";
2282                 }
2283             }
2284
2285             return;
2286         };
2287
2288         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2289
2290         PVE::Tools::wait_for_vnc_port($port);
2291
2292         my $res = {
2293             user => $authuser,
2294             ticket => $ticket,
2295             port => $port,
2296             upid => $upid,
2297             cert => $sslcert,
2298         };
2299         $res->{password} = $password if $param->{'generate-password'};
2300
2301         return $res;
2302     }});
2303
2304 __PACKAGE__->register_method({
2305     name => 'termproxy',
2306     path => '{vmid}/termproxy',
2307     method => 'POST',
2308     protected => 1,
2309     permissions => {
2310         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2311     },
2312     description => "Creates a TCP proxy connections.",
2313     parameters => {
2314         additionalProperties => 0,
2315         properties => {
2316             node => get_standard_option('pve-node'),
2317             vmid => get_standard_option('pve-vmid'),
2318             serial=> {
2319                 optional => 1,
2320                 type => 'string',
2321                 enum => [qw(serial0 serial1 serial2 serial3)],
2322                 description => "opens a serial terminal (defaults to display)",
2323             },
2324         },
2325     },
2326     returns => {
2327         additionalProperties => 0,
2328         properties => {
2329             user => { type => 'string' },
2330             ticket => { type => 'string' },
2331             port => { type => 'integer' },
2332             upid => { type => 'string' },
2333         },
2334     },
2335     code => sub {
2336         my ($param) = @_;
2337
2338         my $rpcenv = PVE::RPCEnvironment::get();
2339
2340         my $authuser = $rpcenv->get_user();
2341
2342         my $vmid = $param->{vmid};
2343         my $node = $param->{node};
2344         my $serial = $param->{serial};
2345
2346         my $conf = PVE::QemuConfig->load_config($vmid, $node); # check if VM exists
2347
2348         if (!defined($serial)) {
2349             if ($conf->{vga}) {
2350                 my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2351                 $serial = $vga->{type} if $vga->{type} =~ m/^serial\d+$/;
2352             }
2353         }
2354
2355         my $authpath = "/vms/$vmid";
2356
2357         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
2358
2359         my $family;
2360         my $remcmd = [];
2361
2362         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
2363             (undef, $family) = PVE::Cluster::remote_node_ip($node);
2364             my $sshinfo = PVE::SSHInfo::get_ssh_info($node);
2365             $remcmd = PVE::SSHInfo::ssh_info_to_command($sshinfo, '-t');
2366             push @$remcmd, '--';
2367         } else {
2368             $family = PVE::Tools::get_host_address_family($node);
2369         }
2370
2371         my $port = PVE::Tools::next_vnc_port($family);
2372
2373         my $termcmd = [ '/usr/sbin/qm', 'terminal', $vmid, '-escape', '0'];
2374         push @$termcmd, '-iface', $serial if $serial;
2375
2376         my $realcmd = sub {
2377             my $upid = shift;
2378
2379             syslog('info', "starting qemu termproxy $upid\n");
2380
2381             my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
2382                        '--perm', 'VM.Console', '--'];
2383             push @$cmd, @$remcmd, @$termcmd;
2384
2385             PVE::Tools::run_command($cmd);
2386         };
2387
2388         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
2389
2390         PVE::Tools::wait_for_vnc_port($port);
2391
2392         return {
2393             user => $authuser,
2394             ticket => $ticket,
2395             port => $port,
2396             upid => $upid,
2397         };
2398     }});
2399
2400 __PACKAGE__->register_method({
2401     name => 'vncwebsocket',
2402     path => '{vmid}/vncwebsocket',
2403     method => 'GET',
2404     permissions => {
2405         description => "You also need to pass a valid ticket (vncticket).",
2406         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2407     },
2408     description => "Opens a weksocket for VNC traffic.",
2409     parameters => {
2410         additionalProperties => 0,
2411         properties => {
2412             node => get_standard_option('pve-node'),
2413             vmid => get_standard_option('pve-vmid'),
2414             vncticket => {
2415                 description => "Ticket from previous call to vncproxy.",
2416                 type => 'string',
2417                 maxLength => 512,
2418             },
2419             port => {
2420                 description => "Port number returned by previous vncproxy call.",
2421                 type => 'integer',
2422                 minimum => 5900,
2423                 maximum => 5999,
2424             },
2425         },
2426     },
2427     returns => {
2428         type => "object",
2429         properties => {
2430             port => { type => 'string' },
2431         },
2432     },
2433     code => sub {
2434         my ($param) = @_;
2435
2436         my $rpcenv = PVE::RPCEnvironment::get();
2437
2438         my $authuser = $rpcenv->get_user();
2439
2440         my $vmid = $param->{vmid};
2441         my $node = $param->{node};
2442
2443         my $authpath = "/vms/$vmid";
2444
2445         PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);
2446
2447         my $conf = PVE::QemuConfig->load_config($vmid, $node); # VM exists ?
2448
2449         # Note: VNC ports are acessible from outside, so we do not gain any
2450         # security if we verify that $param->{port} belongs to VM $vmid. This
2451         # check is done by verifying the VNC ticket (inside VNC protocol).
2452
2453         my $port = $param->{port};
2454
2455         return { port => $port };
2456     }});
2457
2458 __PACKAGE__->register_method({
2459     name => 'spiceproxy',
2460     path => '{vmid}/spiceproxy',
2461     method => 'POST',
2462     protected => 1,
2463     proxyto => 'node',
2464     permissions => {
2465         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
2466     },
2467     description => "Returns a SPICE configuration to connect to the VM.",
2468     parameters => {
2469         additionalProperties => 0,
2470         properties => {
2471             node => get_standard_option('pve-node'),
2472             vmid => get_standard_option('pve-vmid'),
2473             proxy => get_standard_option('spice-proxy', { optional => 1 }),
2474         },
2475     },
2476     returns => get_standard_option('remote-viewer-config'),
2477     code => sub {
2478         my ($param) = @_;
2479
2480         my $rpcenv = PVE::RPCEnvironment::get();
2481
2482         my $authuser = $rpcenv->get_user();
2483
2484         my $vmid = $param->{vmid};
2485         my $node = $param->{node};
2486         my $proxy = $param->{proxy};
2487
2488         my $conf = PVE::QemuConfig->load_config($vmid, $node);
2489         my $title = "VM $vmid";
2490         $title .= " - ". $conf->{name} if $conf->{name};
2491
2492         my $port = PVE::QemuServer::spice_port($vmid);
2493
2494         my ($ticket, undef, $remote_viewer_config) =
2495             PVE::AccessControl::remote_viewer_config($authuser, $vmid, $node, $proxy, $title, $port);
2496
2497         mon_cmd($vmid, "set_password", protocol => 'spice', password => $ticket);
2498         mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
2499
2500         return $remote_viewer_config;
2501     }});
2502
2503 __PACKAGE__->register_method({
2504     name => 'vmcmdidx',
2505     path => '{vmid}/status',
2506     method => 'GET',
2507     proxyto => 'node',
2508     description => "Directory index",
2509     permissions => {
2510         user => 'all',
2511     },
2512     parameters => {
2513         additionalProperties => 0,
2514         properties => {
2515             node => get_standard_option('pve-node'),
2516             vmid => get_standard_option('pve-vmid'),
2517         },
2518     },
2519     returns => {
2520         type => 'array',
2521         items => {
2522             type => "object",
2523             properties => {
2524                 subdir => { type => 'string' },
2525             },
2526         },
2527         links => [ { rel => 'child', href => "{subdir}" } ],
2528     },
2529     code => sub {
2530         my ($param) = @_;
2531
2532         # test if VM exists
2533         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2534
2535         my $res = [
2536             { subdir => 'current' },
2537             { subdir => 'start' },
2538             { subdir => 'stop' },
2539             { subdir => 'reset' },
2540             { subdir => 'shutdown' },
2541             { subdir => 'suspend' },
2542             { subdir => 'reboot' },
2543             ];
2544
2545         return $res;
2546     }});
2547
2548 __PACKAGE__->register_method({
2549     name => 'vm_status',
2550     path => '{vmid}/status/current',
2551     method => 'GET',
2552     proxyto => 'node',
2553     protected => 1, # qemu pid files are only readable by root
2554     description => "Get virtual machine status.",
2555     permissions => {
2556         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
2557     },
2558     parameters => {
2559         additionalProperties => 0,
2560         properties => {
2561             node => get_standard_option('pve-node'),
2562             vmid => get_standard_option('pve-vmid'),
2563         },
2564     },
2565     returns => {
2566         type => 'object',
2567         properties => {
2568             %$PVE::QemuServer::vmstatus_return_properties,
2569             ha => {
2570                 description => "HA manager service status.",
2571                 type => 'object',
2572             },
2573             spice => {
2574                 description => "Qemu VGA configuration supports spice.",
2575                 type => 'boolean',
2576                 optional => 1,
2577             },
2578             agent => {
2579                 description => "Qemu GuestAgent enabled in config.",
2580                 type => 'boolean',
2581                 optional => 1,
2582             },
2583         },
2584     },
2585     code => sub {
2586         my ($param) = @_;
2587
2588         # test if VM exists
2589         my $conf = PVE::QemuConfig->load_config($param->{vmid});
2590
2591         my $vmstatus = PVE::QemuServer::vmstatus($param->{vmid}, 1);
2592         my $status = $vmstatus->{$param->{vmid}};
2593
2594         $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}");
2595
2596         if ($conf->{vga}) {
2597             my $vga = PVE::QemuServer::parse_vga($conf->{vga});
2598             my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/;
2599             $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga});
2600             $status->{spice} = 1 if $spice;
2601         }
2602         $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled');
2603
2604         return $status;
2605     }});
2606
2607 __PACKAGE__->register_method({
2608     name => 'vm_start',
2609     path => '{vmid}/status/start',
2610     method => 'POST',
2611     protected => 1,
2612     proxyto => 'node',
2613     description => "Start virtual machine.",
2614     permissions => {
2615         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2616     },
2617     parameters => {
2618         additionalProperties => 0,
2619         properties => {
2620             node => get_standard_option('pve-node'),
2621             vmid => get_standard_option('pve-vmid',
2622                                         { completion => \&PVE::QemuServer::complete_vmid_stopped }),
2623             skiplock => get_standard_option('skiplock'),
2624             stateuri => get_standard_option('pve-qm-stateuri'),
2625             migratedfrom => get_standard_option('pve-node',{ optional => 1 }),
2626             migration_type => {
2627                 type => 'string',
2628                 enum => ['secure', 'insecure'],
2629                 description => "Migration traffic is encrypted using an SSH " .
2630                   "tunnel by default. On secure, completely private networks " .
2631                   "this can be disabled to increase performance.",
2632                 optional => 1,
2633             },
2634             migration_network => {
2635                 type => 'string', format => 'CIDR',
2636                 description => "CIDR of the (sub) network that is used for migration.",
2637                 optional => 1,
2638             },
2639             machine => get_standard_option('pve-qemu-machine'),
2640             'force-cpu' => {
2641                 description => "Override QEMU's -cpu argument with the given string.",
2642                 type => 'string',
2643                 optional => 1,
2644             },
2645             targetstorage => get_standard_option('pve-targetstorage'),
2646             timeout => {
2647                 description => "Wait maximal timeout seconds.",
2648                 type => 'integer',
2649                 minimum => 0,
2650                 default => 'max(30, vm memory in GiB)',
2651                 optional => 1,
2652             },
2653         },
2654     },
2655     returns => {
2656         type => 'string',
2657     },
2658     code => sub {
2659         my ($param) = @_;
2660
2661         my $rpcenv = PVE::RPCEnvironment::get();
2662         my $authuser = $rpcenv->get_user();
2663
2664         my $node = extract_param($param, 'node');
2665         my $vmid = extract_param($param, 'vmid');
2666         my $timeout = extract_param($param, 'timeout');
2667         my $machine = extract_param($param, 'machine');
2668
2669         my $get_root_param = sub {
2670             my $value = extract_param($param, $_[0]);
2671             raise_param_exc({ "$_[0]" => "Only root may use this option." })
2672                 if $value && $authuser ne 'root@pam';
2673             return $value;
2674         };
2675
2676         my $stateuri = $get_root_param->('stateuri');
2677         my $skiplock = $get_root_param->('skiplock');
2678         my $migratedfrom = $get_root_param->('migratedfrom');
2679         my $migration_type = $get_root_param->('migration_type');
2680         my $migration_network = $get_root_param->('migration_network');
2681         my $targetstorage = $get_root_param->('targetstorage');
2682         my $force_cpu = $get_root_param->('force-cpu');
2683
2684         my $storagemap;
2685
2686         if ($targetstorage) {
2687             raise_param_exc({ targetstorage => "targetstorage can only by used with migratedfrom." })
2688                 if !$migratedfrom;
2689             $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
2690             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
2691                 if $@;
2692         }
2693
2694         # read spice ticket from STDIN
2695         my $spice_ticket;
2696         my $nbd_protocol_version = 0;
2697         my $replicated_volumes = {};
2698         my $offline_volumes = {};
2699         if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) {
2700             while (defined(my $line = <STDIN>)) {
2701                 chomp $line;
2702                 if ($line =~ m/^spice_ticket: (.+)$/) {
2703                     $spice_ticket = $1;
2704                 } elsif ($line =~ m/^nbd_protocol_version: (\d+)$/) {
2705                     $nbd_protocol_version = $1;
2706                 } elsif ($line =~ m/^replicated_volume: (.*)$/) {
2707                     $replicated_volumes->{$1} = 1;
2708                 } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead
2709                     $offline_volumes->{tpmstate0} = $1;
2710                 } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) {
2711                     $offline_volumes->{$1} = $2;
2712                 } elsif (!$spice_ticket) {
2713                     # fallback for old source node
2714                     $spice_ticket = $line;
2715                 } else {
2716                     warn "unknown 'start' parameter on STDIN: '$line'\n";
2717                 }
2718             }
2719         }
2720
2721         PVE::Cluster::check_cfs_quorum();
2722
2723         my $storecfg = PVE::Storage::config();
2724
2725         if (PVE::HA::Config::vm_is_ha_managed($vmid) && !$stateuri &&  $rpcenv->{type} ne 'ha') {
2726             my $hacmd = sub {
2727                 my $upid = shift;
2728
2729                 print "Requesting HA start for VM $vmid\n";
2730
2731                 my $cmd = ['ha-manager', 'set',  "vm:$vmid", '--state', 'started'];
2732                 PVE::Tools::run_command($cmd);
2733                 return;
2734             };
2735
2736             return $rpcenv->fork_worker('hastart', $vmid, $authuser, $hacmd);
2737
2738         } else {
2739
2740             my $realcmd = sub {
2741                 my $upid = shift;
2742
2743                 syslog('info', "start VM $vmid: $upid\n");
2744
2745                 my $migrate_opts = {
2746                     migratedfrom => $migratedfrom,
2747                     spice_ticket => $spice_ticket,
2748                     network => $migration_network,
2749                     type => $migration_type,
2750                     storagemap => $storagemap,
2751                     nbd_proto_version => $nbd_protocol_version,
2752                     replicated_volumes => $replicated_volumes,
2753                     offline_volumes => $offline_volumes,
2754                 };
2755
2756                 my $params = {
2757                     statefile => $stateuri,
2758                     skiplock => $skiplock,
2759                     forcemachine => $machine,
2760                     timeout => $timeout,
2761                     forcecpu => $force_cpu,
2762                 };
2763
2764                 PVE::QemuServer::vm_start($storecfg, $vmid, $params, $migrate_opts);
2765                 return;
2766             };
2767
2768             return $rpcenv->fork_worker('qmstart', $vmid, $authuser, $realcmd);
2769         }
2770     }});
2771
2772 __PACKAGE__->register_method({
2773     name => 'vm_stop',
2774     path => '{vmid}/status/stop',
2775     method => 'POST',
2776     protected => 1,
2777     proxyto => 'node',
2778     description => "Stop virtual machine. The qemu process will exit immediately. This" .
2779         "is akin to pulling the power plug of a running computer and may damage the VM data",
2780     permissions => {
2781         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2782     },
2783     parameters => {
2784         additionalProperties => 0,
2785         properties => {
2786             node => get_standard_option('pve-node'),
2787             vmid => get_standard_option('pve-vmid',
2788                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2789             skiplock => get_standard_option('skiplock'),
2790             migratedfrom => get_standard_option('pve-node', { optional => 1 }),
2791             timeout => {
2792                 description => "Wait maximal timeout seconds.",
2793                 type => 'integer',
2794                 minimum => 0,
2795                 optional => 1,
2796             },
2797             keepActive => {
2798                 description => "Do not deactivate storage volumes.",
2799                 type => 'boolean',
2800                 optional => 1,
2801                 default => 0,
2802             }
2803         },
2804     },
2805     returns => {
2806         type => 'string',
2807     },
2808     code => sub {
2809         my ($param) = @_;
2810
2811         my $rpcenv = PVE::RPCEnvironment::get();
2812         my $authuser = $rpcenv->get_user();
2813
2814         my $node = extract_param($param, 'node');
2815         my $vmid = extract_param($param, 'vmid');
2816
2817         my $skiplock = extract_param($param, 'skiplock');
2818         raise_param_exc({ skiplock => "Only root may use this option." })
2819             if $skiplock && $authuser ne 'root@pam';
2820
2821         my $keepActive = extract_param($param, 'keepActive');
2822         raise_param_exc({ keepActive => "Only root may use this option." })
2823             if $keepActive && $authuser ne 'root@pam';
2824
2825         my $migratedfrom = extract_param($param, 'migratedfrom');
2826         raise_param_exc({ migratedfrom => "Only root may use this option." })
2827             if $migratedfrom && $authuser ne 'root@pam';
2828
2829
2830         my $storecfg = PVE::Storage::config();
2831
2832         if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) {
2833
2834             my $hacmd = sub {
2835                 my $upid = shift;
2836
2837                 print "Requesting HA stop for VM $vmid\n";
2838
2839                 my $cmd = ['ha-manager', 'crm-command', 'stop',  "vm:$vmid", '0'];
2840                 PVE::Tools::run_command($cmd);
2841                 return;
2842             };
2843
2844             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
2845
2846         } else {
2847             my $realcmd = sub {
2848                 my $upid = shift;
2849
2850                 syslog('info', "stop VM $vmid: $upid\n");
2851
2852                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0,
2853                                          $param->{timeout}, 0, 1, $keepActive, $migratedfrom);
2854                 return;
2855             };
2856
2857             return $rpcenv->fork_worker('qmstop', $vmid, $authuser, $realcmd);
2858         }
2859     }});
2860
2861 __PACKAGE__->register_method({
2862     name => 'vm_reset',
2863     path => '{vmid}/status/reset',
2864     method => 'POST',
2865     protected => 1,
2866     proxyto => 'node',
2867     description => "Reset virtual machine.",
2868     permissions => {
2869         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2870     },
2871     parameters => {
2872         additionalProperties => 0,
2873         properties => {
2874             node => get_standard_option('pve-node'),
2875             vmid => get_standard_option('pve-vmid',
2876                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2877             skiplock => get_standard_option('skiplock'),
2878         },
2879     },
2880     returns => {
2881         type => 'string',
2882     },
2883     code => sub {
2884         my ($param) = @_;
2885
2886         my $rpcenv = PVE::RPCEnvironment::get();
2887
2888         my $authuser = $rpcenv->get_user();
2889
2890         my $node = extract_param($param, 'node');
2891
2892         my $vmid = extract_param($param, 'vmid');
2893
2894         my $skiplock = extract_param($param, 'skiplock');
2895         raise_param_exc({ skiplock => "Only root may use this option." })
2896             if $skiplock && $authuser ne 'root@pam';
2897
2898         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
2899
2900         my $realcmd = sub {
2901             my $upid = shift;
2902
2903             PVE::QemuServer::vm_reset($vmid, $skiplock);
2904
2905             return;
2906         };
2907
2908         return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd);
2909     }});
2910
2911 __PACKAGE__->register_method({
2912     name => 'vm_shutdown',
2913     path => '{vmid}/status/shutdown',
2914     method => 'POST',
2915     protected => 1,
2916     proxyto => 'node',
2917     description => "Shutdown virtual machine. This is similar to pressing the power button on a physical machine." .
2918         "This will send an ACPI event for the guest OS, which should then proceed to a clean shutdown.",
2919     permissions => {
2920         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
2921     },
2922     parameters => {
2923         additionalProperties => 0,
2924         properties => {
2925             node => get_standard_option('pve-node'),
2926             vmid => get_standard_option('pve-vmid',
2927                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
2928             skiplock => get_standard_option('skiplock'),
2929             timeout => {
2930                 description => "Wait maximal timeout seconds.",
2931                 type => 'integer',
2932                 minimum => 0,
2933                 optional => 1,
2934             },
2935             forceStop => {
2936                 description => "Make sure the VM stops.",
2937                 type => 'boolean',
2938                 optional => 1,
2939                 default => 0,
2940             },
2941             keepActive => {
2942                 description => "Do not deactivate storage volumes.",
2943                 type => 'boolean',
2944                 optional => 1,
2945                 default => 0,
2946             }
2947         },
2948     },
2949     returns => {
2950         type => 'string',
2951     },
2952     code => sub {
2953         my ($param) = @_;
2954
2955         my $rpcenv = PVE::RPCEnvironment::get();
2956         my $authuser = $rpcenv->get_user();
2957
2958         my $node = extract_param($param, 'node');
2959         my $vmid = extract_param($param, 'vmid');
2960
2961         my $skiplock = extract_param($param, 'skiplock');
2962         raise_param_exc({ skiplock => "Only root may use this option." })
2963             if $skiplock && $authuser ne 'root@pam';
2964
2965         my $keepActive = extract_param($param, 'keepActive');
2966         raise_param_exc({ keepActive => "Only root may use this option." })
2967             if $keepActive && $authuser ne 'root@pam';
2968
2969         my $storecfg = PVE::Storage::config();
2970
2971         my $shutdown = 1;
2972
2973         # if vm is paused, do not shutdown (but stop if forceStop = 1)
2974         # otherwise, we will infer a shutdown command, but run into the timeout,
2975         # then when the vm is resumed, it will instantly shutdown
2976         #
2977         # checking the qmp status here to get feedback to the gui/cli/api
2978         # and the status query should not take too long
2979         if (PVE::QemuServer::vm_is_paused($vmid)) {
2980             if ($param->{forceStop}) {
2981                 warn "VM is paused - stop instead of shutdown\n";
2982                 $shutdown = 0;
2983             } else {
2984                 die "VM is paused - cannot shutdown\n";
2985             }
2986         }
2987
2988         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
2989
2990             my $timeout = $param->{timeout} // 60;
2991             my $hacmd = sub {
2992                 my $upid = shift;
2993
2994                 print "Requesting HA stop for VM $vmid\n";
2995
2996                 my $cmd = ['ha-manager', 'crm-command', 'stop', "vm:$vmid", "$timeout"];
2997                 PVE::Tools::run_command($cmd);
2998                 return;
2999             };
3000
3001             return $rpcenv->fork_worker('hastop', $vmid, $authuser, $hacmd);
3002
3003         } else {
3004
3005             my $realcmd = sub {
3006                 my $upid = shift;
3007
3008                 syslog('info', "shutdown VM $vmid: $upid\n");
3009
3010                 PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0, $param->{timeout},
3011                                          $shutdown, $param->{forceStop}, $keepActive);
3012                 return;
3013             };
3014
3015             return $rpcenv->fork_worker('qmshutdown', $vmid, $authuser, $realcmd);
3016         }
3017     }});
3018
3019 __PACKAGE__->register_method({
3020     name => 'vm_reboot',
3021     path => '{vmid}/status/reboot',
3022     method => 'POST',
3023     protected => 1,
3024     proxyto => 'node',
3025     description => "Reboot the VM by shutting it down, and starting it again. Applies pending changes.",
3026     permissions => {
3027         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3028     },
3029     parameters => {
3030         additionalProperties => 0,
3031         properties => {
3032             node => get_standard_option('pve-node'),
3033             vmid => get_standard_option('pve-vmid',
3034                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3035             timeout => {
3036                 description => "Wait maximal timeout seconds for the shutdown.",
3037                 type => 'integer',
3038                 minimum => 0,
3039                 optional => 1,
3040             },
3041         },
3042     },
3043     returns => {
3044         type => 'string',
3045     },
3046     code => sub {
3047         my ($param) = @_;
3048
3049         my $rpcenv = PVE::RPCEnvironment::get();
3050         my $authuser = $rpcenv->get_user();
3051
3052         my $node = extract_param($param, 'node');
3053         my $vmid = extract_param($param, 'vmid');
3054
3055         die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid);
3056
3057         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3058
3059         my $realcmd = sub {
3060             my $upid = shift;
3061
3062             syslog('info', "requesting reboot of VM $vmid: $upid\n");
3063             PVE::QemuServer::vm_reboot($vmid, $param->{timeout});
3064             return;
3065         };
3066
3067         return $rpcenv->fork_worker('qmreboot', $vmid, $authuser, $realcmd);
3068     }});
3069
3070 __PACKAGE__->register_method({
3071     name => 'vm_suspend',
3072     path => '{vmid}/status/suspend',
3073     method => 'POST',
3074     protected => 1,
3075     proxyto => 'node',
3076     description => "Suspend virtual machine.",
3077     permissions => {
3078         description => "You need 'VM.PowerMgmt' on /vms/{vmid}, and if you have set 'todisk',".
3079             " you need also 'VM.Config.Disk' on /vms/{vmid} and 'Datastore.AllocateSpace'".
3080             " on the storage for the vmstate.",
3081         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3082     },
3083     parameters => {
3084         additionalProperties => 0,
3085         properties => {
3086             node => get_standard_option('pve-node'),
3087             vmid => get_standard_option('pve-vmid',
3088                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3089             skiplock => get_standard_option('skiplock'),
3090             todisk => {
3091                 type => 'boolean',
3092                 default => 0,
3093                 optional => 1,
3094                 description => 'If set, suspends the VM to disk. Will be resumed on next VM start.',
3095             },
3096             statestorage => get_standard_option('pve-storage-id', {
3097                 description => "The storage for the VM state",
3098                 requires => 'todisk',
3099                 optional => 1,
3100                 completion => \&PVE::Storage::complete_storage_enabled,
3101             }),
3102         },
3103     },
3104     returns => {
3105         type => 'string',
3106     },
3107     code => sub {
3108         my ($param) = @_;
3109
3110         my $rpcenv = PVE::RPCEnvironment::get();
3111         my $authuser = $rpcenv->get_user();
3112
3113         my $node = extract_param($param, 'node');
3114         my $vmid = extract_param($param, 'vmid');
3115
3116         my $todisk = extract_param($param, 'todisk') // 0;
3117
3118         my $statestorage = extract_param($param, 'statestorage');
3119
3120         my $skiplock = extract_param($param, 'skiplock');
3121         raise_param_exc({ skiplock => "Only root may use this option." })
3122             if $skiplock && $authuser ne 'root@pam';
3123
3124         die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
3125
3126         die "Cannot suspend HA managed VM to disk\n"
3127             if $todisk && PVE::HA::Config::vm_is_ha_managed($vmid);
3128
3129         # early check for storage permission, for better user feedback
3130         if ($todisk) {
3131             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
3132             my $conf = PVE::QemuConfig->load_config($vmid);
3133
3134             # cannot save the state of a non-virtualized PCIe device, so resume cannot really work
3135             for my $key (keys %$conf) {
3136                 next if $key !~ /^hostpci\d+/;
3137                 die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the"
3138                     ." possibility to save/restore their internal state\n";
3139             }
3140
3141             if (!$statestorage) {
3142                 # get statestorage from config if none is given
3143                 my $storecfg = PVE::Storage::config();
3144                 $statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
3145             }
3146
3147             $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
3148         }
3149
3150         my $realcmd = sub {
3151             my $upid = shift;
3152
3153             syslog('info', "suspend VM $vmid: $upid\n");
3154
3155             PVE::QemuServer::vm_suspend($vmid, $skiplock, $todisk, $statestorage);
3156
3157             return;
3158         };
3159
3160         my $taskname = $todisk ? 'qmsuspend' : 'qmpause';
3161         return $rpcenv->fork_worker($taskname, $vmid, $authuser, $realcmd);
3162     }});
3163
3164 __PACKAGE__->register_method({
3165     name => 'vm_resume',
3166     path => '{vmid}/status/resume',
3167     method => 'POST',
3168     protected => 1,
3169     proxyto => 'node',
3170     description => "Resume virtual machine.",
3171     permissions => {
3172         check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
3173     },
3174     parameters => {
3175         additionalProperties => 0,
3176         properties => {
3177             node => get_standard_option('pve-node'),
3178             vmid => get_standard_option('pve-vmid',
3179                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3180             skiplock => get_standard_option('skiplock'),
3181             nocheck => { type => 'boolean', optional => 1 },
3182
3183         },
3184     },
3185     returns => {
3186         type => 'string',
3187     },
3188     code => sub {
3189         my ($param) = @_;
3190
3191         my $rpcenv = PVE::RPCEnvironment::get();
3192
3193         my $authuser = $rpcenv->get_user();
3194
3195         my $node = extract_param($param, 'node');
3196
3197         my $vmid = extract_param($param, 'vmid');
3198
3199         my $skiplock = extract_param($param, 'skiplock');
3200         raise_param_exc({ skiplock => "Only root may use this option." })
3201             if $skiplock && $authuser ne 'root@pam';
3202
3203         my $nocheck = extract_param($param, 'nocheck');
3204         raise_param_exc({ nocheck => "Only root may use this option." })
3205             if $nocheck && $authuser ne 'root@pam';
3206
3207         my $to_disk_suspended;
3208         eval {
3209             PVE::QemuConfig->lock_config($vmid, sub {
3210                 my $conf = PVE::QemuConfig->load_config($vmid);
3211                 $to_disk_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
3212             });
3213         };
3214
3215         die "VM $vmid not running\n"
3216             if !$to_disk_suspended && !PVE::QemuServer::check_running($vmid, $nocheck);
3217
3218         my $realcmd = sub {
3219             my $upid = shift;
3220
3221             syslog('info', "resume VM $vmid: $upid\n");
3222
3223             if (!$to_disk_suspended) {
3224                 PVE::QemuServer::vm_resume($vmid, $skiplock, $nocheck);
3225             } else {
3226                 my $storecfg = PVE::Storage::config();
3227                 PVE::QemuServer::vm_start($storecfg, $vmid, { skiplock => $skiplock });
3228             }
3229
3230             return;
3231         };
3232
3233         return $rpcenv->fork_worker('qmresume', $vmid, $authuser, $realcmd);
3234     }});
3235
3236 __PACKAGE__->register_method({
3237     name => 'vm_sendkey',
3238     path => '{vmid}/sendkey',
3239     method => 'PUT',
3240     protected => 1,
3241     proxyto => 'node',
3242     description => "Send key event to virtual machine.",
3243     permissions => {
3244         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
3245     },
3246     parameters => {
3247         additionalProperties => 0,
3248         properties => {
3249             node => get_standard_option('pve-node'),
3250             vmid => get_standard_option('pve-vmid',
3251                                         { completion => \&PVE::QemuServer::complete_vmid_running }),
3252             skiplock => get_standard_option('skiplock'),
3253             key => {
3254                 description => "The key (qemu monitor encoding).",
3255                 type => 'string'
3256             }
3257         },
3258     },
3259     returns => { type => 'null'},
3260     code => sub {
3261         my ($param) = @_;
3262
3263         my $rpcenv = PVE::RPCEnvironment::get();
3264
3265         my $authuser = $rpcenv->get_user();
3266
3267         my $node = extract_param($param, 'node');
3268
3269         my $vmid = extract_param($param, 'vmid');
3270
3271         my $skiplock = extract_param($param, 'skiplock');
3272         raise_param_exc({ skiplock => "Only root may use this option." })
3273             if $skiplock && $authuser ne 'root@pam';
3274
3275         PVE::QemuServer::vm_sendkey($vmid, $skiplock, $param->{key});
3276
3277         return;
3278     }});
3279
3280 __PACKAGE__->register_method({
3281     name => 'vm_feature',
3282     path => '{vmid}/feature',
3283     method => 'GET',
3284     proxyto => 'node',
3285     protected => 1,
3286     description => "Check if feature for virtual machine is available.",
3287     permissions => {
3288         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
3289     },
3290     parameters => {
3291         additionalProperties => 0,
3292         properties => {
3293             node => get_standard_option('pve-node'),
3294             vmid => get_standard_option('pve-vmid'),
3295             feature => {
3296                 description => "Feature to check.",
3297                 type => 'string',
3298                 enum => [ 'snapshot', 'clone', 'copy' ],
3299             },
3300             snapname => get_standard_option('pve-snapshot-name', {
3301                 optional => 1,
3302             }),
3303         },
3304     },
3305     returns => {
3306         type => "object",
3307         properties => {
3308             hasFeature => { type => 'boolean' },
3309             nodes => {
3310                 type => 'array',
3311                 items => { type => 'string' },
3312             }
3313         },
3314     },
3315     code => sub {
3316         my ($param) = @_;
3317
3318         my $node = extract_param($param, 'node');
3319
3320         my $vmid = extract_param($param, 'vmid');
3321
3322         my $snapname = extract_param($param, 'snapname');
3323
3324         my $feature = extract_param($param, 'feature');
3325
3326         my $running = PVE::QemuServer::check_running($vmid);
3327
3328         my $conf = PVE::QemuConfig->load_config($vmid);
3329
3330         if($snapname){
3331             my $snap = $conf->{snapshots}->{$snapname};
3332             die "snapshot '$snapname' does not exist\n" if !defined($snap);
3333             $conf = $snap;
3334         }
3335         my $storecfg = PVE::Storage::config();
3336
3337         my $nodelist = PVE::QemuServer::shared_nodes($conf, $storecfg);
3338         my $hasFeature = PVE::QemuConfig->has_feature($feature, $conf, $storecfg, $snapname, $running);
3339
3340         return {
3341             hasFeature => $hasFeature,
3342             nodes => [ keys %$nodelist ],
3343         };
3344     }});
3345
3346 __PACKAGE__->register_method({
3347     name => 'clone_vm',
3348     path => '{vmid}/clone',
3349     method => 'POST',
3350     protected => 1,
3351     proxyto => 'node',
3352     description => "Create a copy of virtual machine/template.",
3353     permissions => {
3354         description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " .
3355             "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
3356             "'Datastore.AllocateSpace' on any used storage.",
3357         check =>
3358         [ 'and',
3359           ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
3360           [ 'or',
3361             [ 'perm', '/vms/{newid}', ['VM.Allocate']],
3362             [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
3363           ],
3364         ]
3365     },
3366     parameters => {
3367         additionalProperties => 0,
3368         properties => {
3369             node => get_standard_option('pve-node'),
3370             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3371             newid => get_standard_option('pve-vmid', {
3372                 completion => \&PVE::Cluster::complete_next_vmid,
3373                 description => 'VMID for the clone.' }),
3374             name => {
3375                 optional => 1,
3376                 type => 'string', format => 'dns-name',
3377                 description => "Set a name for the new VM.",
3378             },
3379             description => {
3380                 optional => 1,
3381                 type => 'string',
3382                 description => "Description for the new VM.",
3383             },
3384             pool => {
3385                 optional => 1,
3386                 type => 'string', format => 'pve-poolid',
3387                 description => "Add the new VM to the specified pool.",
3388             },
3389             snapname => get_standard_option('pve-snapshot-name', {
3390                 optional => 1,
3391             }),
3392             storage => get_standard_option('pve-storage-id', {
3393                 description => "Target storage for full clone.",
3394                 optional => 1,
3395             }),
3396             'format' => {
3397                 description => "Target format for file storage. Only valid for full clone.",
3398                 type => 'string',
3399                 optional => 1,
3400                 enum => [ 'raw', 'qcow2', 'vmdk'],
3401             },
3402             full => {
3403                 optional => 1,
3404                 type => 'boolean',
3405                 description => "Create a full copy of all disks. This is always done when " .
3406                     "you clone a normal VM. For VM templates, we try to create a linked clone by default.",
3407             },
3408             target => get_standard_option('pve-node', {
3409                 description => "Target node. Only allowed if the original VM is on shared storage.",
3410                 optional => 1,
3411             }),
3412             bwlimit => {
3413                 description => "Override I/O bandwidth limit (in KiB/s).",
3414                 optional => 1,
3415                 type => 'integer',
3416                 minimum => '0',
3417                 default => 'clone limit from datacenter or storage config',
3418             },
3419         },
3420     },
3421     returns => {
3422         type => 'string',
3423     },
3424     code => sub {
3425         my ($param) = @_;
3426
3427         my $rpcenv = PVE::RPCEnvironment::get();
3428         my $authuser = $rpcenv->get_user();
3429
3430         my $node = extract_param($param, 'node');
3431         my $vmid = extract_param($param, 'vmid');
3432         my $newid = extract_param($param, 'newid');
3433         my $pool = extract_param($param, 'pool');
3434
3435         my $snapname = extract_param($param, 'snapname');
3436         my $storage = extract_param($param, 'storage');
3437         my $format = extract_param($param, 'format');
3438         my $target = extract_param($param, 'target');
3439
3440         my $localnode = PVE::INotify::nodename();
3441
3442         if ($target && ($target eq $localnode || $target eq 'localhost')) {
3443             undef $target;
3444         }
3445
3446         my $running = PVE::QemuServer::check_running($vmid) || 0;
3447
3448         my $load_and_check = sub {
3449             $rpcenv->check_pool_exist($pool) if defined($pool);
3450             PVE::Cluster::check_node_exists($target) if $target;
3451
3452             my $storecfg = PVE::Storage::config();
3453
3454             if ($storage) {
3455                 # check if storage is enabled on local node
3456                 PVE::Storage::storage_check_enabled($storecfg, $storage);
3457                 if ($target) {
3458                     # check if storage is available on target node
3459                     PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
3460                     # clone only works if target storage is shared
3461                     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
3462                     die "can't clone to non-shared storage '$storage'\n"
3463                         if !$scfg->{shared};
3464                 }
3465             }
3466
3467             PVE::Cluster::check_cfs_quorum();
3468
3469             my $conf = PVE::QemuConfig->load_config($vmid);
3470             PVE::QemuConfig->check_lock($conf);
3471
3472             my $verify_running = PVE::QemuServer::check_running($vmid) || 0;
3473             die "unexpected state change\n" if $verify_running != $running;
3474
3475             die "snapshot '$snapname' does not exist\n"
3476                 if $snapname && !defined( $conf->{snapshots}->{$snapname});
3477
3478             my $full = $param->{full} // !PVE::QemuConfig->is_template($conf);
3479
3480             die "parameter 'storage' not allowed for linked clones\n"
3481                 if defined($storage) && !$full;
3482
3483             die "parameter 'format' not allowed for linked clones\n"
3484                 if defined($format) && !$full;
3485
3486             my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf;
3487
3488             my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage);
3489
3490             die "can't clone VM to node '$target' (VM uses local storage)\n"
3491                 if $target && !$sharedvm;
3492
3493             my $conffile = PVE::QemuConfig->config_file($newid);
3494             die "unable to create VM $newid: config file already exists\n"
3495                 if -f $conffile;
3496
3497             my $newconf = { lock => 'clone' };
3498             my $drives = {};
3499             my $fullclone = {};
3500             my $vollist = [];
3501
3502             foreach my $opt (keys %$oldconf) {
3503                 my $value = $oldconf->{$opt};
3504
3505                 # do not copy snapshot related info
3506                 next if $opt eq 'snapshots' ||  $opt eq 'parent' || $opt eq 'snaptime' ||
3507                     $opt eq 'vmstate' || $opt eq 'snapstate';
3508
3509                 # no need to copy unused images, because VMID(owner) changes anyways
3510                 next if $opt =~ m/^unused\d+$/;
3511
3512                 die "cannot clone TPM state while VM is running\n"
3513                     if $full && $running && !$snapname && $opt eq 'tpmstate0';
3514
3515                 # always change MAC! address
3516                 if ($opt =~ m/^net(\d+)$/) {
3517                     my $net = PVE::QemuServer::parse_net($value);
3518                     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
3519                     $net->{macaddr} =  PVE::Tools::random_ether_addr($dc->{mac_prefix});
3520                     $newconf->{$opt} = PVE::QemuServer::print_net($net);
3521                 } elsif (PVE::QemuServer::is_valid_drivename($opt)) {
3522                     my $drive = PVE::QemuServer::parse_drive($opt, $value);
3523                     die "unable to parse drive options for '$opt'\n" if !$drive;
3524                     if (PVE::QemuServer::drive_is_cdrom($drive, 1)) {
3525                         $newconf->{$opt} = $value; # simply copy configuration
3526                     } else {
3527                         if ($full || PVE::QemuServer::drive_is_cloudinit($drive)) {
3528                             die "Full clone feature is not supported for drive '$opt'\n"
3529                                 if !PVE::Storage::volume_has_feature($storecfg, 'copy', $drive->{file}, $snapname, $running);
3530                             $fullclone->{$opt} = 1;
3531                         } else {
3532                             # not full means clone instead of copy
3533                             die "Linked clone feature is not supported for drive '$opt'\n"
3534                                 if !PVE::Storage::volume_has_feature($storecfg, 'clone', $drive->{file}, $snapname, $running);
3535                         }
3536                         $drives->{$opt} = $drive;
3537                         next if PVE::QemuServer::drive_is_cloudinit($drive);
3538                         push @$vollist, $drive->{file};
3539                     }
3540                 } else {
3541                     # copy everything else
3542                     $newconf->{$opt} = $value;
3543                 }
3544             }
3545
3546             return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone);
3547         };
3548
3549         my $clonefn = sub {
3550             my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->();
3551             my $storecfg = PVE::Storage::config();
3552
3553             # auto generate a new uuid
3554             my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || '');
3555             $smbios1->{uuid} = PVE::QemuServer::generate_uuid();
3556             $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1);
3557             # auto generate a new vmgenid only if the option was set for template
3558             if ($newconf->{vmgenid}) {
3559                 $newconf->{vmgenid} = PVE::QemuServer::generate_uuid();
3560             }
3561
3562             delete $newconf->{template};
3563
3564             if ($param->{name}) {
3565                 $newconf->{name} = $param->{name};
3566             } else {
3567                 $newconf->{name} = "Copy-of-VM-" . ($oldconf->{name} // $vmid);
3568             }
3569
3570             if ($param->{description}) {
3571                 $newconf->{description} = $param->{description};
3572             }
3573
3574             # create empty/temp config - this fails if VM already exists on other node
3575             # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code
3576             PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n");
3577
3578             PVE::Firewall::clone_vmfw_conf($vmid, $newid);
3579
3580             my $newvollist = [];
3581             my $jobs = {};
3582
3583             eval {
3584                 local $SIG{INT} =
3585                     local $SIG{TERM} =
3586                     local $SIG{QUIT} =
3587                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3588
3589                 PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
3590
3591                 my $bwlimit = extract_param($param, 'bwlimit');
3592
3593                 my $total_jobs = scalar(keys %{$drives});
3594                 my $i = 1;
3595
3596                 foreach my $opt (sort keys %$drives) {
3597                     my $drive = $drives->{$opt};
3598                     my $skipcomplete = ($total_jobs != $i); # finish after last drive
3599                     my $completion = $skipcomplete ? 'skip' : 'complete';
3600
3601                     my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
3602                     my $storage_list = [ $src_sid ];
3603                     push @$storage_list, $storage if defined($storage);
3604                     my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
3605
3606                     my $source_info = {
3607                         vmid => $vmid,
3608                         running => $running,
3609                         drivename => $opt,
3610                         drive => $drive,
3611                         snapname => $snapname,
3612                     };
3613
3614                     my $dest_info = {
3615                         vmid => $newid,
3616                         drivename => $opt,
3617                         storage => $storage,
3618                         format => $format,
3619                     };
3620
3621                     $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf)
3622                         if $opt eq 'efidisk0';
3623
3624                     my $newdrive = PVE::QemuServer::clone_disk(
3625                         $storecfg,
3626                         $source_info,
3627                         $dest_info,
3628                         $fullclone->{$opt},
3629                         $newvollist,
3630                         $jobs,
3631                         $completion,
3632                         $oldconf->{agent},
3633                         $clonelimit,
3634                     );
3635
3636                     $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
3637
3638                     PVE::QemuConfig->write_config($newid, $newconf);
3639                     $i++;
3640                 }
3641
3642                 delete $newconf->{lock};
3643
3644                 # do not write pending changes
3645                 if (my @changes = keys %{$newconf->{pending}}) {
3646                     my $pending = join(',', @changes);
3647                     warn "found pending changes for '$pending', discarding for clone\n";
3648                     delete $newconf->{pending};
3649                 }
3650
3651                 PVE::QemuConfig->write_config($newid, $newconf);
3652
3653                 if ($target) {
3654                     # always deactivate volumes - avoid lvm LVs to be active on several nodes
3655                     PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
3656                     PVE::Storage::deactivate_volumes($storecfg, $newvollist);
3657
3658                     my $newconffile = PVE::QemuConfig->config_file($newid, $target);
3659                     die "Failed to move config to node '$target' - rename failed: $!\n"
3660                         if !rename($conffile, $newconffile);
3661                 }
3662
3663                 PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
3664             };
3665             if (my $err = $@) {
3666                 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
3667                 sleep 1; # some storage like rbd need to wait before release volume - really?
3668
3669                 foreach my $volid (@$newvollist) {
3670                     eval { PVE::Storage::vdisk_free($storecfg, $volid); };
3671                     warn $@ if $@;
3672                 }
3673
3674                 PVE::Firewall::remove_vmfw_conf($newid);
3675
3676                 unlink $conffile; # avoid races -> last thing before die
3677
3678                 die "clone failed: $err";
3679             }
3680
3681             return;
3682         };
3683
3684         # Aquire exclusive lock lock for $newid
3685         my $lock_target_vm = sub {
3686             return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn);
3687         };
3688
3689         my $lock_source_vm = sub {
3690             # exclusive lock if VM is running - else shared lock is enough;
3691             if ($running) {
3692                 return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
3693             } else {
3694                 return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
3695             }
3696         };
3697
3698         $load_and_check->(); # early checks before forking/locking
3699
3700         return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm);
3701     }});
3702
3703 __PACKAGE__->register_method({
3704     name => 'move_vm_disk',
3705     path => '{vmid}/move_disk',
3706     method => 'POST',
3707     protected => 1,
3708     proxyto => 'node',
3709     description => "Move volume to different storage or to a different VM.",
3710     permissions => {
3711         description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
3712             "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
3713             "a disk to another VM, you need the permissions on the target VM as well.",
3714         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
3715     },
3716     parameters => {
3717         additionalProperties => 0,
3718         properties => {
3719             node => get_standard_option('pve-node'),
3720             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
3721             'target-vmid' => get_standard_option('pve-vmid', {
3722                 completion => \&PVE::QemuServer::complete_vmid,
3723                 optional => 1,
3724             }),
3725             disk => {
3726                 type => 'string',
3727                 description => "The disk you want to move.",
3728                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3729             },
3730             storage => get_standard_option('pve-storage-id', {
3731                 description => "Target storage.",
3732                 completion => \&PVE::QemuServer::complete_storage,
3733                 optional => 1,
3734             }),
3735             'format' => {
3736                 type => 'string',
3737                 description => "Target Format.",
3738                 enum => [ 'raw', 'qcow2', 'vmdk' ],
3739                 optional => 1,
3740             },
3741             delete => {
3742                 type => 'boolean',
3743                 description => "Delete the original disk after successful copy. By default the"
3744                     ." original disk is kept as unused disk.",
3745                 optional => 1,
3746                 default => 0,
3747             },
3748             digest => {
3749                 type => 'string',
3750                 description => 'Prevent changes if current configuration file has different SHA1"
3751                     ." digest. This can be used to prevent concurrent modifications.',
3752                 maxLength => 40,
3753                 optional => 1,
3754             },
3755             bwlimit => {
3756                 description => "Override I/O bandwidth limit (in KiB/s).",
3757                 optional => 1,
3758                 type => 'integer',
3759                 minimum => '0',
3760                 default => 'move limit from datacenter or storage config',
3761             },
3762             'target-disk' => {
3763                 type => 'string',
3764                 description => "The config key the disk will be moved to on the target VM"
3765                     ." (for example, ide0 or scsi1). Default is the source disk key.",
3766                 enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
3767                 optional => 1,
3768             },
3769             'target-digest' => {
3770                 type => 'string',
3771                 description => 'Prevent changes if the current config file of the target VM has a"
3772                     ." different SHA1 digest. This can be used to detect concurrent modifications.',
3773                 maxLength => 40,
3774                 optional => 1,
3775             },
3776         },
3777     },
3778     returns => {
3779         type => 'string',
3780         description => "the task ID.",
3781     },
3782     code => sub {
3783         my ($param) = @_;
3784
3785         my $rpcenv = PVE::RPCEnvironment::get();
3786         my $authuser = $rpcenv->get_user();
3787
3788         my $node = extract_param($param, 'node');
3789         my $vmid = extract_param($param, 'vmid');
3790         my $target_vmid = extract_param($param, 'target-vmid');
3791         my $digest = extract_param($param, 'digest');
3792         my $target_digest = extract_param($param, 'target-digest');
3793         my $disk = extract_param($param, 'disk');
3794         my $target_disk = extract_param($param, 'target-disk') // $disk;
3795         my $storeid = extract_param($param, 'storage');
3796         my $format = extract_param($param, 'format');
3797
3798         my $storecfg = PVE::Storage::config();
3799
3800         my $load_and_check_move = sub {
3801             my $conf = PVE::QemuConfig->load_config($vmid);
3802             PVE::QemuConfig->check_lock($conf);
3803
3804             PVE::Tools::assert_if_modified($digest, $conf->{digest});
3805
3806             die "disk '$disk' does not exist\n" if !$conf->{$disk};
3807
3808             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
3809
3810             die "disk '$disk' has no associated volume\n" if !$drive->{file};
3811             die "you can't move a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive, 1);
3812
3813             my $old_volid = $drive->{file};
3814             my $oldfmt;
3815             my ($oldstoreid, $oldvolname) = PVE::Storage::parse_volume_id($old_volid);
3816             if ($oldvolname =~ m/\.(raw|qcow2|vmdk)$/){
3817                 $oldfmt = $1;
3818             }
3819
3820             die "you can't move to the same storage with same format\n"
3821                 if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format);
3822
3823             # this only checks snapshots because $disk is passed!
3824             my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use(
3825                 $storecfg,
3826                 $conf,
3827                 $disk,
3828                 $old_volid
3829             );
3830             die "you can't move a disk with snapshots and delete the source\n"
3831                 if $snapshotted && $param->{delete};
3832
3833             return ($conf, $drive, $oldstoreid, $snapshotted);
3834         };
3835
3836         my $move_updatefn = sub {
3837             my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->();
3838             my $old_volid = $drive->{file};
3839
3840             PVE::Cluster::log_msg(
3841                 'info',
3842                 $authuser,
3843                 "move disk VM $vmid: move --disk $disk --storage $storeid"
3844             );
3845
3846             my $running = PVE::QemuServer::check_running($vmid);
3847
3848             PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]);
3849
3850             my $newvollist = [];
3851
3852             eval {
3853                 local $SIG{INT} =
3854                     local $SIG{TERM} =
3855                     local $SIG{QUIT} =
3856                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
3857
3858                 warn "moving disk with snapshots, snapshots will not be moved!\n"
3859                     if $snapshotted;
3860
3861                 my $bwlimit = extract_param($param, 'bwlimit');
3862                 my $movelimit = PVE::Storage::get_bandwidth_limit(
3863                     'move',
3864                     [$oldstoreid, $storeid],
3865                     $bwlimit
3866                 );
3867
3868                 my $source_info = {
3869                     vmid => $vmid,
3870                     running => $running,
3871                     drivename => $disk,
3872                     drive => $drive,
3873                     snapname => undef,
3874                 };
3875
3876                 my $dest_info = {
3877                     vmid => $vmid,
3878                     drivename => $disk,
3879                     storage => $storeid,
3880                     format => $format,
3881                 };
3882
3883                 $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf)
3884                     if $disk eq 'efidisk0';
3885
3886                 my $newdrive = PVE::QemuServer::clone_disk(
3887                     $storecfg,
3888                     $source_info,
3889                     $dest_info,
3890                     1,
3891                     $newvollist,
3892                     undef,
3893                     undef,
3894                     undef,
3895                     $movelimit,
3896                 );
3897                 $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
3898
3899                 PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
3900
3901                 # convert moved disk to base if part of template
3902                 PVE::QemuServer::template_create($vmid, $conf, $disk)
3903                     if PVE::QemuConfig->is_template($conf);
3904
3905                 PVE::QemuConfig->write_config($vmid, $conf);
3906
3907                 my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks');
3908                 if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
3909                     eval { mon_cmd($vmid, "guest-fstrim") };
3910                 }
3911
3912                 eval {
3913                     # try to deactivate volumes - avoid lvm LVs to be active on several nodes
3914                     PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
3915                         if !$running;
3916                 };
3917                 warn $@ if $@;
3918             };
3919             if (my $err = $@) {
3920                 foreach my $volid (@$newvollist) {
3921                     eval { PVE::Storage::vdisk_free($storecfg, $volid) };
3922                     warn $@ if $@;
3923                 }
3924                 die "storage migration failed: $err";
3925             }
3926
3927             if ($param->{delete}) {
3928                 eval {
3929                     PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
3930                     PVE::Storage::vdisk_free($storecfg, $old_volid);
3931                 };
3932                 warn $@ if $@;
3933             }
3934         };
3935
3936         my $load_and_check_reassign_configs = sub {
3937             my $vmlist = PVE::Cluster::get_vmlist()->{ids};
3938
3939             die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid});
3940             die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
3941
3942             my $source_node = $vmlist->{$vmid}->{node};
3943             my $target_node = $vmlist->{$target_vmid}->{node};
3944
3945             die "Both VMs need to be on the same node ($source_node != $target_node)\n"
3946                 if $source_node ne $target_node;
3947
3948             my $source_conf = PVE::QemuConfig->load_config($vmid);
3949             PVE::QemuConfig->check_lock($source_conf);
3950             my $target_conf = PVE::QemuConfig->load_config($target_vmid);
3951             PVE::QemuConfig->check_lock($target_conf);
3952
3953             die "Can't move disks from or to template VMs\n"
3954                 if ($source_conf->{template} || $target_conf->{template});
3955
3956             if ($digest) {
3957                 eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
3958                 die "VM ${vmid}: $@" if $@;
3959             }
3960
3961             if ($target_digest) {
3962                 eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
3963                 die "VM ${target_vmid}: $@" if $@;
3964             }
3965
3966             die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk});
3967
3968             die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n"
3969                 if $target_conf->{$target_disk};
3970
3971             my $drive = PVE::QemuServer::parse_drive(
3972                 $disk,
3973                 $source_conf->{$disk},
3974             );
3975             die "failed to parse source disk - $@\n" if !$drive;
3976
3977             my $source_volid = $drive->{file};
3978
3979             die "disk '${disk}' has no associated volume\n" if !$source_volid;
3980             die "CD drive contents can't be moved to another VM\n"
3981                 if PVE::QemuServer::drive_is_cdrom($drive, 1);
3982
3983             my $storeid = PVE::Storage::parse_volume_id($source_volid, 1);
3984             die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid);
3985
3986             die "Can't move disk used by a snapshot to another VM\n"
3987                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid);
3988             die "Storage does not support moving of this disk to another VM\n"
3989                 if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid));
3990             die "Cannot move disk to another VM while the source VM is running - detach first\n"
3991                 if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/;
3992
3993             # now re-parse using target disk slot format
3994             if ($target_disk =~ /^unused\d+$/) {
3995                 $drive = PVE::QemuServer::parse_drive(
3996                     $target_disk,
3997                     $source_volid,
3998                 );
3999             } else {
4000                 $drive = PVE::QemuServer::parse_drive(
4001                     $target_disk,
4002                     $source_conf->{$disk},
4003                 );
4004             }
4005             die "failed to parse source disk for target disk format - $@\n" if !$drive;
4006
4007             my $repl_conf = PVE::ReplicationConfig->new();
4008             if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
4009                 my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4010                 die "Cannot move disk to a replicated VM. Storage does not support replication!\n"
4011                     if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
4012             }
4013
4014             return ($source_conf, $target_conf, $drive);
4015         };
4016
4017         my $logfunc = sub {
4018             my ($msg) = @_;
4019             print STDERR "$msg\n";
4020         };
4021
4022         my $disk_reassignfn = sub {
4023             return PVE::QemuConfig->lock_config($vmid, sub {
4024                 return PVE::QemuConfig->lock_config($target_vmid, sub {
4025                     my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs();
4026
4027                     my $source_volid = $drive->{file};
4028
4029                     print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n";
4030                     my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
4031
4032                     my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
4033
4034                     my $new_volid = PVE::Storage::rename_volume(
4035                         $storecfg,
4036                         $source_volid,
4037                         $target_vmid,
4038                     );
4039
4040                     $drive->{file} = $new_volid;
4041
4042                     my $boot_order = PVE::QemuServer::device_bootorder($source_conf);
4043                     if (defined(delete $boot_order->{$disk})) {
4044                         print "removing disk '$disk' from boot order config\n";
4045                         my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ];
4046                         $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs);
4047                     }
4048
4049                     delete $source_conf->{$disk};
4050                     print "removing disk '${disk}' from VM '${vmid}' config\n";
4051                     PVE::QemuConfig->write_config($vmid, $source_conf);
4052
4053                     my $drive_string = PVE::QemuServer::print_drive($drive);
4054
4055                     if ($target_disk =~ /^unused\d+$/) {
4056                         $target_conf->{$target_disk} = $drive_string;
4057                         PVE::QemuConfig->write_config($target_vmid, $target_conf);
4058                     } else {
4059                         &$update_vm_api(
4060                             {
4061                                 node => $node,
4062                                 vmid => $target_vmid,
4063                                 digest => $target_digest,
4064                                 $target_disk => $drive_string,
4065                             },
4066                             1,
4067                         );
4068                     }
4069
4070                     # remove possible replication snapshots
4071                     if (PVE::Storage::volume_has_feature(
4072                             $storecfg,
4073                             'replicate',
4074                             $source_volid),
4075                     ) {
4076                         eval {
4077                             PVE::Replication::prepare(
4078                                 $storecfg,
4079                                 [$new_volid],
4080                                 undef,
4081                                 1,
4082                                 undef,
4083                                 $logfunc,
4084                             )
4085                         };
4086                         if (my $err = $@) {
4087                             print "Failed to remove replication snapshots on moved disk " .
4088                                 "'$target_disk'. Manual cleanup could be necessary.\n";
4089                         }
4090                     }
4091                 });
4092             });
4093         };
4094
4095         if ($target_vmid && $storeid) {
4096             my $msg = "either set 'storage' or 'target-vmid', but not both";
4097             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4098         } elsif ($target_vmid) {
4099             $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
4100                 if $authuser ne 'root@pam';
4101
4102             raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" })
4103                 if $vmid eq $target_vmid;
4104
4105             my (undef, undef, $drive) = &$load_and_check_reassign_configs();
4106             my $storage = PVE::Storage::parse_volume_id($drive->{file});
4107             $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
4108
4109             return $rpcenv->fork_worker(
4110                 'qmmove',
4111                 "${vmid}-${disk}>${target_vmid}-${target_disk}",
4112                 $authuser,
4113                 $disk_reassignfn
4114             );
4115         } elsif ($storeid) {
4116             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4117
4118             die "cannot move disk '$disk', only configured disks can be moved to another storage\n"
4119                 if $disk =~ m/^unused\d+$/;
4120
4121             $load_and_check_move->(); # early checks before forking/locking
4122
4123             my $realcmd = sub {
4124                 PVE::QemuConfig->lock_config($vmid, $move_updatefn);
4125             };
4126
4127             return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
4128         } else {
4129             my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
4130             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
4131         }
4132     }});
4133
4134 my $check_vm_disks_local = sub {
4135     my ($storecfg, $vmconf, $vmid) = @_;
4136
4137     my $local_disks = {};
4138
4139     # add some more information to the disks e.g. cdrom
4140     PVE::QemuServer::foreach_volid($vmconf, sub {
4141         my ($volid, $attr) = @_;
4142
4143         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4144         if ($storeid) {
4145             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4146             return if $scfg->{shared};
4147         }
4148         # The shared attr here is just a special case where the vdisk
4149         # is marked as shared manually
4150         return if $attr->{shared};
4151         return if $attr->{cdrom} and $volid eq "none";
4152
4153         if (exists $local_disks->{$volid}) {
4154             @{$local_disks->{$volid}}{keys %$attr} = values %$attr
4155         } else {
4156             $local_disks->{$volid} = $attr;
4157             # ensure volid is present in case it's needed
4158             $local_disks->{$volid}->{volid} = $volid;
4159         }
4160     });
4161
4162     return $local_disks;
4163 };
4164
4165 __PACKAGE__->register_method({
4166     name => 'migrate_vm_precondition',
4167     path => '{vmid}/migrate',
4168     method => 'GET',
4169     protected => 1,
4170     proxyto => 'node',
4171     description => "Get preconditions for migration.",
4172     permissions => {
4173         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4174     },
4175     parameters => {
4176         additionalProperties => 0,
4177         properties => {
4178             node => get_standard_option('pve-node'),
4179             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4180             target => get_standard_option('pve-node', {
4181                 description => "Target node.",
4182                 completion =>  \&PVE::Cluster::complete_migration_target,
4183                 optional => 1,
4184             }),
4185         },
4186     },
4187     returns => {
4188         type => "object",
4189         properties => {
4190             running => { type => 'boolean' },
4191             allowed_nodes => {
4192                 type => 'array',
4193                 optional => 1,
4194                 description => "List nodes allowed for offline migration, only passed if VM is offline"
4195             },
4196             not_allowed_nodes => {
4197                 type => 'object',
4198                 optional => 1,
4199                 description => "List not allowed nodes with additional informations, only passed if VM is offline"
4200             },
4201             local_disks => {
4202                 type => 'array',
4203                 description => "List local disks including CD-Rom, unsused and not referenced disks"
4204             },
4205             local_resources => {
4206                 type => 'array',
4207                 description => "List local resources e.g. pci, usb"
4208             }
4209         },
4210     },
4211     code => sub {
4212         my ($param) = @_;
4213
4214         my $rpcenv = PVE::RPCEnvironment::get();
4215
4216         my $authuser = $rpcenv->get_user();
4217
4218         PVE::Cluster::check_cfs_quorum();
4219
4220         my $res = {};
4221
4222         my $vmid = extract_param($param, 'vmid');
4223         my $target = extract_param($param, 'target');
4224         my $localnode = PVE::INotify::nodename();
4225
4226
4227         # test if VM exists
4228         my $vmconf = PVE::QemuConfig->load_config($vmid);
4229         my $storecfg = PVE::Storage::config();
4230
4231
4232         # try to detect errors early
4233         PVE::QemuConfig->check_lock($vmconf);
4234
4235         $res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0;
4236
4237         # if vm is not running, return target nodes where local storage is available
4238         # for offline migration
4239         if (!$res->{running}) {
4240             $res->{allowed_nodes} = [];
4241             my $checked_nodes = PVE::QemuServer::check_local_storage_availability($vmconf, $storecfg);
4242             delete $checked_nodes->{$localnode};
4243
4244             foreach my $node (keys %$checked_nodes) {
4245                 if (!defined $checked_nodes->{$node}->{unavailable_storages}) {
4246                     push @{$res->{allowed_nodes}}, $node;
4247                 }
4248
4249             }
4250             $res->{not_allowed_nodes} = $checked_nodes;
4251         }
4252
4253
4254         my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid);
4255         $res->{local_disks} = [ values %$local_disks ];;
4256
4257         my $local_resources =  PVE::QemuServer::check_local_resources($vmconf, 1);
4258
4259         $res->{local_resources} = $local_resources;
4260
4261         return $res;
4262
4263
4264     }});
4265
4266 __PACKAGE__->register_method({
4267     name => 'migrate_vm',
4268     path => '{vmid}/migrate',
4269     method => 'POST',
4270     protected => 1,
4271     proxyto => 'node',
4272     description => "Migrate virtual machine. Creates a new migration task.",
4273     permissions => {
4274         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4275     },
4276     parameters => {
4277         additionalProperties => 0,
4278         properties => {
4279             node => get_standard_option('pve-node'),
4280             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4281             target => get_standard_option('pve-node', {
4282                 description => "Target node.",
4283                 completion =>  \&PVE::Cluster::complete_migration_target,
4284             }),
4285             online => {
4286                 type => 'boolean',
4287                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4288                 optional => 1,
4289             },
4290             force => {
4291                 type => 'boolean',
4292                 description => "Allow to migrate VMs which use local devices. Only root may use this option.",
4293                 optional => 1,
4294             },
4295             migration_type => {
4296                 type => 'string',
4297                 enum => ['secure', 'insecure'],
4298                 description => "Migration traffic is encrypted using an SSH tunnel by default. On secure, completely private networks this can be disabled to increase performance.",
4299                 optional => 1,
4300             },
4301             migration_network => {
4302                 type => 'string', format => 'CIDR',
4303                 description => "CIDR of the (sub) network that is used for migration.",
4304                 optional => 1,
4305             },
4306             "with-local-disks" => {
4307                 type => 'boolean',
4308                 description => "Enable live storage migration for local disk",
4309                 optional => 1,
4310             },
4311             targetstorage => get_standard_option('pve-targetstorage', {
4312                 completion => \&PVE::QemuServer::complete_migration_storage,
4313             }),
4314             bwlimit => {
4315                 description => "Override I/O bandwidth limit (in KiB/s).",
4316                 optional => 1,
4317                 type => 'integer',
4318                 minimum => '0',
4319                 default => 'migrate limit from datacenter or storage config',
4320             },
4321         },
4322     },
4323     returns => {
4324         type => 'string',
4325         description => "the task ID.",
4326     },
4327     code => sub {
4328         my ($param) = @_;
4329
4330         my $rpcenv = PVE::RPCEnvironment::get();
4331         my $authuser = $rpcenv->get_user();
4332
4333         my $target = extract_param($param, 'target');
4334
4335         my $localnode = PVE::INotify::nodename();
4336         raise_param_exc({ target => "target is local node."}) if $target eq $localnode;
4337
4338         PVE::Cluster::check_cfs_quorum();
4339
4340         PVE::Cluster::check_node_exists($target);
4341
4342         my $targetip = PVE::Cluster::remote_node_ip($target);
4343
4344         my $vmid = extract_param($param, 'vmid');
4345
4346         raise_param_exc({ force => "Only root may use this option." })
4347             if $param->{force} && $authuser ne 'root@pam';
4348
4349         raise_param_exc({ migration_type => "Only root may use this option." })
4350             if $param->{migration_type} && $authuser ne 'root@pam';
4351
4352         # allow root only until better network permissions are available
4353         raise_param_exc({ migration_network => "Only root may use this option." })
4354             if $param->{migration_network} && $authuser ne 'root@pam';
4355
4356         # test if VM exists
4357         my $conf = PVE::QemuConfig->load_config($vmid);
4358
4359         # try to detect errors early
4360
4361         PVE::QemuConfig->check_lock($conf);
4362
4363         if (PVE::QemuServer::check_running($vmid)) {
4364             die "can't migrate running VM without --online\n" if !$param->{online};
4365
4366             my $repl_conf = PVE::ReplicationConfig->new();
4367             my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
4368             my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target));
4369             if (!$param->{force} && $is_replicated && !$is_replicated_to_target) {
4370                 die "Cannot live-migrate replicated VM to node '$target' - not a replication " .
4371                     "target. Use 'force' to override.\n";
4372             }
4373         } else {
4374             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4375             $param->{online} = 0;
4376         }
4377
4378         my $storecfg = PVE::Storage::config();
4379         if (my $targetstorage = $param->{targetstorage}) {
4380             my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
4381             raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
4382                 if $@;
4383
4384             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
4385                 if !defined($storagemap->{identity});
4386
4387             foreach my $target_sid (values %{$storagemap->{entries}}) {
4388                 $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
4389             }
4390
4391             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
4392                 if $storagemap->{default};
4393
4394             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target)
4395                 if $storagemap->{identity};
4396
4397             $param->{storagemap} = $storagemap;
4398         } else {
4399             PVE::QemuServer::check_storage_availability($storecfg, $conf, $target);
4400         }
4401
4402         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
4403
4404             my $hacmd = sub {
4405                 my $upid = shift;
4406
4407                 print "Requesting HA migration for VM $vmid to node $target\n";
4408
4409                 my $cmd = ['ha-manager', 'migrate', "vm:$vmid", $target];
4410                 PVE::Tools::run_command($cmd);
4411                 return;
4412             };
4413
4414             return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);
4415
4416         } else {
4417
4418             my $realcmd = sub {
4419                 PVE::QemuMigrate->migrate($target, $targetip, $vmid, $param);
4420             };
4421
4422             my $worker = sub {
4423                 return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
4424             };
4425
4426             return $rpcenv->fork_worker('qmigrate', $vmid, $authuser, $worker);
4427         }
4428
4429     }});
4430
4431 __PACKAGE__->register_method({
4432     name => 'remote_migrate_vm',
4433     path => '{vmid}/remote_migrate',
4434     method => 'POST',
4435     protected => 1,
4436     proxyto => 'node',
4437     description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!",
4438     permissions => {
4439         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
4440     },
4441     parameters => {
4442         additionalProperties => 0,
4443         properties => {
4444             node => get_standard_option('pve-node'),
4445             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4446             'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
4447             'target-endpoint' => get_standard_option('proxmox-remote', {
4448                 description => "Remote target endpoint",
4449             }),
4450             online => {
4451                 type => 'boolean',
4452                 description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
4453                 optional => 1,
4454             },
4455             delete => {
4456                 type => 'boolean',
4457                 description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.",
4458                 optional => 1,
4459                 default => 0,
4460             },
4461             'target-storage' => get_standard_option('pve-targetstorage', {
4462                 completion => \&PVE::QemuServer::complete_migration_storage,
4463                 optional => 0,
4464             }),
4465             'target-bridge' => {
4466                 type => 'string',
4467                 description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
4468                 format => 'bridge-pair-list',
4469             },
4470             bwlimit => {
4471                 description => "Override I/O bandwidth limit (in KiB/s).",
4472                 optional => 1,
4473                 type => 'integer',
4474                 minimum => '0',
4475                 default => 'migrate limit from datacenter or storage config',
4476             },
4477         },
4478     },
4479     returns => {
4480         type => 'string',
4481         description => "the task ID.",
4482     },
4483     code => sub {
4484         my ($param) = @_;
4485
4486         my $rpcenv = PVE::RPCEnvironment::get();
4487         my $authuser = $rpcenv->get_user();
4488
4489         my $source_vmid = extract_param($param, 'vmid');
4490         my $target_endpoint = extract_param($param, 'target-endpoint');
4491         my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
4492
4493         my $delete = extract_param($param, 'delete') // 0;
4494
4495         PVE::Cluster::check_cfs_quorum();
4496
4497         # test if VM exists
4498         my $conf = PVE::QemuConfig->load_config($source_vmid);
4499
4500         PVE::QemuConfig->check_lock($conf);
4501
4502         raise_param_exc({ vmid => "cannot migrate HA-managed VM to remote cluster" })
4503             if PVE::HA::Config::vm_is_ha_managed($source_vmid);
4504
4505         my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);
4506
4507         # TODO: move this as helper somewhere appropriate?
4508         my $conn_args = {
4509             protocol => 'https',
4510             host => $remote->{host},
4511             port => $remote->{port} // 8006,
4512             apitoken => $remote->{apitoken},
4513         };
4514
4515         my $fp;
4516         if ($fp = $remote->{fingerprint}) {
4517             $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
4518         }
4519
4520         print "Establishing API connection with remote at '$remote->{host}'\n";
4521
4522         my $api_client = PVE::APIClient::LWP->new(%$conn_args);
4523
4524         if (!defined($fp)) {
4525             my $cert_info = $api_client->get("/nodes/localhost/certificates/info");
4526             foreach my $cert (@$cert_info) {
4527                 my $filename = $cert->{filename};
4528                 next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem';
4529                 $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem';
4530             }
4531             $conn_args->{cached_fingerprints} = { uc($fp) => 1 }
4532                 if defined($fp);
4533         }
4534
4535         my $repl_conf = PVE::ReplicationConfig->new();
4536         my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1);
4537         die "cannot remote-migrate replicated VM\n" if $is_replicated;
4538
4539         if (PVE::QemuServer::check_running($source_vmid)) {
4540             die "can't migrate running VM without --online\n" if !$param->{online};
4541
4542         } else {
4543             warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
4544             $param->{online} = 0;
4545         }
4546
4547         my $storecfg = PVE::Storage::config();
4548         my $target_storage = extract_param($param, 'target-storage');
4549         my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
4550         raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
4551             if $@;
4552
4553         my $target_bridge = extract_param($param, 'target-bridge');
4554         my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') };
4555         raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" })
4556             if $@;
4557
4558         die "remote migration requires explicit storage mapping!\n"
4559             if $storagemap->{identity};
4560
4561         $param->{storagemap} = $storagemap;
4562         $param->{bridgemap} = $bridgemap;
4563         $param->{remote} = {
4564             conn => $conn_args, # re-use fingerprint for tunnel
4565             client => $api_client,
4566             vmid => $target_vmid,
4567         };
4568         $param->{migration_type} = 'websocket';
4569         $param->{'with-local-disks'} = 1;
4570         $param->{delete} = $delete if $delete;
4571
4572         my $cluster_status = $api_client->get("/cluster/status");
4573         my $target_node;
4574         foreach my $entry (@$cluster_status) {
4575             next if $entry->{type} ne 'node';
4576             if ($entry->{local}) {
4577                 $target_node = $entry->{name};
4578                 last;
4579             }
4580         }
4581
4582         die "couldn't determine endpoint's node name\n"
4583             if !defined($target_node);
4584
4585         my $realcmd = sub {
4586             PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param);
4587         };
4588
4589         my $worker = sub {
4590             return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd);
4591         };
4592
4593         return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker);
4594     }});
4595
4596 __PACKAGE__->register_method({
4597     name => 'monitor',
4598     path => '{vmid}/monitor',
4599     method => 'POST',
4600     protected => 1,
4601     proxyto => 'node',
4602     description => "Execute Qemu monitor commands.",
4603     permissions => {
4604         description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')",
4605         check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]],
4606     },
4607     parameters => {
4608         additionalProperties => 0,
4609         properties => {
4610             node => get_standard_option('pve-node'),
4611             vmid => get_standard_option('pve-vmid'),
4612             command => {
4613                 type => 'string',
4614                 description => "The monitor command.",
4615             }
4616         },
4617     },
4618     returns => { type => 'string'},
4619     code => sub {
4620         my ($param) = @_;
4621
4622         my $rpcenv = PVE::RPCEnvironment::get();
4623         my $authuser = $rpcenv->get_user();
4624
4625         my $is_ro = sub {
4626             my $command = shift;
4627             return $command =~ m/^\s*info(\s+|$)/
4628                 || $command =~ m/^\s*help\s*$/;
4629         };
4630
4631         $rpcenv->check_full($authuser, "/", ['Sys.Modify'])
4632             if !&$is_ro($param->{command});
4633
4634         my $vmid = $param->{vmid};
4635
4636         my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists
4637
4638         my $res = '';
4639         eval {
4640             $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, $param->{command});
4641         };
4642         $res = "ERROR: $@" if $@;
4643
4644         return $res;
4645     }});
4646
4647 __PACKAGE__->register_method({
4648     name => 'resize_vm',
4649     path => '{vmid}/resize',
4650     method => 'PUT',
4651     protected => 1,
4652     proxyto => 'node',
4653     description => "Extend volume size.",
4654     permissions => {
4655         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
4656     },
4657     parameters => {
4658         additionalProperties => 0,
4659         properties => {
4660             node => get_standard_option('pve-node'),
4661             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4662             skiplock => get_standard_option('skiplock'),
4663             disk => {
4664                 type => 'string',
4665                 description => "The disk you want to resize.",
4666                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
4667             },
4668             size => {
4669                 type => 'string',
4670                 pattern => '\+?\d+(\.\d+)?[KMGT]?',
4671                 description => "The new size. With the `+` sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
4672             },
4673             digest => {
4674                 type => 'string',
4675                 description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
4676                 maxLength => 40,
4677                 optional => 1,
4678             },
4679         },
4680     },
4681     returns => { type => 'null'},
4682     code => sub {
4683         my ($param) = @_;
4684
4685         my $rpcenv = PVE::RPCEnvironment::get();
4686
4687         my $authuser = $rpcenv->get_user();
4688
4689         my $node = extract_param($param, 'node');
4690
4691         my $vmid = extract_param($param, 'vmid');
4692
4693         my $digest = extract_param($param, 'digest');
4694
4695         my $disk = extract_param($param, 'disk');
4696
4697         my $sizestr = extract_param($param, 'size');
4698
4699         my $skiplock = extract_param($param, 'skiplock');
4700         raise_param_exc({ skiplock => "Only root may use this option." })
4701             if $skiplock && $authuser ne 'root@pam';
4702
4703         my $storecfg = PVE::Storage::config();
4704
4705         my $updatefn =  sub {
4706
4707             my $conf = PVE::QemuConfig->load_config($vmid);
4708
4709             die "checksum missmatch (file change by other user?)\n"
4710                 if $digest && $digest ne $conf->{digest};
4711             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4712
4713             die "disk '$disk' does not exist\n" if !$conf->{$disk};
4714
4715             my $drive = PVE::QemuServer::parse_drive($disk, $conf->{$disk});
4716
4717             my (undef, undef, undef, undef, undef, undef, $format) =
4718                 PVE::Storage::parse_volname($storecfg, $drive->{file});
4719
4720             die "can't resize volume: $disk if snapshot exists\n"
4721                 if %{$conf->{snapshots}} && $format eq 'qcow2';
4722
4723             my $volid = $drive->{file};
4724
4725             die "disk '$disk' has no associated volume\n" if !$volid;
4726
4727             die "you can't resize a cdrom\n" if PVE::QemuServer::drive_is_cdrom($drive);
4728
4729             my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4730
4731             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
4732
4733             PVE::Storage::activate_volumes($storecfg, [$volid]);
4734             my $size = PVE::Storage::volume_size_info($storecfg, $volid, 5);
4735
4736             die "Could not determine current size of volume '$volid'\n" if !defined($size);
4737
4738             die "internal error" if $sizestr !~ m/^(\+)?(\d+(\.\d+)?)([KMGT])?$/;
4739             my ($ext, $newsize, $unit) = ($1, $2, $4);
4740             if ($unit) {
4741                 if ($unit eq 'K') {
4742                     $newsize = $newsize * 1024;
4743                 } elsif ($unit eq 'M') {
4744                     $newsize = $newsize * 1024 * 1024;
4745                 } elsif ($unit eq 'G') {
4746                     $newsize = $newsize * 1024 * 1024 * 1024;
4747                 } elsif ($unit eq 'T') {
4748                     $newsize = $newsize * 1024 * 1024 * 1024 * 1024;
4749                 }
4750             }
4751             $newsize += $size if $ext;
4752             $newsize = int($newsize);
4753
4754             die "shrinking disks is not supported\n" if $newsize < $size;
4755
4756             return if $size == $newsize;
4757
4758             PVE::Cluster::log_msg('info', $authuser, "update VM $vmid: resize --disk $disk --size $sizestr");
4759
4760             PVE::QemuServer::qemu_block_resize($vmid, "drive-$disk", $storecfg, $volid, $newsize);
4761
4762             $drive->{size} = $newsize;
4763             $conf->{$disk} = PVE::QemuServer::print_drive($drive);
4764
4765             PVE::QemuConfig->write_config($vmid, $conf);
4766         };
4767
4768         PVE::QemuConfig->lock_config($vmid, $updatefn);
4769         return;
4770     }});
4771
4772 __PACKAGE__->register_method({
4773     name => 'snapshot_list',
4774     path => '{vmid}/snapshot',
4775     method => 'GET',
4776     description => "List all snapshots.",
4777     permissions => {
4778         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
4779     },
4780     proxyto => 'node',
4781     protected => 1, # qemu pid files are only readable by root
4782     parameters => {
4783         additionalProperties => 0,
4784         properties => {
4785             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4786             node => get_standard_option('pve-node'),
4787         },
4788     },
4789     returns => {
4790         type => 'array',
4791         items => {
4792             type => "object",
4793             properties => {
4794                 name => {
4795                     description => "Snapshot identifier. Value 'current' identifies the current VM.",
4796                     type => 'string',
4797                 },
4798                 vmstate => {
4799                     description => "Snapshot includes RAM.",
4800                     type => 'boolean',
4801                     optional => 1,
4802                 },
4803                 description => {
4804                     description => "Snapshot description.",
4805                     type => 'string',
4806                 },
4807                 snaptime => {
4808                     description => "Snapshot creation time",
4809                     type => 'integer',
4810                     renderer => 'timestamp',
4811                     optional => 1,
4812                 },
4813                 parent => {
4814                     description => "Parent snapshot identifier.",
4815                     type => 'string',
4816                     optional => 1,
4817                 },
4818             },
4819         },
4820         links => [ { rel => 'child', href => "{name}" } ],
4821     },
4822     code => sub {
4823         my ($param) = @_;
4824
4825         my $vmid = $param->{vmid};
4826
4827         my $conf = PVE::QemuConfig->load_config($vmid);
4828         my $snaphash = $conf->{snapshots} || {};
4829
4830         my $res = [];
4831
4832         foreach my $name (keys %$snaphash) {
4833             my $d = $snaphash->{$name};
4834             my $item = {
4835                 name => $name,
4836                 snaptime => $d->{snaptime} || 0,
4837                 vmstate => $d->{vmstate} ? 1 : 0,
4838                 description => $d->{description} || '',
4839             };
4840             $item->{parent} = $d->{parent} if $d->{parent};
4841             $item->{snapstate} = $d->{snapstate} if $d->{snapstate};
4842             push @$res, $item;
4843         }
4844
4845         my $running = PVE::QemuServer::check_running($vmid, 1) ? 1 : 0;
4846         my $current = {
4847             name => 'current',
4848             digest => $conf->{digest},
4849             running => $running,
4850             description => "You are here!",
4851         };
4852         $current->{parent} = $conf->{parent} if $conf->{parent};
4853
4854         push @$res, $current;
4855
4856         return $res;
4857     }});
4858
4859 __PACKAGE__->register_method({
4860     name => 'snapshot',
4861     path => '{vmid}/snapshot',
4862     method => 'POST',
4863     protected => 1,
4864     proxyto => 'node',
4865     description => "Snapshot a VM.",
4866     permissions => {
4867         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4868     },
4869     parameters => {
4870         additionalProperties => 0,
4871         properties => {
4872             node => get_standard_option('pve-node'),
4873             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
4874             snapname => get_standard_option('pve-snapshot-name'),
4875             vmstate => {
4876                 optional => 1,
4877                 type => 'boolean',
4878                 description => "Save the vmstate",
4879             },
4880             description => {
4881                 optional => 1,
4882                 type => 'string',
4883                 description => "A textual description or comment.",
4884             },
4885         },
4886     },
4887     returns => {
4888         type => 'string',
4889         description => "the task ID.",
4890     },
4891     code => sub {
4892         my ($param) = @_;
4893
4894         my $rpcenv = PVE::RPCEnvironment::get();
4895
4896         my $authuser = $rpcenv->get_user();
4897
4898         my $node = extract_param($param, 'node');
4899
4900         my $vmid = extract_param($param, 'vmid');
4901
4902         my $snapname = extract_param($param, 'snapname');
4903
4904         die "unable to use snapshot name 'current' (reserved name)\n"
4905             if $snapname eq 'current';
4906
4907         die "unable to use snapshot name 'pending' (reserved name)\n"
4908             if lc($snapname) eq 'pending';
4909
4910         my $realcmd = sub {
4911             PVE::Cluster::log_msg('info', $authuser, "snapshot VM $vmid: $snapname");
4912             PVE::QemuConfig->snapshot_create($vmid, $snapname, $param->{vmstate},
4913                                              $param->{description});
4914         };
4915
4916         return $rpcenv->fork_worker('qmsnapshot', $vmid, $authuser, $realcmd);
4917     }});
4918
4919 __PACKAGE__->register_method({
4920     name => 'snapshot_cmd_idx',
4921     path => '{vmid}/snapshot/{snapname}',
4922     description => '',
4923     method => 'GET',
4924     permissions => {
4925         user => 'all',
4926     },
4927     parameters => {
4928         additionalProperties => 0,
4929         properties => {
4930             vmid => get_standard_option('pve-vmid'),
4931             node => get_standard_option('pve-node'),
4932             snapname => get_standard_option('pve-snapshot-name'),
4933         },
4934     },
4935     returns => {
4936         type => 'array',
4937         items => {
4938             type => "object",
4939             properties => {},
4940         },
4941         links => [ { rel => 'child', href => "{cmd}" } ],
4942     },
4943     code => sub {
4944         my ($param) = @_;
4945
4946         my $res = [];
4947
4948         push @$res, { cmd => 'rollback' };
4949         push @$res, { cmd => 'config' };
4950
4951         return $res;
4952     }});
4953
4954 __PACKAGE__->register_method({
4955     name => 'update_snapshot_config',
4956     path => '{vmid}/snapshot/{snapname}/config',
4957     method => 'PUT',
4958     protected => 1,
4959     proxyto => 'node',
4960     description => "Update snapshot metadata.",
4961     permissions => {
4962         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
4963     },
4964     parameters => {
4965         additionalProperties => 0,
4966         properties => {
4967             node => get_standard_option('pve-node'),
4968             vmid => get_standard_option('pve-vmid'),
4969             snapname => get_standard_option('pve-snapshot-name'),
4970             description => {
4971                 optional => 1,
4972                 type => 'string',
4973                 description => "A textual description or comment.",
4974             },
4975         },
4976     },
4977     returns => { type => 'null' },
4978     code => sub {
4979         my ($param) = @_;
4980
4981         my $rpcenv = PVE::RPCEnvironment::get();
4982
4983         my $authuser = $rpcenv->get_user();
4984
4985         my $vmid = extract_param($param, 'vmid');
4986
4987         my $snapname = extract_param($param, 'snapname');
4988
4989         return if !defined($param->{description});
4990
4991         my $updatefn =  sub {
4992
4993             my $conf = PVE::QemuConfig->load_config($vmid);
4994
4995             PVE::QemuConfig->check_lock($conf);
4996
4997             my $snap = $conf->{snapshots}->{$snapname};
4998
4999             die "snapshot '$snapname' does not exist\n" if !defined($snap);
5000
5001             $snap->{description} = $param->{description} if defined($param->{description});
5002
5003              PVE::QemuConfig->write_config($vmid, $conf);
5004         };
5005
5006         PVE::QemuConfig->lock_config($vmid, $updatefn);
5007
5008         return;
5009     }});
5010
5011 __PACKAGE__->register_method({
5012     name => 'get_snapshot_config',
5013     path => '{vmid}/snapshot/{snapname}/config',
5014     method => 'GET',
5015     proxyto => 'node',
5016     description => "Get snapshot configuration",
5017     permissions => {
5018         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback', 'VM.Audit' ], any => 1],
5019     },
5020     parameters => {
5021         additionalProperties => 0,
5022         properties => {
5023             node => get_standard_option('pve-node'),
5024             vmid => get_standard_option('pve-vmid'),
5025             snapname => get_standard_option('pve-snapshot-name'),
5026         },
5027     },
5028     returns => { type => "object" },
5029     code => sub {
5030         my ($param) = @_;
5031
5032         my $rpcenv = PVE::RPCEnvironment::get();
5033
5034         my $authuser = $rpcenv->get_user();
5035
5036         my $vmid = extract_param($param, 'vmid');
5037
5038         my $snapname = extract_param($param, 'snapname');
5039
5040         my $conf = PVE::QemuConfig->load_config($vmid);
5041
5042         my $snap = $conf->{snapshots}->{$snapname};
5043
5044         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5045
5046         return $snap;
5047     }});
5048
5049 __PACKAGE__->register_method({
5050     name => 'rollback',
5051     path => '{vmid}/snapshot/{snapname}/rollback',
5052     method => 'POST',
5053     protected => 1,
5054     proxyto => 'node',
5055     description => "Rollback VM state to specified snapshot.",
5056     permissions => {
5057         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot', 'VM.Snapshot.Rollback' ], any => 1],
5058     },
5059     parameters => {
5060         additionalProperties => 0,
5061         properties => {
5062             node => get_standard_option('pve-node'),
5063             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5064             snapname => get_standard_option('pve-snapshot-name'),
5065             start => {
5066                 type => 'boolean',
5067                 description => "Whether the VM should get started after rolling back successfully",
5068                 optional => 1,
5069                 default => 0,
5070             },
5071         },
5072     },
5073     returns => {
5074         type => 'string',
5075         description => "the task ID.",
5076     },
5077     code => sub {
5078         my ($param) = @_;
5079
5080         my $rpcenv = PVE::RPCEnvironment::get();
5081
5082         my $authuser = $rpcenv->get_user();
5083
5084         my $node = extract_param($param, 'node');
5085
5086         my $vmid = extract_param($param, 'vmid');
5087
5088         my $snapname = extract_param($param, 'snapname');
5089
5090         my $realcmd = sub {
5091             PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname");
5092             PVE::QemuConfig->snapshot_rollback($vmid, $snapname);
5093
5094             if ($param->{start}) {
5095                 PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node });
5096             }
5097         };
5098
5099         my $worker = sub {
5100             # hold migration lock, this makes sure that nobody create replication snapshots
5101             return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
5102         };
5103
5104         return $rpcenv->fork_worker('qmrollback', $vmid, $authuser, $worker);
5105     }});
5106
5107 __PACKAGE__->register_method({
5108     name => 'delsnapshot',
5109     path => '{vmid}/snapshot/{snapname}',
5110     method => 'DELETE',
5111     protected => 1,
5112     proxyto => 'node',
5113     description => "Delete a VM snapshot.",
5114     permissions => {
5115         check => ['perm', '/vms/{vmid}', [ 'VM.Snapshot' ]],
5116     },
5117     parameters => {
5118         additionalProperties => 0,
5119         properties => {
5120             node => get_standard_option('pve-node'),
5121             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5122             snapname => get_standard_option('pve-snapshot-name'),
5123             force => {
5124                 optional => 1,
5125                 type => 'boolean',
5126                 description => "For removal from config file, even if removing disk snapshots fails.",
5127             },
5128         },
5129     },
5130     returns => {
5131         type => 'string',
5132         description => "the task ID.",
5133     },
5134     code => sub {
5135         my ($param) = @_;
5136
5137         my $rpcenv = PVE::RPCEnvironment::get();
5138
5139         my $authuser = $rpcenv->get_user();
5140
5141         my $node = extract_param($param, 'node');
5142
5143         my $vmid = extract_param($param, 'vmid');
5144
5145         my $snapname = extract_param($param, 'snapname');
5146
5147         my $lock_obtained;
5148         my $do_delete = sub {
5149             $lock_obtained = 1;
5150             PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname");
5151             PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force});
5152         };
5153
5154         my $realcmd = sub {
5155             if ($param->{force}) {
5156                 $do_delete->();
5157             } else {
5158                 eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); };
5159                 if (my $err = $@) {
5160                     die $err if $lock_obtained;
5161                     die "Failed to obtain guest migration lock - replication running?\n";
5162                 }
5163             }
5164         };
5165
5166         return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd);
5167     }});
5168
5169 __PACKAGE__->register_method({
5170     name => 'template',
5171     path => '{vmid}/template',
5172     method => 'POST',
5173     protected => 1,
5174     proxyto => 'node',
5175     description => "Create a Template.",
5176     permissions => {
5177         description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
5178         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
5179     },
5180     parameters => {
5181         additionalProperties => 0,
5182         properties => {
5183             node => get_standard_option('pve-node'),
5184             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
5185             disk => {
5186                 optional => 1,
5187                 type => 'string',
5188                 description => "If you want to convert only 1 disk to base image.",
5189                 enum => [PVE::QemuServer::Drive::valid_drive_names()],
5190             },
5191
5192         },
5193     },
5194     returns => {
5195         type => 'string',
5196         description => "the task ID.",
5197     },
5198     code => sub {
5199         my ($param) = @_;
5200
5201         my $rpcenv = PVE::RPCEnvironment::get();
5202
5203         my $authuser = $rpcenv->get_user();
5204
5205         my $node = extract_param($param, 'node');
5206
5207         my $vmid = extract_param($param, 'vmid');
5208
5209         my $disk = extract_param($param, 'disk');
5210
5211         my $load_and_check = sub {
5212             my $conf = PVE::QemuConfig->load_config($vmid);
5213
5214             PVE::QemuConfig->check_lock($conf);
5215
5216             die "unable to create template, because VM contains snapshots\n"
5217                 if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});
5218
5219             die "you can't convert a template to a template\n"
5220                 if PVE::QemuConfig->is_template($conf) && !$disk;
5221
5222             die "you can't convert a VM to template if VM is running\n"
5223                 if PVE::QemuServer::check_running($vmid);
5224
5225             return $conf;
5226         };
5227
5228         $load_and_check->();
5229
5230         my $realcmd = sub {
5231             PVE::QemuConfig->lock_config($vmid, sub {
5232                 my $conf = $load_and_check->();
5233
5234                 $conf->{template} = 1;
5235                 PVE::QemuConfig->write_config($vmid, $conf);
5236
5237                 PVE::QemuServer::template_create($vmid, $conf, $disk);
5238             });
5239         };
5240
5241         return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
5242     }});
5243
5244 __PACKAGE__->register_method({
5245     name => 'cloudinit_generated_config_dump',
5246     path => '{vmid}/cloudinit/dump',
5247     method => 'GET',
5248     proxyto => 'node',
5249     description => "Get automatically generated cloudinit config.",
5250     permissions => {
5251         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
5252     },
5253     parameters => {
5254         additionalProperties => 0,
5255         properties => {
5256             node => get_standard_option('pve-node'),
5257             vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
5258             type => {
5259                 description => 'Config type.',
5260                 type => 'string',
5261                 enum => ['user', 'network', 'meta'],
5262             },
5263         },
5264     },
5265     returns => {
5266         type => 'string',
5267     },
5268     code => sub {
5269         my ($param) = @_;
5270
5271         my $conf = PVE::QemuConfig->load_config($param->{vmid});
5272
5273         return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
5274     }});
5275
5276 __PACKAGE__->register_method({
5277     name => 'mtunnel',
5278     path => '{vmid}/mtunnel',
5279     method => 'POST',
5280     protected => 1,
5281     description => 'Migration tunnel endpoint - only for internal use by VM migration.',
5282     permissions => {
5283         check =>
5284         [ 'and',
5285           ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]],
5286           ['perm', '/', [ 'Sys.Incoming' ]],
5287         ],
5288         description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" .
5289                        " on '/'. Further permission checks happen during the actual migration.",
5290     },
5291     parameters => {
5292         additionalProperties => 0,
5293         properties => {
5294             node => get_standard_option('pve-node'),
5295             vmid => get_standard_option('pve-vmid'),
5296             storages => {
5297                 type => 'string',
5298                 format => 'pve-storage-id-list',
5299                 optional => 1,
5300                 description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.',
5301             },
5302             bridges => {
5303                 type => 'string',
5304                 format => 'pve-bridge-id-list',
5305                 optional => 1,
5306                 description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.',
5307             },
5308         },
5309     },
5310     returns => {
5311         additionalProperties => 0,
5312         properties => {
5313             upid => { type => 'string' },
5314             ticket => { type => 'string' },
5315             socket => { type => 'string' },
5316         },
5317     },
5318     code => sub {
5319         my ($param) = @_;
5320
5321         my $rpcenv = PVE::RPCEnvironment::get();
5322         my $authuser = $rpcenv->get_user();
5323
5324         my $node = extract_param($param, 'node');
5325         my $vmid = extract_param($param, 'vmid');
5326
5327         my $storages = extract_param($param, 'storages');
5328         my $bridges = extract_param($param, 'bridges');
5329
5330         my $nodename = PVE::INotify::nodename();
5331
5332         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5333             if $node ne 'localhost' && $node ne $nodename;
5334
5335         $node = $nodename;
5336
5337         my $storecfg = PVE::Storage::config();
5338         foreach my $storeid (PVE::Tools::split_list($storages)) {
5339             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node);
5340         }
5341
5342         foreach my $bridge (PVE::Tools::split_list($bridges)) {
5343             PVE::Network::read_bridge_mtu($bridge);
5344         }
5345
5346         PVE::Cluster::check_cfs_quorum();
5347
5348         my $lock = 'create';
5349         eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); };
5350
5351         raise_param_exc({ vmid => "unable to create empty VM config - $@"})
5352             if $@;
5353
5354         my $realcmd = sub {
5355             my $state = {
5356                 storecfg => PVE::Storage::config(),
5357                 lock => $lock,
5358                 vmid => $vmid,
5359             };
5360
5361             my $run_locked = sub {
5362                 my ($code, $params) = @_;
5363                 return PVE::QemuConfig->lock_config($state->{vmid}, sub {
5364                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5365
5366                     $state->{conf} = $conf;
5367
5368                     die "Encountered wrong lock - aborting mtunnel command handling.\n"
5369                         if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock});
5370
5371                     return $code->($params);
5372                 });
5373             };
5374
5375             my $cmd_desc = {
5376                 config => {
5377                     conf => {
5378                         type => 'string',
5379                         description => 'Full VM config, adapted for target cluster/node',
5380                     },
5381                     'firewall-config' => {
5382                         type => 'string',
5383                         description => 'VM firewall config',
5384                         optional => 1,
5385                     },
5386                 },
5387                 disk => {
5388                     format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'),
5389                     storage => {
5390                         type => 'string',
5391                         format => 'pve-storage-id',
5392                     },
5393                     drive => {
5394                         type => 'object',
5395                         description => 'parsed drive information without volid and format',
5396                     },
5397                 },
5398                 start => {
5399                     start_params => {
5400                         type => 'object',
5401                         description => 'params passed to vm_start_nolock',
5402                     },
5403                     migrate_opts => {
5404                         type => 'object',
5405                         description => 'migrate_opts passed to vm_start_nolock',
5406                     },
5407                 },
5408                 ticket => {
5409                     path => {
5410                         type => 'string',
5411                         description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.',
5412                     },
5413                 },
5414                 quit => {
5415                     cleanup => {
5416                         type => 'boolean',
5417                         description => 'remove VM config and disks, aborting migration',
5418                         default => 0,
5419                     },
5420                 },
5421                 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'},
5422                 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'},
5423                 bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit},
5424             };
5425
5426             my $cmd_handlers = {
5427                 'version' => sub {
5428                     # compared against other end's version
5429                     # bump/reset for breaking changes
5430                     # bump/bump for opt-in changes
5431                     return {
5432                         api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
5433                         age => 0,
5434                     };
5435                 },
5436                 'config' => sub {
5437                     my ($params) = @_;
5438
5439                     # parse and write out VM FW config if given
5440                     if (my $fw_conf = $params->{'firewall-config'}) {
5441                         my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700);
5442
5443                         my $empty_conf = {
5444                             rules => [],
5445                             options => {},
5446                             aliases => {},
5447                             ipset => {} ,
5448                             ipset_comments => {},
5449                         };
5450                         my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf();
5451
5452                         # TODO: add flag for strict parsing?
5453                         # TODO: add import sub that does all this given raw content?
5454                         my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm');
5455                         $vmfw_conf->{vmid} = $state->{vmid};
5456                         PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf);
5457
5458                         $state->{cleanup}->{fw} = 1;
5459                     }
5460
5461                     my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf";
5462                     my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1);
5463                     delete $new_conf->{lock};
5464                     delete $new_conf->{digest};
5465
5466                     # TODO handle properly?
5467                     delete $new_conf->{snapshots};
5468                     delete $new_conf->{parent};
5469                     delete $new_conf->{pending};
5470
5471                     # not handled by update_vm_api
5472                     my $vmgenid = delete $new_conf->{vmgenid};
5473                     my $meta = delete $new_conf->{meta};
5474                     my $cloudinit = delete $new_conf->{cloudinit}; # this is informational only
5475                     $new_conf->{skip_cloud_init} = 1; # re-use image from source side
5476
5477                     $new_conf->{vmid} = $state->{vmid};
5478                     $new_conf->{node} = $node;
5479
5480                     PVE::QemuConfig->remove_lock($state->{vmid}, 'create');
5481
5482                     eval {
5483                         $update_vm_api->($new_conf, 1);
5484                     };
5485                     if (my $err = $@) {
5486                         # revert to locked previous config
5487                         my $conf = PVE::QemuConfig->load_config($state->{vmid});
5488                         $conf->{lock} = 'create';
5489                         PVE::QemuConfig->write_config($state->{vmid}, $conf);
5490
5491                         die $err;
5492                     }
5493
5494                     my $conf = PVE::QemuConfig->load_config($state->{vmid});
5495                     $conf->{lock} = 'migrate';
5496                     $conf->{vmgenid} = $vmgenid if defined($vmgenid);
5497                     $conf->{meta} = $meta if defined($meta);
5498                     $conf->{cloudinit} = $cloudinit if defined($cloudinit);
5499                     PVE::QemuConfig->write_config($state->{vmid}, $conf);
5500
5501                     $state->{lock} = 'migrate';
5502
5503                     return;
5504                 },
5505                 'bwlimit' => sub {
5506                     my ($params) = @_;
5507                     return PVE::StorageTunnel::handle_bwlimit($params);
5508                 },
5509                 'disk' => sub {
5510                     my ($params) = @_;
5511
5512                     my $format = $params->{format};
5513                     my $storeid = $params->{storage};
5514                     my $drive = $params->{drive};
5515
5516                     $check_storage_access_migrate->($rpcenv, $authuser, $state->{storecfg}, $storeid, $node);
5517
5518                     my $storagemap = {
5519                         default => $storeid,
5520                     };
5521
5522                     my $source_volumes = {
5523                         'disk' => [
5524                             undef,
5525                             $storeid,
5526                             undef,
5527                             $drive,
5528                             0,
5529                             $format,
5530                         ],
5531                     };
5532
5533                     my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks($state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap);
5534                     if (defined($res->{disk})) {
5535                         $state->{cleanup}->{volumes}->{$res->{disk}->{volid}} = 1;
5536                         return $res->{disk};
5537                     } else {
5538                         die "failed to allocate NBD disk..\n";
5539                     }
5540                 },
5541                 'disk-import' => sub {
5542                     my ($params) = @_;
5543
5544                     $check_storage_access_migrate->(
5545                         $rpcenv,
5546                         $authuser,
5547                         $state->{storecfg},
5548                         $params->{storage},
5549                         $node
5550                     );
5551
5552                     $params->{unix} = "/run/qemu-server/$state->{vmid}.storage";
5553
5554                     return PVE::StorageTunnel::handle_disk_import($state, $params);
5555                 },
5556                 'query-disk-import' => sub {
5557                     my ($params) = @_;
5558
5559                     return PVE::StorageTunnel::handle_query_disk_import($state, $params);
5560                 },
5561                 'start' => sub {
5562                     my ($params) = @_;
5563
5564                     my $info = PVE::QemuServer::vm_start_nolock(
5565                         $state->{storecfg},
5566                         $state->{vmid},
5567                         $state->{conf},
5568                         $params->{start_params},
5569                         $params->{migrate_opts},
5570                     );
5571
5572
5573                     if ($info->{migrate}->{proto} ne 'unix') {
5574                         PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5575                         die "migration over non-UNIX sockets not possible\n";
5576                     }
5577
5578                     my $socket = $info->{migrate}->{addr};
5579                     chown $state->{socket_uid}, -1, $socket;
5580                     $state->{sockets}->{$socket} = 1;
5581
5582                     my $unix_sockets = $info->{migrate}->{unix_sockets};
5583                     foreach my $socket (@$unix_sockets) {
5584                         chown $state->{socket_uid}, -1, $socket;
5585                         $state->{sockets}->{$socket} = 1;
5586                     }
5587                     return $info;
5588                 },
5589                 'fstrim' => sub {
5590                     if (PVE::QemuServer::qga_check_running($state->{vmid})) {
5591                         eval { mon_cmd($state->{vmid}, "guest-fstrim") };
5592                         warn "fstrim failed: $@\n" if $@;
5593                     }
5594                     return;
5595                 },
5596                 'stop' => sub {
5597                     PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
5598                     return;
5599                 },
5600                 'nbdstop' => sub {
5601                     PVE::QemuServer::nbd_stop($state->{vmid});
5602                     return;
5603                 },
5604                 'resume' => sub {
5605                     if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) {
5606                         PVE::QemuServer::vm_resume($state->{vmid}, 1, 1);
5607                     } else {
5608                         die "VM $state->{vmid} not running\n";
5609                     }
5610                     return;
5611                 },
5612                 'unlock' => sub {
5613                     PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock});
5614                     delete $state->{lock};
5615                     return;
5616                 },
5617                 'ticket' => sub {
5618                     my ($params) = @_;
5619
5620                     my $path = $params->{path};
5621
5622                     die "Not allowed to generate ticket for unknown socket '$path'\n"
5623                         if !defined($state->{sockets}->{$path});
5624
5625                     return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") };
5626                 },
5627                 'quit' => sub {
5628                     my ($params) = @_;
5629
5630                     if ($params->{cleanup}) {
5631                         if ($state->{cleanup}->{fw}) {
5632                             PVE::Firewall::remove_vmfw_conf($state->{vmid});
5633                         }
5634
5635                         for my $volid (keys $state->{cleanup}->{volumes}->%*) {
5636                             print "freeing volume '$volid' as part of cleanup\n";
5637                             eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) };
5638                             warn $@ if $@;
5639                         }
5640
5641                         PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5642                     }
5643
5644                     print "switching to exit-mode, waiting for client to disconnect\n";
5645                     $state->{exit} = 1;
5646                     return;
5647                 },
5648             };
5649
5650             $run_locked->(sub {
5651                 my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel";
5652                 unlink $socket_addr;
5653
5654                 $state->{socket} = IO::Socket::UNIX->new(
5655                     Type => SOCK_STREAM(),
5656                     Local => $socket_addr,
5657                     Listen => 1,
5658                 );
5659
5660                 $state->{socket_uid} = getpwnam('www-data')
5661                     or die "Failed to resolve user 'www-data' to numeric UID\n";
5662                 chown $state->{socket_uid}, -1, $socket_addr;
5663             });
5664
5665             print "mtunnel started\n";
5666
5667             my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) };
5668             if ($@) {
5669                 warn "Failed to accept tunnel connection - $@\n";
5670
5671                 warn "Removing tunnel socket..\n";
5672                 unlink $state->{socket};
5673
5674                 warn "Removing temporary VM config..\n";
5675                 $run_locked->(sub {
5676                     PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
5677                 });
5678
5679                 die "Exiting mtunnel\n";
5680             }
5681
5682             $state->{conn} = $conn;
5683
5684             my $reply_err = sub {
5685                 my ($msg) = @_;
5686
5687                 my $reply = JSON::encode_json({
5688                     success => JSON::false,
5689                     msg => $msg,
5690                 });
5691                 $conn->print("$reply\n");
5692                 $conn->flush();
5693             };
5694
5695             my $reply_ok = sub {
5696                 my ($res) = @_;
5697
5698                 $res->{success} = JSON::true;
5699                 my $reply = JSON::encode_json($res);
5700                 $conn->print("$reply\n");
5701                 $conn->flush();
5702             };
5703
5704             while (my $line = <$conn>) {
5705                 chomp $line;
5706
5707                 # untaint, we validate below if needed
5708                 ($line) = $line =~ /^(.*)$/;
5709                 my $parsed = eval { JSON::decode_json($line) };
5710                 if ($@) {
5711                     $reply_err->("failed to parse command - $@");
5712                     next;
5713                 }
5714
5715                 my $cmd = delete $parsed->{cmd};
5716                 if (!defined($cmd)) {
5717                     $reply_err->("'cmd' missing");
5718                 } elsif ($state->{exit}) {
5719                     $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible");
5720                     next;
5721                 } elsif (my $handler = $cmd_handlers->{$cmd}) {
5722                     print "received command '$cmd'\n";
5723                     eval {
5724                         if ($cmd_desc->{$cmd}) {
5725                             PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd});
5726                         } else {
5727                             $parsed = {};
5728                         }
5729                         my $res = $run_locked->($handler, $parsed);
5730                         $reply_ok->($res);
5731                     };
5732                     $reply_err->("failed to handle '$cmd' command - $@")
5733                         if $@;
5734                 } else {
5735                     $reply_err->("unknown command '$cmd' given");
5736                 }
5737             }
5738
5739             if ($state->{exit}) {
5740                 print "mtunnel exited\n";
5741             } else {
5742                 die "mtunnel exited unexpectedly\n";
5743             }
5744         };
5745
5746         my $socket_addr = "/run/qemu-server/$vmid.mtunnel";
5747         my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr");
5748         my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd);
5749
5750         return {
5751             ticket => $ticket,
5752             upid => $upid,
5753             socket => $socket_addr,
5754         };
5755     }});
5756
5757 __PACKAGE__->register_method({
5758     name => 'mtunnelwebsocket',
5759     path => '{vmid}/mtunnelwebsocket',
5760     method => 'GET',
5761     permissions => {
5762         description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
5763         user => 'all', # check inside
5764     },
5765     description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
5766     parameters => {
5767         additionalProperties => 0,
5768         properties => {
5769             node => get_standard_option('pve-node'),
5770             vmid => get_standard_option('pve-vmid'),
5771             socket => {
5772                 type => "string",
5773                 description => "unix socket to forward to",
5774             },
5775             ticket => {
5776                 type => "string",
5777                 description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command",
5778             },
5779         },
5780     },
5781     returns => {
5782         type => "object",
5783         properties => {
5784             port => { type => 'string', optional => 1 },
5785             socket => { type => 'string', optional => 1 },
5786         },
5787     },
5788     code => sub {
5789         my ($param) = @_;
5790
5791         my $rpcenv = PVE::RPCEnvironment::get();
5792         my $authuser = $rpcenv->get_user();
5793
5794         my $nodename = PVE::INotify::nodename();
5795         my $node = extract_param($param, 'node');
5796
5797         raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
5798             if $node ne 'localhost' && $node ne $nodename;
5799
5800         my $vmid = $param->{vmid};
5801         # check VM exists
5802         PVE::QemuConfig->load_config($vmid);
5803
5804         my $socket = $param->{socket};
5805         PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket");
5806
5807         return { socket => $socket };
5808     }});
5809
5810 1;