src/PVE/API2/LXC.pm

   1 package PVE::API2::LXC;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::SafeSyslog;
   7 use PVE::Tools qw(extract_param run_command);
   8 use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
   9 use PVE::INotify;
  10 use PVE::Cluster qw(cfs_read_file);
  11 use PVE::RRD;
  12 use PVE::DataCenterConfig;
  13 use PVE::AccessControl;
  14 use PVE::Firewall;
  15 use PVE::Storage;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::ReplicationConfig;
  19 use PVE::LXC;
  20 use PVE::LXC::Create;
  21 use PVE::LXC::Migrate;
  22 use PVE::GuestHelpers;
  23 use PVE::VZDump::Plugin;
  24 use PVE::API2::LXC::Config;
  25 use PVE::API2::LXC::Status;
  26 use PVE::API2::LXC::Snapshot;
  27 use PVE::JSONSchema qw(get_standard_option);
  28 use base qw(PVE::RESTHandler);
  29
  30 BEGIN {
  31     if (!$ENV{PVE_GENERATING_DOCS}) {
  32         require PVE::HA::Env::PVE2;
  33         import  PVE::HA::Env::PVE2;
  34         require PVE::HA::Config;
  35         import  PVE::HA::Config;
  36     }
  37 }
  38
  39 my $check_storage_access_migrate = sub {
  40     my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
  41
  42     PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
  43
  44     $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
  45
  46     my $scfg = PVE::Storage::storage_config($storecfg, $storage);
  47     die "storage '$storage' does not support CT rootdirs\n"
  48         if !$scfg->{content}->{rootdir};
  49 };
  50
  51 __PACKAGE__->register_method ({
  52     subclass => "PVE::API2::LXC::Config",
  53     path => '{vmid}/config',
  54 });
  55
  56 __PACKAGE__->register_method ({
  57     subclass => "PVE::API2::LXC::Status",
  58     path => '{vmid}/status',
  59 });
  60
  61 __PACKAGE__->register_method ({
  62     subclass => "PVE::API2::LXC::Snapshot",
  63     path => '{vmid}/snapshot',
  64 });
  65
  66 __PACKAGE__->register_method ({
  67     subclass => "PVE::API2::Firewall::CT",
  68     path => '{vmid}/firewall',
  69 });
  70
  71 __PACKAGE__->register_method({
  72     name => 'vmlist',
  73     path => '',
  74     method => 'GET',
  75     description => "LXC container index (per node).",
  76     permissions => {
  77         description => "Only list CTs where you have VM.Audit permissons on /vms/<vmid>.",
  78         user => 'all',
  79     },
  80     proxyto => 'node',
  81     protected => 1, # /proc files are only readable by root
  82     parameters => {
  83         additionalProperties => 0,
  84         properties => {
  85             node => get_standard_option('pve-node'),
  86         },
  87     },
  88     returns => {
  89         type => 'array',
  90         items => {
  91             type => "object",
  92             properties => $PVE::LXC::vmstatus_return_properties,
  93         },
  94         links => [ { rel => 'child', href => "{vmid}" } ],
  95     },
  96     code => sub {
  97         my ($param) = @_;
  98
  99         my $rpcenv = PVE::RPCEnvironment::get();
 100         my $authuser = $rpcenv->get_user();
 101
 102         my $vmstatus = PVE::LXC::vmstatus();
 103
 104         my $res = [];
 105         foreach my $vmid (keys %$vmstatus) {
 106             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);
 107
 108             my $data = $vmstatus->{$vmid};
 109             push @$res, $data;
 110         }
 111
 112         return $res;
 113
 114     }});
 115
 116 __PACKAGE__->register_method({
 117     name => 'create_vm',
 118     path => '',
 119     method => 'POST',
 120     description => "Create or restore a container.",
 121     permissions => {
 122         user => 'all', # check inside
 123         description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
 124             "For restore, it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
 125             "You also need 'Datastore.AllocateSpace' permissions on the storage.",
 126     },
 127     protected => 1,
 128     proxyto => 'node',
 129     parameters => {
 130         additionalProperties => 0,
 131         properties => PVE::LXC::Config->json_config_properties({
 132             node => get_standard_option('pve-node'),
 133             vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
 134             ostemplate => {
 135                 description => "The OS template or backup file.",
 136                 type => 'string',
 137                 maxLength => 255,
 138                 completion => \&PVE::LXC::complete_os_templates,
 139             },
 140             password => {
 141                 optional => 1,
 142                 type => 'string',
 143                 description => "Sets root password inside container.",
 144                 minLength => 5,
 145             },
 146             storage => get_standard_option('pve-storage-id', {
 147                 description => "Default Storage.",
 148                 default => 'local',
 149                 optional => 1,
 150                 completion => \&PVE::Storage::complete_storage_enabled,
 151             }),
 152             force => {
 153                 optional => 1,
 154                 type => 'boolean',
 155                 description => "Allow to overwrite existing container.",
 156             },
 157             restore => {
 158                 optional => 1,
 159                 type => 'boolean',
 160                 description => "Mark this as restore task.",
 161             },
 162             unique => {
 163                 optional => 1,
 164                 type => 'boolean',
 165                 description => "Assign a unique random ethernet address.",
 166                 requires => 'restore',
 167             },
 168             pool => {
 169                 optional => 1,
 170                 type => 'string', format => 'pve-poolid',
 171                 description => "Add the VM to the specified pool.",
 172             },
 173             'ignore-unpack-errors' => {
 174                 optional => 1,
 175                 type => 'boolean',
 176                 description => "Ignore errors when extracting the template.",
 177             },
 178             'ssh-public-keys' => {
 179                 optional => 1,
 180                 type => 'string',
 181                 description => "Setup public SSH keys (one key per line, " .
 182                                 "OpenSSH format).",
 183             },
 184             bwlimit => {
 185                 description => "Override I/O bandwidth limit (in KiB/s).",
 186                 optional => 1,
 187                 type => 'number',
 188                 minimum => '0',
 189                 default => 'restore limit from datacenter or storage config',
 190             },
 191             start => {
 192                 optional => 1,
 193                 type => 'boolean',
 194                 default => 0,
 195                 description => "Start the CT after its creation finished successfully.",
 196             },
 197         }),
 198     },
 199     returns => {
 200         type => 'string',
 201     },
 202     code => sub {
 203         my ($param) = @_;
 204
 205         PVE::Cluster::check_cfs_quorum();
 206
 207         my $rpcenv = PVE::RPCEnvironment::get();
 208         my $authuser = $rpcenv->get_user();
 209
 210         my $node = extract_param($param, 'node');
 211         my $vmid = extract_param($param, 'vmid');
 212         my $ignore_unpack_errors = extract_param($param, 'ignore-unpack-errors');
 213         my $bwlimit = extract_param($param, 'bwlimit');
 214         my $start_after_create = extract_param($param, 'start');
 215
 216         my $basecfg_fn = PVE::LXC::Config->config_file($vmid);
 217         my $same_container_exists = -f $basecfg_fn;
 218
 219         # 'unprivileged' is read-only, so we can't pass it to update_pct_config
 220         my $unprivileged = extract_param($param, 'unprivileged');
 221         my $restore = extract_param($param, 'restore');
 222         my $unique = extract_param($param, 'unique');
 223
 224         # used to skip firewall config restore if user lacks permission
 225         my $skip_fw_config_restore = 0;
 226
 227         if ($restore) {
 228             # fixme: limit allowed parameters
 229         }
 230
 231         my $force = extract_param($param, 'force');
 232
 233         if (!($same_container_exists && $restore && $force)) {
 234             PVE::Cluster::check_vmid_unused($vmid);
 235         } else {
 236             die "can't overwrite running container\n" if PVE::LXC::check_running($vmid);
 237             my $conf = PVE::LXC::Config->load_config($vmid);
 238             PVE::LXC::Config->check_protection($conf, "unable to restore CT $vmid");
 239         }
 240
 241         my $password = extract_param($param, 'password');
 242         my $ssh_keys = extract_param($param, 'ssh-public-keys');
 243         PVE::Tools::validate_ssh_public_keys($ssh_keys) if defined($ssh_keys);
 244
 245         my $pool = extract_param($param, 'pool');
 246         $rpcenv->check_pool_exist($pool) if defined($pool);
 247
 248         if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
 249             # OK
 250         } elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
 251             # OK
 252         } elsif ($restore && $force && $same_container_exists &&
 253                  $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
 254             # OK: user has VM.Backup permissions, and want to restore an existing VM
 255
 256             # we don't want to restore a container-provided FW conf in this case
 257             # since the user is lacking permission to configure the container's FW
 258             $skip_fw_config_restore = 1;
 259
 260             # error out if a user tries to change from unprivileged to privileged
 261             # explicit change is checked here, implicit is checked down below or happening in root-only paths
 262             my $conf = PVE::LXC::Config->load_config($vmid);
 263             if ($conf->{unprivileged} && defined($unprivileged) && !$unprivileged) {
 264                 raise_perm_exc("cannot change from unprivileged to privileged without VM.Allocate");
 265             }
 266         } else {
 267             raise_perm_exc();
 268         }
 269
 270         my $ostemplate = extract_param($param, 'ostemplate');
 271         my $storage = extract_param($param, 'storage') // 'local';
 272
 273         PVE::LXC::check_ct_modify_config_perm($rpcenv, $authuser, $vmid, $pool, undef, $param, [], $unprivileged);
 274
 275         my $storage_cfg = cfs_read_file("storage.cfg");
 276
 277         my $archive;
 278         if ($ostemplate eq '-') {
 279             die "pipe requires cli environment\n"
 280                 if $rpcenv->{type} ne 'cli';
 281             die "pipe can only be used with restore tasks\n"
 282                 if !$restore;
 283             $archive = '-';
 284             die "restore from pipe requires rootfs parameter\n" if !defined($param->{rootfs});
 285         } else {
 286             my $content_type = $restore ? 'backup' : 'vztmpl';
 287             PVE::Storage::check_volume_access(
 288                 $rpcenv,
 289                 $authuser,
 290                 $storage_cfg,
 291                 $vmid,
 292                 $ostemplate,
 293                 $content_type,
 294             );
 295             $archive = $ostemplate;
 296         }
 297
 298         my %used_storages;
 299         my $check_and_activate_storage = sub {
 300             my ($sid) = @_;
 301
 302             my $scfg = PVE::Storage::storage_check_enabled($storage_cfg, $sid, $node);
 303
 304             raise_param_exc({ storage => "storage '$sid' does not support container directories"})
 305                 if !$scfg->{content}->{rootdir};
 306
 307             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
 308
 309             PVE::Storage::activate_storage($storage_cfg, $sid);
 310             $used_storages{$sid} = 1;
 311         };
 312
 313         my $conf = {};
 314
 315         my $is_root = $authuser eq 'root@pam';
 316
 317         my $no_disk_param = {};
 318         my $mp_param = {};
 319         my $storage_only_mode = 1;
 320         foreach my $opt (keys %$param) {
 321             my $value = $param->{$opt};
 322             if ($opt eq 'rootfs' || $opt =~ m/^mp\d+$/) {
 323                 # allow to use simple numbers (add default storage in that case)
 324                 if ($value =~ m/^\d+(\.\d+)?$/) {
 325                     $mp_param->{$opt} = "$storage:$value";
 326                 } else {
 327                     $mp_param->{$opt} = $value;
 328                 }
 329                 $storage_only_mode = 0;
 330             } elsif ($opt =~ m/^unused\d+$/) {
 331                 warn "ignoring '$opt', cannot create/restore with unused volume\n";
 332                 delete $param->{$opt};
 333             } else {
 334                 $no_disk_param->{$opt} = $value;
 335             }
 336         }
 337
 338         die "mount points configured, but 'rootfs' not set - aborting\n"
 339             if !$storage_only_mode && !defined($mp_param->{rootfs});
 340
 341         # check storage access, activate storage
 342         my $delayed_mp_param = {};
 343         PVE::LXC::Config->foreach_volume($mp_param, sub {
 344             my ($ms, $mountpoint) = @_;
 345
 346             my $volid = $mountpoint->{volume};
 347             my $mp = $mountpoint->{mp};
 348
 349             if ($mountpoint->{type} ne 'volume') { # bind or device
 350                 die "Only root can pass arbitrary filesystem paths.\n"
 351                     if !$is_root;
 352             } else {
 353                 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 354                 &$check_and_activate_storage($sid);
 355             }
 356         });
 357
 358         # check/activate default storage
 359         &$check_and_activate_storage($storage) if !defined($mp_param->{rootfs});
 360
 361         PVE::LXC::Config->update_pct_config($vmid, $conf, 0, $no_disk_param);
 362
 363         $conf->{unprivileged} = 1 if $unprivileged;
 364
 365         my $emsg = $restore ? "unable to restore CT $vmid -" : "unable to create CT $vmid -";
 366
 367         eval { PVE::LXC::Config->create_and_lock_config($vmid, $force) };
 368         die "$emsg $@" if $@;
 369
 370         my $remove_lock = 1;
 371
 372         my $code = sub {
 373             my $old_conf = PVE::LXC::Config->load_config($vmid);
 374             my $was_template;
 375
 376             eval {
 377                 my $orig_mp_param; # only used if $restore
 378                 if ($restore) {
 379                     die "can't overwrite running container\n" if PVE::LXC::check_running($vmid);
 380                     if ($archive ne '-') {
 381                         my $orig_conf;
 382                         print "recovering backed-up configuration from '$archive'\n";
 383                         ($orig_conf, $orig_mp_param) = PVE::LXC::Create::recover_config($storage_cfg, $archive, $vmid);
 384
 385                         $was_template = delete $orig_conf->{template};
 386
 387                         # When we're root call 'restore_configuration' with restricted=0,
 388                         # causing it to restore the raw lxc entries, among which there may be
 389                         # 'lxc.idmap' entries. We need to make sure that the extracted contents
 390                         # of the container match up with the restored configuration afterwards:
 391                         $conf->{lxc} = $orig_conf->{lxc} if $is_root;
 392
 393                         $conf->{unprivileged} = $orig_conf->{unprivileged}
 394                             if !defined($unprivileged) && defined($orig_conf->{unprivileged});
 395
 396                         # implicit privileged change is checked here
 397                         if ($old_conf->{unprivileged} && !$conf->{unprivileged}) {
 398                             $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Allocate']);
 399                         }
 400                     }
 401                 }
 402                 if ($storage_only_mode) {
 403                     if ($restore) {
 404                         if (!defined($orig_mp_param)) {
 405                             print "recovering backed-up configuration from '$archive'\n";
 406                             (undef, $orig_mp_param) = PVE::LXC::Create::recover_config($storage_cfg, $archive, $vmid);
 407                         }
 408                         $mp_param = $orig_mp_param;
 409                         die "rootfs configuration could not be recovered, please check and specify manually!\n"
 410                             if !defined($mp_param->{rootfs});
 411                         PVE::LXC::Config->foreach_volume($mp_param, sub {
 412                             my ($ms, $mountpoint) = @_;
 413                             my $type = $mountpoint->{type};
 414                             if ($type eq 'volume') {
 415                                 die "unable to detect disk size - please specify $ms (size)\n"
 416                                     if !defined($mountpoint->{size});
 417                                 my $disksize = $mountpoint->{size} / (1024 * 1024 * 1024); # create_disks expects GB as unit size
 418                                 delete $mountpoint->{size};
 419                                 $mountpoint->{volume} = "$storage:$disksize";
 420                                 $mp_param->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
 421                             } else {
 422                                 my $type = $mountpoint->{type};
 423                                 die "restoring rootfs to $type mount is only possible by specifying -rootfs manually!\n"
 424                                     if ($ms eq 'rootfs');
 425                                 die "restoring '$ms' to $type mount is only possible for root\n"
 426                                     if !$is_root;
 427
 428                                 if ($mountpoint->{backup}) {
 429                                     warn "WARNING - unsupported configuration!\n";
 430                                     warn "backup was enabled for $type mount point $ms ('$mountpoint->{mp}')\n";
 431                                     warn "mount point configuration will be restored after archive extraction!\n";
 432                                     warn "contained files will be restored to wrong directory!\n";
 433                                 }
 434                                 delete $mp_param->{$ms}; # actually delay bind/dev mps
 435                                 $delayed_mp_param->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
 436                             }
 437                         });
 438                     } else {
 439                         $mp_param->{rootfs} = "$storage:4"; # defaults to 4GB
 440                     }
 441                 }
 442             };
 443             die "$emsg $@" if $@;
 444
 445             # up until here we did not modify the container, besides the lock
 446             $remove_lock = 0;
 447
 448             my $vollist = [];
 449             eval {
 450                 $vollist = PVE::LXC::create_disks($storage_cfg, $vmid, $mp_param, $conf);
 451
 452                 # we always have the 'create' lock so check for more than 1 entry
 453                 if (scalar(keys %$old_conf) > 1) {
 454                     # destroy old container volumes
 455                     PVE::LXC::destroy_lxc_container($storage_cfg, $vmid, $old_conf, { lock => 'create' });
 456                 }
 457
 458                 eval {
 459                     my $rootdir = PVE::LXC::mount_all($vmid, $storage_cfg, $conf, 1);
 460                     $bwlimit = PVE::Storage::get_bandwidth_limit('restore', [keys %used_storages], $bwlimit);
 461                     print "restoring '$archive' now..\n"
 462                         if $restore && $archive ne '-';
 463                     PVE::LXC::Create::restore_archive($storage_cfg, $archive, $rootdir, $conf, $ignore_unpack_errors, $bwlimit);
 464
 465                     if ($restore) {
 466                         print "merging backed-up and given configuration..\n";
 467                         PVE::LXC::Create::restore_configuration($vmid, $storage_cfg, $archive, $rootdir, $conf, !$is_root, $unique, $skip_fw_config_restore);
 468                         my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
 469                         $lxc_setup->template_fixup($conf);
 470                     } else {
 471                         my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir); # detect OS
 472                         PVE::LXC::Config->write_config($vmid, $conf); # safe config (after OS detection)
 473                         $lxc_setup->post_create_hook($password, $ssh_keys);
 474                     }
 475                 };
 476                 my $err = $@;
 477                 PVE::LXC::umount_all($vmid, $storage_cfg, $conf, $err ? 1 : 0);
 478                 PVE::Storage::deactivate_volumes($storage_cfg, PVE::LXC::Config->get_vm_volumes($conf));
 479                 die $err if $err;
 480                 # set some defaults
 481                 $conf->{hostname} ||= "CT$vmid";
 482                 $conf->{memory} ||= 512;
 483                 $conf->{swap} //= 512;
 484                 foreach my $mp (keys %$delayed_mp_param) {
 485                     $conf->{$mp} = $delayed_mp_param->{$mp};
 486                 }
 487                 # If the template flag was set, we try to convert again to template after restore
 488                 if ($was_template) {
 489                     print STDERR "Convert restored container to template...\n";
 490                     PVE::LXC::template_create($vmid, $conf);
 491                     $conf->{template} = 1;
 492                 }
 493                 PVE::LXC::Config->write_config($vmid, $conf);
 494             };
 495             if (my $err = $@) {
 496                 PVE::LXC::destroy_disks($storage_cfg, $vollist);
 497                 eval { PVE::LXC::Config->destroy_config($vmid) };
 498                 warn $@ if $@;
 499                 die "$emsg $err";
 500             }
 501             PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
 502
 503             PVE::API2::LXC::Status->vm_start({ vmid => $vmid, node => $node })
 504                 if $start_after_create;
 505         };
 506
 507         my $workername = $restore ? 'vzrestore' : 'vzcreate';
 508         my $realcmd = sub {
 509             eval {
 510                 PVE::LXC::Config->lock_config($vmid, $code);
 511             };
 512             if (my $err = $@) {
 513                 # if we aborted before changing the container, we must remove the create lock
 514                 if ($remove_lock) {
 515                     PVE::LXC::Config->remove_lock($vmid, 'create');
 516                 }
 517                 die $err;
 518             }
 519         };
 520
 521         return $rpcenv->fork_worker($workername, $vmid, $authuser, $realcmd);
 522     }});
 523
 524 __PACKAGE__->register_method({
 525     name => 'vmdiridx',
 526     path => '{vmid}',
 527     method => 'GET',
 528     proxyto => 'node',
 529     description => "Directory index",
 530     permissions => {
 531         user => 'all',
 532     },
 533     parameters => {
 534         additionalProperties => 0,
 535         properties => {
 536             node => get_standard_option('pve-node'),
 537             vmid => get_standard_option('pve-vmid'),
 538         },
 539     },
 540     returns => {
 541         type => 'array',
 542         items => {
 543             type => "object",
 544             properties => {
 545                 subdir => { type => 'string' },
 546             },
 547         },
 548         links => [ { rel => 'child', href => "{subdir}" } ],
 549     },
 550     code => sub {
 551         my ($param) = @_;
 552
 553         # test if VM exists
 554         my $conf = PVE::LXC::Config->load_config($param->{vmid});
 555
 556         my $res = [
 557             { subdir => 'config' },
 558             { subdir => 'pending' },
 559             { subdir => 'status' },
 560             { subdir => 'vncproxy' },
 561             { subdir => 'termproxy' },
 562             { subdir => 'vncwebsocket' },
 563             { subdir => 'spiceproxy' },
 564             { subdir => 'migrate' },
 565             { subdir => 'clone' },
 566 #           { subdir => 'initlog' },
 567             { subdir => 'rrd' },
 568             { subdir => 'rrddata' },
 569             { subdir => 'firewall' },
 570             { subdir => 'snapshot' },
 571             { subdir => 'resize' },
 572             ];
 573
 574         return $res;
 575     }});
 576
 577
 578 __PACKAGE__->register_method({
 579     name => 'rrd',
 580     path => '{vmid}/rrd',
 581     method => 'GET',
 582     protected => 1, # fixme: can we avoid that?
 583     permissions => {
 584         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
 585     },
 586     description => "Read VM RRD statistics (returns PNG)",
 587     parameters => {
 588         additionalProperties => 0,
 589         properties => {
 590             node => get_standard_option('pve-node'),
 591             vmid => get_standard_option('pve-vmid'),
 592             timeframe => {
 593                 description => "Specify the time frame you are interested in.",
 594                 type => 'string',
 595                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 596             },
 597             ds => {
 598                 description => "The list of datasources you want to display.",
 599                 type => 'string', format => 'pve-configid-list',
 600             },
 601             cf => {
 602                 description => "The RRD consolidation function",
 603                 type => 'string',
 604                 enum => [ 'AVERAGE', 'MAX' ],
 605                 optional => 1,
 606             },
 607         },
 608     },
 609     returns => {
 610         type => "object",
 611         properties => {
 612             filename => { type => 'string' },
 613         },
 614     },
 615     code => sub {
 616         my ($param) = @_;
 617
 618         return PVE::RRD::create_rrd_graph(
 619             "pve2-vm/$param->{vmid}", $param->{timeframe},
 620             $param->{ds}, $param->{cf});
 621
 622     }});
 623
 624 __PACKAGE__->register_method({
 625     name => 'rrddata',
 626     path => '{vmid}/rrddata',
 627     method => 'GET',
 628     protected => 1, # fixme: can we avoid that?
 629     permissions => {
 630         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
 631     },
 632     description => "Read VM RRD statistics",
 633     parameters => {
 634         additionalProperties => 0,
 635         properties => {
 636             node => get_standard_option('pve-node'),
 637             vmid => get_standard_option('pve-vmid'),
 638             timeframe => {
 639                 description => "Specify the time frame you are interested in.",
 640                 type => 'string',
 641                 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
 642             },
 643             cf => {
 644                 description => "The RRD consolidation function",
 645                 type => 'string',
 646                 enum => [ 'AVERAGE', 'MAX' ],
 647                 optional => 1,
 648             },
 649         },
 650     },
 651     returns => {
 652         type => "array",
 653         items => {
 654             type => "object",
 655             properties => {},
 656         },
 657     },
 658     code => sub {
 659         my ($param) = @_;
 660
 661         return PVE::RRD::create_rrd_data(
 662             "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
 663     }});
 664
 665 __PACKAGE__->register_method({
 666     name => 'destroy_vm',
 667     path => '{vmid}',
 668     method => 'DELETE',
 669     protected => 1,
 670     proxyto => 'node',
 671     description => "Destroy the container (also delete all uses files).",
 672     permissions => {
 673         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
 674     },
 675     parameters => {
 676         additionalProperties => 0,
 677         properties => {
 678             node => get_standard_option('pve-node'),
 679             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid_stopped }),
 680             force => {
 681                 type => 'boolean',
 682                 description => "Force destroy, even if running.",
 683                 default => 0,
 684                 optional => 1,
 685             },
 686             purge => {
 687                 type => 'boolean',
 688                 description => "Remove container from all related configurations."
 689                     ." For example, backup jobs, replication jobs or HA."
 690                     ." Related ACLs and Firewall entries will *always* be removed.",
 691                 default => 0,
 692                 optional => 1,
 693             },
 694             'destroy-unreferenced-disks' => {
 695                 type => 'boolean',
 696                 description => "If set, destroy additionally all disks with the VMID from all"
 697                     ." enabled storages which are not referenced in the config.",
 698                 optional => 1,
 699             },
 700         },
 701     },
 702     returns => {
 703         type => 'string',
 704     },
 705     code => sub {
 706         my ($param) = @_;
 707
 708         my $rpcenv = PVE::RPCEnvironment::get();
 709         my $authuser = $rpcenv->get_user();
 710         my $vmid = $param->{vmid};
 711
 712         # test if container exists
 713
 714         my $conf = PVE::LXC::Config->load_config($vmid);
 715         my $early_checks = sub {
 716             my ($conf) = @_;
 717             PVE::LXC::Config->check_protection($conf, "can't remove CT $vmid");
 718             PVE::LXC::Config->check_lock($conf);
 719
 720             my $ha_managed = PVE::HA::Config::service_is_configured("ct:$vmid");
 721
 722             if (!$param->{purge}) {
 723                 die "unable to remove CT $vmid - used in HA resources and purge parameter not set.\n"
 724                     if $ha_managed;
 725
 726                 # do not allow destroy if there are replication jobs without purge
 727                 my $repl_conf = PVE::ReplicationConfig->new();
 728                 $repl_conf->check_for_existing_jobs($vmid);
 729             }
 730
 731             return $ha_managed;
 732         };
 733
 734         $early_checks->($conf);
 735
 736         my $running_error_msg = "unable to destroy CT $vmid - container is running\n";
 737         die $running_error_msg if !$param->{force} && PVE::LXC::check_running($vmid); # check early
 738
 739         my $code = sub {
 740             # reload config after lock
 741             $conf = PVE::LXC::Config->load_config($vmid);
 742             my $ha_managed = $early_checks->($conf);
 743
 744             if (PVE::LXC::check_running($vmid)) {
 745                 die $running_error_msg if !$param->{force};
 746                 warn "forced to stop CT $vmid before destroying!\n";
 747                 if (!$ha_managed) {
 748                     PVE::LXC::vm_stop($vmid, 1);
 749                 } else {
 750                     run_command(['ha-manager', 'crm-command', 'stop',  "ct:$vmid", '120']);
 751                 }
 752             }
 753
 754             my $storage_cfg = cfs_read_file("storage.cfg");
 755             PVE::LXC::destroy_lxc_container(
 756                 $storage_cfg,
 757                 $vmid,
 758                 $conf,
 759                 { lock => 'destroyed' },
 760                 $param->{'destroy-unreferenced-disks'},
 761             );
 762
 763             PVE::AccessControl::remove_vm_access($vmid);
 764             PVE::Firewall::remove_vmfw_conf($vmid);
 765             if ($param->{purge}) {
 766                 print "purging CT $vmid from related configurations..\n";
 767                 PVE::ReplicationConfig::remove_vmid_jobs($vmid);
 768                 PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);
 769
 770                 if ($ha_managed) {
 771                     PVE::HA::Config::delete_service_from_config("ct:$vmid");
 772                     print "NOTE: removed CT $vmid from HA resource configuration.\n";
 773                 }
 774             }
 775
 776             # only now remove the zombie config, else we can have reuse race
 777             PVE::LXC::Config->destroy_config($vmid);
 778         };
 779
 780         my $realcmd = sub { PVE::LXC::Config->lock_config($vmid, $code); };
 781
 782         return $rpcenv->fork_worker('vzdestroy', $vmid, $authuser, $realcmd);
 783     }});
 784
 785 my $sslcert;
 786
 787 __PACKAGE__->register_method ({
 788     name => 'vncproxy',
 789     path => '{vmid}/vncproxy',
 790     method => 'POST',
 791     protected => 1,
 792     permissions => {
 793         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
 794     },
 795     description => "Creates a TCP VNC proxy connections.",
 796     parameters => {
 797         additionalProperties => 0,
 798         properties => {
 799             node => get_standard_option('pve-node'),
 800             vmid => get_standard_option('pve-vmid'),
 801             websocket => {
 802                 optional => 1,
 803                 type => 'boolean',
 804                 description => "use websocket instead of standard VNC.",
 805             },
 806             width => {
 807                 optional => 1,
 808                 description => "sets the width of the console in pixels.",
 809                 type => 'integer',
 810                 minimum => 16,
 811                 maximum => 4096,
 812             },
 813             height => {
 814                 optional => 1,
 815                 description => "sets the height of the console in pixels.",
 816                 type => 'integer',
 817                 minimum => 16,
 818                 maximum => 2160,
 819             },
 820         },
 821     },
 822     returns => {
 823         additionalProperties => 0,
 824         properties => {
 825             user => { type => 'string' },
 826             ticket => { type => 'string' },
 827             cert => { type => 'string' },
 828             port => { type => 'integer' },
 829             upid => { type => 'string' },
 830         },
 831     },
 832     code => sub {
 833         my ($param) = @_;
 834
 835         my $rpcenv = PVE::RPCEnvironment::get();
 836
 837         my $authuser = $rpcenv->get_user();
 838
 839         my $vmid = $param->{vmid};
 840         my $node = $param->{node};
 841
 842         my $authpath = "/vms/$vmid";
 843
 844         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
 845
 846         $sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
 847             if !$sslcert;
 848
 849         my ($remip, $family);
 850
 851         if ($node ne PVE::INotify::nodename()) {
 852             ($remip, $family) = PVE::Cluster::remote_node_ip($node);
 853         } else {
 854             $family = PVE::Tools::get_host_address_family($node);
 855         }
 856
 857         my $port = PVE::Tools::next_vnc_port($family);
 858
 859         # NOTE: vncterm VNC traffic is already TLS encrypted,
 860         # so we select the fastest chipher here (or 'none'?)
 861         my $remcmd = $remip ?
 862             ['/usr/bin/ssh', '-e', 'none', '-t', $remip] : [];
 863
 864         my $conf = PVE::LXC::Config->load_config($vmid, $node);
 865         my $concmd = PVE::LXC::get_console_command($vmid, $conf, -1);
 866
 867         my $shcmd = [ '/usr/bin/dtach', '-A',
 868                       "/var/run/dtach/vzctlconsole$vmid",
 869                       '-r', 'winch', '-z', @$concmd];
 870
 871         my $realcmd = sub {
 872             my $upid = shift;
 873
 874             syslog ('info', "starting lxc vnc proxy $upid\n");
 875
 876             my $timeout = 10;
 877
 878             my $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
 879                        '-timeout', $timeout, '-authpath', $authpath,
 880                        '-perm', 'VM.Console'];
 881
 882             if ($param->{width}) {
 883                 push @$cmd, '-width', $param->{width};
 884             }
 885
 886             if ($param->{height}) {
 887                 push @$cmd, '-height', $param->{height};
 888             }
 889
 890             if ($param->{websocket}) {
 891                 $ENV{PVE_VNC_TICKET} = $ticket; # pass ticket to vncterm
 892                 push @$cmd, '-notls', '-listen', 'localhost';
 893             }
 894
 895             push @$cmd, '-c', @$remcmd, @$shcmd;
 896
 897             run_command($cmd, keeplocale => 1);
 898
 899             return;
 900         };
 901
 902         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd);
 903
 904         PVE::Tools::wait_for_vnc_port($port);
 905
 906         return {
 907             user => $authuser,
 908             ticket => $ticket,
 909             port => $port,
 910             upid => $upid,
 911             cert => $sslcert,
 912         };
 913     }});
 914
 915 __PACKAGE__->register_method ({
 916     name => 'termproxy',
 917     path => '{vmid}/termproxy',
 918     method => 'POST',
 919     protected => 1,
 920     permissions => {
 921         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
 922     },
 923     description => "Creates a TCP proxy connection.",
 924     parameters => {
 925         additionalProperties => 0,
 926         properties => {
 927             node => get_standard_option('pve-node'),
 928             vmid => get_standard_option('pve-vmid'),
 929         },
 930     },
 931     returns => {
 932         additionalProperties => 0,
 933         properties => {
 934             user => { type => 'string' },
 935             ticket => { type => 'string' },
 936             port => { type => 'integer' },
 937             upid => { type => 'string' },
 938         },
 939     },
 940     code => sub {
 941         my ($param) = @_;
 942
 943         my $rpcenv = PVE::RPCEnvironment::get();
 944
 945         my $authuser = $rpcenv->get_user();
 946
 947         my $vmid = $param->{vmid};
 948         my $node = $param->{node};
 949
 950         my $authpath = "/vms/$vmid";
 951
 952         my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);
 953
 954         my ($remip, $family);
 955
 956         if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
 957             ($remip, $family) = PVE::Cluster::remote_node_ip($node);
 958         } else {
 959             $family = PVE::Tools::get_host_address_family($node);
 960         }
 961
 962         my $port = PVE::Tools::next_vnc_port($family);
 963
 964         my $remcmd = $remip ?
 965             ['/usr/bin/ssh', '-e', 'none', '-t', $remip, '--'] : [];
 966
 967         my $conf = PVE::LXC::Config->load_config($vmid, $node);
 968         my $concmd = PVE::LXC::get_console_command($vmid, $conf, -1);
 969
 970         my $shcmd = [ '/usr/bin/dtach', '-A',
 971                       "/var/run/dtach/vzctlconsole$vmid",
 972                       '-r', 'winch', '-z', @$concmd];
 973
 974         my $realcmd = sub {
 975             my $upid = shift;
 976
 977             syslog ('info', "starting lxc termproxy $upid\n");
 978
 979             my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
 980                        '--perm', 'VM.Console', '--'];
 981             push @$cmd, @$remcmd, @$shcmd;
 982
 983             PVE::Tools::run_command($cmd);
 984         };
 985
 986         my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);
 987
 988         PVE::Tools::wait_for_vnc_port($port);
 989
 990         return {
 991             user => $authuser,
 992             ticket => $ticket,
 993             port => $port,
 994             upid => $upid,
 995         };
 996     }});
 997
 998 __PACKAGE__->register_method({
 999     name => 'vncwebsocket',
1000     path => '{vmid}/vncwebsocket',
1001     method => 'GET',
1002     permissions => {
1003         description => "You also need to pass a valid ticket (vncticket).",
1004         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
1005     },
1006     description => "Opens a weksocket for VNC traffic.",
1007     parameters => {
1008         additionalProperties => 0,
1009         properties => {
1010             node => get_standard_option('pve-node'),
1011             vmid => get_standard_option('pve-vmid'),
1012             vncticket => {
1013                 description => "Ticket from previous call to vncproxy.",
1014                 type => 'string',
1015                 maxLength => 512,
1016             },
1017             port => {
1018                 description => "Port number returned by previous vncproxy call.",
1019                 type => 'integer',
1020                 minimum => 5900,
1021                 maximum => 5999,
1022             },
1023         },
1024     },
1025     returns => {
1026         type => "object",
1027         properties => {
1028             port => { type => 'string' },
1029         },
1030     },
1031     code => sub {
1032         my ($param) = @_;
1033
1034         my $rpcenv = PVE::RPCEnvironment::get();
1035
1036         my $authuser = $rpcenv->get_user();
1037
1038         my $authpath = "/vms/$param->{vmid}";
1039
1040         PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);
1041
1042         my $port = $param->{port};
1043
1044         return { port => $port };
1045     }});
1046
1047 __PACKAGE__->register_method ({
1048     name => 'spiceproxy',
1049     path => '{vmid}/spiceproxy',
1050     method => 'POST',
1051     protected => 1,
1052     proxyto => 'node',
1053     permissions => {
1054         check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
1055     },
1056     description => "Returns a SPICE configuration to connect to the CT.",
1057     parameters => {
1058         additionalProperties => 0,
1059         properties => {
1060             node => get_standard_option('pve-node'),
1061             vmid => get_standard_option('pve-vmid'),
1062             proxy => get_standard_option('spice-proxy', { optional => 1 }),
1063         },
1064     },
1065     returns => get_standard_option('remote-viewer-config'),
1066     code => sub {
1067         my ($param) = @_;
1068
1069         my $vmid = $param->{vmid};
1070         my $node = $param->{node};
1071         my $proxy = $param->{proxy};
1072
1073         my $authpath = "/vms/$vmid";
1074         my $permissions = 'VM.Console';
1075
1076         my $conf = PVE::LXC::Config->load_config($vmid);
1077
1078         die "CT $vmid not running\n" if !PVE::LXC::check_running($vmid);
1079
1080         my $concmd = PVE::LXC::get_console_command($vmid, $conf);
1081
1082         my $shcmd = ['/usr/bin/dtach', '-A',
1083                      "/var/run/dtach/vzctlconsole$vmid",
1084                      '-r', 'winch', '-z', @$concmd];
1085
1086         my $title = "CT $vmid";
1087
1088         return PVE::API2Tools::run_spiceterm($authpath, $permissions, $vmid, $node, $proxy, $title, $shcmd);
1089     }});
1090
1091
1092 __PACKAGE__->register_method({
1093     name => 'migrate_vm',
1094     path => '{vmid}/migrate',
1095     method => 'POST',
1096     protected => 1,
1097     proxyto => 'node',
1098     description => "Migrate the container to another node. Creates a new migration task.",
1099     permissions => {
1100         check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
1101     },
1102     parameters => {
1103         additionalProperties => 0,
1104         properties => {
1105             node => get_standard_option('pve-node'),
1106             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
1107             target => get_standard_option('pve-node', {
1108                 description => "Target node.",
1109                 completion => \&PVE::Cluster::complete_migration_target,
1110             }),
1111             'target-storage' => get_standard_option('pve-targetstorage'),
1112             online => {
1113                 type => 'boolean',
1114                 description => "Use online/live migration.",
1115                 optional => 1,
1116             },
1117             restart => {
1118                 type => 'boolean',
1119                 description => "Use restart migration",
1120                 optional => 1,
1121             },
1122             timeout => {
1123                 type => 'integer',
1124                 description => "Timeout in seconds for shutdown for restart migration",
1125                 optional => 1,
1126                 default => 180,
1127             },
1128             bwlimit => {
1129                 description => "Override I/O bandwidth limit (in KiB/s).",
1130                 optional => 1,
1131                 type => 'number',
1132                 minimum => '0',
1133                 default => 'migrate limit from datacenter or storage config',
1134             },
1135         },
1136     },
1137     returns => {
1138         type => 'string',
1139         description => "the task ID.",
1140     },
1141     code => sub {
1142         my ($param) = @_;
1143
1144         my $rpcenv = PVE::RPCEnvironment::get();
1145
1146         my $authuser = $rpcenv->get_user();
1147
1148         my $target = extract_param($param, 'target');
1149
1150         my $localnode = PVE::INotify::nodename();
1151         raise_param_exc({ target => "target is local node."}) if $target eq $localnode;
1152
1153         PVE::Cluster::check_cfs_quorum();
1154
1155         PVE::Cluster::check_node_exists($target);
1156
1157         my $targetip = PVE::Cluster::remote_node_ip($target);
1158
1159         my $vmid = extract_param($param, 'vmid');
1160
1161         # test if VM exists
1162         PVE::LXC::Config->load_config($vmid);
1163
1164         # try to detect errors early
1165         if (PVE::LXC::check_running($vmid)) {
1166             die "can't migrate running container without --online or --restart\n"
1167                 if !$param->{online} && !$param->{restart};
1168         }
1169
1170         if (my $targetstorage = delete $param->{'target-storage'}) {
1171             my $storecfg = PVE::Storage::config();
1172             my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
1173             raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
1174                 if $@;
1175
1176             $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
1177                 if !defined($storagemap->{identity});
1178
1179             foreach my $target_sid (values %{$storagemap->{entries}}) {
1180                 $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
1181             }
1182
1183             $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
1184                 if $storagemap->{default};
1185
1186             $param->{storagemap} = $storagemap;
1187         }
1188
1189         if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {
1190
1191             my $hacmd = sub {
1192                 my $upid = shift;
1193
1194                 my $service = "ct:$vmid";
1195
1196                 my $cmd = ['ha-manager', 'migrate', $service, $target];
1197
1198                 print "Requesting HA migration for CT $vmid to node $target\n";
1199
1200                 PVE::Tools::run_command($cmd);
1201
1202                 return;
1203             };
1204
1205             return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);
1206
1207         } else {
1208
1209             my $realcmd = sub {
1210                 PVE::LXC::Migrate->migrate($target, $targetip, $vmid, $param);
1211             };
1212
1213             my $worker = sub {
1214                 return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
1215             };
1216
1217             return $rpcenv->fork_worker('vzmigrate', $vmid, $authuser, $worker);
1218         }
1219     }});
1220
1221 __PACKAGE__->register_method({
1222     name => 'vm_feature',
1223     path => '{vmid}/feature',
1224     method => 'GET',
1225     proxyto => 'node',
1226     protected => 1,
1227     description => "Check if feature for virtual machine is available.",
1228     permissions => {
1229         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
1230     },
1231     parameters => {
1232         additionalProperties => 0,
1233         properties => {
1234             node => get_standard_option('pve-node'),
1235             vmid => get_standard_option('pve-vmid'),
1236             feature => {
1237                 description => "Feature to check.",
1238                 type => 'string',
1239                 enum => [ 'snapshot', 'clone', 'copy' ],
1240             },
1241             snapname => get_standard_option('pve-snapshot-name', {
1242                 optional => 1,
1243             }),
1244         },
1245     },
1246     returns => {
1247         type => "object",
1248         properties => {
1249             hasFeature => { type => 'boolean' },
1250             #nodes => {
1251                 #type => 'array',
1252                 #items => { type => 'string' },
1253             #}
1254         },
1255     },
1256     code => sub {
1257         my ($param) = @_;
1258
1259         my $node = extract_param($param, 'node');
1260
1261         my $vmid = extract_param($param, 'vmid');
1262
1263         my $snapname = extract_param($param, 'snapname');
1264
1265         my $feature = extract_param($param, 'feature');
1266
1267         my $conf = PVE::LXC::Config->load_config($vmid);
1268
1269         if($snapname){
1270             my $snap = $conf->{snapshots}->{$snapname};
1271             die "snapshot '$snapname' does not exist\n" if !defined($snap);
1272             $conf = $snap;
1273         }
1274         my $storage_cfg = PVE::Storage::config();
1275         #Maybe include later
1276         #my $nodelist = PVE::LXC::shared_nodes($conf, $storage_cfg);
1277         my $hasFeature = PVE::LXC::Config->has_feature($feature, $conf, $storage_cfg, $snapname);
1278
1279         return {
1280             hasFeature => $hasFeature,
1281             #nodes => [ keys %$nodelist ],
1282         };
1283     }});
1284
1285 __PACKAGE__->register_method({
1286     name => 'template',
1287     path => '{vmid}/template',
1288     method => 'POST',
1289     protected => 1,
1290     proxyto => 'node',
1291     description => "Create a Template.",
1292     permissions => {
1293         description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
1294         check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
1295     },
1296     parameters => {
1297         additionalProperties => 0,
1298         properties => {
1299             node => get_standard_option('pve-node'),
1300             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid_stopped }),
1301         },
1302     },
1303     returns => { type => 'null'},
1304     code => sub {
1305         my ($param) = @_;
1306
1307         my $rpcenv = PVE::RPCEnvironment::get();
1308
1309         my $authuser = $rpcenv->get_user();
1310
1311         my $node = extract_param($param, 'node');
1312
1313         my $vmid = extract_param($param, 'vmid');
1314
1315         my $updatefn =  sub {
1316
1317             my $conf = PVE::LXC::Config->load_config($vmid);
1318             PVE::LXC::Config->check_lock($conf);
1319
1320             die "unable to create template, because CT contains snapshots\n"
1321                 if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});
1322
1323             die "you can't convert a template to a template\n"
1324                 if PVE::LXC::Config->is_template($conf);
1325
1326             die "you can't convert a CT to template if the CT is running\n"
1327                 if PVE::LXC::check_running($vmid);
1328
1329             my $realcmd = sub {
1330                 PVE::LXC::template_create($vmid, $conf);
1331
1332                 $conf->{template} = 1;
1333
1334                 PVE::LXC::Config->write_config($vmid, $conf);
1335                 # and remove lxc config
1336                 PVE::LXC::update_lxc_config($vmid, $conf);
1337             };
1338
1339             return $rpcenv->fork_worker('vztemplate', $vmid, $authuser, $realcmd);
1340         };
1341
1342         PVE::LXC::Config->lock_config($vmid, $updatefn);
1343
1344         return undef;
1345     }});
1346
1347 __PACKAGE__->register_method({
1348     name => 'clone_vm',
1349     path => '{vmid}/clone',
1350     method => 'POST',
1351     protected => 1,
1352     proxyto => 'node',
1353     description => "Create a container clone/copy",
1354     permissions => {
1355         description => "You need 'VM.Clone' permissions on /vms/{vmid}, " .
1356             "and 'VM.Allocate' permissions " .
1357             "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
1358             "'Datastore.AllocateSpace' on any used storage.",
1359         check =>
1360         [ 'and',
1361           ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
1362           [ 'or',
1363             [ 'perm', '/vms/{newid}', ['VM.Allocate']],
1364             [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
1365           ],
1366         ]
1367     },
1368     parameters => {
1369         additionalProperties => 0,
1370         properties => {
1371             node => get_standard_option('pve-node'),
1372             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
1373             newid => get_standard_option('pve-vmid', {
1374                 completion => \&PVE::Cluster::complete_next_vmid,
1375                 description => 'VMID for the clone.' }),
1376             hostname => {
1377                 optional => 1,
1378                 type => 'string', format => 'dns-name',
1379                 description => "Set a hostname for the new CT.",
1380             },
1381             description => {
1382                 optional => 1,
1383                 type => 'string',
1384                 description => "Description for the new CT.",
1385             },
1386             pool => {
1387                 optional => 1,
1388                 type => 'string', format => 'pve-poolid',
1389                 description => "Add the new CT to the specified pool.",
1390             },
1391             snapname => get_standard_option('pve-snapshot-name', {
1392                 optional => 1,
1393             }),
1394             storage => get_standard_option('pve-storage-id', {
1395                 description => "Target storage for full clone.",
1396                 optional => 1,
1397             }),
1398             full => {
1399                 optional => 1,
1400                 type => 'boolean',
1401                 description => "Create a full copy of all disks. This is always done when " .
1402                     "you clone a normal CT. For CT templates, we try to create a linked clone by default.",
1403             },
1404             target => get_standard_option('pve-node', {
1405                 description => "Target node. Only allowed if the original VM is on shared storage.",
1406                 optional => 1,
1407             }),
1408             bwlimit => {
1409                 description => "Override I/O bandwidth limit (in KiB/s).",
1410                 optional => 1,
1411                 type => 'number',
1412                 minimum => '0',
1413                 default => 'clone limit from datacenter or storage config',
1414             },
1415         },
1416     },
1417     returns => {
1418         type => 'string',
1419     },
1420     code => sub {
1421         my ($param) = @_;
1422
1423         my $rpcenv = PVE::RPCEnvironment::get();
1424         my $authuser = $rpcenv->get_user();
1425
1426         my $node = extract_param($param, 'node');
1427         my $vmid = extract_param($param, 'vmid');
1428         my $newid = extract_param($param, 'newid');
1429         my $pool = extract_param($param, 'pool');
1430         if (defined($pool)) {
1431             $rpcenv->check_pool_exist($pool);
1432         }
1433         my $snapname = extract_param($param, 'snapname');
1434         my $storage = extract_param($param, 'storage');
1435         my $target = extract_param($param, 'target');
1436         my $localnode = PVE::INotify::nodename();
1437
1438         undef $target if $target && ($target eq $localnode || $target eq 'localhost');
1439
1440         PVE::Cluster::check_node_exists($target) if $target;
1441
1442         my $storecfg = PVE::Storage::config();
1443
1444         if ($storage) {
1445             # check if storage is enabled on local node
1446             PVE::Storage::storage_check_enabled($storecfg, $storage);
1447             if ($target) {
1448                 # check if storage is available on target node
1449                 PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
1450                 # clone only works if target storage is shared
1451                 my $scfg = PVE::Storage::storage_config($storecfg, $storage);
1452                 die "can't clone to non-shared storage '$storage'\n" if !$scfg->{shared};
1453             }
1454         }
1455
1456         PVE::Cluster::check_cfs_quorum();
1457
1458         my $newconf = {};
1459         my $mountpoints = {};
1460         my $fullclone = {};
1461         my $vollist = [];
1462         my $running;
1463
1464         PVE::LXC::Config->create_and_lock_config($newid, 0);
1465         PVE::Firewall::clone_vmfw_conf($vmid, $newid);
1466
1467         my $lock_and_reload = sub {
1468             my ($vmid, $code) = @_;
1469             return PVE::LXC::Config->lock_config($vmid, sub {
1470                 my $conf = PVE::LXC::Config->load_config($vmid);
1471                 die "Lost 'create' config lock, aborting.\n"
1472                     if !PVE::LXC::Config->has_lock($conf, 'create');
1473
1474                 return $code->($conf);
1475             });
1476         };
1477
1478         my $src_conf = PVE::LXC::Config->set_lock($vmid, 'disk');
1479
1480         $running = PVE::LXC::check_running($vmid) || 0;
1481
1482         my $full = extract_param($param, 'full');
1483         if (!defined($full)) {
1484             $full = !PVE::LXC::Config->is_template($src_conf);
1485         }
1486
1487         eval {
1488             die "parameter 'storage' not allowed for linked clones\n"
1489                 if defined($storage) && !$full;
1490
1491             die "snapshot '$snapname' does not exist\n"
1492                 if $snapname && !defined($src_conf->{snapshots}->{$snapname});
1493
1494             my $src_conf = $snapname ? $src_conf->{snapshots}->{$snapname} : $src_conf;
1495
1496             my $sharedvm = 1;
1497             for my $opt (sort keys %$src_conf) {
1498                 next if $opt =~ m/^unused\d+$/;
1499
1500                 my $value = $src_conf->{$opt};
1501
1502                 if (($opt eq 'rootfs') || ($opt =~ m/^mp\d+$/)) {
1503                     my $mp = PVE::LXC::Config->parse_volume($opt, $value);
1504
1505                     if ($mp->{type} eq 'volume') {
1506                         my $volid = $mp->{volume};
1507
1508                         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
1509                         $sid = $storage if defined($storage);
1510                         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
1511                         if (!$scfg->{shared}) {
1512                             $sharedvm = 0;
1513                             warn "found non-shared volume: $volid\n" if $target;
1514                         }
1515
1516                         $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
1517
1518                         if ($full) {
1519                             die "Cannot do full clones on a running container without snapshots\n"
1520                                 if $running && !defined($snapname);
1521                             $fullclone->{$opt} = 1;
1522                         } else {
1523                             # not full means clone instead of copy
1524                             die "Linked clone feature for '$volid' is not available\n"
1525                                 if !PVE::Storage::volume_has_feature($storecfg, 'clone', $volid, $snapname, $running, {'valid_target_formats' => ['raw', 'subvol']});
1526                         }
1527
1528                         $mountpoints->{$opt} = $mp;
1529                         push @$vollist, $volid;
1530
1531                     } else {
1532                         # TODO: allow bind mounts?
1533                         die "unable to clone mountpoint '$opt' (type $mp->{type})\n";
1534                     }
1535                 } elsif ($opt =~ m/^net(\d+)$/) {
1536                     # always change MAC! address
1537                     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1538                     my $net = PVE::LXC::Config->parse_lxc_network($value);
1539                     $net->{hwaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1540                     $newconf->{$opt} = PVE::LXC::Config->print_lxc_network($net);
1541                 } else {
1542                     # copy everything else
1543                     $newconf->{$opt} = $value;
1544                 }
1545             }
1546             die "can't clone CT to node '$target' (CT uses local storage)\n"
1547                 if $target && !$sharedvm;
1548
1549             # Replace the 'disk' lock with a 'create' lock.
1550             $newconf->{lock} = 'create';
1551
1552             # delete all snapshot related config options
1553             delete $newconf->@{qw(snapshots parent snaptime snapstate)};
1554
1555             delete $newconf->{pending};
1556             delete $newconf->{template};
1557
1558             $newconf->{hostname} = $param->{hostname} if $param->{hostname};
1559             $newconf->{description} = $param->{description} if $param->{description};
1560
1561             $lock_and_reload->($newid, sub {
1562                 PVE::LXC::Config->write_config($newid, $newconf);
1563             });
1564         };
1565         if (my $err = $@) {
1566             eval { PVE::LXC::Config->remove_lock($vmid, 'disk') };
1567             warn "Failed to remove source CT config lock - $@\n" if $@;
1568
1569             eval {
1570                 $lock_and_reload->($newid, sub {
1571                     PVE::LXC::Config->destroy_config($newid);
1572                     PVE::Firewall::remove_vmfw_conf($newid);
1573                 });
1574             };
1575             warn "Failed to remove target CT config - $@\n" if $@;
1576
1577             die $err;
1578         }
1579
1580         my $update_conf = sub {
1581             my ($key, $value) = @_;
1582             return $lock_and_reload->($newid, sub {
1583                 my $conf = shift;
1584                 $conf->{$key} = $value;
1585                 PVE::LXC::Config->write_config($newid, $conf);
1586             });
1587         };
1588
1589         my $realcmd = sub {
1590             my ($upid) = @_;
1591
1592             my $newvollist = [];
1593
1594             my $verify_running = PVE::LXC::check_running($vmid) || 0;
1595             die "unexpected state change\n" if $verify_running != $running;
1596
1597             eval {
1598                 local $SIG{INT} =
1599                     local $SIG{TERM} =
1600                     local $SIG{QUIT} =
1601                     local $SIG{HUP} = sub { die "interrupted by signal\n"; };
1602
1603                 PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
1604                 my $bwlimit = extract_param($param, 'bwlimit');
1605
1606                 foreach my $opt (keys %$mountpoints) {
1607                     my $mp = $mountpoints->{$opt};
1608                     my $volid = $mp->{volume};
1609
1610                     my $newvolid;
1611                     if ($fullclone->{$opt}) {
1612                         print "create full clone of mountpoint $opt ($volid)\n";
1613                         my $source_storage = PVE::Storage::parse_volume_id($volid);
1614                         my $target_storage = $storage // $source_storage;
1615                         my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', [$source_storage, $target_storage], $bwlimit);
1616                         $newvolid = PVE::LXC::copy_volume($mp, $newid, $target_storage, $storecfg, $newconf, $snapname, $clonelimit);
1617                     } else {
1618                         print "create linked clone of mount point $opt ($volid)\n";
1619                         $newvolid = PVE::Storage::vdisk_clone($storecfg, $volid, $newid, $snapname);
1620                     }
1621
1622                     push @$newvollist, $newvolid;
1623                     $mp->{volume} = $newvolid;
1624
1625                     $update_conf->($opt, PVE::LXC::Config->print_ct_mountpoint($mp, $opt eq 'rootfs'));
1626                 }
1627
1628                 PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
1629
1630                 $lock_and_reload->($newid, sub {
1631                     my $conf = shift;
1632                     my $rootdir = PVE::LXC::mount_all($newid, $storecfg, $conf, 1);
1633                     eval {
1634                         my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
1635                         $lxc_setup->post_clone_hook($conf);
1636                     };
1637                     my $err = $@;
1638                     eval { PVE::LXC::umount_all($newid, $storecfg, $conf, 1); };
1639                     if ($err) {
1640                         warn "$@\n" if $@;
1641                         die $err;
1642                     } else {
1643                         die $@ if $@;
1644                     }
1645                 });
1646             };
1647             my $err = $@;
1648             # Unlock the source config in any case:
1649             eval { PVE::LXC::Config->remove_lock($vmid, 'disk') };
1650             warn $@ if $@;
1651
1652             if ($err) {
1653                 # Now cleanup the config & disks:
1654                 sleep 1; # some storages like rbd need to wait before release volume - really?
1655
1656                 foreach my $volid (@$newvollist) {
1657                     eval { PVE::Storage::vdisk_free($storecfg, $volid); };
1658                     warn $@ if $@;
1659                 }
1660
1661                 eval {
1662                     $lock_and_reload->($newid, sub {
1663                         PVE::LXC::Config->destroy_config($newid);
1664                         PVE::Firewall::remove_vmfw_conf($newid);
1665                     });
1666                 };
1667                 warn "Failed to remove target CT config - $@\n" if $@;
1668
1669                 die "clone failed: $err";
1670             }
1671
1672             $lock_and_reload->($newid, sub {
1673                 PVE::LXC::Config->remove_lock($newid, 'create');
1674
1675                 if ($target) {
1676                     # always deactivate volumes - avoid lvm LVs to be active on several nodes
1677                     PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
1678                     PVE::Storage::deactivate_volumes($storecfg, $newvollist);
1679
1680                     PVE::LXC::Config->move_config_to_node($newid, $target);
1681                 }
1682             });
1683
1684             return;
1685         };
1686
1687         return $rpcenv->fork_worker('vzclone', $vmid, $authuser, $realcmd);
1688     }});
1689
1690
1691 __PACKAGE__->register_method({
1692     name => 'resize_vm',
1693     path => '{vmid}/resize',
1694     method => 'PUT',
1695     protected => 1,
1696     proxyto => 'node',
1697     description => "Resize a container mount point.",
1698     permissions => {
1699         check => ['perm', '/vms/{vmid}', ['VM.Config.Disk'], any => 1],
1700     },
1701     parameters => {
1702         additionalProperties => 0,
1703         properties => {
1704             node => get_standard_option('pve-node'),
1705             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
1706             disk => {
1707                 type => 'string',
1708                 description => "The disk you want to resize.",
1709                 enum => [PVE::LXC::Config->valid_volume_keys()],
1710             },
1711             size => {
1712                 type => 'string',
1713                 pattern => '\+?\d+(\.\d+)?[KMGT]?',
1714                 description => "The new size. With the '+' sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
1715             },
1716             digest => {
1717                 type => 'string',
1718                 description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
1719                 maxLength => 40,
1720                 optional => 1,
1721             }
1722         },
1723     },
1724     returns => {
1725         type => 'string',
1726         description => "the task ID.",
1727     },
1728     code => sub {
1729         my ($param) = @_;
1730
1731         my $rpcenv = PVE::RPCEnvironment::get();
1732
1733         my $authuser = $rpcenv->get_user();
1734
1735         my $node = extract_param($param, 'node');
1736
1737         my $vmid = extract_param($param, 'vmid');
1738
1739         my $digest = extract_param($param, 'digest');
1740
1741         my $sizestr = extract_param($param, 'size');
1742         my $ext = ($sizestr =~ s/^\+//);
1743         my $newsize = PVE::JSONSchema::parse_size($sizestr);
1744         die "invalid size string" if !defined($newsize);
1745
1746         die "no options specified\n" if !scalar(keys %$param);
1747
1748         my $storage_cfg = cfs_read_file("storage.cfg");
1749
1750         my $code = sub {
1751
1752             my $conf = PVE::LXC::Config->load_config($vmid);
1753             PVE::LXC::Config->check_lock($conf);
1754
1755             PVE::LXC::check_ct_modify_config_perm($rpcenv, $authuser, $vmid, undef, $conf, $param, [], $conf->{unprivileged});
1756
1757             PVE::Tools::assert_if_modified($digest, $conf->{digest});
1758
1759             my $running = PVE::LXC::check_running($vmid);
1760
1761             my $disk = $param->{disk};
1762             my $mp = PVE::LXC::Config->parse_volume($disk, $conf->{$disk});
1763
1764             my $volid = $mp->{volume};
1765
1766             my (undef, undef, $owner, undef, undef, undef, $format) =
1767                 PVE::Storage::parse_volname($storage_cfg, $volid);
1768
1769             die "can't resize mount point owned by another container ($owner)"
1770                 if $vmid != $owner;
1771
1772             die "can't resize volume: $disk if snapshot exists\n"
1773                 if %{$conf->{snapshots}} && $format eq 'qcow2';
1774
1775             my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
1776
1777             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
1778
1779             PVE::Storage::activate_volumes($storage_cfg, [$volid]);
1780
1781             my $size = PVE::Storage::volume_size_info($storage_cfg, $volid, 5);
1782
1783             die "Could not determine current size of volume '$volid'\n" if !defined($size);
1784
1785             $newsize += $size if $ext;
1786             $newsize = int($newsize);
1787
1788             die "unable to shrink disk size\n" if $newsize < $size;
1789
1790             die "disk is already at specified size\n" if $size == $newsize;
1791
1792             PVE::Cluster::log_msg('info', $authuser, "update CT $vmid: resize --disk $disk --size $sizestr");
1793             my $realcmd = sub {
1794                 # Note: PVE::Storage::volume_resize doesn't do anything if $running=1, so
1795                 # we pass 0 here (parameter only makes sense for qemu)
1796                 PVE::Storage::volume_resize($storage_cfg, $volid, $newsize, 0);
1797
1798                 $mp->{size} = $newsize;
1799                 $conf->{$disk} = PVE::LXC::Config->print_ct_mountpoint($mp, $disk eq 'rootfs');
1800
1801                 PVE::LXC::Config->write_config($vmid, $conf);
1802
1803                 if ($format eq 'raw') {
1804                     # we need to ensure that the volume is mapped, if not needed this is a NOP
1805                     my $path = PVE::Storage::map_volume($storage_cfg, $volid);
1806                     $path = PVE::Storage::path($storage_cfg, $volid) if !defined($path);
1807                     if ($running) {
1808
1809                         $mp->{mp} = '/';
1810                         my $use_loopdev = (PVE::LXC::mountpoint_mount_path($mp, $storage_cfg))[1];
1811                         $path = PVE::LXC::query_loopdev($path) if $use_loopdev;
1812                         die "internal error: CT running but mount point not attached to a loop device"
1813                             if !$path;
1814                         PVE::Tools::run_command(['losetup', '--set-capacity', $path]) if $use_loopdev;
1815
1816                         # In order for resize2fs to know that we need online-resizing a mountpoint needs
1817                         # to be visible to it in its namespace.
1818                         # To not interfere with the rest of the system we unshare the current mount namespace,
1819                         # mount over /tmp and then run resize2fs.
1820
1821                         # interestingly we don't need to e2fsck on mounted systems...
1822                         my $quoted = PVE::Tools::shellquote($path);
1823                         my $cmd = "mount --make-rprivate / && mount $quoted /tmp && resize2fs $quoted";
1824                         eval {
1825                             PVE::Tools::run_command(['unshare', '-m', '--', 'sh', '-c', $cmd]);
1826                         };
1827                         warn "Failed to update the container's filesystem: $@\n" if $@;
1828                     } else {
1829                         eval {
1830                             PVE::Tools::run_command(['e2fsck', '-f', '-y', $path]);
1831                             PVE::Tools::run_command(['resize2fs', $path]);
1832                         };
1833                         warn "Failed to update the container's filesystem: $@\n" if $@;
1834
1835                         # always un-map if not running, this is a NOP if not needed
1836                         PVE::Storage::unmap_volume($storage_cfg, $volid);
1837                     }
1838                 }
1839             };
1840
1841             return $rpcenv->fork_worker('resize', $vmid, $authuser, $realcmd);
1842         };
1843
1844         return PVE::LXC::Config->lock_config($vmid, $code);;
1845     }});
1846
1847 __PACKAGE__->register_method({
1848     name => 'move_volume',
1849     path => '{vmid}/move_volume',
1850     method => 'POST',
1851     protected => 1,
1852     proxyto => 'node',
1853     description => "Move a rootfs-/mp-volume to a different storage or to a different container.",
1854     permissions => {
1855         description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
1856             "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
1857             "a volume to another container, you need the permissions on the ".
1858             "target container as well.",
1859         check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
1860     },
1861     parameters => {
1862         additionalProperties => 0,
1863         properties => {
1864             node => get_standard_option('pve-node'),
1865             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
1866             'target-vmid' => get_standard_option('pve-vmid', {
1867                 completion => \&PVE::LXC::complete_ctid,
1868                 optional => 1,
1869             }),
1870             volume => {
1871                 type => 'string',
1872                 #TODO: check how to handle unused mount points as the mp parameter is not configured
1873                 enum => [ PVE::LXC::Config->valid_volume_keys_with_unused() ],
1874                 description => "Volume which will be moved.",
1875             },
1876             storage => get_standard_option('pve-storage-id', {
1877                 description => "Target Storage.",
1878                 completion => \&PVE::Storage::complete_storage_enabled,
1879                 optional => 1,
1880             }),
1881             delete => {
1882                 type => 'boolean',
1883                 description => "Delete the original volume after successful copy. By default the " .
1884                     "original is kept as an unused volume entry.",
1885                 optional => 1,
1886                 default => 0,
1887             },
1888             digest => {
1889                 type => 'string',
1890                 description => 'Prevent changes if current configuration file has different SHA1 " .
1891                     "digest. This can be used to prevent concurrent modifications.',
1892                 maxLength => 40,
1893                 optional => 1,
1894             },
1895             bwlimit => {
1896                 description => "Override I/O bandwidth limit (in KiB/s).",
1897                 optional => 1,
1898                 type => 'number',
1899                 minimum => '0',
1900                 default => 'clone limit from datacenter or storage config',
1901             },
1902             'target-volume' => {
1903                 type => 'string',
1904                 description => "The config key the volume will be moved to. Default is the " .
1905                     "source volume key.",
1906                 enum => [PVE::LXC::Config->valid_volume_keys_with_unused()],
1907                 optional => 1,
1908             },
1909             'target-digest' => {
1910                 type => 'string',
1911                 description => 'Prevent changes if current configuration file of the target " .
1912                     "container has a different SHA1 digest. This can be used to prevent " .
1913                     "concurrent modifications.',
1914                 maxLength => 40,
1915                 optional => 1,
1916             },
1917         },
1918     },
1919     returns => {
1920         type => 'string',
1921     },
1922     code => sub {
1923         my ($param) = @_;
1924
1925         my $rpcenv = PVE::RPCEnvironment::get();
1926
1927         my $authuser = $rpcenv->get_user();
1928
1929         my $vmid = extract_param($param, 'vmid');
1930
1931         my $target_vmid = extract_param($param, 'target-vmid');
1932
1933         my $storage = extract_param($param, 'storage');
1934
1935         my $mpkey = extract_param($param, 'volume');
1936
1937         my $target_mpkey = extract_param($param, 'target-volume') // $mpkey;
1938
1939         my $digest = extract_param($param, 'digest');
1940
1941         my $target_digest = extract_param($param, 'target-digest');
1942
1943         my $lockname = 'disk';
1944
1945         my ($mpdata, $old_volid);
1946
1947         die "either set storage or target-vmid, but not both\n"
1948             if $storage && $target_vmid;
1949
1950         my $storecfg = PVE::Storage::config();
1951
1952         my $move_to_storage_checks = sub {
1953             PVE::LXC::Config->lock_config($vmid, sub {
1954                 my $conf = PVE::LXC::Config->load_config($vmid);
1955                 PVE::LXC::Config->check_lock($conf);
1956
1957                 die "cannot move volumes of a running container\n"
1958                     if PVE::LXC::check_running($vmid);
1959
1960                 if ($mpkey =~ m/^unused\d+$/) {
1961                     die "cannot move volume '$mpkey', only configured volumes can be moved to ".
1962                         "another storage\n";
1963                 }
1964
1965                 $mpdata = PVE::LXC::Config->parse_volume($mpkey, $conf->{$mpkey});
1966                 $old_volid = $mpdata->{volume};
1967
1968                 die "you can't move a volume with snapshots and delete the source\n"
1969                     if $param->{delete} && PVE::LXC::Config->is_volume_in_use_by_snapshots($conf, $old_volid);
1970
1971                 PVE::Tools::assert_if_modified($digest, $conf->{digest});
1972
1973                 PVE::LXC::Config->set_lock($vmid, $lockname);
1974             });
1975         };
1976
1977         my $storage_realcmd = sub {
1978             eval {
1979                 PVE::Cluster::log_msg(
1980                     'info',
1981                     $authuser,
1982                     "move volume CT $vmid: move --volume $mpkey --storage $storage"
1983                 );
1984
1985                 my $conf = PVE::LXC::Config->load_config($vmid);
1986                 my $storage_cfg = PVE::Storage::config();
1987
1988                 my $new_volid;
1989
1990                 eval {
1991                     PVE::Storage::activate_volumes($storage_cfg, [ $old_volid ]);
1992                     my $bwlimit = extract_param($param, 'bwlimit');
1993                     my $source_storage = PVE::Storage::parse_volume_id($old_volid);
1994                     my $movelimit = PVE::Storage::get_bandwidth_limit(
1995                         'move',
1996                         [$source_storage, $storage],
1997                         $bwlimit
1998                     );
1999                     $new_volid = PVE::LXC::copy_volume(
2000                         $mpdata,
2001                         $vmid,
2002                         $storage,
2003                         $storage_cfg,
2004                         $conf,
2005                         undef,
2006                         $movelimit
2007                     );
2008                     if (PVE::LXC::Config->is_template($conf)) {
2009                         PVE::Storage::activate_volumes($storage_cfg, [ $new_volid ]);
2010                         my $template_volid = PVE::Storage::vdisk_create_base($storage_cfg, $new_volid);
2011                         $mpdata->{volume} = $template_volid;
2012                     } else {
2013                         $mpdata->{volume} = $new_volid;
2014                     }
2015
2016                     PVE::LXC::Config->lock_config($vmid, sub {
2017                         my $digest = $conf->{digest};
2018                         $conf = PVE::LXC::Config->load_config($vmid);
2019                         PVE::Tools::assert_if_modified($digest, $conf->{digest});
2020
2021                         $conf->{$mpkey} = PVE::LXC::Config->print_ct_mountpoint(
2022                             $mpdata,
2023                             $mpkey eq 'rootfs'
2024                         );
2025
2026                         PVE::LXC::Config->add_unused_volume($conf, $old_volid) if !$param->{delete};
2027
2028                         PVE::LXC::Config->write_config($vmid, $conf);
2029                     });
2030
2031                     eval {
2032                         # try to deactivate volumes - avoid lvm LVs to be active on several nodes
2033                         PVE::Storage::deactivate_volumes($storage_cfg, [ $new_volid ])
2034                     };
2035                     warn $@ if $@;
2036                 };
2037                 if (my $err = $@) {
2038                     eval {
2039                         PVE::Storage::vdisk_free($storage_cfg, $new_volid)
2040                             if defined($new_volid);
2041                     };
2042                     warn $@ if $@;
2043                     die $err;
2044                 }
2045
2046                 my $deactivated = 0;
2047                 eval {
2048                     PVE::Storage::deactivate_volumes($storage_cfg, [ $old_volid ]);
2049                     $deactivated = 1;
2050                 };
2051                 warn $@ if $@;
2052
2053                 if ($param->{delete}) {
2054                     my $removed = 0;
2055                     if ($deactivated) {
2056                         eval {
2057                             PVE::Storage::vdisk_free($storage_cfg, $old_volid);
2058                             $removed = 1;
2059                         };
2060                         warn $@ if $@;
2061                     }
2062                     if (!$removed) {
2063                         PVE::LXC::Config->lock_config($vmid, sub {
2064                             my $conf = PVE::LXC::Config->load_config($vmid);
2065                             PVE::LXC::Config->add_unused_volume($conf, $old_volid);
2066                             PVE::LXC::Config->write_config($vmid, $conf);
2067                         });
2068                     }
2069                 }
2070             };
2071             my $err = $@;
2072             eval { PVE::LXC::Config->remove_lock($vmid, $lockname) };
2073             warn $@ if $@;
2074             die $err if $err;
2075         };
2076
2077         my $load_and_check_reassign_configs = sub {
2078             my $vmlist = PVE::Cluster::get_vmlist()->{ids};
2079
2080             die "Cannot move to/from 'rootfs'\n" if $mpkey eq "rootfs" || $target_mpkey eq "rootfs";
2081
2082             if ($mpkey =~ m/^unused\d+$/ && $target_mpkey !~ m/^unused\d+$/) {
2083                 die "Moving an unused volume to a used one is not possible\n";
2084             }
2085             die "could not find CT ${vmid}\n" if !exists($vmlist->{$vmid});
2086             die "could not find CT ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
2087
2088             my $source_node = $vmlist->{$vmid}->{node};
2089             my $target_node = $vmlist->{$target_vmid}->{node};
2090
2091             die "Both containers need to be on the same node ($source_node != $target_node)\n"
2092                 if $source_node ne $target_node;
2093
2094             my $source_conf = PVE::LXC::Config->load_config($vmid);
2095             PVE::LXC::Config->check_lock($source_conf);
2096             my $target_conf;
2097             if ($target_vmid eq $vmid) {
2098                 $target_conf = $source_conf;
2099             } else {
2100                 $target_conf = PVE::LXC::Config->load_config($target_vmid);
2101                 PVE::LXC::Config->check_lock($target_conf);
2102             }
2103
2104             die "Can't move volumes from or to template CT\n"
2105                 if ($source_conf->{template} || $target_conf->{template});
2106
2107             if ($digest) {
2108                 eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
2109                 die "Container ${vmid}: $@" if $@;
2110             }
2111
2112             if ($target_digest) {
2113                 eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
2114                 die "Container ${target_vmid}: $@" if $@;
2115             }
2116
2117             die "volume '${mpkey}' for container '$vmid' does not exist\n"
2118                 if !defined($source_conf->{$mpkey});
2119
2120             die "Target volume key '${target_mpkey}' is already in use for container '$target_vmid'\n"
2121                 if exists $target_conf->{$target_mpkey};
2122
2123             my $drive = PVE::LXC::Config->parse_volume($mpkey, $source_conf->{$mpkey});
2124             my $source_volid = $drive->{volume} or die "Volume '${mpkey}' has no associated image\n";
2125             die "Cannot move volume used by a snapshot to another container\n"
2126                 if PVE::LXC::Config->is_volume_in_use_by_snapshots($source_conf, $source_volid);
2127             die "Storage does not support moving of this disk to another container\n"
2128                 if !PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid);
2129             die "Cannot move a bindmount or device mount to another container\n"
2130                 if $drive->{type} ne "volume";
2131             die "Cannot move in-use volume while the source CT is running - detach or shutdown first\n"
2132                 if PVE::LXC::check_running($vmid) && $mpkey !~ m/^unused\d+$/;
2133
2134             my $repl_conf = PVE::ReplicationConfig->new();
2135             if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
2136                 my ($storeid, undef) = PVE::Storage::parse_volume_id($source_volid);
2137                 my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
2138
2139                 die "Cannot move volume on storage '$storeid' to a replicated container - missing replication support\n"
2140                     if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
2141             }
2142
2143             return ($source_conf, $target_conf, $drive);
2144         };
2145
2146         my $logfunc = sub { print STDERR "$_[0]\n"; };
2147
2148         my $volume_reassignfn = sub {
2149             return PVE::LXC::Config->lock_config($vmid, sub {
2150                 return PVE::LXC::Config->lock_config($target_vmid, sub {
2151                     my ($source_conf, $target_conf, $drive) = $load_and_check_reassign_configs->();
2152                     my $source_volid = $drive->{volume};
2153
2154                     my $target_unused = $target_mpkey =~ m/^unused\d+$/;
2155
2156                     print "moving volume '$mpkey' from container '$vmid' to '$target_vmid'\n";
2157
2158                     my ($storage, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
2159
2160                     my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
2161
2162                     my $new_volid = PVE::Storage::rename_volume(
2163                         $storecfg,
2164                         $source_volid,
2165                         $target_vmid,
2166                     );
2167
2168                     $drive->{volume} = $new_volid;
2169
2170                     delete $source_conf->{$mpkey};
2171                     print "removing volume '${mpkey}' from container '${vmid}' config\n";
2172                     PVE::LXC::Config->write_config($vmid, $source_conf);
2173
2174                     my $drive_string;
2175                     if ($target_unused) {
2176                         $drive_string = $new_volid;
2177                     } else {
2178                         $drive_string = PVE::LXC::Config->print_volume($target_mpkey, $drive);
2179                     }
2180
2181                     if ($target_unused) {
2182                         $target_conf->{$target_mpkey} = $drive_string;
2183                     } else {
2184                         my $running = PVE::LXC::check_running($target_vmid);
2185                         my $param = { $target_mpkey => $drive_string };
2186                         my $errors = PVE::LXC::Config->update_pct_config(
2187                             $target_vmid,
2188                             $target_conf,
2189                             $running,
2190                             $param
2191                         );
2192                         $rpcenv->warn($errors->{$_}) for keys $errors->%*;
2193                     }
2194
2195                     PVE::LXC::Config->write_config($target_vmid, $target_conf);
2196                     $target_conf = PVE::LXC::Config->load_config($target_vmid);
2197
2198                     PVE::LXC::update_lxc_config($target_vmid, $target_conf) if !$target_unused;
2199                     print "target container '$target_vmid' updated with '$target_mpkey'\n";
2200
2201                     # remove possible replication snapshots
2202                     if (PVE::Storage::volume_has_feature($storecfg,'replicate', $source_volid)) {
2203                         eval {
2204                             PVE::Replication::prepare(
2205                                 $storecfg,
2206                                 [$new_volid],
2207                                 undef,
2208                                 1,
2209                                 undef,
2210                                 $logfunc,
2211                             )
2212                         };
2213                         if (my $err = $@) {
2214                             $rpcenv->warn("Failed to remove replication snapshots on volume ".
2215                                 "'${target_mpkey}'. Manual cleanup could be necessary. " .
2216                                 "Error: ${err}\n");
2217                         }
2218                     }
2219                 });
2220             });
2221         };
2222
2223         if ($target_vmid && $storage) {
2224             my $msg = "either set 'storage' or 'target-vmid', but not both";
2225             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
2226         } elsif ($target_vmid) {
2227             $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
2228                 if $authuser ne 'root@pam';
2229
2230             my (undef, undef, $drive) = $load_and_check_reassign_configs->();
2231             my $storeid = PVE::Storage::parse_volume_id($drive->{volume});
2232             $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
2233             return $rpcenv->fork_worker(
2234                 'move_volume',
2235                 "${vmid}-${mpkey}>${target_vmid}-${target_mpkey}",
2236                 $authuser,
2237                 $volume_reassignfn
2238             );
2239         } elsif ($storage) {
2240             $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
2241             &$move_to_storage_checks();
2242             my $task = eval {
2243                 $rpcenv->fork_worker('move_volume', $vmid, $authuser, $storage_realcmd);
2244             };
2245             if (my $err = $@) {
2246                 eval { PVE::LXC::Config->remove_lock($vmid, $lockname) };
2247                 warn $@ if $@;
2248                 die $err;
2249             }
2250             return $task;
2251         } else {
2252             my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
2253             raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
2254         }
2255   }});
2256
2257 __PACKAGE__->register_method({
2258     name => 'vm_pending',
2259     path => '{vmid}/pending',
2260     method => 'GET',
2261     proxyto => 'node',
2262     description => 'Get container configuration, including pending changes.',
2263     permissions => {
2264         check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
2265     },
2266     parameters => {
2267         additionalProperties => 0,
2268         properties => {
2269             node => get_standard_option('pve-node'),
2270             vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
2271         },
2272     },
2273     returns => {
2274         type => "array",
2275         items => {
2276             type => "object",
2277             properties => {
2278                 key => {
2279                     description => 'Configuration option name.',
2280                     type => 'string',
2281                 },
2282                 value => {
2283                     description => 'Current value.',
2284                     type => 'string',
2285                     optional => 1,
2286                 },
2287                 pending => {
2288                     description => 'Pending value.',
2289                     type => 'string',
2290                     optional => 1,
2291                 },
2292                 delete => {
2293                     description => "Indicates a pending delete request if present and not 0.",
2294                     type => 'integer',
2295                     minimum => 0,
2296                     maximum => 2,
2297                     optional => 1,
2298                 },
2299             },
2300         },
2301     },
2302     code => sub {
2303         my ($param) = @_;
2304
2305         my $conf = PVE::LXC::Config->load_config($param->{vmid});
2306
2307         my $pending_delete_hash = PVE::LXC::Config->parse_pending_delete($conf->{pending}->{delete});
2308
2309         return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
2310     }});
2311
2312 1;