vm_create|restore: create ips in ipam

[pve-container.git] / src / PVE / API2 / LXC.pm

package PVE::API2::LXC;

use strict;
use warnings;

use Socket qw(SOCK_STREAM);

use PVE::SafeSyslog;
use PVE::Tools qw(extract_param run_command);
use PVE::Exception qw(raise raise_param_exc raise_perm_exc);
use PVE::INotify;
use PVE::Cluster qw(cfs_read_file);
use PVE::RRD;
use PVE::DataCenterConfig;
use PVE::AccessControl;
use PVE::Firewall;
use PVE::Storage;
use PVE::RESTHandler;
use PVE::RPCEnvironment;
use PVE::ReplicationConfig;
use PVE::LXC;
use PVE::LXC::Create;
use PVE::LXC::Migrate;
use PVE::GuestHelpers;
use PVE::VZDump::Plugin;
use PVE::API2::LXC::Config;
use PVE::API2::LXC::Status;
use PVE::API2::LXC::Snapshot;
use PVE::JSONSchema qw(get_standard_option);
use base qw(PVE::RESTHandler);

BEGIN {
    if (!$ENV{PVE_GENERATING_DOCS}) {
	require PVE::HA::Env::PVE2;
	import  PVE::HA::Env::PVE2;
	require PVE::HA::Config;
	import  PVE::HA::Config;
    }
}

my $check_storage_access_migrate = sub {
    my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;

    PVE::Storage::storage_check_enabled($storecfg, $storage, $node);

    $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);

    my $scfg = PVE::Storage::storage_config($storecfg, $storage);
    die "storage '$storage' does not support CT rootdirs\n"
	if !$scfg->{content}->{rootdir};
};

__PACKAGE__->register_method ({
    subclass => "PVE::API2::LXC::Config",
    path => '{vmid}/config',
});

__PACKAGE__->register_method ({
    subclass => "PVE::API2::LXC::Status",
    path => '{vmid}/status',
});

__PACKAGE__->register_method ({
    subclass => "PVE::API2::LXC::Snapshot",
    path => '{vmid}/snapshot',
});

__PACKAGE__->register_method ({
    subclass => "PVE::API2::Firewall::CT",
    path => '{vmid}/firewall',
});

__PACKAGE__->register_method({
    name => 'vmlist',
    path => '',
    method => 'GET',
    description => "LXC container index (per node).",
    permissions => {
	description => "Only list CTs where you have VM.Audit permissons on /vms/<vmid>.",
	user => 'all',
    },
    proxyto => 'node',
    protected => 1, # /proc files are only readable by root
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => $PVE::LXC::vmstatus_return_properties,
	},
	links => [ { rel => 'child', href => "{vmid}" } ],
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $vmstatus = PVE::LXC::vmstatus();

	my $res = [];
	foreach my $vmid (keys %$vmstatus) {
	    next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ], 1);

	    my $data = $vmstatus->{$vmid};
	    push @$res, $data;
	}

	return $res;

    }});

__PACKAGE__->register_method({
    name => 'create_vm',
    path => '',
    method => 'POST',
    description => "Create or restore a container.",
    permissions => {
	user => 'all', # check inside
 	description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
	    "For restore, it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
	    "You also need 'Datastore.AllocateSpace' permissions on the storage.",
    },
    protected => 1,
    proxyto => 'node',
    parameters => {
    	additionalProperties => 0,
	properties => PVE::LXC::Config->json_config_properties({
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
	    ostemplate => {
		description => "The OS template or backup file.",
		type => 'string',
		maxLength => 255,
		completion => \&PVE::LXC::complete_os_templates,
	    },
	    password => {
		optional => 1,
		type => 'string',
		description => "Sets root password inside container.",
		minLength => 5,
	    },
	    storage => get_standard_option('pve-storage-id', {
		description => "Default Storage.",
		default => 'local',
		optional => 1,
		completion => \&PVE::Storage::complete_storage_enabled,
	    }),
	    force => {
		optional => 1,
		type => 'boolean',
		description => "Allow to overwrite existing container.",
	    },
	    restore => {
		optional => 1,
		type => 'boolean',
		description => "Mark this as restore task.",
	    },
	    unique => {
		optional => 1,
		type => 'boolean',
		description => "Assign a unique random ethernet address.",
		requires => 'restore',
	    },
	    pool => {
		optional => 1,
		type => 'string', format => 'pve-poolid',
		description => "Add the VM to the specified pool.",
	    },
	    'ignore-unpack-errors' => {
		optional => 1,
		type => 'boolean',
		description => "Ignore errors when extracting the template.",
	    },
	    'ssh-public-keys' => {
		optional => 1,
		type => 'string',
		description => "Setup public SSH keys (one key per line, " .
				"OpenSSH format).",
	    },
	    bwlimit => {
		description => "Override I/O bandwidth limit (in KiB/s).",
		optional => 1,
		type => 'number',
		minimum => '0',
		default => 'restore limit from datacenter or storage config',
	    },
	    start => {
		optional => 1,
		type => 'boolean',
		default => 0,
		description => "Start the CT after its creation finished successfully.",
	    },
	}),
    },
    returns => {
	type => 'string',
    },
    code => sub {
	my ($param) = @_;

	PVE::Cluster::check_cfs_quorum();

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $node = extract_param($param, 'node');
	my $vmid = extract_param($param, 'vmid');
	my $ignore_unpack_errors = extract_param($param, 'ignore-unpack-errors');
	my $bwlimit = extract_param($param, 'bwlimit');
	my $start_after_create = extract_param($param, 'start');

	my $basecfg_fn = PVE::LXC::Config->config_file($vmid);
	my $same_container_exists = -f $basecfg_fn;

	# 'unprivileged' is read-only, so we can't pass it to update_pct_config
	my $unprivileged = extract_param($param, 'unprivileged');
	my $restore = extract_param($param, 'restore');
	my $unique = extract_param($param, 'unique');

	$param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
	    if defined($param->{cpuunits}); # clamp value depending on cgroup version

	# used to skip firewall config restore if user lacks permission
	my $skip_fw_config_restore = 0;

	if ($restore) {
	    # fixme: limit allowed parameters
	}

	my $force = extract_param($param, 'force');

	if (!($same_container_exists && $restore && $force)) {
	    PVE::Cluster::check_vmid_unused($vmid);
	} else {
	    die "can't overwrite running container\n" if PVE::LXC::check_running($vmid);
	    my $conf = PVE::LXC::Config->load_config($vmid);
	    PVE::LXC::Config->check_protection($conf, "unable to restore CT $vmid");
	}

	my $password = extract_param($param, 'password');
	my $ssh_keys = extract_param($param, 'ssh-public-keys');
	PVE::Tools::validate_ssh_public_keys($ssh_keys) if defined($ssh_keys);

	my $pool = extract_param($param, 'pool');
	$rpcenv->check_pool_exist($pool) if defined($pool);

	if ($rpcenv->check($authuser, "/vms/$vmid", ['VM.Allocate'], 1)) {
	    # OK
	} elsif ($pool && $rpcenv->check($authuser, "/pool/$pool", ['VM.Allocate'], 1)) {
	    # OK
	} elsif ($restore && $force && $same_container_exists &&
		 $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
	    # OK: user has VM.Backup permissions, and want to restore an existing VM

	    # we don't want to restore a container-provided FW conf in this case
	    # since the user is lacking permission to configure the container's FW
	    $skip_fw_config_restore = 1;

	    # error out if a user tries to change from unprivileged to privileged
	    # explicit change is checked here, implicit is checked down below or happening in root-only paths
	    my $conf = PVE::LXC::Config->load_config($vmid);
	    if ($conf->{unprivileged} && defined($unprivileged) && !$unprivileged) {
		raise_perm_exc("cannot change from unprivileged to privileged without VM.Allocate");
	    }
	} else {
	    raise_perm_exc();
	}

	my $ostemplate = extract_param($param, 'ostemplate');
	my $storage = extract_param($param, 'storage') // 'local';

	PVE::LXC::check_ct_modify_config_perm($rpcenv, $authuser, $vmid, $pool, undef, $param, [], $unprivileged);

	my $storage_cfg = cfs_read_file("storage.cfg");

	my $archive;
	if ($ostemplate eq '-') {
	    die "pipe requires cli environment\n"
		if $rpcenv->{type} ne 'cli';
	    die "pipe can only be used with restore tasks\n"
		if !$restore;
	    $archive = '-';
	    die "restore from pipe requires rootfs parameter\n" if !defined($param->{rootfs});
	} else {
	    my $content_type = $restore ? 'backup' : 'vztmpl';
	    PVE::Storage::check_volume_access(
		$rpcenv,
		$authuser,
		$storage_cfg,
		$vmid,
		$ostemplate,
		$content_type,
	    );
	    $archive = $ostemplate;
	}

	my %used_storages;
	my $check_and_activate_storage = sub {
	    my ($sid) = @_;

	    my $scfg = PVE::Storage::storage_check_enabled($storage_cfg, $sid, $node);

	    raise_param_exc({ storage => "storage '$sid' does not support container directories"})
		if !$scfg->{content}->{rootdir};

	    $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);

	    PVE::Storage::activate_storage($storage_cfg, $sid);
	    $used_storages{$sid} = 1;
	};

	my $conf = {};

	my $is_root = $authuser eq 'root@pam';

	my $no_disk_param = {};
	my $mp_param = {};
	my $storage_only_mode = 1;
	foreach my $opt (keys %$param) {
	    my $value = $param->{$opt};
	    if ($opt eq 'rootfs' || $opt =~ m/^mp\d+$/) {
		# allow to use simple numbers (add default storage in that case)
		if ($value =~ m/^\d+(\.\d+)?$/) {
		    $mp_param->{$opt} = "$storage:$value";
		} else {
		    $mp_param->{$opt} = $value;
		}
		$storage_only_mode = 0;
	    } elsif ($opt =~ m/^unused\d+$/) {
		warn "ignoring '$opt', cannot create/restore with unused volume\n";
		delete $param->{$opt};
	    } else {
		$no_disk_param->{$opt} = $value;
	    }
	}

	die "mount points configured, but 'rootfs' not set - aborting\n"
	    if !$storage_only_mode && !defined($mp_param->{rootfs});

	# check storage access, activate storage
	my $delayed_mp_param = {};
	PVE::LXC::Config->foreach_volume($mp_param, sub {
	    my ($ms, $mountpoint) = @_;

	    my $volid = $mountpoint->{volume};
	    my $mp = $mountpoint->{mp};

	    if ($mountpoint->{type} ne 'volume') { # bind or device
		die "Only root can pass arbitrary filesystem paths.\n"
		    if !$is_root;
	    } else {
		my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
		&$check_and_activate_storage($sid);
	    }
	});

	# check/activate default storage
	&$check_and_activate_storage($storage) if !defined($mp_param->{rootfs});

	PVE::LXC::Config->update_pct_config($vmid, $conf, 0, $no_disk_param);

	$conf->{unprivileged} = 1 if $unprivileged;

	my $emsg = $restore ? "unable to restore CT $vmid -" : "unable to create CT $vmid -";

	eval { PVE::LXC::Config->create_and_lock_config($vmid, $force) };
	die "$emsg $@" if $@;

	my $destroy_config_on_error = !$same_container_exists;

	my $code = sub {
	    my $old_conf = PVE::LXC::Config->load_config($vmid);
	    my $was_template;

	    my $vollist = [];
	    eval {
		my $orig_mp_param; # only used if $restore
		if ($restore) {
		    die "can't overwrite running container\n" if PVE::LXC::check_running($vmid);
		    if ($archive ne '-') {
			my $orig_conf;
			print "recovering backed-up configuration from '$archive'\n";
			($orig_conf, $orig_mp_param) = PVE::LXC::Create::recover_config($storage_cfg, $archive, $vmid);

			for my $opt (keys %$orig_conf) {
			    # early check before disks are created
			    # the "real" check is in later on when actually merging the configs
			    if ($opt =~ /^net\d+$/ && !defined($param->{$opt})) {
				PVE::LXC::check_bridge_access($rpcenv, $authuser, $orig_conf->{$opt});
			    }
			}

			$was_template = delete $orig_conf->{template};

			# When we're root call 'restore_configuration' with restricted=0,
			# causing it to restore the raw lxc entries, among which there may be
			# 'lxc.idmap' entries. We need to make sure that the extracted contents
			# of the container match up with the restored configuration afterwards:
			$conf->{lxc} = $orig_conf->{lxc} if $is_root;

			$conf->{unprivileged} = $orig_conf->{unprivileged}
			    if !defined($unprivileged) && defined($orig_conf->{unprivileged});

			# implicit privileged change is checked here
			if ($old_conf->{unprivileged} && !$conf->{unprivileged}) {
			    $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Allocate']);
			}
		    }
		}
		if ($storage_only_mode) {
		    if ($restore) {
			if (!defined($orig_mp_param)) {
			    print "recovering backed-up configuration from '$archive'\n";
			    (undef, $orig_mp_param) = PVE::LXC::Create::recover_config($storage_cfg, $archive, $vmid);
			}
			$mp_param = $orig_mp_param;
			die "rootfs configuration could not be recovered, please check and specify manually!\n"
			    if !defined($mp_param->{rootfs});
			PVE::LXC::Config->foreach_volume($mp_param, sub {
			    my ($ms, $mountpoint) = @_;
			    my $type = $mountpoint->{type};
			    if ($type eq 'volume') {
				die "unable to detect disk size - please specify $ms (size)\n"
				    if !defined($mountpoint->{size});
				my $disksize = $mountpoint->{size} / (1024 * 1024 * 1024); # create_disks expects GB as unit size
				delete $mountpoint->{size};
				$mountpoint->{volume} = "$storage:$disksize";
				$mp_param->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
			    } else {
				my $type = $mountpoint->{type};
				die "restoring rootfs to $type mount is only possible by specifying -rootfs manually!\n"
				    if ($ms eq 'rootfs');
				die "restoring '$ms' to $type mount is only possible for root\n"
				    if !$is_root;

				if ($mountpoint->{backup}) {
				    warn "WARNING - unsupported configuration!\n";
				    warn "backup was enabled for $type mount point $ms ('$mountpoint->{mp}')\n";
				    warn "mount point configuration will be restored after archive extraction!\n";
				    warn "contained files will be restored to wrong directory!\n";
				}
				delete $mp_param->{$ms}; # actually delay bind/dev mps
				$delayed_mp_param->{$ms} = PVE::LXC::Config->print_ct_mountpoint($mountpoint, $ms eq 'rootfs');
			    }
			});
		    } else {
			$mp_param->{rootfs} = "$storage:4"; # defaults to 4GB
		    }
		}

		# up until here we did not modify the container, besides the lock
		$destroy_config_on_error = 1;

		$vollist = PVE::LXC::create_disks($storage_cfg, $vmid, $mp_param, $conf);

		# we always have the 'create' lock so check for more than 1 entry
		if (scalar(keys %$old_conf) > 1) {
		    # destroy old container volumes
		    PVE::LXC::destroy_lxc_container($storage_cfg, $vmid, $old_conf, { lock => 'create' });
		}

		eval {
		    my $rootdir = PVE::LXC::mount_all($vmid, $storage_cfg, $conf, 1);
		    $bwlimit = PVE::Storage::get_bandwidth_limit('restore', [keys %used_storages], $bwlimit);
		    print "restoring '$archive' now..\n"
			if $restore && $archive ne '-';
		    PVE::LXC::Create::restore_archive($storage_cfg, $archive, $rootdir, $conf, $ignore_unpack_errors, $bwlimit);

		    if ($restore) {
			print "merging backed-up and given configuration..\n";
			PVE::LXC::Create::restore_configuration($vmid, $storage_cfg, $archive, $rootdir, $conf, !$is_root, $unique, $skip_fw_config_restore);
			PVE::LXC::create_ifaces_ipams_ips($conf, $vmid) if $unique;
			my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
			$lxc_setup->template_fixup($conf);
		    } else {
			PVE::LXC::create_ifaces_ipams_ips($conf, $vmid);
			my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir); # detect OS
			PVE::LXC::Config->write_config($vmid, $conf); # safe config (after OS detection)
			$lxc_setup->post_create_hook($password, $ssh_keys);
		    }
		};
		my $err = $@;
		PVE::LXC::umount_all($vmid, $storage_cfg, $conf, $err ? 1 : 0);
		PVE::Storage::deactivate_volumes($storage_cfg, PVE::LXC::Config->get_vm_volumes($conf));
		die $err if $err;
		# set some defaults
		$conf->{hostname} ||= "CT$vmid";
		$conf->{memory} ||= 512;
		$conf->{swap} //= 512;
		foreach my $mp (keys %$delayed_mp_param) {
		    $conf->{$mp} = $delayed_mp_param->{$mp};
		}
		# If the template flag was set, we try to convert again to template after restore
		if ($was_template) {
		    print STDERR "Convert restored container to template...\n";
		    PVE::LXC::template_create($vmid, $conf);
		    $conf->{template} = 1;
		}
		PVE::LXC::Config->write_config($vmid, $conf);
	    };
	    if (my $err = $@) {
		eval { PVE::LXC::delete_ifaces_ipams_ips($conf, $vmid) };
		warn $@ if $@;
		PVE::LXC::destroy_disks($storage_cfg, $vollist);
		if ($destroy_config_on_error) {
		    eval { PVE::LXC::Config->destroy_config($vmid) };
		    warn $@ if $@;

		    if (!$skip_fw_config_restore) { # Only if user has permission to change the fw
			PVE::Firewall::remove_vmfw_conf($vmid);
			warn $@ if $@;
		    }
		}
		die "$emsg $err";
	    }
	    PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;

	    PVE::API2::LXC::Status->vm_start({ vmid => $vmid, node => $node })
		if $start_after_create;
	};

	my $workername = $restore ? 'vzrestore' : 'vzcreate';
	my $realcmd = sub {
	    eval {
		PVE::LXC::Config->lock_config($vmid, $code);
	    };
	    if (my $err = $@) {
		# if we aborted before changing the container, we must remove the create lock
		if (!$destroy_config_on_error) {
		    PVE::LXC::Config->remove_lock($vmid, 'create');
		}
		die $err;
	    }
	};

	return $rpcenv->fork_worker($workername, $vmid, $authuser, $realcmd);
    }});

__PACKAGE__->register_method({
    name => 'vmdiridx',
    path => '{vmid}',
    method => 'GET',
    proxyto => 'node',
    description => "Directory index",
    permissions => {
	user => 'all',
    },
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {
		subdir => { type => 'string' },
	    },
	},
	links => [ { rel => 'child', href => "{subdir}" } ],
    },
    code => sub {
	my ($param) = @_;

	# test if VM exists
	my $conf = PVE::LXC::Config->load_config($param->{vmid});

	my $res = [
	    { subdir => 'config' },
	    { subdir => 'pending' },
	    { subdir => 'status' },
	    { subdir => 'vncproxy' },
	    { subdir => 'termproxy' },
	    { subdir => 'vncwebsocket' },
	    { subdir => 'spiceproxy' },
	    { subdir => 'migrate' },
	    { subdir => 'clone' },
#	    { subdir => 'initlog' },
	    { subdir => 'rrd' },
	    { subdir => 'rrddata' },
	    { subdir => 'firewall' },
	    { subdir => 'snapshot' },
	    { subdir => 'resize' },
	    ];

	return $res;
    }});


__PACKAGE__->register_method({
    name => 'rrd',
    path => '{vmid}/rrd',
    method => 'GET',
    protected => 1, # fixme: can we avoid that?
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
    },
    description => "Read VM RRD statistics (returns PNG)",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    timeframe => {
		description => "Specify the time frame you are interested in.",
		type => 'string',
		enum => [ 'hour', 'day', 'week', 'month', 'year' ],
	    },
	    ds => {
		description => "The list of datasources you want to display.",
 		type => 'string', format => 'pve-configid-list',
	    },
	    cf => {
		description => "The RRD consolidation function",
 		type => 'string',
		enum => [ 'AVERAGE', 'MAX' ],
		optional => 1,
	    },
	},
    },
    returns => {
	type => "object",
	properties => {
	    filename => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	return PVE::RRD::create_rrd_graph(
	    "pve2-vm/$param->{vmid}", $param->{timeframe},
	    $param->{ds}, $param->{cf});

    }});

__PACKAGE__->register_method({
    name => 'rrddata',
    path => '{vmid}/rrddata',
    method => 'GET',
    protected => 1, # fixme: can we avoid that?
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
    },
    description => "Read VM RRD statistics",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    timeframe => {
		description => "Specify the time frame you are interested in.",
		type => 'string',
		enum => [ 'hour', 'day', 'week', 'month', 'year' ],
	    },
	    cf => {
		description => "The RRD consolidation function",
 		type => 'string',
		enum => [ 'AVERAGE', 'MAX' ],
		optional => 1,
	    },
	},
    },
    returns => {
	type => "array",
	items => {
	    type => "object",
	    properties => {},
	},
    },
    code => sub {
	my ($param) = @_;

	return PVE::RRD::create_rrd_data(
	    "pve2-vm/$param->{vmid}", $param->{timeframe}, $param->{cf});
    }});

__PACKAGE__->register_method({
    name => 'destroy_vm',
    path => '{vmid}',
    method => 'DELETE',
    protected => 1,
    proxyto => 'node',
    description => "Destroy the container (also delete all uses files).",
    permissions => {
	check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
    },
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid_stopped }),
	    force => {
		type => 'boolean',
		description => "Force destroy, even if running.",
		default => 0,
		optional => 1,
	    },
	    purge => {
		type => 'boolean',
		description => "Remove container from all related configurations."
		    ." For example, backup jobs, replication jobs or HA."
		    ." Related ACLs and Firewall entries will *always* be removed.",
		default => 0,
		optional => 1,
	    },
	    'destroy-unreferenced-disks' => {
		type => 'boolean',
		description => "If set, destroy additionally all disks with the VMID from all"
		    ." enabled storages which are not referenced in the config.",
		optional => 1,
	    },
	},
    },
    returns => {
	type => 'string',
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();
	my $vmid = $param->{vmid};

	# test if container exists

	my $conf = PVE::LXC::Config->load_config($vmid);
	my $early_checks = sub {
	    my ($conf) = @_;
	    PVE::LXC::Config->check_protection($conf, "can't remove CT $vmid");
	    PVE::LXC::Config->check_lock($conf);

	    my $ha_managed = PVE::HA::Config::service_is_configured("ct:$vmid");

	    if (!$param->{purge}) {
		die "unable to remove CT $vmid - used in HA resources and purge parameter not set.\n"
		    if $ha_managed;

		# do not allow destroy if there are replication jobs without purge
		my $repl_conf = PVE::ReplicationConfig->new();
		$repl_conf->check_for_existing_jobs($vmid);
	    }

	    return $ha_managed;
	};

	$early_checks->($conf);

	my $running_error_msg = "unable to destroy CT $vmid - container is running\n";
	die $running_error_msg if !$param->{force} && PVE::LXC::check_running($vmid); # check early

	my $code = sub {
	    # reload config after lock
	    $conf = PVE::LXC::Config->load_config($vmid);
	    my $ha_managed = $early_checks->($conf);

	    if (PVE::LXC::check_running($vmid)) {
		die $running_error_msg if !$param->{force};
		warn "forced to stop CT $vmid before destroying!\n";
		if (!$ha_managed) {
		    PVE::LXC::vm_stop($vmid, 1);
		} else {
		    run_command(['ha-manager', 'crm-command', 'stop',  "ct:$vmid", '120']);
		}
	    }

	    my $storage_cfg = cfs_read_file("storage.cfg");
	    PVE::LXC::destroy_lxc_container(
		$storage_cfg,
		$vmid,
		$conf,
		{ lock => 'destroyed' },
		$param->{'destroy-unreferenced-disks'},
	    );

	    PVE::AccessControl::remove_vm_access($vmid);
	    PVE::Firewall::remove_vmfw_conf($vmid);
	    if ($param->{purge}) {
	        print "purging CT $vmid from related configurations..\n";
		PVE::ReplicationConfig::remove_vmid_jobs($vmid);
		PVE::VZDump::Plugin::remove_vmid_from_backup_jobs($vmid);

		if ($ha_managed) {
		    PVE::HA::Config::delete_service_from_config("ct:$vmid");
		    print "NOTE: removed CT $vmid from HA resource configuration.\n";
		}
	    }

	    # only now remove the zombie config, else we can have reuse race
	    PVE::LXC::Config->destroy_config($vmid);
	};

	my $realcmd = sub { PVE::LXC::Config->lock_config($vmid, $code); };

	return $rpcenv->fork_worker('vzdestroy', $vmid, $authuser, $realcmd);
    }});

my $sslcert;

__PACKAGE__->register_method ({
    name => 'vncproxy',
    path => '{vmid}/vncproxy',
    method => 'POST',
    protected => 1,
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
    },
    description => "Creates a TCP VNC proxy connections.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    websocket => {
		optional => 1,
		type => 'boolean',
		description => "use websocket instead of standard VNC.",
	    },
	    width => {
		optional => 1,
		description => "sets the width of the console in pixels.",
		type => 'integer',
		minimum => 16,
		maximum => 4096,
	    },
	    height => {
		optional => 1,
		description => "sets the height of the console in pixels.",
		type => 'integer',
		minimum => 16,
		maximum => 2160,
	    },
	},
    },
    returns => {
    	additionalProperties => 0,
	properties => {
	    user => { type => 'string' },
	    ticket => { type => 'string' },
	    cert => { type => 'string' },
	    port => { type => 'integer' },
	    upid => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $vmid = $param->{vmid};
	my $node = $param->{node};

	my $authpath = "/vms/$vmid";

	my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);

	$sslcert = PVE::Tools::file_get_contents("/etc/pve/pve-root-ca.pem", 8192)
	    if !$sslcert;

	my ($remip, $family);

	if ($node ne PVE::INotify::nodename()) {
	    ($remip, $family) = PVE::Cluster::remote_node_ip($node);
	} else {
	    $family = PVE::Tools::get_host_address_family($node);
	}

	my $port = PVE::Tools::next_vnc_port($family);

	# NOTE: vncterm VNC traffic is already TLS encrypted,
	# so we select the fastest chipher here (or 'none'?)
	my $remcmd = $remip ?
	    ['/usr/bin/ssh', '-e', 'none', '-t', $remip] : [];

	my $conf = PVE::LXC::Config->load_config($vmid, $node);
	my $concmd = PVE::LXC::get_console_command($vmid, $conf, -1);

	my $shcmd = [ '/usr/bin/dtach', '-A',
		      "/var/run/dtach/vzctlconsole$vmid",
		      '-r', 'winch', '-z', @$concmd];

	my $realcmd = sub {
	    my $upid = shift;

	    syslog ('info', "starting lxc vnc proxy $upid\n");

	    my $timeout = 10;

	    my $cmd = ['/usr/bin/vncterm', '-rfbport', $port,
		       '-timeout', $timeout, '-authpath', $authpath,
		       '-perm', 'VM.Console'];

	    if ($param->{width}) {
		push @$cmd, '-width', $param->{width};
	    }

	    if ($param->{height}) {
		push @$cmd, '-height', $param->{height};
	    }

	    if ($param->{websocket}) {
		$ENV{PVE_VNC_TICKET} = $ticket; # pass ticket to vncterm
		push @$cmd, '-notls', '-listen', 'localhost';
	    }

	    push @$cmd, '-c', @$remcmd, @$shcmd;

	    run_command($cmd, keeplocale => 1);

	    return;
	};

	my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd);

	PVE::Tools::wait_for_vnc_port($port);

	return {
	    user => $authuser,
	    ticket => $ticket,
	    port => $port,
	    upid => $upid,
	    cert => $sslcert,
	};
    }});

__PACKAGE__->register_method ({
    name => 'termproxy',
    path => '{vmid}/termproxy',
    method => 'POST',
    protected => 1,
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
    },
    description => "Creates a TCP proxy connection.",
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	},
    },
    returns => {
	additionalProperties => 0,
	properties => {
	    user => { type => 'string' },
	    ticket => { type => 'string' },
	    port => { type => 'integer' },
	    upid => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $vmid = $param->{vmid};
	my $node = $param->{node};

	my $authpath = "/vms/$vmid";

	my $ticket = PVE::AccessControl::assemble_vnc_ticket($authuser, $authpath);

	my ($remip, $family);

	if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
	    ($remip, $family) = PVE::Cluster::remote_node_ip($node);
	} else {
	    $family = PVE::Tools::get_host_address_family($node);
	}

	my $port = PVE::Tools::next_vnc_port($family);

	my $remcmd = $remip ?
	    ['/usr/bin/ssh', '-e', 'none', '-t', $remip, '--'] : [];

	my $conf = PVE::LXC::Config->load_config($vmid, $node);
	my $concmd = PVE::LXC::get_console_command($vmid, $conf, -1);

	my $shcmd = [ '/usr/bin/dtach', '-A',
		      "/var/run/dtach/vzctlconsole$vmid",
		      '-r', 'winch', '-z', @$concmd];

	my $realcmd = sub {
	    my $upid = shift;

	    syslog ('info', "starting lxc termproxy $upid\n");

	    my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
		       '--perm', 'VM.Console', '--'];
	    push @$cmd, @$remcmd, @$shcmd;

	    PVE::Tools::run_command($cmd);
	};

	my $upid = $rpcenv->fork_worker('vncproxy', $vmid, $authuser, $realcmd, 1);

	PVE::Tools::wait_for_vnc_port($port);

	return {
	    user => $authuser,
	    ticket => $ticket,
	    port => $port,
	    upid => $upid,
	};
    }});

__PACKAGE__->register_method({
    name => 'vncwebsocket',
    path => '{vmid}/vncwebsocket',
    method => 'GET',
    permissions => {
	description => "You also need to pass a valid ticket (vncticket).",
	check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
    },
    description => "Opens a weksocket for VNC traffic.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    vncticket => {
		description => "Ticket from previous call to vncproxy.",
		type => 'string',
		maxLength => 512,
	    },
	    port => {
		description => "Port number returned by previous vncproxy call.",
		type => 'integer',
		minimum => 5900,
		maximum => 5999,
	    },
	},
    },
    returns => {
	type => "object",
	properties => {
	    port => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $authpath = "/vms/$param->{vmid}";

	PVE::AccessControl::verify_vnc_ticket($param->{vncticket}, $authuser, $authpath);

	my $port = $param->{port};

	return { port => $port };
    }});

__PACKAGE__->register_method ({
    name => 'spiceproxy',
    path => '{vmid}/spiceproxy',
    method => 'POST',
    protected => 1,
    proxyto => 'node',
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
    },
    description => "Returns a SPICE configuration to connect to the CT.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    proxy => get_standard_option('spice-proxy', { optional => 1 }),
	},
    },
    returns => get_standard_option('remote-viewer-config'),
    code => sub {
	my ($param) = @_;

	my $vmid = $param->{vmid};
	my $node = $param->{node};
	my $proxy = $param->{proxy};

	my $authpath = "/vms/$vmid";
	my $permissions = 'VM.Console';

	my $conf = PVE::LXC::Config->load_config($vmid);

	die "CT $vmid not running\n" if !PVE::LXC::check_running($vmid);

	my $concmd = PVE::LXC::get_console_command($vmid, $conf);

	my $shcmd = ['/usr/bin/dtach', '-A',
		     "/var/run/dtach/vzctlconsole$vmid",
		     '-r', 'winch', '-z', @$concmd];

	my $title = "CT $vmid";

	return PVE::API2Tools::run_spiceterm($authpath, $permissions, $vmid, $node, $proxy, $title, $shcmd);
    }});


__PACKAGE__->register_method({
    name => 'remote_migrate_vm',
    path => '{vmid}/remote_migrate',
    method => 'POST',
    protected => 1,
    proxyto => 'node',
    description => "Migrate the container to another cluster. Creates a new migration task. EXPERIMENTAL feature!",
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
    },
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	    'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
	    'target-endpoint' => get_standard_option('proxmox-remote', {
		description => "Remote target endpoint",
	    }),
	    online => {
		type => 'boolean',
		description => "Use online/live migration.",
		optional => 1,
	    },
	    restart => {
		type => 'boolean',
		description => "Use restart migration",
		optional => 1,
	    },
	    timeout => {
		type => 'integer',
		description => "Timeout in seconds for shutdown for restart migration",
		optional => 1,
		default => 180,
	    },
	    delete => {
		type => 'boolean',
		description => "Delete the original CT and related data after successful migration. By default the original CT is kept on the source cluster in a stopped state.",
		optional => 1,
		default => 0,
	    },
	    'target-storage' => get_standard_option('pve-targetstorage', {
		optional => 0,
	    }),
	    'target-bridge' => {
		type => 'string',
		description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
		format => 'bridge-pair-list',
	    },
	    bwlimit => {
		description => "Override I/O bandwidth limit (in KiB/s).",
		optional => 1,
		type => 'number',
		minimum => '0',
		default => 'migrate limit from datacenter or storage config',
	    },
	},
    },
    returns => {
	type => 'string',
	description => "the task ID.",
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $source_vmid = extract_param($param, 'vmid');
	my $target_endpoint = extract_param($param, 'target-endpoint');
	my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;

	my $delete = extract_param($param, 'delete') // 0;

	PVE::Cluster::check_cfs_quorum();

	# test if CT exists
	my $conf = PVE::LXC::Config->load_config($source_vmid);
	PVE::LXC::Config->check_lock($conf);

	# try to detect errors early
	if (PVE::LXC::check_running($source_vmid)) {
	    die "can't migrate running container without --online or --restart\n"
		if !$param->{online} && !$param->{restart};
	}

	raise_param_exc({ vmid => "cannot migrate HA-managed CT to remote cluster" })
	    if PVE::HA::Config::vm_is_ha_managed($source_vmid);

	my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);

	# TODO: move this as helper somewhere appropriate?
	my $conn_args = {
	    protocol => 'https',
	    host => $remote->{host},
	    port => $remote->{port} // 8006,
	    apitoken => $remote->{apitoken},
	};

	my $fp;
	if ($fp = $remote->{fingerprint}) {
	    $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
	}

	print "Establishing API connection with remote at '$remote->{host}'\n";

	my $api_client = PVE::APIClient::LWP->new(%$conn_args);

	if (!defined($fp)) {
	    my $cert_info = $api_client->get("/nodes/localhost/certificates/info");
	    foreach my $cert (@$cert_info) {
		my $filename = $cert->{filename};
		next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem';
		$fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem';
	    }
	    $conn_args->{cached_fingerprints} = { uc($fp) => 1 }
		if defined($fp);
	}

	my $storecfg = PVE::Storage::config();
	my $target_storage = extract_param($param, 'target-storage');
	my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
	raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
	    if $@;

	my $target_bridge = extract_param($param, 'target-bridge');
	my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') };
	raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" })
	    if $@;

	die "remote migration requires explicit storage mapping!\n"
	    if $storagemap->{identity};

	$param->{storagemap} = $storagemap;
	$param->{bridgemap} = $bridgemap;
	$param->{remote} = {
	    conn => $conn_args, # re-use fingerprint for tunnel
	    client => $api_client,
	    vmid => $target_vmid,
	};
	$param->{migration_type} = 'websocket';
	$param->{delete} = $delete if $delete;

	my $cluster_status = $api_client->get("/cluster/status");
	my $target_node;
	foreach my $entry (@$cluster_status) {
	    next if $entry->{type} ne 'node';
	    if ($entry->{local}) {
		$target_node = $entry->{name};
		last;
	    }
	}

	die "couldn't determine endpoint's node name\n"
	    if !defined($target_node);

	my $realcmd = sub {
	    PVE::LXC::Migrate->migrate($target_node, $remote->{host}, $source_vmid, $param);
	};

	my $worker = sub {
	    return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd);
	};

	return $rpcenv->fork_worker('vzmigrate', $source_vmid, $authuser, $worker);
    }});


__PACKAGE__->register_method({
    name => 'migrate_vm',
    path => '{vmid}/migrate',
    method => 'POST',
    protected => 1,
    proxyto => 'node',
    description => "Migrate the container to another node. Creates a new migration task.",
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
    },
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	    target => get_standard_option('pve-node', {
		description => "Target node.",
		completion => \&PVE::Cluster::complete_migration_target,
	    }),
	    'target-storage' => get_standard_option('pve-targetstorage'),
	    online => {
		type => 'boolean',
		description => "Use online/live migration.",
		optional => 1,
	    },
	    restart => {
		type => 'boolean',
		description => "Use restart migration",
		optional => 1,
	    },
	    timeout => {
		type => 'integer',
		description => "Timeout in seconds for shutdown for restart migration",
		optional => 1,
		default => 180,
	    },
	    bwlimit => {
		description => "Override I/O bandwidth limit (in KiB/s).",
		optional => 1,
		type => 'number',
		minimum => '0',
		default => 'migrate limit from datacenter or storage config',
	    },
	},
    },
    returns => {
	type => 'string',
	description => "the task ID.",
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $target = extract_param($param, 'target');

	my $localnode = PVE::INotify::nodename();
	raise_param_exc({ target => "target is local node."}) if $target eq $localnode;

	PVE::Cluster::check_cfs_quorum();

	PVE::Cluster::check_node_exists($target);

	my $targetip = PVE::Cluster::remote_node_ip($target);

	my $vmid = extract_param($param, 'vmid');

	# test if VM exists
	PVE::LXC::Config->load_config($vmid);

	# try to detect errors early
	if (PVE::LXC::check_running($vmid)) {
	    die "can't migrate running container without --online or --restart\n"
		if !$param->{online} && !$param->{restart};
	}

	if (my $targetstorage = delete $param->{'target-storage'}) {
	    my $storecfg = PVE::Storage::config();
	    my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
	    raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
		if $@;

	    $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
		if !defined($storagemap->{identity});

	    foreach my $target_sid (values %{$storagemap->{entries}}) {
		$check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
	    }

	    $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
		if $storagemap->{default};

	    $param->{storagemap} = $storagemap;
	}

	if (PVE::HA::Config::vm_is_ha_managed($vmid) && $rpcenv->{type} ne 'ha') {

	    my $hacmd = sub {
		my $upid = shift;

		my $service = "ct:$vmid";

		my $cmd = ['ha-manager', 'migrate', $service, $target];

		print "Requesting HA migration for CT $vmid to node $target\n";

		PVE::Tools::run_command($cmd);

		return;
	    };

	    return $rpcenv->fork_worker('hamigrate', $vmid, $authuser, $hacmd);

	} else {

	    my $realcmd = sub {
		PVE::LXC::Migrate->migrate($target, $targetip, $vmid, $param);
	    };

	    my $worker = sub {
		return PVE::GuestHelpers::guest_migration_lock($vmid, 10, $realcmd);
	    };

	    return $rpcenv->fork_worker('vzmigrate', $vmid, $authuser, $worker);
	}
    }});

__PACKAGE__->register_method({
    name => 'vm_feature',
    path => '{vmid}/feature',
    method => 'GET',
    proxyto => 'node',
    protected => 1,
    description => "Check if feature for virtual machine is available.",
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
            feature => {
                description => "Feature to check.",
                type => 'string',
                enum => [ 'snapshot', 'clone', 'copy' ],
            },
            snapname => get_standard_option('pve-snapshot-name', {
                optional => 1,
            }),
	},
    },
    returns => {
	type => "object",
	properties => {
	    hasFeature => { type => 'boolean' },
	    #nodes => {
		#type => 'array',
		#items => { type => 'string' },
	    #}
	},
    },
    code => sub {
	my ($param) = @_;

	my $node = extract_param($param, 'node');

	my $vmid = extract_param($param, 'vmid');

	my $snapname = extract_param($param, 'snapname');

	my $feature = extract_param($param, 'feature');

	my $conf = PVE::LXC::Config->load_config($vmid);

	if($snapname){
	    my $snap = $conf->{snapshots}->{$snapname};
	    die "snapshot '$snapname' does not exist\n" if !defined($snap);
	    $conf = $snap;
	}
	my $storage_cfg = PVE::Storage::config();
	#Maybe include later
	#my $nodelist = PVE::LXC::shared_nodes($conf, $storage_cfg);
	my $hasFeature = PVE::LXC::Config->has_feature($feature, $conf, $storage_cfg, $snapname);

	return {
	    hasFeature => $hasFeature,
	    #nodes => [ keys %$nodelist ],
	};
    }});

__PACKAGE__->register_method({
    name => 'template',
    path => '{vmid}/template',
    method => 'POST',
    protected => 1,
    proxyto => 'node',
    description => "Create a Template.",
    permissions => {
	description => "You need 'VM.Allocate' permissions on /vms/{vmid}",
	check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid_stopped }),
	},
    },
    returns => { type => 'null'},
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $node = extract_param($param, 'node');

	my $vmid = extract_param($param, 'vmid');

	my $updatefn =  sub {

	    my $conf = PVE::LXC::Config->load_config($vmid);
	    PVE::LXC::Config->check_lock($conf);

	    die "unable to create template, because CT contains snapshots\n"
		if $conf->{snapshots} && scalar(keys %{$conf->{snapshots}});

	    die "you can't convert a template to a template\n"
		if PVE::LXC::Config->is_template($conf);

	    die "you can't convert a CT to template if the CT is running\n"
		if PVE::LXC::check_running($vmid);

	    my $realcmd = sub {
		PVE::LXC::template_create($vmid, $conf);

		$conf->{template} = 1;

		PVE::LXC::Config->write_config($vmid, $conf);
		# and remove lxc config
		PVE::LXC::update_lxc_config($vmid, $conf);
	    };

	    return $rpcenv->fork_worker('vztemplate', $vmid, $authuser, $realcmd);
	};

	PVE::LXC::Config->lock_config($vmid, $updatefn);

	return undef;
    }});

__PACKAGE__->register_method({
    name => 'clone_vm',
    path => '{vmid}/clone',
    method => 'POST',
    protected => 1,
    proxyto => 'node',
    description => "Create a container clone/copy",
    permissions => {
	description => "You need 'VM.Clone' permissions on /vms/{vmid}, " .
	    "and 'VM.Allocate' permissions " .
	    "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
	    "'Datastore.AllocateSpace' on any used storage, and 'SDN.Use' on any bridge.",
	check =>
	[ 'and',
	  ['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
	  [ 'or',
	    [ 'perm', '/vms/{newid}', ['VM.Allocate']],
	    [ 'perm', '/pool/{pool}', ['VM.Allocate'], require_param => 'pool'],
	  ],
	]
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	    newid => get_standard_option('pve-vmid', {
		completion => \&PVE::Cluster::complete_next_vmid,
		description => 'VMID for the clone.' }),
	    hostname => {
		optional => 1,
		type => 'string', format => 'dns-name',
		description => "Set a hostname for the new CT.",
	    },
	    description => {
		optional => 1,
		type => 'string',
		description => "Description for the new CT.",
	    },
	    pool => {
		optional => 1,
		type => 'string', format => 'pve-poolid',
		description => "Add the new CT to the specified pool.",
	    },
	    snapname => get_standard_option('pve-snapshot-name', {
		optional => 1,
            }),
	    storage => get_standard_option('pve-storage-id', {
		description => "Target storage for full clone.",
		optional => 1,
	    }),
	    full => {
		optional => 1,
	        type => 'boolean',
	        description => "Create a full copy of all disks. This is always done when " .
		    "you clone a normal CT. For CT templates, we try to create a linked clone by default.",
	    },
	    target => get_standard_option('pve-node', {
		description => "Target node. Only allowed if the original VM is on shared storage.",
		optional => 1,
	    }),
	    bwlimit => {
		description => "Override I/O bandwidth limit (in KiB/s).",
		optional => 1,
		type => 'number',
		minimum => '0',
		default => 'clone limit from datacenter or storage config',
	    },
        },
    },
    returns => {
	type => 'string',
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $node = extract_param($param, 'node');
	my $vmid = extract_param($param, 'vmid');
	my $newid = extract_param($param, 'newid');
	my $pool = extract_param($param, 'pool');
	if (defined($pool)) {
	    $rpcenv->check_pool_exist($pool);
	}
	my $snapname = extract_param($param, 'snapname');
	my $storage = extract_param($param, 'storage');
	my $target = extract_param($param, 'target');
        my $localnode = PVE::INotify::nodename();

	undef $target if $target && ($target eq $localnode || $target eq 'localhost');

	PVE::Cluster::check_node_exists($target) if $target;

	my $storecfg = PVE::Storage::config();

	if ($storage) {
	    # check if storage is enabled on local node
	    PVE::Storage::storage_check_enabled($storecfg, $storage);
	    if ($target) {
		# check if storage is available on target node
		PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
		# clone only works if target storage is shared
		my $scfg = PVE::Storage::storage_config($storecfg, $storage);
		die "can't clone to non-shared storage '$storage'\n" if !$scfg->{shared};
	    }
	}

	PVE::Cluster::check_cfs_quorum();

	my $newconf = {};
	my $mountpoints = {};
	my $fullclone = {};
	my $vollist = [];
	my $running;

	my $lock_and_reload = sub {
	    my ($vmid, $code) = @_;
	    return PVE::LXC::Config->lock_config($vmid, sub {
		my $conf = PVE::LXC::Config->load_config($vmid);
		die "Lost 'create' config lock, aborting.\n"
		    if !PVE::LXC::Config->has_lock($conf, 'create');

		return $code->($conf);
	    });
	};

	my $src_conf = PVE::LXC::Config->set_lock($vmid, 'disk');

	eval {
	    PVE::LXC::Config->create_and_lock_config($newid, 0);
	};
	if (my $err = $@) {
	    eval { PVE::LXC::Config->remove_lock($vmid, 'disk') };
	    warn "Failed to remove source CT config lock - $@\n" if $@;

	    die $err;
	}

	eval {
	    $running = PVE::LXC::check_running($vmid) || 0;

	    my $full = extract_param($param, 'full');
	    if (!defined($full)) {
		$full = !PVE::LXC::Config->is_template($src_conf);
	    }

	    PVE::Firewall::clone_vmfw_conf($vmid, $newid);

	    die "parameter 'storage' not allowed for linked clones\n"
		if defined($storage) && !$full;

	    die "snapshot '$snapname' does not exist\n"
		if $snapname && !defined($src_conf->{snapshots}->{$snapname});

	    my $src_conf = $snapname ? $src_conf->{snapshots}->{$snapname} : $src_conf;

	    my $sharedvm = 1;
	    for my $opt (sort keys %$src_conf) {
		next if $opt =~ m/^unused\d+$/;

		my $value = $src_conf->{$opt};

		if (($opt eq 'rootfs') || ($opt =~ m/^mp\d+$/)) {
		    my $mp = PVE::LXC::Config->parse_volume($opt, $value);

		    if ($mp->{type} eq 'volume') {
			my $volid = $mp->{volume};

			my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
			$sid = $storage if defined($storage);
			my $scfg = PVE::Storage::storage_config($storecfg, $sid);
			if (!$scfg->{shared}) {
			    $sharedvm = 0;
			    warn "found non-shared volume: $volid\n" if $target;
			}

			$rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);

			if ($full) {
			    die "Cannot do full clones on a running container without snapshots\n"
				if $running && !defined($snapname);
			    $fullclone->{$opt} = 1;
			} else {
			    # not full means clone instead of copy
			    die "Linked clone feature for '$volid' is not available\n"
				if !PVE::Storage::volume_has_feature($storecfg, 'clone', $volid, $snapname, $running, {'valid_target_formats' => ['raw', 'subvol']});
			}

			$mountpoints->{$opt} = $mp;
			push @$vollist, $volid;

		    } else {
			# TODO: allow bind mounts?
			die "unable to clone mountpoint '$opt' (type $mp->{type})\n";
		    }
		} elsif ($opt =~ m/^net(\d+)$/) {
		    # always change MAC! address
		    my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
		    my $net = PVE::LXC::Config->parse_lxc_network($value);
		    $net->{hwaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
		    $newconf->{$opt} = PVE::LXC::Config->print_lxc_network($net);

		    PVE::LXC::check_bridge_access($rpcenv, $authuser, $newconf->{$opt});
		} else {
		    # copy everything else
		    $newconf->{$opt} = $value;
		}
	    }
	    die "can't clone CT to node '$target' (CT uses local storage)\n"
		if $target && !$sharedvm;

	    # Replace the 'disk' lock with a 'create' lock.
	    $newconf->{lock} = 'create';

	    # delete all snapshot related config options
	    delete $newconf->@{qw(snapshots parent snaptime snapstate)};

	    delete $newconf->{pending};
	    delete $newconf->{template};

	    $newconf->{hostname} = $param->{hostname} if $param->{hostname};
	    $newconf->{description} = $param->{description} if $param->{description};

	    $lock_and_reload->($newid, sub {
		PVE::LXC::Config->write_config($newid, $newconf);
	    });
	};
	if (my $err = $@) {
	    eval { PVE::LXC::Config->remove_lock($vmid, 'disk') };
	    warn "Failed to remove source CT config lock - $@\n" if $@;

	    eval {
		$lock_and_reload->($newid, sub {
		    PVE::LXC::Config->destroy_config($newid);
		    PVE::Firewall::remove_vmfw_conf($newid);
		});
	    };
	    warn "Failed to remove target CT config - $@\n" if $@;

	    die $err;
	}

	my $update_conf = sub {
	    my ($key, $value) = @_;
	    return $lock_and_reload->($newid, sub {
		my $conf = shift;
		$conf->{$key} = $value;
		PVE::LXC::Config->write_config($newid, $conf);
	    });
	};

	my $realcmd = sub {
	    my ($upid) = @_;

	    my $newvollist = [];

	    my $verify_running = PVE::LXC::check_running($vmid) || 0;
	    die "unexpected state change\n" if $verify_running != $running;

	    eval {
		local $SIG{INT} =
		    local $SIG{TERM} =
		    local $SIG{QUIT} =
		    local $SIG{HUP} = sub { die "interrupted by signal\n"; };

		PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
		my $bwlimit = extract_param($param, 'bwlimit');

		foreach my $opt (keys %$mountpoints) {
		    my $mp = $mountpoints->{$opt};
		    my $volid = $mp->{volume};

		    my $newvolid;
		    if ($fullclone->{$opt}) {
			print "create full clone of mountpoint $opt ($volid)\n";
			my $source_storage = PVE::Storage::parse_volume_id($volid);
			my $target_storage = $storage // $source_storage;
			my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', [$source_storage, $target_storage], $bwlimit);
			$newvolid = PVE::LXC::copy_volume($mp, $newid, $target_storage, $storecfg, $newconf, $snapname, $clonelimit);
		    } else {
			print "create linked clone of mount point $opt ($volid)\n";
			$newvolid = PVE::Storage::vdisk_clone($storecfg, $volid, $newid, $snapname);
		    }

		    push @$newvollist, $newvolid;
		    $mp->{volume} = $newvolid;

		    $update_conf->($opt, PVE::LXC::Config->print_ct_mountpoint($mp, $opt eq 'rootfs'));
		}

		PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;

		$lock_and_reload->($newid, sub {
		    my $conf = shift;
		    my $rootdir = PVE::LXC::mount_all($newid, $storecfg, $conf, 1);
		    eval {
			my $lxc_setup = PVE::LXC::Setup->new($conf, $rootdir);
			$lxc_setup->post_clone_hook($conf);
		    };
		    my $err = $@;
		    eval { PVE::LXC::umount_all($newid, $storecfg, $conf, 1); };
		    if ($err) {
			warn "$@\n" if $@;
			die $err;
		    } else {
			die $@ if $@;
		    }
		});
	    };
	    my $err = $@;
	    # Unlock the source config in any case:
	    eval { PVE::LXC::Config->remove_lock($vmid, 'disk') };
	    warn $@ if $@;

	    if ($err) {
		# Now cleanup the config & disks:
		sleep 1; # some storages like rbd need to wait before release volume - really?

		foreach my $volid (@$newvollist) {
		    eval { PVE::Storage::vdisk_free($storecfg, $volid); };
		    warn $@ if $@;
		}

		eval {
		    $lock_and_reload->($newid, sub {
			PVE::LXC::Config->destroy_config($newid);
			PVE::Firewall::remove_vmfw_conf($newid);
		    });
		};
		warn "Failed to remove target CT config - $@\n" if $@;

		die "clone failed: $err";
	    }

	    $lock_and_reload->($newid, sub {
		PVE::LXC::Config->remove_lock($newid, 'create');

		if ($target) {
		    # always deactivate volumes - avoid lvm LVs to be active on several nodes
		    PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
		    PVE::Storage::deactivate_volumes($storecfg, $newvollist);

		    PVE::LXC::Config->move_config_to_node($newid, $target);
		}
	    });

	    return;
	};

	return $rpcenv->fork_worker('vzclone', $vmid, $authuser, $realcmd);
    }});


__PACKAGE__->register_method({
    name => 'resize_vm',
    path => '{vmid}/resize',
    method => 'PUT',
    protected => 1,
    proxyto => 'node',
    description => "Resize a container mount point.",
    permissions => {
	check => ['perm', '/vms/{vmid}', ['VM.Config.Disk'], any => 1],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	    disk => {
		type => 'string',
		description => "The disk you want to resize.",
		enum => [PVE::LXC::Config->valid_volume_keys()],
	    },
	    size => {
		type => 'string',
		pattern => '\+?\d+(\.\d+)?[KMGT]?',
		description => "The new size. With the '+' sign the value is added to the actual size of the volume and without it, the value is taken as an absolute one. Shrinking disk size is not supported.",
	    },
	    digest => {
		type => 'string',
		description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
		maxLength => 40,
		optional => 1,
	    }
	},
    },
    returns => {
	type => 'string',
	description => "the task ID.",
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $node = extract_param($param, 'node');

	my $vmid = extract_param($param, 'vmid');

	my $digest = extract_param($param, 'digest');

	my $sizestr = extract_param($param, 'size');
	my $ext = ($sizestr =~ s/^\+//);
	my $request_size = PVE::JSONSchema::parse_size($sizestr);
	die "invalid size string" if !defined($request_size);

	die "no options specified\n" if !scalar(keys %$param);

	my $storage_cfg = cfs_read_file("storage.cfg");

	my $load_and_check = sub {
	    my $conf = PVE::LXC::Config->load_config($vmid);
	    PVE::LXC::Config->check_lock($conf);

	    PVE::LXC::check_ct_modify_config_perm($rpcenv, $authuser, $vmid, undef, $conf, $param, [], $conf->{unprivileged});

	    PVE::Tools::assert_if_modified($digest, $conf->{digest});

	    my $disk = $param->{disk};
	    my $mp = PVE::LXC::Config->parse_volume($disk, $conf->{$disk});

	    my $volid = $mp->{volume};

	    my (undef, undef, $owner, undef, undef, undef, $format) =
		PVE::Storage::parse_volname($storage_cfg, $volid);

	    die "can't resize mount point owned by another container ($owner)"
		if $vmid != $owner;

	    my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);

	    $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);

	    PVE::Storage::activate_volumes($storage_cfg, [$volid]);

	    my $size = PVE::Storage::volume_size_info($storage_cfg, $volid, 5);

	    die "Could not determine current size of volume '$volid'\n" if !defined($size);

	    my $newsize = $ext ? $size + $request_size : $request_size;
	    $newsize = int($newsize);

	    die "unable to shrink disk size\n" if $newsize < $size;

	    die "disk is already at specified size\n" if $size == $newsize;

	    return ($conf, $disk, $mp, $volid, $format, $newsize);
	};

	my $code = sub {
	    my ($conf, $disk, $mp, $volid, $format, $newsize) = $load_and_check->();

	    my $running = PVE::LXC::check_running($vmid);

	    PVE::Cluster::log_msg('info', $authuser, "update CT $vmid: resize --disk $disk --size $sizestr");

	    # Note: PVE::Storage::volume_resize doesn't do anything if $running=1, so
	    # we pass 0 here (parameter only makes sense for qemu)
	    PVE::Storage::volume_resize($storage_cfg, $volid, $newsize, 0);

	    $mp->{size} = $newsize;
	    $conf->{$disk} = PVE::LXC::Config->print_ct_mountpoint($mp, $disk eq 'rootfs');

	    PVE::LXC::Config->write_config($vmid, $conf);

	    if ($format eq 'raw') {
		# we need to ensure that the volume is mapped, if not needed this is a NOP
		my $path = PVE::Storage::map_volume($storage_cfg, $volid);
		$path = PVE::Storage::path($storage_cfg, $volid) if !defined($path);
		if ($running) {

		    $mp->{mp} = '/';
		    my $use_loopdev = (PVE::LXC::mountpoint_mount_path($mp, $storage_cfg))[1];
		    $path = PVE::LXC::query_loopdev($path) if $use_loopdev;
		    die "internal error: CT running but mount point not attached to a loop device"
			if !$path;
		    PVE::Tools::run_command(['losetup', '--set-capacity', $path]) if $use_loopdev;

		    # In order for resize2fs to know that we need online-resizing a mountpoint needs
		    # to be visible to it in its namespace.
		    # To not interfere with the rest of the system we unshare the current mount namespace,
		    # mount over /tmp and then run resize2fs.

		    # interestingly we don't need to e2fsck on mounted systems...
		    my $quoted = PVE::Tools::shellquote($path);
		    my $cmd = "mount --make-rprivate / && mount $quoted /tmp && resize2fs $quoted";
		    eval {
			PVE::Tools::run_command(['unshare', '-m', '--', 'sh', '-c', $cmd]);
		    };
		    warn "Failed to update the container's filesystem: $@\n" if $@;
		} else {
		    eval {
			PVE::Tools::run_command(['e2fsck', '-f', '-y', $path]);
			PVE::Tools::run_command(['resize2fs', $path]);
		    };
		    warn "Failed to update the container's filesystem: $@\n" if $@;

		    # always un-map if not running, this is a NOP if not needed
		    PVE::Storage::unmap_volume($storage_cfg, $volid);
		}
	    }
	};

	my $worker = sub {
	    PVE::LXC::Config->lock_config($vmid, $code);;
	};

	$load_and_check->(); # early checks before forking+locking

	return $rpcenv->fork_worker('resize', $vmid, $authuser, $worker);
    }});

__PACKAGE__->register_method({
    name => 'move_volume',
    path => '{vmid}/move_volume',
    method => 'POST',
    protected => 1,
    proxyto => 'node',
    description => "Move a rootfs-/mp-volume to a different storage or to a different container.",
    permissions => {
	description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
	    "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
	    "a volume to another container, you need the permissions on the ".
	    "target container as well.",
	check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	    'target-vmid' => get_standard_option('pve-vmid', {
		completion => \&PVE::LXC::complete_ctid,
		optional => 1,
	    }),
	    volume => {
		type => 'string',
		#TODO: check how to handle unused mount points as the mp parameter is not configured
		enum => [ PVE::LXC::Config->valid_volume_keys_with_unused() ],
		description => "Volume which will be moved.",
	    },
	    storage => get_standard_option('pve-storage-id', {
		description => "Target Storage.",
		completion => \&PVE::Storage::complete_storage_enabled,
		optional => 1,
	    }),
	    delete => {
		type => 'boolean',
		description => "Delete the original volume after successful copy. By default the " .
		    "original is kept as an unused volume entry.",
		optional => 1,
		default => 0,
	    },
	    digest => {
		type => 'string',
		description => 'Prevent changes if current configuration file has different SHA1 " .
		    "digest. This can be used to prevent concurrent modifications.',
		maxLength => 40,
		optional => 1,
	    },
	    bwlimit => {
		description => "Override I/O bandwidth limit (in KiB/s).",
		optional => 1,
		type => 'number',
		minimum => '0',
		default => 'clone limit from datacenter or storage config',
	    },
	    'target-volume' => {
	        type => 'string',
		description => "The config key the volume will be moved to. Default is the " .
		    "source volume key.",
		enum => [PVE::LXC::Config->valid_volume_keys_with_unused()],
		optional => 1,
	    },
	    'target-digest' => {
		type => 'string',
		description => 'Prevent changes if current configuration file of the target " .
		    "container has a different SHA1 digest. This can be used to prevent " .
		    "concurrent modifications.',
		maxLength => 40,
		optional => 1,
	    },
	},
    },
    returns => {
	type => 'string',
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $authuser = $rpcenv->get_user();

	my $vmid = extract_param($param, 'vmid');

	my $target_vmid = extract_param($param, 'target-vmid');

	my $storage = extract_param($param, 'storage');

	my $mpkey = extract_param($param, 'volume');

	my $target_mpkey = extract_param($param, 'target-volume') // $mpkey;

	my $digest = extract_param($param, 'digest');

	my $target_digest = extract_param($param, 'target-digest');

	my $lockname = 'disk';

	my ($mpdata, $old_volid);

	die "either set storage or target-vmid, but not both\n"
	    if $storage && $target_vmid;

	my $storecfg = PVE::Storage::config();

	my $move_to_storage_checks = sub {
	    PVE::LXC::Config->lock_config($vmid, sub {
		my $conf = PVE::LXC::Config->load_config($vmid);
		PVE::LXC::Config->check_lock($conf);

		die "cannot move volumes of a running container\n"
		    if PVE::LXC::check_running($vmid);

		if ($mpkey =~ m/^unused\d+$/) {
		    die "cannot move volume '$mpkey', only configured volumes can be moved to ".
			"another storage\n";
		}

		$mpdata = PVE::LXC::Config->parse_volume($mpkey, $conf->{$mpkey});
		$old_volid = $mpdata->{volume};

		die "you can't move a volume with snapshots and delete the source\n"
		    if $param->{delete} && PVE::LXC::Config->is_volume_in_use_by_snapshots($conf, $old_volid);

		PVE::Tools::assert_if_modified($digest, $conf->{digest});

		PVE::LXC::Config->set_lock($vmid, $lockname);
	    });
	};

	my $storage_realcmd = sub {
	    eval {
		PVE::Cluster::log_msg(
		    'info',
		    $authuser,
		    "move volume CT $vmid: move --volume $mpkey --storage $storage"
		);

		my $conf = PVE::LXC::Config->load_config($vmid);
		my $storage_cfg = PVE::Storage::config();

		my $new_volid;

		eval {
		    PVE::Storage::activate_volumes($storage_cfg, [ $old_volid ]);
		    my $bwlimit = extract_param($param, 'bwlimit');
		    my $source_storage = PVE::Storage::parse_volume_id($old_volid);
		    my $movelimit = PVE::Storage::get_bandwidth_limit(
			'move',
			[$source_storage, $storage],
			$bwlimit
		    );
		    $new_volid = PVE::LXC::copy_volume(
			$mpdata,
			$vmid,
			$storage,
			$storage_cfg,
			$conf,
			undef,
			$movelimit
		    );
		    if (PVE::LXC::Config->is_template($conf)) {
			PVE::Storage::activate_volumes($storage_cfg, [ $new_volid ]);
			my $template_volid = PVE::Storage::vdisk_create_base($storage_cfg, $new_volid);
			$mpdata->{volume} = $template_volid;
		    } else {
			$mpdata->{volume} = $new_volid;
		    }

		    PVE::LXC::Config->lock_config($vmid, sub {
			my $digest = $conf->{digest};
			$conf = PVE::LXC::Config->load_config($vmid);
			PVE::Tools::assert_if_modified($digest, $conf->{digest});

			$conf->{$mpkey} = PVE::LXC::Config->print_ct_mountpoint(
			    $mpdata,
			    $mpkey eq 'rootfs'
			);

			PVE::LXC::Config->add_unused_volume($conf, $old_volid) if !$param->{delete};

			PVE::LXC::Config->write_config($vmid, $conf);
		    });

		    eval {
			# try to deactivate volumes - avoid lvm LVs to be active on several nodes
			PVE::Storage::deactivate_volumes($storage_cfg, [ $new_volid ])
		    };
		    warn $@ if $@;
		};
		if (my $err = $@) {
		    eval {
			PVE::Storage::vdisk_free($storage_cfg, $new_volid)
			    if defined($new_volid);
		    };
		    warn $@ if $@;
		    die $err;
		}

		my $deactivated = 0;
		eval {
		    PVE::Storage::deactivate_volumes($storage_cfg, [ $old_volid ]);
		    $deactivated = 1;
		};
		warn $@ if $@;

		if ($param->{delete}) {
		    my $removed = 0;
		    if ($deactivated) {
			eval {
			    PVE::Storage::vdisk_free($storage_cfg, $old_volid);
			    $removed = 1;
			};
			warn $@ if $@;
		    }
		    if (!$removed) {
			PVE::LXC::Config->lock_config($vmid, sub {
			    my $conf = PVE::LXC::Config->load_config($vmid);
			    PVE::LXC::Config->add_unused_volume($conf, $old_volid);
			    PVE::LXC::Config->write_config($vmid, $conf);
			});
		    }
		}
	    };
	    my $err = $@;
	    eval { PVE::LXC::Config->remove_lock($vmid, $lockname) };
	    warn $@ if $@;
	    die $err if $err;
	};

	my $load_and_check_reassign_configs = sub {
	    my $vmlist = PVE::Cluster::get_vmlist()->{ids};

	    die "Cannot move to/from 'rootfs'\n" if $mpkey eq "rootfs" || $target_mpkey eq "rootfs";

	    if ($mpkey =~ m/^unused\d+$/ && $target_mpkey !~ m/^unused\d+$/) {
		die "Moving an unused volume to a used one is not possible\n";
	    }
	    die "could not find CT ${vmid}\n" if !exists($vmlist->{$vmid});
	    die "could not find CT ${target_vmid}\n" if !exists($vmlist->{$target_vmid});

	    my $source_node = $vmlist->{$vmid}->{node};
	    my $target_node = $vmlist->{$target_vmid}->{node};

	    die "Both containers need to be on the same node ($source_node != $target_node)\n"
		if $source_node ne $target_node;

	    my $source_conf = PVE::LXC::Config->load_config($vmid);
	    PVE::LXC::Config->check_lock($source_conf);
	    my $target_conf;
	    if ($target_vmid eq $vmid) {
		$target_conf = $source_conf;
	    } else {
		$target_conf = PVE::LXC::Config->load_config($target_vmid);
		PVE::LXC::Config->check_lock($target_conf);
	    }

	    die "Can't move volumes from or to template CT\n"
		if ($source_conf->{template} || $target_conf->{template});

	    if ($digest) {
		eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
		die "Container ${vmid}: $@" if $@;
	    }

	    if ($target_digest) {
		eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
		die "Container ${target_vmid}: $@" if $@;
	    }

	    die "volume '${mpkey}' for container '$vmid' does not exist\n"
		if !defined($source_conf->{$mpkey});

	    die "Target volume key '${target_mpkey}' is already in use for container '$target_vmid'\n"
		if exists $target_conf->{$target_mpkey};

	    my $drive = PVE::LXC::Config->parse_volume($mpkey, $source_conf->{$mpkey});
	    my $source_volid = $drive->{volume} or die "Volume '${mpkey}' has no associated image\n";
	    die "Cannot move volume used by a snapshot to another container\n"
		if PVE::LXC::Config->is_volume_in_use_by_snapshots($source_conf, $source_volid);
	    die "Storage does not support moving of this disk to another container\n"
		if !PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid);
	    die "Cannot move a bindmount or device mount to another container\n"
		if $drive->{type} ne "volume";
	    die "Cannot move in-use volume while the source CT is running - detach or shutdown first\n"
		if PVE::LXC::check_running($vmid) && $mpkey !~ m/^unused\d+$/;

	    my $repl_conf = PVE::ReplicationConfig->new();
	    if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
		my ($storeid, undef) = PVE::Storage::parse_volume_id($source_volid);
		my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];

		die "Cannot move volume on storage '$storeid' to a replicated container - missing replication support\n"
		    if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
	    }

	    return ($source_conf, $target_conf, $drive);
	};

	my $logfunc = sub { print STDERR "$_[0]\n"; };

	my $volume_reassignfn = sub {
	    return PVE::LXC::Config->lock_config($vmid, sub {
		return PVE::LXC::Config->lock_config($target_vmid, sub {
		    my ($source_conf, $target_conf, $drive) = $load_and_check_reassign_configs->();
		    my $source_volid = $drive->{volume};

		    my $target_unused = $target_mpkey =~ m/^unused\d+$/;

		    print "moving volume '$mpkey' from container '$vmid' to '$target_vmid'\n";

		    my ($storage, $source_volname) = PVE::Storage::parse_volume_id($source_volid);

		    my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];

		    my $new_volid = PVE::Storage::rename_volume(
			$storecfg,
			$source_volid,
			$target_vmid,
		    );

		    $drive->{volume} = $new_volid;

		    delete $source_conf->{$mpkey};
		    print "removing volume '${mpkey}' from container '${vmid}' config\n";
		    PVE::LXC::Config->write_config($vmid, $source_conf);

		    my $drive_string;
		    if ($target_unused) {
			$drive_string = $new_volid;
		    } else {
			$drive_string = PVE::LXC::Config->print_volume($target_mpkey, $drive);
		    }

		    if ($target_unused) {
			$target_conf->{$target_mpkey} = $drive_string;
		    } else {
			my $running = PVE::LXC::check_running($target_vmid);
			my $param = { $target_mpkey => $drive_string };
		        my $errors = PVE::LXC::Config->update_pct_config(
			    $target_vmid,
			    $target_conf,
			    $running,
			    $param
			);
			$rpcenv->warn($errors->{$_}) for keys $errors->%*;
		    }

		    PVE::LXC::Config->write_config($target_vmid, $target_conf);
		    $target_conf = PVE::LXC::Config->load_config($target_vmid);

		    PVE::LXC::update_lxc_config($target_vmid, $target_conf) if !$target_unused;
		    print "target container '$target_vmid' updated with '$target_mpkey'\n";

		    # remove possible replication snapshots
		    if (PVE::Storage::volume_has_feature($storecfg,'replicate', $source_volid)) {
			eval {
			    PVE::Replication::prepare(
				$storecfg,
				[$new_volid],
				undef,
				1,
				undef,
				$logfunc,
			    )
			};
			if (my $err = $@) {
			    $rpcenv->warn("Failed to remove replication snapshots on volume ".
				"'${target_mpkey}'. Manual cleanup could be necessary. " .
				"Error: ${err}\n");
			}
		    }
		});
	    });
	};

	if ($target_vmid && $storage) {
	    my $msg = "either set 'storage' or 'target-vmid', but not both";
	    raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
	} elsif ($target_vmid) {
	    $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
		if $authuser ne 'root@pam';

	    my (undef, undef, $drive) = $load_and_check_reassign_configs->();
	    my $storeid = PVE::Storage::parse_volume_id($drive->{volume});
	    $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
	    return $rpcenv->fork_worker(
		'move_volume',
		"${vmid}-${mpkey}>${target_vmid}-${target_mpkey}",
		$authuser,
		$volume_reassignfn
	    );
	} elsif ($storage) {
	    $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
	    &$move_to_storage_checks();
	    my $task = eval {
		$rpcenv->fork_worker('move_volume', $vmid, $authuser, $storage_realcmd);
	    };
	    if (my $err = $@) {
		eval { PVE::LXC::Config->remove_lock($vmid, $lockname) };
		warn $@ if $@;
		die $err;
	    }
	    return $task;
	} else {
	    my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
	    raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
	}
  }});

__PACKAGE__->register_method({
    name => 'vm_pending',
    path => '{vmid}/pending',
    method => 'GET',
    proxyto => 'node',
    description => 'Get container configuration, including pending changes.',
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	},
    },
    returns => {
	type => "array",
	items => {
	    type => "object",
	    properties => {
		key => {
		    description => 'Configuration option name.',
		    type => 'string',
		},
		value => {
		    description => 'Current value.',
		    type => 'string',
		    optional => 1,
		},
		pending => {
		    description => 'Pending value.',
		    type => 'string',
		    optional => 1,
		},
		delete => {
		    description => "Indicates a pending delete request if present and not 0.",
		    type => 'integer',
		    minimum => 0,
		    maximum => 2,
		    optional => 1,
		},
	    },
	},
    },
    code => sub {
	my ($param) = @_;

	my $conf = PVE::LXC::Config->load_config($param->{vmid});

	my $pending_delete_hash = PVE::LXC::Config->parse_pending_delete($conf->{pending}->{delete});

	return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
    }});

__PACKAGE__->register_method({
    name => 'ip',
    path => '{vmid}/interfaces',
    method => 'GET',
    protected => 1,
    permissions => {
	check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
    },
    description => 'Get IP addresses of the specified container interface.',
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid', { completion => \&PVE::LXC::complete_ctid }),
	},
    },
    returns => {
	type => "array",
	items => {
	    type => 'object',
	    properties => {
		name => {
		    type => 'string',
		    description => 'The name of the interface',
		    optional => 0,
		},
		hwaddr => {
		    type => 'string',
		    description => 'The MAC address of the interface',
		    optional => 0,
		},
		inet => {
		    type => 'string',
		    description => 'The IPv4 address of the interface',
		    optional => 1,
		},
		inet6 => {
		    type => 'string',
		    description => 'The IPv6 address of the interface',
		    optional => 1,
		},
	    }
	},
    },
    code => sub {
	my ($param) = @_;

	return PVE::LXC::get_interfaces($param->{vmid});
    }});

__PACKAGE__->register_method({
    name => 'mtunnel',
    path => '{vmid}/mtunnel',
    method => 'POST',
    protected => 1,
    description => 'Migration tunnel endpoint - only for internal use by CT migration.',
    permissions => {
	check =>
	[ 'and',
	  ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]],
	  ['perm', '/', [ 'Sys.Incoming' ]],
	],
	description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" .
	               " on '/'. Further permission checks happen during the actual migration.",
    },
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    storages => {
		type => 'string',
		format => 'pve-storage-id-list',
		optional => 1,
		description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.',
	    },
	    bridges => {
		type => 'string',
		format => 'pve-bridge-id-list',
		optional => 1,
		description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.',
	    },
	},
    },
    returns => {
	additionalProperties => 0,
	properties => {
	    upid => { type => 'string' },
	    ticket => { type => 'string' },
	    socket => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $node = extract_param($param, 'node');
	my $vmid = extract_param($param, 'vmid');

	my $storages = extract_param($param, 'storages');
	my $bridges = extract_param($param, 'bridges');

	my $nodename = PVE::INotify::nodename();

	raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
	    if $node ne 'localhost' && $node ne $nodename;

	$node = $nodename;

	my $storecfg = PVE::Storage::config();
	foreach my $storeid (PVE::Tools::split_list($storages)) {
	    $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node);
	}

	foreach my $bridge (PVE::Tools::split_list($bridges)) {
	    PVE::Network::read_bridge_mtu($bridge);
	}

	PVE::Cluster::check_cfs_quorum();

	my $socket_addr = "/run/pve/ct-$vmid.mtunnel";

	my $lock = 'create';
	eval { PVE::LXC::Config->create_and_lock_config($vmid, 0, $lock); };

	raise_param_exc({ vmid => "unable to create empty CT config - $@"})
	    if $@;

	my $realcmd = sub {
	    my $state = {
		storecfg => PVE::Storage::config(),
		lock => $lock,
		vmid => $vmid,
	    };

	    my $run_locked = sub {
		my ($code, $params) = @_;
		return PVE::LXC::Config->lock_config($state->{vmid}, sub {
		    my $conf = PVE::LXC::Config->load_config($state->{vmid});

		    $state->{conf} = $conf;

		    die "Encountered wrong lock - aborting mtunnel command handling.\n"
			if $state->{lock} && !PVE::LXC::Config->has_lock($conf, $state->{lock});

		    return $code->($params);
		});
	    };

	    my $cmd_desc = {
		config => {
		    conf => {
			type => 'string',
			description => 'Full CT config, adapted for target cluster/node',
		    },
		    'firewall-config' => {
			type => 'string',
			description => 'CT firewall config',
			optional => 1,
		    },
		},
		ticket => {
		    path => {
			type => 'string',
			description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.',
		    },
		},
		quit => {
		    cleanup => {
			type => 'boolean',
			description => 'remove CT config and volumes, aborting migration',
			default => 0,
		    },
		},
		'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'},
		'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'},
		bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit},
	    };

	    my $cmd_handlers = {
		'version' => sub {
		    # compared against other end's version
		    # bump/reset for breaking changes
		    # bump/bump for opt-in changes
		    return {
			api => $PVE::LXC::Migrate::WS_TUNNEL_VERSION,
			age => 0,
		    };
		},
		'config' => sub {
		    my ($params) = @_;

		    # parse and write out VM FW config if given
		    if (my $fw_conf = $params->{'firewall-config'}) {
			my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700);

			my $empty_conf = {
			    rules => [],
			    options => {},
			    aliases => {},
			    ipset => {} ,
			    ipset_comments => {},
			};
			my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf();

			# TODO: add flag for strict parsing?
			# TODO: add import sub that does all this given raw content?
			my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm');
			$vmfw_conf->{vmid} = $state->{vmid};
			PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf);

			$state->{cleanup}->{fw} = 1;
		    }

		    my $conf_fn = "incoming/lxc/$state->{vmid}.conf";
		    my $new_conf = PVE::LXC::Config::parse_pct_config($conf_fn, $params->{conf}, 1);
		    delete $new_conf->{lock};
		    delete $new_conf->{digest};

		    my $unprivileged = delete $new_conf->{unprivileged};
		    my $arch = delete $new_conf->{arch};

		    # TODO handle properly?
		    delete $new_conf->{snapshots};
		    delete $new_conf->{parent};
		    delete $new_conf->{pending};
		    delete $new_conf->{lxc};

		    PVE::LXC::Config->remove_lock($state->{vmid}, 'create');

		    eval {
			my $conf = {
			    unprivileged => $unprivileged,
			    arch => $arch,
			};
			PVE::LXC::check_ct_modify_config_perm(
			    $rpcenv,
			    $authuser,
			    $state->{vmid},
			    undef,
			    $conf,
			    $new_conf,
			    undef,
			    $unprivileged,
			);
			my $errors = PVE::LXC::Config->update_pct_config(
			    $state->{vmid},
			    $conf,
			    0,
			    $new_conf,
			    [],
			    [],
			);
			raise_param_exc($errors) if scalar(keys %$errors);
			PVE::LXC::Config->write_config($state->{vmid}, $conf);
			PVE::LXC::update_lxc_config($vmid, $conf);
		    };
		    if (my $err = $@) {
			# revert to locked previous config
			my $conf = PVE::LXC::Config->load_config($state->{vmid});
			$conf->{lock} = 'create';
			PVE::LXC::Config->write_config($state->{vmid}, $conf);

			die $err;
		    }

		    my $conf = PVE::LXC::Config->load_config($state->{vmid});
		    $conf->{lock} = 'migrate';
		    PVE::LXC::Config->write_config($state->{vmid}, $conf);

		    $state->{lock} = 'migrate';

		    return;
		},
		'bwlimit' => sub {
		    my ($params) = @_;
		    return PVE::StorageTunnel::handle_bwlimit($params);
		},
		'disk-import' => sub {
		    my ($params) = @_;

		    $check_storage_access_migrate->(
			$rpcenv,
			$authuser,
			$state->{storecfg},
			$params->{storage},
			$node
		    );

		    $params->{unix} = "/run/pve/ct-$state->{vmid}.storage";

		    return PVE::StorageTunnel::handle_disk_import($state, $params);
		},
		'query-disk-import' => sub {
		    my ($params) = @_;

		    return PVE::StorageTunnel::handle_query_disk_import($state, $params);
		},
		'unlock' => sub {
		    PVE::LXC::Config->remove_lock($state->{vmid}, $state->{lock});
		    delete $state->{lock};
		    return;
		},
		'start' => sub {
		    PVE::LXC::vm_start(
			$state->{vmid},
			$state->{conf},
			0
		    );

		    return;
		},
		'stop' => sub {
		    PVE::LXC::vm_stop($state->{vmid}, 1, 10, 1);
		    return;
		},
		'ticket' => sub {
		    my ($params) = @_;

		    my $path = $params->{path};

		    die "Not allowed to generate ticket for unknown socket '$path'\n"
			if !defined($state->{sockets}->{$path});

		    return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") };
		},
		'quit' => sub {
		    my ($params) = @_;

		    if ($params->{cleanup}) {
			if ($state->{cleanup}->{fw}) {
			    PVE::Firewall::remove_vmfw_conf($state->{vmid});
			}

			for my $volid (keys $state->{cleanup}->{volumes}->%*) {
			    print "freeing volume '$volid' as part of cleanup\n";
			    eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) };
			    warn $@ if $@;
			}

			PVE::LXC::destroy_lxc_container(
			    $state->{storecfg},
			    $state->{vmid},
			    $state->{conf},
			    undef,
			    0,
			);
		    }

		    print "switching to exit-mode, waiting for client to disconnect\n";
		    $state->{exit} = 1;
		    return;
		},
	    };

	    $run_locked->(sub {
		my $socket_addr = "/run/pve/ct-$state->{vmid}.mtunnel";
		unlink $socket_addr;

		$state->{socket} = IO::Socket::UNIX->new(
	            Type => SOCK_STREAM(),
		    Local => $socket_addr,
		    Listen => 1,
		);

		$state->{socket_uid} = getpwnam('www-data')
		    or die "Failed to resolve user 'www-data' to numeric UID\n";
		chown $state->{socket_uid}, -1, $socket_addr;
	    });

	    print "mtunnel started\n";

	    my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) };
	    if ($@) {
		warn "Failed to accept tunnel connection - $@\n";

		warn "Removing tunnel socket..\n";
		unlink $state->{socket};

		warn "Removing temporary VM config..\n";
		$run_locked->(sub {
		    PVE::LXC::destroy_config($state->{vmid});
		});

		die "Exiting mtunnel\n";
	    }

	    $state->{conn} = $conn;

	    my $reply_err = sub {
		my ($msg) = @_;

		my $reply = JSON::encode_json({
		    success => JSON::false,
		    msg => $msg,
		});
		$conn->print("$reply\n");
		$conn->flush();
	    };

	    my $reply_ok = sub {
		my ($res) = @_;

		$res->{success} = JSON::true;
		my $reply = JSON::encode_json($res);
		$conn->print("$reply\n");
		$conn->flush();
	    };

	    while (my $line = <$conn>) {
		chomp $line;

		# untaint, we validate below if needed
		($line) = $line =~ /^(.*)$/;
		my $parsed = eval { JSON::decode_json($line) };
		if ($@) {
		    $reply_err->("failed to parse command - $@");
		    next;
		}

		my $cmd = delete $parsed->{cmd};
		if (!defined($cmd)) {
		    $reply_err->("'cmd' missing");
		} elsif ($state->{exit}) {
		    $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible");
		    next;
		} elsif (my $handler = $cmd_handlers->{$cmd}) {
		    print "received command '$cmd'\n";
		    eval {
			if ($cmd_desc->{$cmd}) {
			    PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd});
			} else {
			    $parsed = {};
			}
			my $res = $run_locked->($handler, $parsed);
			$reply_ok->($res);
		    };
		    $reply_err->("failed to handle '$cmd' command - $@")
			if $@;
		} else {
		    $reply_err->("unknown command '$cmd' given");
		}
	    }

	    if ($state->{exit}) {
		print "mtunnel exited\n";
	    } else {
		die "mtunnel exited unexpectedly\n";
	    }
	};

	my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr");
	my $upid = $rpcenv->fork_worker('vzmtunnel', $vmid, $authuser, $realcmd);

	return {
	    ticket => $ticket,
	    upid => $upid,
	    socket => $socket_addr,
	};
    }});

__PACKAGE__->register_method({
    name => 'mtunnelwebsocket',
    path => '{vmid}/mtunnelwebsocket',
    method => 'GET',
    permissions => {
	description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
        user => 'all', # check inside
    },
    description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    vmid => get_standard_option('pve-vmid'),
	    socket => {
		type => "string",
		description => "unix socket to forward to",
	    },
	    ticket => {
		type => "string",
		description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command",
	    },
	},
    },
    returns => {
	type => "object",
	properties => {
	    port => { type => 'string', optional => 1 },
	    socket => { type => 'string', optional => 1 },
	},
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $nodename = PVE::INotify::nodename();
	my $node = extract_param($param, 'node');

	raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
	    if $node ne 'localhost' && $node ne $nodename;

	my $vmid = $param->{vmid};
	# check VM exists
	PVE::LXC::Config->load_config($vmid);

	my $socket = $param->{socket};
	PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket");

	return { socket => $socket };
    }});
1;