[pve-cluster.git] / data / PVE / pvecm

#!/usr/bin/perl

use strict;
use warnings;
use Getopt::Long;
use Socket;
use IO::File;
use File::Path;
use File::Basename;
use Data::Dumper; # fixme: remove 
use PVE::Tools;
use PVE::Cluster;
use PVE::INotify;
use PVE::JSONSchema;
use PVE::CLIHandler;

use base qw(PVE::CLIHandler);

$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
$ENV{LC_ALL} = 'C';
$ENV{HOME} = '/root'; # for ssh-copy-id

die "please run as root\n" if $> != 0;

my $nodename = PVE::INotify::nodename();
# trigger check that we have resolvable name
my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);

my $basedir = "/etc/pve";
my $clusterconf = "$basedir/cluster.conf";
my $libdir = "/var/lib/pve-cluster";
my $backupdir = "/var/lib/pve-cluster/backup";
my $dbfile = "$libdir/config.db";
my $authfile = "$libdir/corosync.authkey";


sub backup_database {

    print "backup old database\n";

    mkdir $backupdir;
    
    my $ctime = time();
    my $cmd = "echo '.dump' |";
    $cmd .= "sqlite3 '$dbfile' |";
    $cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";

    system($cmd) == 0 ||
	die "can't backup old database: $!\n";

    # purge older backup
    my $maxfiles = 10;

    my @bklist = ();
    foreach my $fn (<$backupdir/config-*.sql.gz>) {
	if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
	    push @bklist, [$fn, $1];
	}
    }
	
    @bklist = sort { $b->[1] <=> $a->[1] } @bklist;

    while (scalar (@bklist) >= $maxfiles) {
	my $d = pop @bklist;
	print "delete old backup '$d->[0]'\n";
	unlink $d->[0];
    }
}

__PACKAGE__->register_method ({
    name => 'keygen', 
    path => 'keygen',
    method => 'PUT',
    description => "Generate new cryptographic key for corosync.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    filename => {
		type => 'string',
		description => "Output file name"
	    }
	},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	my $filename = $param->{filename};

	# test EUID
	$> == 0 || die "Error: Authorization key must be generated as root user.\n";
	my $dirname = dirname($filename);
	my $basename = basename($filename);

	File::Path::make_path($dirname) if $dirname;

	my $fh = IO::File->new ("/dev/urandom", 'r') ||
	    die "can't open /dev/urandom - $!\n";

	my $keysize = 128;
	my $key = '';

	my $bytes = 0;
	while ($bytes < $keysize) {
	    my $rb = sysread($fh, $key, $keysize - $bytes, $bytes);
	    ($rb > 0) || die "Could not read /dev/urandom - $!\n";
	    $bytes += $rb;
	}

	close($fh);

	my $tmpfn = "$filename.tmp.$$";
	$fh = IO::File->new ($tmpfn, O_CREAT|O_WRONLY, 0400);
	die "can't open temporary file '$tmpfn' - $!\n" if !$fh;

	eval {
	    my $wb;
	    (($wb = syswrite($fh, $key)) == $keysize) || 
		die "writing key failed - short write $wb\n;"
	};
	my $err = $@;
    
	$fh->close();

	if ($err) {
	    unlink $tmpfn;
	    die $err;
	}

	if (!rename($tmpfn, $filename)) {
	    my $err = $!;
	    unlink $tmpfn;	
	    die "rename $tmpfn to $filename failed - $err\n";
	}

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'create', 
    path => 'create',
    method => 'PUT',
    description => "Generate new cluster configuration.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    clustername => {
		description => "The name of the cluster.",
		type => 'string', format => 'pve-node',
		maxLength => 15,
	    },
	    nodeid => {
		type => 'integer',
		description => "Node id for this node.",
		minimum => 1,
		optional => 1,
	    },
	    votes => {
		type => 'integer',
		description => "Number of votes for this node",
		minimum => 1,
		optional => 1,
	    },
	},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	-f $clusterconf && die "cluster config '$clusterconf' already exists\n";

	PVE::Cluster::setup_rootsshconfig();
	PVE::Cluster::setup_ssh_keys();

	-f $authfile || __PACKAGE__->keygen({filename => $authfile});

	-f $authfile || die "no authentication key available\n";

	my $clustername = $param->{clustername};

	$param->{nodeid} = 1 if !$param->{nodeid};

	$param->{votes} = 1 if !defined($param->{votes});

	my $config = <<_EOD;
<?xml version="1.0"?>
<cluster name="$clustername" config_version="1">

  <cman keyfile="$authfile">
  </cman>

  <clusternodes>
  <clusternode name="${nodename}" votes="$param->{votes}" nodeid="$param->{nodeid}"/>
  </clusternodes>

</cluster>
_EOD
;
	PVE::Tools::file_set_contents($clusterconf, $config);

	PVE::Cluster::ssh_merge_keys();

	PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);

	PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);

	PVE::Tools::run_command('/etc/init.d/pve-cluster restart'); # restart

	# that cman init script returns strange values - simply ignore for now
	system('/etc/init.d/cman start');

	# also start clvm
	system('/etc/init.d/clvm start');
	
	return undef;
}});

__PACKAGE__->register_method ({
    name => 'addnode', 
    path => 'addnode',
    method => 'PUT',
    description => "Adds a node to the cluster configuration.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => PVE::JSONSchema::get_standard_option('pve-node'),
	    nodeid => {
		type => 'integer',
		description => "Node id for this node.",
		minimum => 1,
		optional => 1,
	    },
	    votes => {
		type => 'integer',
		description => "Number of votes for this node",
		minimum => 0,
		optional => 1,
	    },
	    force => {
		type => 'boolean',
		description => "Do not throw error if node already exists.",
		optional => 1,
	    },
	},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	PVE::Cluster::check_cfs_quorum();

	my $lst = lsnode();

	my $name = $param->{node};

	if (defined(my $res = $lst->{$name})) {
	    $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
	    $param->{votes} = $res->{votes} if !defined($param->{votes});

	    if ($res->{votes} == $param->{votes} &&
		$res->{nodeid} == $param->{nodeid}) {
		print "node $name already defined\n";
		if ($param->{force}) {
		    exit (0);
		} else {
		    exit (-1);
		}
	    }
	} elsif (!$param->{nodeid}) {
	    my $nodeid = 1;
	    
	    while(1) {
		my $found = 0; 
		foreach my $v (values %$lst) {
		    if ($v->{nodeid} eq $nodeid) {
			$found = 1;
			$nodeid++;
			last;
		    }
		}
		last if !$found;
	    };

	    $param->{nodeid} = $nodeid;
	}

	$param->{votes} = 1 if !defined($param->{votes});

	PVE::Cluster::gen_local_dirs($name);

	eval { 	PVE::Cluster::ssh_merge_keys(); };
	warn $@ if $@;

	my $cmd = ['ccs_tool', 'addnode', '-c', $clusterconf];

	# NOTE: cman does not like votes="0"
	if ($param->{votes}) {
	    push @$cmd, '-v', $param->{votes};
	}

	push @$cmd, '-n', $param->{nodeid}, $name;

	system(@$cmd) == 0 || exit(-1);

	exit (0);
    }});


__PACKAGE__->register_method ({
    name => 'delnode', 
    path => 'delnode',
    method => 'PUT',
    description => "Removes a node to the cluster configuration.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => PVE::JSONSchema::get_standard_option('pve-node'),
	},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	PVE::Cluster::check_cfs_quorum();

	my $cmd = ['ccs_tool', 'delnode', '-c', $clusterconf, $param->{node}];

	exec (@$cmd);

	exit (-1); # should not be reached
    }});

__PACKAGE__->register_method ({
    name => 'add', 
    path => 'add',
    method => 'PUT',
    description => "Adds the current node to an existing cluster.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    hostname => {
		type => 'string',
		description => "Hostname (or IP) of an existing cluster member."
	    },
	    nodeid => {
		type => 'integer',
		description => "Node id for this node.",
		minimum => 1,
		optional => 1,
	    },
	    votes => {
		type => 'integer',
		description => "Number of votes for this node",
		minimum => 0,
		optional => 1,
	    },
	    force => {
		type => 'boolean',
		description => "Do not throw error if node already exists.",
		optional => 1,
	    },
	},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	PVE::Cluster::setup_rootsshconfig();
	PVE::Cluster::setup_ssh_keys();

	my $host = $param->{hostname};

	if (!$param->{force}) {
	    
	    if (-f $authfile) {
		die "authentication key already exists\n";
	    }

	    if (-f $clusterconf)  {
		die "cluster config '$clusterconf' already exists\n";
	    }

	    my $vmlist = PVE::Cluster::get_vmlist();
	    if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
		die "this host already contains virtual machines - please remove the first\n";
	    }

	    if (system("cman_tool status >/dev/null 2>&1") == 0) {
		die "cman is already running\n";
	    }
	}

	# make sure known_hosts is on local filesystem
	PVE::Cluster::ssh_unmerge_known_hosts();

	my $cmd = "ssh-copy-id -i /root/.ssh/id_rsa 'root\@$host' >/dev/null 2>&1";
	system ($cmd) == 0 ||
	    die "unable to copy ssh ID\n";

	$cmd = ['ssh', $host, '-o', 'BatchMode=yes',
		'pvecm', 'addnode', $nodename, '--force', 1];

	push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};

	push @$cmd, '--votes', $param->{votes} if defined($param->{votes});

	if (system (@$cmd) != 0) {
	    my $cmdtxt = join (' ', @$cmd);
	    die "unable to add node: command failed ($cmdtxt)\n";
	}

	my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
	mkdir $tmpdir;

	eval {
	    print "copy corosync auth key\n";
	    $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq', 
		    "$host:$authfile $clusterconf", $tmpdir];

	    system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";

	    mkdir "/etc/cluster";
	    my $confbase = basename($clusterconf);

	    $cmd = "cp '$tmpdir/$confbase' '/etc/cluster/$confbase'";
	    system($cmd) == 0 || die "can't copy cluster configuration\n";

	    my $keybase = basename($authfile);
	    system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
		die "can't copy '$tmpdir/$keybase' to '$authfile'\n";

	    print "stopping pve-cluster service\n";

	    system("umount $basedir -f >/dev/null 2>&1");
	    system("/etc/init.d/pve-cluster stop") == 0 ||
		die "can't stop pve-cluster service\n";

	    backup_database();

	    unlink $dbfile;

	    system("/etc/init.d/pve-cluster start") == 0 ||
		die "starting pve-cluster failed\n";

	    system("/etc/init.d/cman start");

	    # wait for quorum
	    my $printqmsg = 1;
	    while (!PVE::Cluster::check_cfs_quorum(1)) {
		if ($printqmsg) {
		    print "waiting for quorum...";
		    STDOUT->flush();
		    $printqmsg = 0;
		}
		sleep(1);
	    }
	    print "OK\n" if !$printqmsg;

	    system("/etc/init.d/clvm start");

	    print "generating node certificates\n";
	    PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); 

	    print "merge known_hosts file\n";
	    PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);

	    print "restart services\n";
	    # restart pvedaemon (changed certs)
	    system("/etc/init.d/pvedaemon restart");
	    # restart pveproxy (changed certs)
	    system("/etc/init.d/pveproxy restart");

	    print "successfully added node '$nodename' to cluster.\n";
	};
	my $err = $@;

	rmtree $tmpdir;

	die $err if $err;

	return undef;
    }});

__PACKAGE__->register_method ({
    name => 'status', 
    path => 'status',
    method => 'GET',
    description => "Displays the local view of the cluster status.",
    parameters => {
    	additionalProperties => 0,
	properties => {},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	my $cmd = ['cman_tool', 'status'];

	exec (@$cmd);

	exit (-1); # should not be reached
    }});

__PACKAGE__->register_method ({
    name => 'nodes', 
    path => 'nodes',
    method => 'GET',
    description => "Displays the local view of the cluster nodes.",
    parameters => {
    	additionalProperties => 0,
	properties => {},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	my $cmd = ['cman_tool', 'nodes'];

	exec (@$cmd);

	exit (-1); # should not be reached
    }});

__PACKAGE__->register_method ({
    name => 'expected', 
    path => 'expected',
    method => 'PUT',
    description => "Tells CMAN a new value of expected votes.",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    expected => {
		type => 'integer',
		description => "Expected votes",
		minimum => 1,
	    },
	},
    },
    returns => { type => 'null' },
    
    code => sub {
	my ($param) = @_;

	my $cmd = ['cman_tool', 'expected', '-e', $param->{expected}];

	exec (@$cmd);

	exit (-1); # should not be reached

    }});

sub lsnode {

    my $res = {};

    my $cmd = ['ccs_tool',  'lsnode', '-c', $clusterconf];

    my $scan = 0;
    my $parser = sub {
	my $line = shift;

	if ($line =~ m/^Nodename\s+Votes\s+Nodeid\s/i) {
	    $scan = 1;
	    return;
	}
	return if !$scan;
	if ($line =~ m/^(\S+)\s+(\d+)\s+(\d+)\s/) {
	    $res->{$1} = {
		name => $1,
		votes => $2,
		nodeid => $3 };
	}
    };

    PVE::Tools::run_command($cmd, outfunc => $parser);
    return $res;
}

__PACKAGE__->register_method ({
    name => 'updatecerts', 
    path => 'updatecerts',
    method => 'PUT',
    description => "Update node certificates (and generate all needed files/directories).",
    parameters => {
    	additionalProperties => 0,
	properties => {
	    force => {
		description => "Force generation of new SSL certifate.",
		type => 'boolean',
		optional => 1,
	    },
	    silent => {
		description => "Ignore errors (i.e. when cluster has no quorum).",
		type => 'boolean',
		optional => 1,
	    },
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	PVE::Cluster::setup_rootsshconfig();

	PVE::Cluster::gen_pve_vzdump_symlink();

	if (!PVE::Cluster::check_cfs_quorum(1)) {
	    return undef if $param->{silent};
	    die "no quorum - unable to update files\n";
	}

	PVE::Cluster::setup_ssh_keys();

	PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
	PVE::Cluster::ssh_merge_keys();
	PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
	PVE::Cluster::gen_pve_vzdump_files();

	return undef;
    }});


my $cmddef = {
    keygen => [ __PACKAGE__, 'keygen', ['filename']],
    create => [ __PACKAGE__, 'create', ['clustername']],
    add => [ __PACKAGE__, 'add', ['hostname']],
    addnode => [ __PACKAGE__, 'addnode', ['node']],
    delnode => [ __PACKAGE__, 'delnode', ['node']],
    status => [ __PACKAGE__, 'status' ],
    nodes => [ __PACKAGE__, 'nodes' ],
    expected => [ __PACKAGE__, 'expected', ['expected']],
    updatecerts => [ __PACKAGE__, 'updatecerts', []],
};

my $cmd = shift;

if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
    PVE::Cluster::check_cfs_is_mounted();
    PVE::Cluster::cfs_update();
}

PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);

exit 0;

__END__

=head1 NAME

pvecm - Proxmox VE cluster manager toolkit

=head1 SYNOPSIS

=include synopsis

=head1 DESCRIPTION

pvecm is a program to manage the cluster configuration. It can be used
to create a new cluster, join nodes to a cluster, leave the cluster,
get status information and do various other cluster related tasks.

=include pve_copyright
Commit	Line	Data
7181f622 DM	1	#!/usr/bin/perl
7181f622 DM	2
fe000966	3	use strict;
7181f622	4	use warnings;
fe000966 DM	5	use Getopt::Long;
	6	use Socket;
	7	use IO::File;
	8	use File::Path;
	9	use File::Basename;
	10	use Data::Dumper; # fixme: remove
	11	use PVE::Tools;
	12	use PVE::Cluster;
	13	use PVE::INotify;
	14	use PVE::JSONSchema;
	15	use PVE::CLIHandler;
	16
	17	use base qw(PVE::CLIHandler);
	18
	19	$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
c25b167f DM	20	$ENV{LC_ALL} = 'C';
c25b167f DM	21	$ENV{HOME} = '/root'; # for ssh-copy-id
fe000966 DM	22
	23	die "please run as root\n" if $> != 0;
	24
	25	my $nodename = PVE::INotify::nodename();
	26	# trigger check that we have resolvable name
	27	my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
	28
	29	my $basedir = "/etc/pve";
	30	my $clusterconf = "$basedir/cluster.conf";
	31	my $libdir = "/var/lib/pve-cluster";
	32	my $backupdir = "/var/lib/pve-cluster/backup";
	33	my $dbfile = "$libdir/config.db";
	34	my $authfile = "$libdir/corosync.authkey";
	35
	36
	37	sub backup_database {
	38
	39	print "backup old database\n";
	40
	41	mkdir $backupdir;
	42
	43	my $ctime = time();
	44	my $cmd = "echo '.dump' \|";
	45	$cmd .= "sqlite3 '$dbfile' \|";
	46	$cmd .= "gzip - >'${backupdir}/config-${ctime}.sql.gz'";
	47
	48	system($cmd) == 0 \|\|
	49	die "can't backup old database: $!\n";
	50
	51	# purge older backup
	52	my $maxfiles = 10;
	53
	54	my @bklist = ();
	55	foreach my $fn (<$backupdir/config-*.sql.gz>) {
	56	if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
	57	push @bklist, [$fn, $1];
	58	}
	59	}
	60
	61	@bklist = sort { $b->[1] <=> $a->[1] } @bklist;
	62
	63	while (scalar (@bklist) >= $maxfiles) {
	64	my $d = pop @bklist;
	65	print "delete old backup '$d->[0]'\n";
	66	unlink $d->[0];
	67	}
	68	}
	69
	70	__PACKAGE__->register_method ({
	71	name => 'keygen',
	72	path => 'keygen',
	73	method => 'PUT',
	74	description => "Generate new cryptographic key for corosync.",
	75	parameters => {
	76	additionalProperties => 0,
	77	properties => {
	78	filename => {
	79	type => 'string',
	80	description => "Output file name"
	81	}
	82	},
	83	},
	84	returns => { type => 'null' },
	85
86	code => sub {
87	my ($param) = @_;
88
89	my $filename = $param->{filename};
90
91	# test EUID
92	$> == 0 \|\| die "Error: Authorization key must be generated as root user.\n";
93	my $dirname = dirname($filename);
94	my $basename = basename($filename);
95
96	File::Path::make_path($dirname) if $dirname;
97
98	my $fh = IO::File->new ("/dev/urandom", 'r') \|\|
99	die "can't open /dev/urandom - $!\n";
100
101	my $keysize = 128;
102	my $key = '';
103
104	my $bytes = 0;
105	while ($bytes < $keysize) {
106	my $rb = sysread($fh, $key, $keysize - $bytes, $bytes);
107	($rb > 0) \|\| die "Could not read /dev/urandom - $!\n";
108	$bytes += $rb;
109	}
110
111	close($fh);
112
113	my $tmpfn = "$filename.tmp.$$";
114	$fh = IO::File->new ($tmpfn, O_CREAT\|O_WRONLY, 0400);
115	die "can't open temporary file '$tmpfn' - $!\n" if !$fh;
116
117	eval {
118	my $wb;
119	(($wb = syswrite($fh, $key)) == $keysize) \|\|
120	die "writing key failed - short write $wb\n;"
121	};
122	my $err = $@;
123
124	$fh->close();
125
126	if ($err) {
127	unlink $tmpfn;
128	die $err;
129	}
130
131	if (!rename($tmpfn, $filename)) {
132	my $err = $!;
133	unlink $tmpfn;
134	die "rename $tmpfn to $filename failed - $err\n";
135	}
136
137	return undef;
138	}});
139
140	__PACKAGE__->register_method ({
141	name => 'create',
142	path => 'create',
143	method => 'PUT',
144	description => "Generate new cluster configuration.",
145	parameters => {
146	additionalProperties => 0,
147	properties => {
148	clustername => {
149	description => "The name of the cluster.",
150	type => 'string', format => 'pve-node',
e51796ba	151	maxLength => 15,
fe000966 DM	152	},
	153	nodeid => {
	154	type => 'integer',
	155	description => "Node id for this node.",
	156	minimum => 1,
	157	optional => 1,
	158	},
	159	votes => {
	160	type => 'integer',
	161	description => "Number of votes for this node",
	162	minimum => 1,
	163	optional => 1,
	164	},
	165	},
	166	},
	167	returns => { type => 'null' },
	168
	169	code => sub {
	170	my ($param) = @_;
	171
	172	-f $clusterconf && die "cluster config '$clusterconf' already exists\n";
	173
f666cdde	174	PVE::Cluster::setup_rootsshconfig();
fe000966 DM	175	PVE::Cluster::setup_ssh_keys();
	176
	177	-f $authfile \|\| __PACKAGE__->keygen({filename => $authfile});
	178
	179	-f $authfile \|\| die "no authentication key available\n";
	180
	181	my $clustername = $param->{clustername};
	182
	183	$param->{nodeid} = 1 if !$param->{nodeid};
	184
	185	$param->{votes} = 1 if !defined($param->{votes});
	186
	187	my $config = <<_EOD;
	188	<?xml version="1.0"?>
	189	<cluster name="$clustername" config_version="1">
	190
	191	<cman keyfile="$authfile">
	192	</cman>
	193
	194	<clusternodes>
	195	<clusternode name="${nodename}" votes="$param->{votes}" nodeid="$param->{nodeid}"/>
	196	</clusternodes>
	197
	198	</cluster>
	199	_EOD
	200	;
	201	PVE::Tools::file_set_contents($clusterconf, $config);
	202
	203	PVE::Cluster::ssh_merge_keys();
	204
	205	PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
	206
	207	PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
	208
	209	PVE::Tools::run_command('/etc/init.d/pve-cluster restart'); # restart
	210
	211	# that cman init script returns strange values - simply ignore for now
	212	system('/etc/init.d/cman start');
4e433fe4 DM	213
	214	# also start clvm
	215	system('/etc/init.d/clvm start');
fe000966 DM	216
	217	return undef;
	218	}});
	219
	220	__PACKAGE__->register_method ({
	221	name => 'addnode',
	222	path => 'addnode',
	223	method => 'PUT',
	224	description => "Adds a node to the cluster configuration.",
	225	parameters => {
	226	additionalProperties => 0,
	227	properties => {
	228	node => PVE::JSONSchema::get_standard_option('pve-node'),
	229	nodeid => {
	230	type => 'integer',
	231	description => "Node id for this node.",
	232	minimum => 1,
	233	optional => 1,
	234	},
	235	votes => {
	236	type => 'integer',
	237	description => "Number of votes for this node",
	238	minimum => 0,
	239	optional => 1,
	240	},
	241	force => {
	242	type => 'boolean',
	243	description => "Do not throw error if node already exists.",
	244	optional => 1,
	245	},
	246	},
	247	},
	248	returns => { type => 'null' },
	249
	250	code => sub {
	251	my ($param) = @_;
	252
01dddfb9	253	PVE::Cluster::check_cfs_quorum();
fe000966 DM	254
	255	my $lst = lsnode();
	256
	257	my $name = $param->{node};
	258
	259	if (defined(my $res = $lst->{$name})) {
	260	$param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
	261	$param->{votes} = $res->{votes} if !defined($param->{votes});
	262
	263	if ($res->{votes} == $param->{votes} &&
	264	$res->{nodeid} == $param->{nodeid}) {
	265	print "node $name already defined\n";
	266	if ($param->{force}) {
	267	exit (0);
	268	} else {
	269	exit (-1);
	270	}
	271	}
	272	} elsif (!$param->{nodeid}) {
	273	my $nodeid = 1;
	274
	275	while(1) {
	276	my $found = 0;
	277	foreach my $v (values %$lst) {
	278	if ($v->{nodeid} eq $nodeid) {
	279	$found = 1;
	280	$nodeid++;
	281	last;
	282	}
	283	}
	284	last if !$found;
	285	};
	286
	287	$param->{nodeid} = $nodeid;
	288	}
	289
	290	$param->{votes} = 1 if !defined($param->{votes});
	291
	292	PVE::Cluster::gen_local_dirs($name);
	293
	294	eval { PVE::Cluster::ssh_merge_keys(); };
	295	warn $@ if $@;
	296
	297	my $cmd = ['ccs_tool', 'addnode', '-c', $clusterconf];
	298
	299	# NOTE: cman does not like votes="0"
	300	if ($param->{votes}) {
	301	push @$cmd, '-v', $param->{votes};
	302	}
	303
	304	push @$cmd, '-n', $param->{nodeid}, $name;
	305
	306	system(@$cmd) == 0 \|\| exit(-1);
	307
	308	exit (0);
	309	}});
	310
	311
	312	__PACKAGE__->register_method ({
	313	name => 'delnode',
	314	path => 'delnode',
	315	method => 'PUT',
	316	description => "Removes a node to the cluster configuration.",
	317	parameters => {
318	additionalProperties => 0,
319	properties => {
320	node => PVE::JSONSchema::get_standard_option('pve-node'),
321	},
322	},
323	returns => { type => 'null' },
324
325	code => sub {
326	my ($param) = @_;
327
1bf30626	328	PVE::Cluster::check_cfs_quorum();
fe000966 DM	329
	330	my $cmd = ['ccs_tool', 'delnode', '-c', $clusterconf, $param->{node}];
	331
	332	exec (@$cmd);
	333
	334	exit (-1); # should not be reached
	335	}});
	336
	337	__PACKAGE__->register_method ({
	338	name => 'add',
	339	path => 'add',
	340	method => 'PUT',
	341	description => "Adds the current node to an existing cluster.",
	342	parameters => {
	343	additionalProperties => 0,
	344	properties => {
	345	hostname => {
	346	type => 'string',
	347	description => "Hostname (or IP) of an existing cluster member."
	348	},
	349	nodeid => {
	350	type => 'integer',
	351	description => "Node id for this node.",
	352	minimum => 1,
	353	optional => 1,
	354	},
	355	votes => {
	356	type => 'integer',
	357	description => "Number of votes for this node",
	358	minimum => 0,
	359	optional => 1,
	360	},
	361	force => {
	362	type => 'boolean',
	363	description => "Do not throw error if node already exists.",
	364	optional => 1,
	365	},
	366	},
	367	},
	368	returns => { type => 'null' },
	369
	370	code => sub {
	371	my ($param) = @_;
	372
f666cdde	373	PVE::Cluster::setup_rootsshconfig();
fe000966 DM	374	PVE::Cluster::setup_ssh_keys();
	375
	376	my $host = $param->{hostname};
	377
	378	if (!$param->{force}) {
	379
	380	if (-f $authfile) {
	381	die "authentication key already exists\n";
	382	}
	383
	384	if (-f $clusterconf) {
	385	die "cluster config '$clusterconf' already exists\n";
	386	}
	387
	388	my $vmlist = PVE::Cluster::get_vmlist();
	389	if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
	390	die "this host already contains virtual machines - please remove the first\n";
	391	}
	392
	393	if (system("cman_tool status >/dev/null 2>&1") == 0) {
	394	die "cman is already running\n";
	395	}
	396	}
	397
	398	# make sure known_hosts is on local filesystem
	399	PVE::Cluster::ssh_unmerge_known_hosts();
	400
4bd5d833	401	my $cmd = "ssh-copy-id -i /root/.ssh/id_rsa 'root\@$host' >/dev/null 2>&1";
fe000966 DM	402	system ($cmd) == 0 \|\|
	403	die "unable to copy ssh ID\n";
	404
	405	$cmd = ['ssh', $host, '-o', 'BatchMode=yes',
	406	'pvecm', 'addnode', $nodename, '--force', 1];
	407
849bf59f	408	push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
fe000966	409
849bf59f	410	push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
fe000966 DM	411
	412	if (system (@$cmd) != 0) {
	413	my $cmdtxt = join (' ', @$cmd);
	414	die "unable to add node: command failed ($cmdtxt)\n";
	415	}
	416
	417	my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
	418	mkdir $tmpdir;
	419
	420	eval {
	421	print "copy corosync auth key\n";
	422	$cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
	423	"$host:$authfile $clusterconf", $tmpdir];
	424
	425	system(@$cmd) == 0 \|\| die "can't rsync data from host '$host'\n";
	426
	427	mkdir "/etc/cluster";
	428	my $confbase = basename($clusterconf);
	429
	430	$cmd = "cp '$tmpdir/$confbase' '/etc/cluster/$confbase'";
	431	system($cmd) == 0 \|\| die "can't copy cluster configuration\n";
	432
	433	my $keybase = basename($authfile);
	434	system ("cp '$tmpdir/$keybase' '$authfile'") == 0 \|\|
	435	die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
	436
	437	print "stopping pve-cluster service\n";
	438
	439	system("umount $basedir -f >/dev/null 2>&1");
	440	system("/etc/init.d/pve-cluster stop") == 0 \|\|
	441	die "can't stop pve-cluster service\n";
	442
	443	backup_database();
	444
	445	unlink $dbfile;
	446
	447	system("/etc/init.d/pve-cluster start") == 0 \|\|
	448	die "starting pve-cluster failed\n";
	449
	450	system("/etc/init.d/cman start");
	451
	452	# wait for quorum
	453	my $printqmsg = 1;
1bf30626	454	while (!PVE::Cluster::check_cfs_quorum(1)) {
fe000966 DM	455	if ($printqmsg) {
	456	print "waiting for quorum...";
	457	STDOUT->flush();
	458	$printqmsg = 0;
	459	}
	460	sleep(1);
	461	}
	462	print "OK\n" if !$printqmsg;
	463
4e433fe4 DM	464	system("/etc/init.d/clvm start");
4e433fe4 DM	465
fe000966 DM	466	print "generating node certificates\n";
	467	PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
	468
	469	print "merge known_hosts file\n";
	470	PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
	471
	472	print "restart services\n";
c56c240f DM	473	# restart pvedaemon (changed certs)
c56c240f DM	474	system("/etc/init.d/pvedaemon restart");
0c112dde DM	475	# restart pveproxy (changed certs)
0c112dde DM	476	system("/etc/init.d/pveproxy restart");
fe000966 DM	477
	478	print "successfully added node '$nodename' to cluster.\n";
	479	};
	480	my $err = $@;
	481
	482	rmtree $tmpdir;
	483
	484	die $err if $err;
	485
	486	return undef;
	487	}});
	488
	489	__PACKAGE__->register_method ({
	490	name => 'status',
	491	path => 'status',
	492	method => 'GET',
	493	description => "Displays the local view of the cluster status.",
	494	parameters => {
	495	additionalProperties => 0,
	496	properties => {},
	497	},
	498	returns => { type => 'null' },
	499
	500	code => sub {
	501	my ($param) = @_;
	502
	503	my $cmd = ['cman_tool', 'status'];
	504
	505	exec (@$cmd);
	506
	507	exit (-1); # should not be reached
	508	}});
	509
	510	__PACKAGE__->register_method ({
	511	name => 'nodes',
	512	path => 'nodes',
	513	method => 'GET',
	514	description => "Displays the local view of the cluster nodes.",
	515	parameters => {
	516	additionalProperties => 0,
	517	properties => {},
	518	},
	519	returns => { type => 'null' },
	520
	521	code => sub {
	522	my ($param) = @_;
	523
	524	my $cmd = ['cman_tool', 'nodes'];
	525
	526	exec (@$cmd);
	527
	528	exit (-1); # should not be reached
	529	}});
	530
	531	__PACKAGE__->register_method ({
	532	name => 'expected',
	533	path => 'expected',
	534	method => 'PUT',
	535	description => "Tells CMAN a new value of expected votes.",
	536	parameters => {
	537	additionalProperties => 0,
	538	properties => {
	539	expected => {
	540	type => 'integer',
541	description => "Expected votes",
542	minimum => 1,
543	},
544	},
545	},
546	returns => { type => 'null' },
547
548	code => sub {
549	my ($param) = @_;
550
551	my $cmd = ['cman_tool', 'expected', '-e', $param->{expected}];
552
553	exec (@$cmd);
554
555	exit (-1); # should not be reached
556
557	}});
558
559	sub lsnode {
560
561	my $res = {};
562
563	my $cmd = ['ccs_tool', 'lsnode', '-c', $clusterconf];
564
565	my $scan = 0;
566	my $parser = sub {
567	my $line = shift;
568
569	if ($line =~ m/^Nodename\s+Votes\s+Nodeid\s/i) {
570	$scan = 1;
571	return;
572	}
573	return if !$scan;
574	if ($line =~ m/^(\S+)\s+(\d+)\s+(\d+)\s/) {
575	$res->{$1} = {
576	name => $1,
577	votes => $2,
578	nodeid => $3 };
579	}
580	};
581
582	PVE::Tools::run_command($cmd, outfunc => $parser);
583	return $res;
584	}
585
586	__PACKAGE__->register_method ({
587	name => 'updatecerts',
588	path => 'updatecerts',
589	method => 'PUT',
bd0ae7ff	590	description => "Update node certificates (and generate all needed files/directories).",
fe000966 DM	591	parameters => {
	592	additionalProperties => 0,
	593	properties => {
	594	force => {
	595	description => "Force generation of new SSL certifate.",
	596	type => 'boolean',
	597	optional => 1,
	598	},
	599	silent => {
	600	description => "Ignore errors (i.e. when cluster has no quorum).",
	601	type => 'boolean',
	602	optional => 1,
	603	},
	604	},
	605	},
	606	returns => { type => 'null' },
	607	code => sub {
	608	my ($param) = @_;
	609
c1eb6bdf	610	PVE::Cluster::setup_rootsshconfig();
c1eb6bdf	611
bd0ae7ff DM	612	PVE::Cluster::gen_pve_vzdump_symlink();
bd0ae7ff DM	613
5bf1b615	614	if (!PVE::Cluster::check_cfs_quorum(1)) {
fe000966 DM	615	return undef if $param->{silent};
	616	die "no quorum - unable to update files\n";
	617	}
	618
39df71df DM	619	PVE::Cluster::setup_ssh_keys();
39df71df DM	620
fe000966 DM	621	PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
	622	PVE::Cluster::ssh_merge_keys();
	623	PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
bd0ae7ff	624	PVE::Cluster::gen_pve_vzdump_files();
fe000966 DM	625
	626	return undef;
	627	}});
	628
	629
	630	my $cmddef = {
	631	keygen => [ __PACKAGE__, 'keygen', ['filename']],
	632	create => [ __PACKAGE__, 'create', ['clustername']],
	633	add => [ __PACKAGE__, 'add', ['hostname']],
	634	addnode => [ __PACKAGE__, 'addnode', ['node']],
	635	delnode => [ __PACKAGE__, 'delnode', ['node']],
	636	status => [ __PACKAGE__, 'status' ],
	637	nodes => [ __PACKAGE__, 'nodes' ],
	638	expected => [ __PACKAGE__, 'expected', ['expected']],
	639	updatecerts => [ __PACKAGE__, 'updatecerts', []],
	640	};
	641
	642	my $cmd = shift;
	643
	644	if ($cmd && $cmd ne 'printmanpod' && $cmd ne 'verifyapi') {
	645	PVE::Cluster::check_cfs_is_mounted();
	646	PVE::Cluster::cfs_update();
	647	}
	648
	649	PVE::CLIHandler::handle_cmd($cmddef, "pvecm", $cmd, \@ARGV, undef, $0);
	650
	651	exit 0;
	652
	653	__END__
	654
	655	=head1 NAME
	656
	657	pvecm - Proxmox VE cluster manager toolkit
	658
	659	=head1 SYNOPSIS
	660
	661	=include synopsis
	662
	663	=head1 DESCRIPTION
	664
	665	pvecm is a program to manage the cluster configuration. It can be used
	666	to create a new cluster, join nodes to a cluster, leave the cluster,
	667	get status information and do various other cluster related tasks.
	668
	669	=include pve_copyright
	670
	671