]>
git.proxmox.com Git - pve-cluster.git/blob - data/PVE/CLI/pvecm.pm
1 package PVE
::CLI
::pvecm
;
9 use PVE
::Tools
qw(run_command);
16 use base
qw(PVE::CLIHandler);
18 $ENV{HOME
} = '/root'; # for ssh-copy-id
20 my $basedir = "/etc/pve";
21 my $clusterconf = "$basedir/corosync.conf";
22 my $libdir = "/var/lib/pve-cluster";
23 my $backupdir = "/var/lib/pve-cluster/backup";
24 my $dbfile = "$libdir/config.db";
25 my $authfile = "/etc/corosync/authkey";
29 print "backup old database\n";
37 ['gzip', '-', \
">${backupdir}/config-${ctime}.sql.gz"],
40 run_command
($cmd, 'errmsg' => "cannot backup old database\n");
46 foreach my $fn (<$backupdir/config-*.sql
.gz
>) {
47 if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
48 push @bklist, [$fn, $1];
52 @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
54 while (scalar (@bklist) >= $maxfiles) {
56 print "delete old backup '$d->[0]'\n";
61 __PACKAGE__-
>register_method ({
65 description
=> "Generate new cryptographic key for corosync.",
67 additionalProperties
=> 0,
71 description
=> "Output file name"
75 returns
=> { type
=> 'null' },
80 my $filename = $param->{filename
};
83 $> == 0 || die "Error: Authorization key must be generated as root user.\n";
84 my $dirname = dirname
($filename);
86 die "key file '$filename' already exists\n" if -e
$filename;
88 File
::Path
::make_path
($dirname) if $dirname;
90 run_command
(['corosync-keygen', '-l', '-k', $filename]);
95 __PACKAGE__-
>register_method ({
99 description
=> "Generate new cluster configuration.",
101 additionalProperties
=> 0,
104 description
=> "The name of the cluster.",
105 type
=> 'string', format
=> 'pve-node',
110 description
=> "Node id for this node.",
116 description
=> "Number of votes for this node.",
121 type
=> 'string', format
=> 'ip',
122 description
=> "This specifies the network address the corosync ring 0".
123 " executive should bind to and defaults to the local IP address of the node.",
127 type
=> 'string', format
=> 'address',
128 description
=> "Hostname (or IP) of the corosync ring0 address of this node.".
129 " Defaults to the hostname of the node.",
133 type
=> 'string', format
=> 'ip',
134 description
=> "This specifies the network address the corosync ring 1".
135 " executive should bind to and is optional.",
139 type
=> 'string', format
=> 'address',
140 description
=> "Hostname (or IP) of the corosync ring1 address, this".
141 " needs an valid bindnet1_addr.",
146 returns
=> { type
=> 'null' },
151 -f
$clusterconf && die "cluster config '$clusterconf' already exists\n";
153 PVE
::Cluster
::setup_sshd_config
(1);
154 PVE
::Cluster
::setup_rootsshconfig
();
155 PVE
::Cluster
::setup_ssh_keys
();
157 -f
$authfile || __PACKAGE__-
>keygen({filename
=> $authfile});
159 -f
$authfile || die "no authentication key available\n";
161 my $clustername = $param->{clustername
};
163 $param->{nodeid
} = 1 if !$param->{nodeid
};
165 $param->{votes
} = 1 if !defined($param->{votes
});
167 my $nodename = PVE
::INotify
::nodename
();
169 my $local_ip_address = PVE
::Cluster
::remote_node_ip
($nodename);
171 $param->{bindnet0_addr
} = $local_ip_address
172 if !defined($param->{bindnet0_addr
});
174 $param->{ring0_addr
} = $nodename if !defined($param->{ring0_addr
});
176 die "Param bindnet1_addr and ring1_addr are dependend, use both or none!\n"
177 if (defined($param->{bindnet1_addr
}) != defined($param->{ring1_addr
}));
179 my $bind_is_ipv6 = Net
::IP
::ip_is_ipv6
($param->{bindnet0_addr
});
181 # use string as here-doc format distracts more
182 my $interfaces = "interface {\n ringnumber: 0\n" .
183 " bindnetaddr: $param->{bindnet0_addr}\n }";
185 my $ring_addresses = "ring0_addr: $param->{ring0_addr}" ;
187 # allow use of multiple rings (rrp) at cluster creation time
188 if ($param->{bindnet1_addr
}) {
189 die "IPv6 and IPv4 cannot be mixed, use one or the other!\n"
190 if Net
::IP
::ip_is_ipv6
($param->{bindnet1_addr
}) != $bind_is_ipv6;
192 $interfaces .= "\n interface {\n ringnumber: 1\n" .
193 " bindnetaddr: $param->{bindnet1_addr}\n }\n";
195 $interfaces .= "rrp_mode: passive\n"; # only passive is stable and tested
197 $ring_addresses .= "\n ring1_addr: $param->{ring1_addr}";
199 } elsif($param->{rrp_mode
} && $param->{rrp_mode
} ne 'none') {
201 warn "rrp_mode '$param->{rrp_mode}' useless when using only one".
202 " ring, using 'none' instead";
203 # corosync defaults to none if only one interface is configured
204 $param->{rrp_mode
} = undef;
208 # No, corosync cannot deduce this on its own
209 my $ipversion = $bind_is_ipv6 ?
'ipv6' : 'ipv4';
215 cluster_name: $clustername
217 ip_version: $ipversion
225 nodeid: $param->{nodeid}
226 quorum_votes: $param->{votes}
231 provider: corosync_votequorum
240 PVE
::Tools
::file_set_contents
($clusterconf, $config);
242 PVE
::Cluster
::ssh_merge_keys
();
244 PVE
::Cluster
::gen_pve_node_files
($nodename, $local_ip_address);
246 PVE
::Cluster
::ssh_merge_known_hosts
($nodename, $local_ip_address, 1);
248 run_command
('systemctl restart pve-cluster'); # restart
250 run_command
('systemctl restart corosync'); # restart
255 __PACKAGE__-
>register_method ({
259 description
=> "Adds a node to the cluster configuration.",
261 additionalProperties
=> 0,
263 node
=> PVE
::JSONSchema
::get_standard_option
('pve-node'),
266 description
=> "Node id for this node.",
272 description
=> "Number of votes for this node",
278 description
=> "Do not throw error if node already exists.",
282 type
=> 'string', format
=> 'address',
283 description
=> "Hostname (or IP) of the corosync ring0 address of this node.".
284 " Defaults to nodes hostname.",
288 type
=> 'string', format
=> 'address',
289 description
=> "Hostname (or IP) of the corosync ring1 address, this".
290 " needs an valid bindnet1_addr.",
295 returns
=> { type
=> 'null' },
300 if (!$param->{force
} && (-t STDIN
|| -t STDOUT
)) {
301 die "error: `addnode` should not get called interactively!\nUse ".
302 "`pvecm add <cluster-node>` to add a node to a cluster!\n";
305 PVE
::Cluster
::check_cfs_quorum
();
308 my $conf = PVE
::Cluster
::cfs_read_file
("corosync.conf");
309 my $nodelist = PVE
::Corosync
::nodelist
($conf);
310 my $totem_cfg = PVE
::Corosync
::totem_config
($conf);
312 my $name = $param->{node
};
314 # ensure we do not reuse an address, that can crash the whole cluster!
315 my $check_duplicate_addr = sub {
317 return if !defined($addr);
319 while (my ($k, $v) = each %$nodelist) {
320 next if $k eq $name; # allows re-adding a node if force is set
321 if ($v->{ring0_addr
} eq $addr || ($v->{ring1_addr
} && $v->{ring1_addr
} eq $addr)) {
322 die "corosync: address '$addr' already defined by node '$k'\n";
327 &$check_duplicate_addr($param->{ring0_addr
});
328 &$check_duplicate_addr($param->{ring1_addr
});
330 $param->{ring0_addr
} = $name if !$param->{ring0_addr
};
332 die "corosync: using 'ring1_addr' parameter needs a configured ring 1 interface!\n"
333 if $param->{ring1_addr
} && !defined($totem_cfg->{interface
}->{1});
335 die "corosync: ring 1 interface configured but 'ring1_addr' parameter not defined!\n"
336 if defined($totem_cfg->{interface
}->{1}) && !defined($param->{ring1_addr
});
338 if (defined(my $res = $nodelist->{$name})) {
339 $param->{nodeid
} = $res->{nodeid
} if !$param->{nodeid
};
340 $param->{votes
} = $res->{quorum_votes
} if !defined($param->{votes
});
342 if ($res->{quorum_votes
} == $param->{votes
} &&
343 $res->{nodeid
} == $param->{nodeid
}) {
344 print "node $name already defined\n";
345 if ($param->{force
}) {
351 die "can't add existing node\n";
353 } elsif (!$param->{nodeid
}) {
358 foreach my $v (values %$nodelist) {
359 if ($v->{nodeid
} eq $nodeid) {
368 $param->{nodeid
} = $nodeid;
371 $param->{votes
} = 1 if !defined($param->{votes
});
373 PVE
::Cluster
::gen_local_dirs
($name);
375 eval { PVE
::Cluster
::ssh_merge_keys
(); };
378 $nodelist->{$name} = {
379 ring0_addr
=> $param->{ring0_addr
},
380 nodeid
=> $param->{nodeid
},
383 $nodelist->{$name}->{ring1_addr
} = $param->{ring1_addr
} if $param->{ring1_addr
};
384 $nodelist->{$name}->{quorum_votes
} = $param->{votes
} if $param->{votes
};
386 PVE
::Corosync
::update_nodelist
($conf, $nodelist);
389 PVE
::Cluster
::cfs_lock_file
('corosync.conf', 10, $code);
396 __PACKAGE__-
>register_method ({
400 description
=> "Removes a node to the cluster configuration.",
402 additionalProperties
=> 0,
406 description
=> "Hostname or IP of the corosync ring0 address of this node.",
410 returns
=> { type
=> 'null' },
415 my $local_node = PVE
::INotify
::nodename
();
416 die "Cannot delete myself from cluster!\n" if $param->{node
} eq $local_node;
418 PVE
::Cluster
::check_cfs_quorum
();
421 my $conf = PVE
::Cluster
::cfs_read_file
("corosync.conf");
422 my $nodelist = PVE
::Corosync
::nodelist
($conf);
427 foreach my $tmp_node (keys %$nodelist) {
428 my $d = $nodelist->{$tmp_node};
429 my $ring0_addr = $d->{ring0_addr
};
430 my $ring1_addr = $d->{ring1_addr
};
431 if (($tmp_node eq $param->{node
}) ||
432 (defined($ring0_addr) && ($ring0_addr eq $param->{node
})) ||
433 (defined($ring1_addr) && ($ring1_addr eq $param->{node
}))) {
435 $nodeid = $d->{nodeid
};
440 die "Node/IP: $param->{node} is not a known host of the cluster.\n"
443 delete $nodelist->{$node};
445 PVE
::Corosync
::update_nodelist
($conf, $nodelist);
447 run_command
(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
450 PVE
::Cluster
::cfs_lock_file
('corosync.conf', 10, $code);
456 __PACKAGE__-
>register_method ({
460 description
=> "Adds the current node to an existing cluster.",
462 additionalProperties
=> 0,
466 description
=> "Hostname (or IP) of an existing cluster member."
470 description
=> "Node id for this node.",
476 description
=> "Number of votes for this node",
482 description
=> "Do not throw error if node already exists.",
486 type
=> 'string', format
=> 'address',
487 description
=> "Hostname (or IP) of the corosync ring0 address of this node.".
488 " Defaults to nodes hostname.",
492 type
=> 'string', format
=> 'address',
493 description
=> "Hostname (or IP) of the corosync ring1 address, this".
494 " needs an valid configured ring 1 interface in the cluster.",
499 returns
=> { type
=> 'null' },
504 my $nodename = PVE
::INotify
::nodename
();
506 PVE
::Cluster
::setup_sshd_config
(1);
507 PVE
::Cluster
::setup_rootsshconfig
();
508 PVE
::Cluster
::setup_ssh_keys
();
510 my $host = $param->{hostname
};
512 my ($errors, $warnings) = ('', '');
515 my ($msg, $suppress) = @_;
518 $warnings .= "* $msg\n";
520 $errors .= "* $msg\n";
524 if (!$param->{force
}) {
527 &$error("authentication key '$authfile' already exists", $param->{force
});
530 if (-f
$clusterconf) {
531 &$error("cluster config '$clusterconf' already exists", $param->{force
});
534 my $vmlist = PVE
::Cluster
::get_vmlist
();
535 if ($vmlist && $vmlist->{ids
} && scalar(keys %{$vmlist->{ids
}})) {
536 &$error("this host already contains virtual guests", $param->{force
});
539 if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) {
540 &$error("corosync is already running, is this node already in a cluster?!", $param->{force
});
544 # check if corosync ring IPs are configured on the current nodes interfaces
548 if (!PVE
::JSONSchema
::pve_verify_ip
($ip, 1)) {
550 eval { $ip = PVE
::Network
::get_ip_from_hostname
($host); };
552 &$error("cannot use '$host': $@\n") ;
557 my $cidr = (Net
::IP
::ip_is_ipv6
($ip)) ?
"$ip/128" : "$ip/32";
558 my $configured_ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
560 &$error("cannot use IP '$ip', it must be configured exactly once on local node!\n")
561 if (scalar(@$configured_ips) != 1);
565 &$check_ip($param->{ring0_addr
});
566 &$check_ip($param->{ring1_addr
});
568 warn "warning, ignore the following errors:\n$warnings" if $warnings;
569 die "detected the following error(s):\n$errors" if $errors;
571 # make sure known_hosts is on local filesystem
572 PVE
::Cluster
::ssh_unmerge_known_hosts
();
574 my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
575 run_command
($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
576 'errmsg' => "unable to copy ssh ID");
578 $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
579 'pvecm', 'addnode', $nodename, '--force', 1];
581 push @$cmd, '--nodeid', $param->{nodeid
} if $param->{nodeid
};
583 push @$cmd, '--votes', $param->{votes
} if defined($param->{votes
});
585 push @$cmd, '--ring0_addr', $param->{ring0_addr
} if defined($param->{ring0_addr
});
587 push @$cmd, '--ring1_addr', $param->{ring1_addr
} if defined($param->{ring1_addr
});
589 if (system (@$cmd) != 0) {
590 my $cmdtxt = join (' ', @$cmd);
591 die "unable to add node: command failed ($cmdtxt)\n";
594 my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
598 print "copy corosync auth key\n";
599 $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
600 "[$host]:$authfile $clusterconf", $tmpdir];
602 system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
604 mkdir "/etc/corosync";
605 my $confbase = basename
($clusterconf);
607 $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
608 system($cmd) == 0 || die "can't copy cluster configuration\n";
610 my $keybase = basename
($authfile);
611 system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
612 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
614 print "stopping pve-cluster service\n";
616 system("umount $basedir -f >/dev/null 2>&1");
617 system("systemctl stop pve-cluster") == 0 ||
618 die "can't stop pve-cluster service\n";
624 system("systemctl start pve-cluster") == 0 ||
625 die "starting pve-cluster failed\n";
627 system("systemctl start corosync");
631 while (!PVE
::Cluster
::check_cfs_quorum
(1)) {
633 print "waiting for quorum...";
639 print "OK\n" if !$printqmsg;
641 my $local_ip_address = PVE
::Cluster
::remote_node_ip
($nodename);
643 print "generating node certificates\n";
644 PVE
::Cluster
::gen_pve_node_files
($nodename, $local_ip_address);
646 print "merge known_hosts file\n";
647 PVE
::Cluster
::ssh_merge_known_hosts
($nodename, $local_ip_address, 1);
649 print "restart services\n";
650 # restart pvedaemon (changed certs)
651 system("systemctl restart pvedaemon");
652 # restart pveproxy (changed certs)
653 system("systemctl restart pveproxy");
655 print "successfully added node '$nodename' to cluster.\n";
666 __PACKAGE__-
>register_method ({
670 description
=> "Displays the local view of the cluster status.",
672 additionalProperties
=> 0,
675 returns
=> { type
=> 'null' },
680 PVE
::Corosync
::check_conf_exists
();
682 my $cmd = ['corosync-quorumtool', '-siH'];
686 exit (-1); # should not be reached
689 __PACKAGE__-
>register_method ({
693 description
=> "Displays the local view of the cluster nodes.",
695 additionalProperties
=> 0,
698 returns
=> { type
=> 'null' },
703 PVE
::Corosync
::check_conf_exists
();
705 my $cmd = ['corosync-quorumtool', '-l'];
709 exit (-1); # should not be reached
712 __PACKAGE__-
>register_method ({
716 description
=> "Tells corosync a new value of expected votes.",
718 additionalProperties
=> 0,
722 description
=> "Expected votes",
727 returns
=> { type
=> 'null' },
732 PVE
::Corosync
::check_conf_exists
();
734 my $cmd = ['corosync-quorumtool', '-e', $param->{expected
}];
738 exit (-1); # should not be reached
742 __PACKAGE__-
>register_method ({
743 name
=> 'updatecerts',
744 path
=> 'updatecerts',
746 description
=> "Update node certificates (and generate all needed files/directories).",
748 additionalProperties
=> 0,
751 description
=> "Force generation of new SSL certifate.",
756 description
=> "Ignore errors (i.e. when cluster has no quorum).",
762 returns
=> { type
=> 'null' },
766 PVE
::Cluster
::setup_rootsshconfig
();
768 PVE
::Cluster
::gen_pve_vzdump_symlink
();
770 if (!PVE
::Cluster
::check_cfs_quorum
(1)) {
771 return undef if $param->{silent
};
772 die "no quorum - unable to update files\n";
775 PVE
::Cluster
::setup_ssh_keys
();
777 my $nodename = PVE
::INotify
::nodename
();
779 my $local_ip_address = PVE
::Cluster
::remote_node_ip
($nodename);
781 PVE
::Cluster
::gen_pve_node_files
($nodename, $local_ip_address, $param->{force
});
782 PVE
::Cluster
::ssh_merge_keys
();
783 PVE
::Cluster
::ssh_merge_known_hosts
($nodename, $local_ip_address);
784 PVE
::Cluster
::gen_pve_vzdump_files
();
789 __PACKAGE__-
>register_method ({
793 description
=> "Used by VM/CT migration - do not use manually.",
795 additionalProperties
=> 0,
797 get_migration_ip
=> {
800 description
=> 'return the migration IP, if configured',
803 migration_network
=> {
806 description
=> 'the migration network used to detect the local migration IP',
811 description
=> 'Run a command with a tcp socket as standard input.'
812 .' The IP address and port are printed via this'
813 ." command's stdandard output first, each on a separate line.",
816 'extra-args' => PVE
::JSONSchema
::get_standard_option
('extra-args'),
819 returns
=> { type
=> 'null'},
823 if (!PVE
::Cluster
::check_cfs_quorum
(1)) {
828 my $network = $param->{migration_network
};
829 if ($param->{get_migration_ip
}) {
830 die "cannot use --run-command with --get_migration_ip\n"
831 if $param->{'run-command'};
832 if (my $ip = PVE
::Cluster
::get_local_migration_ip
($network)) {
837 # do not keep tunnel open when asked for migration ip
841 if ($param->{'run-command'}) {
842 my $cmd = $param->{'extra-args'};
843 die "missing command\n"
844 if !$cmd || !scalar(@$cmd);
846 # Get an ip address to listen on, and find a free migration port
848 if (defined($network)) {
849 $ip = PVE
::Cluster
::get_local_migration_ip
($network)
850 or die "failed to get migration IP address to listen on\n";
851 $family = PVE
::Tools
::get_host_address_family
($ip);
853 my $nodename = PVE
::INotify
::nodename
();
854 ($ip, $family) = PVE
::Network
::get_ip_from_hostname
($nodename, 0);
856 my $port = PVE
::Tools
::next_migrate_port
($family, $ip);
858 PVE
::Tools
::pipe_socket_to_command
($cmd, $ip, $port);
862 print "tunnel online\n";
865 while (my $line = <>) {
867 last if $line =~ m/^quit$/;
875 keygen
=> [ __PACKAGE__
, 'keygen', ['filename']],
876 create
=> [ __PACKAGE__
, 'create', ['clustername']],
877 add
=> [ __PACKAGE__
, 'add', ['hostname']],
878 addnode
=> [ __PACKAGE__
, 'addnode', ['node']],
879 delnode
=> [ __PACKAGE__
, 'delnode', ['node']],
880 status
=> [ __PACKAGE__
, 'status' ],
881 nodes
=> [ __PACKAGE__
, 'nodes' ],
882 expected
=> [ __PACKAGE__
, 'expected', ['expected']],
883 updatecerts
=> [ __PACKAGE__
, 'updatecerts', []],
884 mtunnel
=> [ __PACKAGE__
, 'mtunnel', ['extra-args']],