1 package PVE
::CLI
::pvecm
;
11 use Data
::Dumper
; # fixme: remove
18 use base
qw(PVE::CLIHandler);
20 $ENV{HOME
} = '/root'; # for ssh-copy-id
22 my $basedir = "/etc/pve";
23 my $clusterconf = "$basedir/corosync.conf";
24 my $libdir = "/var/lib/pve-cluster";
25 my $backupdir = "/var/lib/pve-cluster/backup";
26 my $dbfile = "$libdir/config.db";
27 my $authfile = "/etc/corosync/authkey";
31 print "backup old database\n";
39 ['gzip', '-', \
">${backupdir}/config-${ctime}.sql.gz"],
42 PVE
::Tools
::run_command
($cmd, 'errmsg' => "cannot backup old database\n");
48 foreach my $fn (<$backupdir/config-*.sql
.gz
>) {
49 if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
50 push @bklist, [$fn, $1];
54 @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
56 while (scalar (@bklist) >= $maxfiles) {
58 print "delete old backup '$d->[0]'\n";
63 __PACKAGE__-
>register_method ({
67 description
=> "Generate new cryptographic key for corosync.",
69 additionalProperties
=> 0,
73 description
=> "Output file name"
77 returns
=> { type
=> 'null' },
82 my $filename = $param->{filename
};
85 $> == 0 || die "Error: Authorization key must be generated as root user.\n";
86 my $dirname = dirname
($filename);
87 my $basename = basename
($filename);
89 die "key file '$filename' already exists\n" if -e
$filename;
91 File
::Path
::make_path
($dirname) if $dirname;
93 my $cmd = ['corosync-keygen', '-l', '-k', $filename];
94 PVE
::Tools
::run_command
($cmd);
99 __PACKAGE__-
>register_method ({
103 description
=> "Generate new cluster configuration.",
105 additionalProperties
=> 0,
108 description
=> "The name of the cluster.",
109 type
=> 'string', format
=> 'pve-node',
114 description
=> "Node id for this node.",
120 description
=> "Number of votes for this node.",
125 type
=> 'string', format
=> 'ip',
126 description
=> "This specifies the network address the corosync ring 0".
127 " executive should bind to and defaults to the local IP address of the node.",
131 type
=> 'string', format
=> 'address',
132 description
=> "Hostname (or IP) of the corosync ring0 address of this node.".
133 " Defaults to the hostname of the node.",
138 enum
=> ['none', 'active', 'passive'],
139 description
=> "This specifies the mode of redundant ring, which" .
140 " may be none, active or passive. Using multiple interfaces".
141 " only allows 'active' or 'passive'.",
146 type
=> 'string', format
=> 'ip',
147 description
=> "This specifies the network address the corosync ring 1".
148 " executive should bind to and is optional.",
152 type
=> 'string', format
=> 'address',
153 description
=> "Hostname (or IP) of the corosync ring1 address, this".
154 " needs an valid bindnet1_addr.",
159 returns
=> { type
=> 'null' },
164 -f
$clusterconf && die "cluster config '$clusterconf' already exists\n";
166 PVE
::Cluster
::setup_sshd_config
();
167 PVE
::Cluster
::setup_rootsshconfig
();
168 PVE
::Cluster
::setup_ssh_keys
();
170 -f
$authfile || __PACKAGE__-
>keygen({filename
=> $authfile});
172 -f
$authfile || die "no authentication key available\n";
174 my $clustername = $param->{clustername
};
176 $param->{nodeid
} = 1 if !$param->{nodeid
};
178 $param->{votes
} = 1 if !defined($param->{votes
});
180 my $nodename = PVE
::INotify
::nodename
();
182 my $local_ip_address = PVE
::Cluster
::remote_node_ip
($nodename);
184 $param->{bindnet0_addr
} = $local_ip_address
185 if !defined($param->{bindnet0_addr
});
187 $param->{ring0_addr
} = $nodename if !defined($param->{ring0_addr
});
189 die "Param bindnet1_addr and ring1_addr are dependend, use both or none!\n"
190 if (defined($param->{bindnet1_addr
}) != defined($param->{ring1_addr
}));
192 my $bind_is_ipv6 = Net
::IP
::ip_is_ipv6
($param->{bindnet0_addr
});
194 # use string as here-doc format distracts more
195 my $interfaces = "interface {\n ringnumber: 0\n" .
196 " bindnetaddr: $param->{bindnet0_addr}\n }";
198 my $ring_addresses = "ring0_addr: $param->{ring0_addr}" ;
200 # allow use of multiple rings (rrp) at cluster creation time
201 if ($param->{bindnet1_addr
}) {
202 die "IPv6 and IPv4 cannot be mixed, use one or the other!\n"
203 if Net
::IP
::ip_is_ipv6
($param->{bindnet1_addr
}) != $bind_is_ipv6;
205 die "rrp_mode 'none' is not allowed when using multiple interfaces,".
206 " use 'active' or 'passive'!\n"
207 if !$param->{rrp_mode
} || $param->{rrp_mode
} eq 'none';
209 $interfaces .= "\n interface {\n ringnumber: 1\n" .
210 " bindnetaddr: $param->{bindnet1_addr}\n }\n";
212 $ring_addresses .= "\n ring1_addr: $param->{ring1_addr}";
214 } elsif($param->{rrp_mode
} && $param->{rrp_mode
} ne 'none') {
216 warn "rrp_mode '$param->{rrp_mode}' useless when using only one".
217 " ring, using 'none' instead";
218 # corosync defaults to none if only one interface is configured
219 $param->{rrp_mode
} = undef;
223 $interfaces = "rrp_mode: $param->{rrp_mode}\n " . $interfaces
224 if $param->{rrp_mode
};
226 # No, corosync cannot deduce this on its own
227 my $ipversion = $bind_is_ipv6 ?
'ipv6' : 'ipv4';
233 cluster_name: $clustername
235 ip_version: $ipversion
243 nodeid: $param->{nodeid}
244 quorum_votes: $param->{votes}
249 provider: corosync_votequorum
258 PVE
::Tools
::file_set_contents
($clusterconf, $config);
260 PVE
::Cluster
::ssh_merge_keys
();
262 PVE
::Cluster
::gen_pve_node_files
($nodename, $local_ip_address);
264 PVE
::Cluster
::ssh_merge_known_hosts
($nodename, $local_ip_address, 1);
266 PVE
::Tools
::run_command
('systemctl restart pve-cluster'); # restart
268 PVE
::Tools
::run_command
('systemctl restart corosync'); # restart
273 __PACKAGE__-
>register_method ({
277 description
=> "Adds a node to the cluster configuration.",
279 additionalProperties
=> 0,
281 node
=> PVE
::JSONSchema
::get_standard_option
('pve-node'),
284 description
=> "Node id for this node.",
290 description
=> "Number of votes for this node",
296 description
=> "Do not throw error if node already exists.",
300 type
=> 'string', format
=> 'address',
301 description
=> "Hostname (or IP) of the corosync ring0 address of this node.".
302 " Defaults to nodes hostname.",
306 type
=> 'string', format
=> 'address',
307 description
=> "Hostname (or IP) of the corosync ring1 address, this".
308 " needs an valid bindnet1_addr.",
313 returns
=> { type
=> 'null' },
318 PVE
::Cluster
::check_cfs_quorum
();
320 my $conf = PVE
::Cluster
::cfs_read_file
("corosync.conf");
322 my $nodelist = corosync_nodelist
($conf);
324 my $totem_cfg = corosync_totem_config
($conf);
326 my $name = $param->{node
};
328 $param->{ring0_addr
} = $name if !$param->{ring0_addr
};
330 die " ring1_addr needs a configured ring 1 interface!\n"
331 if $param->{ring1_addr
} && !defined($totem_cfg->{interface
}->{1});
333 if (defined(my $res = $nodelist->{$name})) {
334 $param->{nodeid
} = $res->{nodeid
} if !$param->{nodeid
};
335 $param->{votes
} = $res->{quorum_votes
} if !defined($param->{votes
});
337 if ($res->{quorum_votes
} == $param->{votes
} &&
338 $res->{nodeid
} == $param->{nodeid
}) {
339 print "node $name already defined\n";
340 if ($param->{force
}) {
346 die "can't add existing node\n";
348 } elsif (!$param->{nodeid
}) {
353 foreach my $v (values %$nodelist) {
354 if ($v->{nodeid
} eq $nodeid) {
363 $param->{nodeid
} = $nodeid;
366 $param->{votes
} = 1 if !defined($param->{votes
});
368 PVE
::Cluster
::gen_local_dirs
($name);
370 eval { PVE
::Cluster
::ssh_merge_keys
(); };
373 $nodelist->{$name} = {
374 ring0_addr
=> $param->{ring0_addr
},
375 nodeid
=> $param->{nodeid
},
378 $nodelist->{$name}->{ring1_addr
} = $param->{ring1_addr
} if $param->{ring1_addr
};
379 $nodelist->{$name}->{quorum_votes
} = $param->{votes
} if $param->{votes
};
381 corosync_update_nodelist
($conf, $nodelist);
387 __PACKAGE__-
>register_method ({
391 description
=> "Removes a node to the cluster configuration.",
393 additionalProperties
=> 0,
397 description
=> "Hostname or IP of the corosync ring0 address of this node.",
401 returns
=> { type
=> 'null' },
406 PVE
::Cluster
::check_cfs_quorum
();
408 my $conf = PVE
::Cluster
::cfs_read_file
("corosync.conf");
410 my $nodelist = corosync_nodelist
($conf);
415 foreach my $tmp_node (keys %$nodelist) {
416 my $d = $nodelist->{$tmp_node};
417 my $ring0_addr = $d->{ring0_addr
};
418 my $ring1_addr = $d->{ring1_addr
};
419 if (($tmp_node eq $param->{node
}) ||
420 (defined($ring0_addr) && ($ring0_addr eq $param->{node
})) ||
421 (defined($ring1_addr) && ($ring1_addr eq $param->{node
}))) {
423 $nodeid = $d->{nodeid
};
428 die "Node/IP: $param->{node} is not a known host of the cluster.\n"
431 delete $nodelist->{$node};
433 corosync_update_nodelist
($conf, $nodelist);
435 PVE
::Tools
::run_command
(['corosync-cfgtool','-k', $nodeid])
441 __PACKAGE__-
>register_method ({
445 description
=> "Adds the current node to an existing cluster.",
447 additionalProperties
=> 0,
451 description
=> "Hostname (or IP) of an existing cluster member."
455 description
=> "Node id for this node.",
461 description
=> "Number of votes for this node",
467 description
=> "Do not throw error if node already exists.",
471 type
=> 'string', format
=> 'address',
472 description
=> "Hostname (or IP) of the corosync ring0 address of this node.".
473 " Defaults to nodes hostname.",
477 type
=> 'string', format
=> 'address',
478 description
=> "Hostname (or IP) of the corosync ring1 address, this".
479 " needs an valid configured ring 1 interface in the cluster.",
484 returns
=> { type
=> 'null' },
489 my $nodename = PVE
::INotify
::nodename
();
491 PVE
::Cluster
::setup_sshd_config
();
492 PVE
::Cluster
::setup_rootsshconfig
();
493 PVE
::Cluster
::setup_ssh_keys
();
495 my $host = $param->{hostname
};
497 if (!$param->{force
}) {
500 die "authentication key already exists\n";
503 if (-f
$clusterconf) {
504 die "cluster config '$clusterconf' already exists\n";
507 my $vmlist = PVE
::Cluster
::get_vmlist
();
508 if ($vmlist && $vmlist->{ids
} && scalar(keys %{$vmlist->{ids
}})) {
509 die "this host already contains virtual machines - please remove them first\n";
512 if (system("corosync-quorumtool >/dev/null 2>&1") == 0) {
513 die "corosync is already running\n";
517 # make sure known_hosts is on local filesystem
518 PVE
::Cluster
::ssh_unmerge_known_hosts
();
520 my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
521 PVE
::Tools
::run_command
($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
522 'errmsg' => "unable to copy ssh ID");
524 $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
525 'pvecm', 'addnode', $nodename, '--force', 1];
527 push @$cmd, '--nodeid', $param->{nodeid
} if $param->{nodeid
};
529 push @$cmd, '--votes', $param->{votes
} if defined($param->{votes
});
531 push @$cmd, '--ring0_addr', $param->{ring0_addr
} if defined($param->{ring0_addr
});
533 push @$cmd, '--ring1_addr', $param->{ring1_addr
} if defined($param->{ring1_addr
});
535 if (system (@$cmd) != 0) {
536 my $cmdtxt = join (' ', @$cmd);
537 die "unable to add node: command failed ($cmdtxt)\n";
540 my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
544 print "copy corosync auth key\n";
545 $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
546 "[$host]:$authfile $clusterconf", $tmpdir];
548 system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
550 mkdir "/etc/corosync";
551 my $confbase = basename
($clusterconf);
553 $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
554 system($cmd) == 0 || die "can't copy cluster configuration\n";
556 my $keybase = basename
($authfile);
557 system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
558 die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
560 print "stopping pve-cluster service\n";
562 system("umount $basedir -f >/dev/null 2>&1");
563 system("systemctl stop pve-cluster") == 0 ||
564 die "can't stop pve-cluster service\n";
570 system("systemctl start pve-cluster") == 0 ||
571 die "starting pve-cluster failed\n";
573 system("systemctl start corosync");
577 while (!PVE
::Cluster
::check_cfs_quorum
(1)) {
579 print "waiting for quorum...";
585 print "OK\n" if !$printqmsg;
587 # system("systemctl start clvm");
589 my $local_ip_address = PVE
::Cluster
::remote_node_ip
($nodename);
591 print "generating node certificates\n";
592 PVE
::Cluster
::gen_pve_node_files
($nodename, $local_ip_address);
594 print "merge known_hosts file\n";
595 PVE
::Cluster
::ssh_merge_known_hosts
($nodename, $local_ip_address, 1);
597 print "restart services\n";
598 # restart pvedaemon (changed certs)
599 system("systemctl restart pvedaemon");
600 # restart pveproxy (changed certs)
601 system("systemctl restart pveproxy");
603 print "successfully added node '$nodename' to cluster.\n";
614 __PACKAGE__-
>register_method ({
618 description
=> "Displays the local view of the cluster status.",
620 additionalProperties
=> 0,
623 returns
=> { type
=> 'null' },
628 PVE
::Cluster
::check_corosync_conf_exists
();
630 my $cmd = ['corosync-quorumtool', '-siH'];
634 exit (-1); # should not be reached
637 __PACKAGE__-
>register_method ({
641 description
=> "Displays the local view of the cluster nodes.",
643 additionalProperties
=> 0,
646 returns
=> { type
=> 'null' },
651 PVE
::Cluster
::check_corosync_conf_exists
();
653 my $cmd = ['corosync-quorumtool', '-l'];
657 exit (-1); # should not be reached
660 __PACKAGE__-
>register_method ({
664 description
=> "Tells corosync a new value of expected votes.",
666 additionalProperties
=> 0,
670 description
=> "Expected votes",
675 returns
=> { type
=> 'null' },
680 PVE
::Cluster
::check_corosync_conf_exists
();
682 my $cmd = ['corosync-quorumtool', '-e', $param->{expected
}];
686 exit (-1); # should not be reached
690 sub corosync_update_nodelist
{
691 my ($conf, $nodelist) = @_;
693 delete $conf->{digest
};
695 my $version = PVE
::Cluster
::corosync_conf_version
($conf);
696 PVE
::Cluster
::corosync_conf_version
($conf, undef, $version + 1);
699 foreach my $v (values %$nodelist) {
700 next if !($v->{ring0_addr
} || $v->{name
});
702 foreach my $k (keys %$v) {
703 push @$kv, { key
=> $k, value
=> $v->{$k} };
705 my $ns = { section
=> 'node', children
=> $kv };
706 push @$children, $ns;
709 foreach my $main (@{$conf->{children
}}) {
710 next if !defined($main->{section
});
711 if ($main->{section
} eq 'nodelist') {
712 $main->{children
} = $children;
718 PVE
::Cluster
::cfs_write_file
("corosync.conf.new", $conf);
720 rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
721 || die "activate corosync.conf.new failed - $!\n";
724 sub corosync_nodelist
{
729 foreach my $main (@{$conf->{children
}}) {
730 next if !defined($main->{section
});
731 if ($main->{section
} eq 'nodelist') {
732 foreach my $ne (@{$main->{children
}}) {
733 next if !defined($ne->{section
}) || ($ne->{section
} ne 'node');
734 my $node = { quorum_votes
=> 1 };
736 foreach my $child (@{$ne->{children
}}) {
737 next if !defined($child->{key
});
738 $node->{$child->{key
}} = $child->{value
};
739 # use 'name' over 'ring0_addr' if set
740 if ($child->{key
} eq 'name') {
741 delete $nodelist->{$name} if $name;
742 $name = $child->{value
};
743 $nodelist->{$name} = $node;
744 } elsif(!$name && $child->{key
} eq 'ring0_addr') {
745 $name = $child->{value
};
746 $nodelist->{$name} = $node;
756 # get a hash representation of the corosync config totem section
757 sub corosync_totem_config
{
762 foreach my $main (@{$conf->{children
}}) {
763 next if !defined($main->{section
}) ||
764 $main->{section
} ne 'totem';
766 foreach my $e (@{$main->{children
}}) {
768 if ($e->{section
} && $e->{section
} eq 'interface') {
771 $res->{interface
} = {};
773 foreach my $child (@{$e->{children
}}) {
774 next if !defined($child->{key
});
775 $entry->{$child->{key
}} = $child->{value
};
776 if($child->{key
} eq 'ringnumber') {
777 $res->{interface
}->{$child->{value
}} = $entry;
781 } elsif ($e->{key
}) {
782 $res->{$e->{key
}} = $e->{value
};
790 __PACKAGE__-
>register_method ({
791 name
=> 'updatecerts',
792 path
=> 'updatecerts',
794 description
=> "Update node certificates (and generate all needed files/directories).",
796 additionalProperties
=> 0,
799 description
=> "Force generation of new SSL certifate.",
804 description
=> "Ignore errors (i.e. when cluster has no quorum).",
810 returns
=> { type
=> 'null' },
814 PVE
::Cluster
::setup_rootsshconfig
();
816 PVE
::Cluster
::gen_pve_vzdump_symlink
();
818 if (!PVE
::Cluster
::check_cfs_quorum
(1)) {
819 return undef if $param->{silent
};
820 die "no quorum - unable to update files\n";
823 PVE
::Cluster
::setup_ssh_keys
();
825 my $nodename = PVE
::INotify
::nodename
();
827 my $local_ip_address = PVE
::Cluster
::remote_node_ip
($nodename);
829 PVE
::Cluster
::gen_pve_node_files
($nodename, $local_ip_address, $param->{force
});
830 PVE
::Cluster
::ssh_merge_keys
();
831 PVE
::Cluster
::ssh_merge_known_hosts
($nodename, $local_ip_address);
832 PVE
::Cluster
::gen_pve_vzdump_files
();
837 __PACKAGE__-
>register_method ({
841 description
=> "Used by VM/CT migration - do not use manually.",
843 additionalProperties
=> 0,
845 get_migration_ip
=> {
848 description
=> 'return the migration IP, if configured',
851 migration_network
=> {
854 description
=> 'the migration network used to detect the local migration IP',
859 returns
=> { type
=> 'null'},
863 if (!PVE
::Cluster
::check_cfs_quorum
(1)) {
868 if ($param->{get_migration_ip
}) {
869 my $network = $param->{migration_network
};
870 if (my $ip = PVE
::Cluster
::get_local_migration_ip
($network)) {
875 # do not keep tunnel open when asked for migration ip
879 print "tunnel online\n";
882 while (my $line = <>) {
884 last if $line =~ m/^quit$/;
892 keygen
=> [ __PACKAGE__
, 'keygen', ['filename']],
893 create
=> [ __PACKAGE__
, 'create', ['clustername']],
894 add
=> [ __PACKAGE__
, 'add', ['hostname']],
895 addnode
=> [ __PACKAGE__
, 'addnode', ['node']],
896 delnode
=> [ __PACKAGE__
, 'delnode', ['node']],
897 status
=> [ __PACKAGE__
, 'status' ],
898 nodes
=> [ __PACKAGE__
, 'nodes' ],
899 expected
=> [ __PACKAGE__
, 'expected', ['expected']],
900 updatecerts
=> [ __PACKAGE__
, 'updatecerts', []],
901 mtunnel
=> [ __PACKAGE__
, 'mtunnel', []],