data/PVE/API2/ClusterConfig.pm

   1 package PVE::API2::ClusterConfig;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Tools;
   7 use PVE::SafeSyslog;
   8 use PVE::RESTHandler;
   9 use PVE::RPCEnvironment;
  10 use PVE::JSONSchema qw(get_standard_option);
  11 use PVE::Cluster;
  12 use PVE::APIClient::LWP;
  13 use PVE::Corosync;
  14
  15 use base qw(PVE::RESTHandler);
  16
  17 my $clusterconf = "/etc/pve/corosync.conf";
  18 my $authfile = "/etc/corosync/authkey";
  19 my $local_cluster_lock = "/var/lock/pvecm.lock";
  20
  21 my $nodeid_desc = {
  22     type => 'integer',
  23     description => "Node id for this node.",
  24     minimum => 1,
  25     optional => 1,
  26 };
  27 PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);
  28
  29 __PACKAGE__->register_method({
  30     name => 'index',
  31     path => '',
  32     method => 'GET',
  33     description => "Directory index.",
  34     permissions => {
  35         check => ['perm', '/', [ 'Sys.Audit' ]],
  36     },
  37     parameters => {
  38         additionalProperties => 0,
  39         properties => {},
  40     },
  41     returns => {
  42         type => 'array',
  43         items => {
  44             type => "object",
  45             properties => {},
  46         },
  47         links => [ { rel => 'child', href => "{name}" } ],
  48     },
  49     code => sub {
  50         my ($param) = @_;
  51
  52         my $result = [
  53             { name => 'nodes' },
  54             { name => 'totem' },
  55             { name => 'join' },
  56         ];
  57
  58         return $result;
  59     }});
  60
  61 __PACKAGE__->register_method ({
  62     name => 'create',
  63     path => '',
  64     method => 'POST',
  65     protected => 1,
  66     description => "Generate new cluster configuration.",
  67     parameters => {
  68         additionalProperties => 0,
  69         properties => {
  70             clustername => {
  71                 description => "The name of the cluster.",
  72                 type => 'string', format => 'pve-node',
  73                 maxLength => 15,
  74             },
  75             nodeid => get_standard_option('corosync-nodeid'),
  76             votes => {
  77                 type => 'integer',
  78                 description => "Number of votes for this node.",
  79                 minimum => 1,
  80                 optional => 1,
  81             },
  82             link0 => get_standard_option('corosync-link'),
  83             link1 => get_standard_option('corosync-link'),
  84         },
  85     },
  86     returns => { type => 'string' },
  87     code => sub {
  88         my ($param) = @_;
  89
  90         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
  91
  92         my $rpcenv = PVE::RPCEnvironment::get();
  93         my $authuser = $rpcenv->get_user();
  94
  95         my $code = sub {
  96             STDOUT->autoflush();
  97             PVE::Cluster::setup_sshd_config(1);
  98             PVE::Cluster::setup_rootsshconfig();
  99             PVE::Cluster::setup_ssh_keys();
 100
 101             PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
 102                 if !-f $authfile;
 103             die "no authentication key available\n" if -f !$authfile;
 104
 105             my $nodename = PVE::INotify::nodename();
 106
 107             # get the corosync basis config for the new cluster
 108             my $config = PVE::Corosync::create_conf($nodename, %$param);
 109
 110             print "Writing corosync config to /etc/pve/corosync.conf\n";
 111             PVE::Corosync::atomic_write_conf($config);
 112
 113             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 114             PVE::Cluster::ssh_merge_keys();
 115             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 116             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 117
 118             print "Restart corosync and cluster filesystem\n";
 119             PVE::Tools::run_command('systemctl restart corosync pve-cluster');
 120         };
 121
 122         my $worker = sub {
 123             PVE::Tools::lock_file($local_cluster_lock, 10, $code);
 124             die $@ if $@;
 125         };
 126
 127         return $rpcenv->fork_worker('clustercreate', $param->{clustername},  $authuser, $worker);
 128 }});
 129
 130 __PACKAGE__->register_method({
 131     name => 'nodes',
 132     path => 'nodes',
 133     method => 'GET',
 134     description => "Corosync node list.",
 135     permissions => {
 136         check => ['perm', '/', [ 'Sys.Audit' ]],
 137     },
 138     parameters => {
 139         additionalProperties => 0,
 140         properties => {},
 141     },
 142     returns => {
 143         type => 'array',
 144         items => {
 145             type => "object",
 146             properties => {
 147                 node => { type => 'string' },
 148             },
 149         },
 150         links => [ { rel => 'child', href => "{node}" } ],
 151     },
 152     code => sub {
 153         my ($param) = @_;
 154
 155
 156         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 157         my $nodelist = PVE::Corosync::nodelist($conf);
 158
 159         return PVE::RESTHandler::hash_to_array($nodelist, 'node');
 160     }});
 161
 162 # lock method to ensure local and cluster wide atomicity
 163 # if we're a single node cluster just lock locally, we have no other cluster
 164 # node which we could contend with, else also acquire a cluster wide lock
 165 my $config_change_lock = sub {
 166     my ($code) = @_;
 167
 168     PVE::Tools::lock_file($local_cluster_lock, 10, sub {
 169         PVE::Cluster::cfs_update(1);
 170         my $members = PVE::Cluster::get_members();
 171         if (scalar(keys %$members) > 1) {
 172             return PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 173         } else {
 174             return $code->();
 175         }
 176     });
 177 };
 178
 179 __PACKAGE__->register_method ({
 180     name => 'addnode',
 181     path => 'nodes/{node}',
 182     method => 'POST',
 183     protected => 1,
 184     description => "Adds a node to the cluster configuration. This call is for internal use.",
 185     parameters => {
 186         additionalProperties => 0,
 187         properties => {
 188             node => get_standard_option('pve-node'),
 189             nodeid => get_standard_option('corosync-nodeid'),
 190             votes => {
 191                 type => 'integer',
 192                 description => "Number of votes for this node",
 193                 minimum => 0,
 194                 optional => 1,
 195             },
 196             force => {
 197                 type => 'boolean',
 198                 description => "Do not throw error if node already exists.",
 199                 optional => 1,
 200             },
 201             link0 => get_standard_option('corosync-link'),
 202             link1 => get_standard_option('corosync-link'),
 203         },
 204     },
 205     returns => {
 206         type => "object",
 207         properties => {
 208             corosync_authkey => {
 209                 type => 'string',
 210             },
 211             corosync_conf => {
 212                 type => 'string',
 213             }
 214         },
 215     },
 216     code => sub {
 217         my ($param) = @_;
 218
 219         PVE::Cluster::check_cfs_quorum();
 220
 221         my $code = sub {
 222             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 223             my $nodelist = PVE::Corosync::nodelist($conf);
 224             my $totem_cfg = PVE::Corosync::totem_config($conf);
 225
 226             my $name = $param->{node};
 227
 228             # ensure we do not reuse an address, that can crash the whole cluster!
 229             my $check_duplicate_addr = sub {
 230                 my $link = shift;
 231                 return if !defined($link) || !defined($link->{address});
 232                 my $addr = $link->{address};
 233
 234                 while (my ($k, $v) = each %$nodelist) {
 235                     next if $k eq $name; # allows re-adding a node if force is set
 236
 237                     for my $linknumber (0..1) {
 238                         my $id = "ring${linknumber}_addr";
 239                         next if !defined($v->{$id});
 240
 241                         die "corosync: address '$addr' already used on link $id by node '$k'\n"
 242                             if $v->{$id} eq $addr;
 243                     }
 244                 }
 245             };
 246
 247             my $link0 = PVE::Cluster::parse_corosync_link($param->{link0});
 248             my $link1 = PVE::Cluster::parse_corosync_link($param->{link1});
 249
 250             $check_duplicate_addr->($link0);
 251             $check_duplicate_addr->($link1);
 252
 253             # FIXME: handle all links (0-7), they're all independent now
 254             $link0->{address} //= $name if exists($totem_cfg->{interface}->{0});
 255
 256             die "corosync: using 'link1' parameter needs a interface with linknumber '1' configured!\n"
 257                 if $link1 && !defined($totem_cfg->{interface}->{1});
 258
 259             die "corosync: totem interface with linknumber 1 configured but 'link1' parameter not defined!\n"
 260                 if defined($totem_cfg->{interface}->{1}) && !defined($link1);
 261
 262             if (defined(my $res = $nodelist->{$name})) {
 263                 $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 264                 $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 265
 266                 if ($res->{quorum_votes} == $param->{votes} &&
 267                     $res->{nodeid} == $param->{nodeid} && $param->{force}) {
 268                     print "forcing overwrite of configured node '$name'\n";
 269                 } else {
 270                     die "can't add existing node '$name'\n";
 271                 }
 272             } elsif (!$param->{nodeid}) {
 273                 my $nodeid = 1;
 274
 275                 while(1) {
 276                     my $found = 0;
 277                     foreach my $v (values %$nodelist) {
 278                         if ($v->{nodeid} eq $nodeid) {
 279                             $found = 1;
 280                             $nodeid++;
 281                             last;
 282                         }
 283                     }
 284                     last if !$found;
 285                 };
 286
 287                 $param->{nodeid} = $nodeid;
 288             }
 289
 290             $param->{votes} = 1 if !defined($param->{votes});
 291
 292             PVE::Cluster::gen_local_dirs($name);
 293
 294             eval { PVE::Cluster::ssh_merge_keys(); };
 295             warn $@ if $@;
 296
 297             $nodelist->{$name} = {
 298                 ring0_addr => $link0->{address},
 299                 nodeid => $param->{nodeid},
 300                 name => $name,
 301             };
 302             $nodelist->{$name}->{ring1_addr} = $link1->{address} if defined($link1);
 303             $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 304
 305             PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");
 306
 307             PVE::Corosync::update_nodelist($conf, $nodelist);
 308         };
 309
 310         $config_change_lock->($code);
 311         die $@ if $@;
 312
 313         my $res = {
 314             corosync_authkey => PVE::Tools::file_get_contents($authfile),
 315             corosync_conf => PVE::Tools::file_get_contents($clusterconf),
 316         };
 317
 318         return $res;
 319     }});
 320
 321
 322 __PACKAGE__->register_method ({
 323     name => 'delnode',
 324     path => 'nodes/{node}',
 325     method => 'DELETE',
 326     protected => 1,
 327     description => "Removes a node from the cluster configuration.",
 328     parameters => {
 329         additionalProperties => 0,
 330         properties => {
 331             node => get_standard_option('pve-node'),
 332         },
 333     },
 334     returns => { type => 'null' },
 335     code => sub {
 336         my ($param) = @_;
 337
 338         my $local_node = PVE::INotify::nodename();
 339         die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;
 340
 341         PVE::Cluster::check_cfs_quorum();
 342
 343         my $code = sub {
 344             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 345             my $nodelist = PVE::Corosync::nodelist($conf);
 346
 347             my $node;
 348             my $nodeid;
 349
 350             foreach my $tmp_node (keys %$nodelist) {
 351                 my $d = $nodelist->{$tmp_node};
 352                 my $ring0_addr = $d->{ring0_addr};
 353                 my $ring1_addr = $d->{ring1_addr};
 354                 if (($tmp_node eq $param->{node}) ||
 355                     (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
 356                     (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
 357                     $node = $tmp_node;
 358                     $nodeid = $d->{nodeid};
 359                     last;
 360                 }
 361             }
 362
 363             die "Node/IP: $param->{node} is not a known host of the cluster.\n"
 364                 if !defined($node);
 365
 366             PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");
 367
 368             delete $nodelist->{$node};
 369
 370             PVE::Corosync::update_nodelist($conf, $nodelist);
 371
 372             PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
 373         };
 374
 375         $config_change_lock->($code);
 376         die $@ if $@;
 377
 378         return undef;
 379     }});
 380
 381 __PACKAGE__->register_method ({
 382     name => 'join_info',
 383     path => 'join',
 384     permissions => {
 385         check => ['perm', '/', [ 'Sys.Audit' ]],
 386     },
 387     method => 'GET',
 388     description => "Get information needed to join this cluster over the connected node.",
 389     parameters => {
 390         additionalProperties => 0,
 391         properties => {
 392             node => get_standard_option('pve-node', {
 393                 description => "The node for which the joinee gets the nodeinfo. ",
 394                 default => "current connected node",
 395                 optional => 1,
 396             }),
 397         },
 398     },
 399     returns => {
 400         type => 'object',
 401         additionalProperties => 0,
 402         properties => {
 403             nodelist => {
 404                 type => 'array',
 405                 items => {
 406                     type => "object",
 407                     additionalProperties => 1,
 408                     properties => {
 409                         name => get_standard_option('pve-node'),
 410                         nodeid => get_standard_option('corosync-nodeid'),
 411                         ring0_addr => get_standard_option('corosync-link'),
 412                         quorum_votes => { type => 'integer', minimum => 0 },
 413                         pve_addr => { type => 'string', format => 'ip' },
 414                         pve_fp => get_standard_option('fingerprint-sha256'),
 415                     },
 416                 },
 417             },
 418             preferred_node => get_standard_option('pve-node'),
 419             totem => { type => 'object' },
 420             config_digest => { type => 'string' },
 421         },
 422     },
 423     code => sub {
 424         my ($param) = @_;
 425
 426         my $nodename = $param->{node} // PVE::INotify::nodename();
 427
 428         PVE::Cluster::cfs_update(1);
 429         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 430
 431         die "node is not in a cluster, no join info available!\n"
 432             if !($conf && $conf->{main});
 433
 434         my $totem_cfg = $conf->{main}->{totem} // {};
 435         my $nodelist = $conf->{main}->{nodelist}->{node} // {};
 436         my $corosync_config_digest = $conf->{digest};
 437
 438         die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};
 439
 440         foreach my $name (keys %$nodelist) {
 441             my $node = $nodelist->{$name};
 442             $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
 443             $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
 444         }
 445
 446         my $res = {
 447             nodelist => [ values %$nodelist ],
 448             preferred_node => $nodename,
 449             totem => $totem_cfg,
 450             config_digest => $corosync_config_digest,
 451         };
 452
 453         return $res;
 454     }});
 455
 456 __PACKAGE__->register_method ({
 457     name => 'join',
 458     path => 'join',
 459     method => 'POST',
 460     protected => 1,
 461     description => "Joins this node into an existing cluster.",
 462     parameters => {
 463         additionalProperties => 0,
 464         properties => {
 465             hostname => {
 466                 type => 'string',
 467                 description => "Hostname (or IP) of an existing cluster member."
 468             },
 469             nodeid => get_standard_option('corosync-nodeid'),
 470             votes => {
 471                 type => 'integer',
 472                 description => "Number of votes for this node",
 473                 minimum => 0,
 474                 optional => 1,
 475             },
 476             force => {
 477                 type => 'boolean',
 478                 description => "Do not throw error if node already exists.",
 479                 optional => 1,
 480             },
 481             link0 => get_standard_option('corosync-link', {
 482                 default => "IP resolved by node's hostname",
 483             }),
 484             link1 => get_standard_option('corosync-link'),
 485             fingerprint => get_standard_option('fingerprint-sha256'),
 486             password => {
 487                 description => "Superuser (root) password of peer node.",
 488                 type => 'string',
 489                 maxLength => 128,
 490             },
 491         },
 492     },
 493     returns => { type => 'string' },
 494     code => sub {
 495         my ($param) = @_;
 496
 497         my $rpcenv = PVE::RPCEnvironment::get();
 498         my $authuser = $rpcenv->get_user();
 499
 500         my $worker = sub {
 501             STDOUT->autoflush();
 502             PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
 503             die $@ if $@;
 504         };
 505
 506         return $rpcenv->fork_worker('clusterjoin', undef,  $authuser, $worker);
 507     }});
 508
 509
 510 __PACKAGE__->register_method({
 511     name => 'totem',
 512     path => 'totem',
 513     method => 'GET',
 514     description => "Get corosync totem protocol settings.",
 515     permissions => {
 516         check => ['perm', '/', [ 'Sys.Audit' ]],
 517     },
 518     parameters => {
 519         additionalProperties => 0,
 520         properties => {},
 521     },
 522     returns => {
 523         type => "object",
 524         properties => {},
 525     },
 526     code => sub {
 527         my ($param) = @_;
 528
 529
 530         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 531
 532         my $totem_cfg = $conf->{main}->{totem};
 533
 534         return $totem_cfg;
 535     }});
 536
 537 1;