data/PVE/API2/ClusterConfig.pm

   1 package PVE::API2::ClusterConfig;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Tools;
   7 use PVE::SafeSyslog;
   8 use PVE::RESTHandler;
   9 use PVE::RPCEnvironment;
  10 use PVE::JSONSchema qw(get_standard_option);
  11 use PVE::Cluster;
  12 use PVE::APIClient::LWP;
  13 use PVE::Corosync;
  14
  15 use base qw(PVE::RESTHandler);
  16
  17 my $clusterconf = "/etc/pve/corosync.conf";
  18 my $authfile = "/etc/corosync/authkey";
  19 my $local_cluster_lock = "/var/lock/pvecm.lock";
  20
  21 my $ring0_desc = {
  22     type => 'string', format => 'address',
  23     description => "Hostname (or IP) of the corosync ring0 address of this node.",
  24     default => "Hostname of the node",
  25     optional => 1,
  26 };
  27 PVE::JSONSchema::register_standard_option("corosync-ring0-addr", $ring0_desc);
  28
  29 my $ring1_desc = {
  30     type => 'string', format => 'address',
  31     description => "Hostname (or IP) of the corosync ring1 address of this node.".
  32         " Requires a valid configured ring 1 (bindnet1_addr) in the cluster.",
  33     optional => 1,
  34 };
  35 PVE::JSONSchema::register_standard_option("corosync-ring1-addr", $ring1_desc);
  36
  37 my $nodeid_desc = {
  38     type => 'integer',
  39     description => "Node id for this node.",
  40     minimum => 1,
  41     optional => 1,
  42 };
  43 PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);
  44
  45 __PACKAGE__->register_method({
  46     name => 'index',
  47     path => '',
  48     method => 'GET',
  49     description => "Directory index.",
  50     permissions => {
  51         check => ['perm', '/', [ 'Sys.Audit' ]],
  52     },
  53     parameters => {
  54         additionalProperties => 0,
  55         properties => {},
  56     },
  57     returns => {
  58         type => 'array',
  59         items => {
  60             type => "object",
  61             properties => {},
  62         },
  63         links => [ { rel => 'child', href => "{name}" } ],
  64     },
  65     code => sub {
  66         my ($param) = @_;
  67
  68         my $result = [
  69             { name => 'nodes' },
  70             { name => 'totem' },
  71             { name => 'join' },
  72         ];
  73
  74         return $result;
  75     }});
  76
  77 __PACKAGE__->register_method ({
  78     name => 'create',
  79     path => '',
  80     method => 'POST',
  81     protected => 1,
  82     description => "Generate new cluster configuration.",
  83     parameters => {
  84         additionalProperties => 0,
  85         properties => {
  86             clustername => {
  87                 description => "The name of the cluster.",
  88                 type => 'string', format => 'pve-node',
  89                 maxLength => 15,
  90             },
  91             nodeid => get_standard_option('corosync-nodeid'),
  92             votes => {
  93                 type => 'integer',
  94                 description => "Number of votes for this node.",
  95                 minimum => 1,
  96                 optional => 1,
  97             },
  98             bindnet0_addr => {
  99                 type => 'string', format => 'ip',
 100                 description => "This specifies the network address the corosync ring 0".
 101                     " executive should bind to and defaults to the local IP address of the node.",
 102                 optional => 1,
 103             },
 104             ring0_addr => get_standard_option('corosync-ring0-addr'),
 105             bindnet1_addr => {
 106                 type => 'string', format => 'ip',
 107                 description => "This specifies the network address the corosync ring 1".
 108                     " executive should bind to and is optional.",
 109                 optional => 1,
 110             },
 111             ring1_addr => get_standard_option('corosync-ring1-addr'),
 112         },
 113     },
 114     returns => { type => 'string' },
 115     code => sub {
 116         my ($param) = @_;
 117
 118         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
 119
 120         my $rpcenv = PVE::RPCEnvironment::get();
 121         my $authuser = $rpcenv->get_user();
 122
 123         my $code = sub {
 124             PVE::Cluster::setup_sshd_config(1);
 125             PVE::Cluster::setup_rootsshconfig();
 126             PVE::Cluster::setup_ssh_keys();
 127
 128             PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
 129                 if !-f $authfile;
 130             die "no authentication key available\n" if -f !$authfile;
 131
 132             my $nodename = PVE::INotify::nodename();
 133
 134             # get the corosync basis config for the new cluster
 135             my $config = PVE::Corosync::create_conf($nodename, %$param);
 136
 137             print "Writing corosync config to /etc/pve/corosync.conf\n";
 138             PVE::Corosync::atomic_write_conf($config);
 139
 140             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 141             PVE::Cluster::ssh_merge_keys();
 142             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 143             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 144
 145             print "Restart corosync and cluster filesystem\n";
 146             PVE::Tools::run_command('systemctl restart corosync pve-cluster');
 147         };
 148
 149         my $worker = sub {
 150             PVE::Tools::lock_file($local_cluster_lock, 10, $code);
 151             die $@ if $@;
 152         };
 153
 154         return $rpcenv->fork_worker('clustercreate', $param->{clustername},  $authuser, $worker);
 155 }});
 156
 157 __PACKAGE__->register_method({
 158     name => 'nodes',
 159     path => 'nodes',
 160     method => 'GET',
 161     description => "Corosync node list.",
 162     permissions => {
 163         check => ['perm', '/', [ 'Sys.Audit' ]],
 164     },
 165     parameters => {
 166         additionalProperties => 0,
 167         properties => {},
 168     },
 169     returns => {
 170         type => 'array',
 171         items => {
 172             type => "object",
 173             properties => {
 174                 node => { type => 'string' },
 175             },
 176         },
 177         links => [ { rel => 'child', href => "{node}" } ],
 178     },
 179     code => sub {
 180         my ($param) = @_;
 181
 182
 183         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 184         my $nodelist = PVE::Corosync::nodelist($conf);
 185
 186         return PVE::RESTHandler::hash_to_array($nodelist, 'node');
 187     }});
 188
 189 # lock method to ensure local and cluster wide atomicity
 190 # if we're a single node cluster just lock locally, we have no other cluster
 191 # node which we could contend with, else also acquire a cluster wide lock
 192 my $config_change_lock = sub {
 193     my ($code) = @_;
 194
 195     PVE::Tools::lock_file($local_cluster_lock, 10, sub {
 196         PVE::Cluster::cfs_update(1);
 197         my $members = PVE::Cluster::get_members();
 198         if (scalar(keys %$members) > 1) {
 199             return PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 200         } else {
 201             return $code->();
 202         }
 203     });
 204 };
 205
 206 __PACKAGE__->register_method ({
 207     name => 'addnode',
 208     path => 'nodes/{node}',
 209     method => 'POST',
 210     protected => 1,
 211     description => "Adds a node to the cluster configuration.",
 212     parameters => {
 213         additionalProperties => 0,
 214         properties => {
 215             node => get_standard_option('pve-node'),
 216             nodeid => get_standard_option('corosync-nodeid'),
 217             votes => {
 218                 type => 'integer',
 219                 description => "Number of votes for this node",
 220                 minimum => 0,
 221                 optional => 1,
 222             },
 223             force => {
 224                 type => 'boolean',
 225                 description => "Do not throw error if node already exists.",
 226                 optional => 1,
 227             },
 228             ring0_addr => get_standard_option('corosync-ring0-addr'),
 229             ring1_addr => get_standard_option('corosync-ring1-addr'),
 230         },
 231     },
 232     returns => {
 233         type => "object",
 234         properties => {
 235             corosync_authkey => {
 236                 type => 'string',
 237             },
 238             corosync_conf => {
 239                 type => 'string',
 240             }
 241         },
 242     },
 243     code => sub {
 244         my ($param) = @_;
 245
 246         PVE::Cluster::check_cfs_quorum();
 247
 248         my $code = sub {
 249             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 250             my $nodelist = PVE::Corosync::nodelist($conf);
 251             my $totem_cfg = PVE::Corosync::totem_config($conf);
 252
 253             my $name = $param->{node};
 254
 255             # ensure we do not reuse an address, that can crash the whole cluster!
 256             my $check_duplicate_addr = sub {
 257                 my $addr = shift;
 258                 return if !defined($addr);
 259
 260                 while (my ($k, $v) = each %$nodelist) {
 261                     next if $k eq $name; # allows re-adding a node if force is set
 262                         if ($v->{ring0_addr} eq $addr || ($v->{ring1_addr} && $v->{ring1_addr} eq $addr)) {
 263                             die "corosync: address '$addr' already defined by node '$k'\n";
 264                         }
 265                 }
 266             };
 267
 268             &$check_duplicate_addr($param->{ring0_addr});
 269             &$check_duplicate_addr($param->{ring1_addr});
 270
 271             $param->{ring0_addr} = $name if !$param->{ring0_addr};
 272
 273             die "corosync: using 'ring1_addr' parameter needs a configured ring 1 interface!\n"
 274                 if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1});
 275
 276             die "corosync: ring 1 interface configured but 'ring1_addr' parameter not defined!\n"
 277                 if defined($totem_cfg->{interface}->{1}) && !defined($param->{ring1_addr});
 278
 279             if (defined(my $res = $nodelist->{$name})) {
 280                 $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 281                 $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 282
 283                 if ($res->{quorum_votes} == $param->{votes} &&
 284                     $res->{nodeid} == $param->{nodeid} && $param->{force}) {
 285                     print "forcing overwrite of configured node '$name'\n";
 286                 } else {
 287                     die "can't add existing node '$name'\n";
 288                 }
 289             } elsif (!$param->{nodeid}) {
 290                 my $nodeid = 1;
 291
 292                 while(1) {
 293                     my $found = 0;
 294                     foreach my $v (values %$nodelist) {
 295                         if ($v->{nodeid} eq $nodeid) {
 296                             $found = 1;
 297                             $nodeid++;
 298                             last;
 299                         }
 300                     }
 301                     last if !$found;
 302                 };
 303
 304                 $param->{nodeid} = $nodeid;
 305             }
 306
 307             $param->{votes} = 1 if !defined($param->{votes});
 308
 309             PVE::Cluster::gen_local_dirs($name);
 310
 311             eval { PVE::Cluster::ssh_merge_keys(); };
 312             warn $@ if $@;
 313
 314             $nodelist->{$name} = {
 315                 ring0_addr => $param->{ring0_addr},
 316                 nodeid => $param->{nodeid},
 317                 name => $name,
 318             };
 319             $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr};
 320             $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 321
 322             PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");
 323
 324             PVE::Corosync::update_nodelist($conf, $nodelist);
 325         };
 326
 327         $config_change_lock->($code);
 328         die $@ if $@;
 329
 330         my $res = {
 331             corosync_authkey => PVE::Tools::file_get_contents($authfile),
 332             corosync_conf => PVE::Tools::file_get_contents($clusterconf),
 333         };
 334
 335         return $res;
 336     }});
 337
 338
 339 __PACKAGE__->register_method ({
 340     name => 'delnode',
 341     path => 'nodes/{node}',
 342     method => 'DELETE',
 343     protected => 1,
 344     description => "Removes a node from the cluster configuration.",
 345     parameters => {
 346         additionalProperties => 0,
 347         properties => {
 348             node => get_standard_option('pve-node'),
 349         },
 350     },
 351     returns => { type => 'null' },
 352     code => sub {
 353         my ($param) = @_;
 354
 355         my $local_node = PVE::INotify::nodename();
 356         die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;
 357
 358         PVE::Cluster::check_cfs_quorum();
 359
 360         my $code = sub {
 361             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 362             my $nodelist = PVE::Corosync::nodelist($conf);
 363
 364             my $node;
 365             my $nodeid;
 366
 367             foreach my $tmp_node (keys %$nodelist) {
 368                 my $d = $nodelist->{$tmp_node};
 369                 my $ring0_addr = $d->{ring0_addr};
 370                 my $ring1_addr = $d->{ring1_addr};
 371                 if (($tmp_node eq $param->{node}) ||
 372                     (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
 373                     (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
 374                     $node = $tmp_node;
 375                     $nodeid = $d->{nodeid};
 376                     last;
 377                 }
 378             }
 379
 380             die "Node/IP: $param->{node} is not a known host of the cluster.\n"
 381                 if !defined($node);
 382
 383             PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");
 384
 385             delete $nodelist->{$node};
 386
 387             PVE::Corosync::update_nodelist($conf, $nodelist);
 388
 389             PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
 390         };
 391
 392         $config_change_lock->($code);
 393         die $@ if $@;
 394
 395         return undef;
 396     }});
 397
 398 __PACKAGE__->register_method ({
 399     name => 'join_info',
 400     path => 'join',
 401     method => 'GET',
 402     description => "Get information needed to join this cluster over the connected node.",
 403     parameters => {
 404         additionalProperties => 0,
 405         properties => {
 406             node => get_standard_option('pve-node', {
 407                 description => "The node for which the joinee gets the nodeinfo. ",
 408                 default => "current connected node",
 409                 optional => 1,
 410             }),
 411         },
 412     },
 413     returns => {
 414         type => 'object',
 415         additionalProperties => 0,
 416         properties => {
 417             nodelist => {
 418                 type => 'array',
 419                 items => {
 420                     type => "object",
 421                     additionalProperties => 1,
 422                     properties => {
 423                         name => get_standard_option('pve-node'),
 424                         nodeid => get_standard_option('corosync-nodeid'),
 425                         ring0_addr => get_standard_option('corosync-ring0-addr'),
 426                         quorum_votes => { type => 'integer', minimum => 0 },
 427                         pve_addr => { type => 'string', format => 'ip' },
 428                         pve_fp => get_standard_option('fingerprint-sha256'),
 429                     },
 430                 },
 431             },
 432             preferred_node => get_standard_option('pve-node'),
 433             totem => { type => 'object' },
 434             config_digest => { type => 'string' },
 435         },
 436     },
 437     code => sub {
 438         my ($param) = @_;
 439
 440         my $nodename = $param->{node} // PVE::INotify::nodename();
 441
 442         PVE::Cluster::cfs_update(1);
 443         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 444
 445         die "node is not in a cluster, no join info available!\n"
 446             if !($conf && $conf->{main});
 447
 448         my $totem_cfg = $conf->{main}->{totem} // {};
 449         my $nodelist = $conf->{main}->{nodelist}->{node} // {};
 450         my $corosync_config_digest = $conf->{digest};
 451
 452         die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};
 453
 454         foreach my $name (keys %$nodelist) {
 455             my $node = $nodelist->{$name};
 456             $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
 457             $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
 458         }
 459
 460         my $res = {
 461             nodelist => [ values %$nodelist ],
 462             preferred_node => $nodename,
 463             totem => $totem_cfg,
 464             config_digest => $corosync_config_digest,
 465         };
 466
 467         return $res;
 468     }});
 469
 470 __PACKAGE__->register_method ({
 471     name => 'join',
 472     path => 'join',
 473     method => 'POST',
 474     protected => 1,
 475     description => "Joins this node into an existing cluster.",
 476     parameters => {
 477         additionalProperties => 0,
 478         properties => {
 479             hostname => {
 480                 type => 'string',
 481                 description => "Hostname (or IP) of an existing cluster member."
 482             },
 483             nodeid => get_standard_option('corosync-nodeid'),
 484             votes => {
 485                 type => 'integer',
 486                 description => "Number of votes for this node",
 487                 minimum => 0,
 488                 optional => 1,
 489             },
 490             force => {
 491                 type => 'boolean',
 492                 description => "Do not throw error if node already exists.",
 493                 optional => 1,
 494             },
 495             ring0_addr => get_standard_option('corosync-ring0-addr', {
 496                 default => "IP resolved by node's hostname",
 497             }),
 498             ring1_addr => get_standard_option('corosync-ring1-addr'),
 499             fingerprint => get_standard_option('fingerprint-sha256'),
 500             password => {
 501                 description => "Superuser (root) password of peer node.",
 502                 type => 'string',
 503                 maxLength => 128,
 504             },
 505         },
 506     },
 507     returns => { type => 'string' },
 508     code => sub {
 509         my ($param) = @_;
 510
 511         my $rpcenv = PVE::RPCEnvironment::get();
 512         my $authuser = $rpcenv->get_user();
 513
 514         my $worker = sub {
 515             PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
 516             die $@ if $@;
 517         };
 518
 519         return $rpcenv->fork_worker('clusterjoin', $param->{hostname},  $authuser, $worker);
 520     }});
 521
 522
 523 __PACKAGE__->register_method({
 524     name => 'totem',
 525     path => 'totem',
 526     method => 'GET',
 527     description => "Get corosync totem protocol settings.",
 528     permissions => {
 529         check => ['perm', '/', [ 'Sys.Audit' ]],
 530     },
 531     parameters => {
 532         additionalProperties => 0,
 533         properties => {},
 534     },
 535     returns => {
 536         type => "object",
 537         properties => {},
 538     },
 539     code => sub {
 540         my ($param) = @_;
 541
 542
 543         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 544
 545         my $totem_cfg = $conf->{main}->{totem};
 546
 547         return $totem_cfg;
 548     }});
 549
 550 1;