data/PVE/API2/ClusterConfig.pm

   1 package PVE::API2::ClusterConfig;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Tools;
   7 use PVE::SafeSyslog;
   8 use PVE::RESTHandler;
   9 use PVE::RPCEnvironment;
  10 use PVE::JSONSchema qw(get_standard_option);
  11 use PVE::Cluster;
  12 use PVE::APIClient::LWP;
  13 use PVE::Corosync;
  14
  15 use IO::Socket::UNIX;
  16
  17 use base qw(PVE::RESTHandler);
  18
  19 my $clusterconf = "/etc/pve/corosync.conf";
  20 my $authfile = "/etc/corosync/authkey";
  21 my $local_cluster_lock = "/var/lock/pvecm.lock";
  22
  23 my $nodeid_desc = {
  24     type => 'integer',
  25     description => "Node id for this node.",
  26     minimum => 1,
  27     optional => 1,
  28 };
  29 PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);
  30
  31 __PACKAGE__->register_method({
  32     name => 'index',
  33     path => '',
  34     method => 'GET',
  35     description => "Directory index.",
  36     permissions => {
  37         check => ['perm', '/', [ 'Sys.Audit' ]],
  38     },
  39     parameters => {
  40         additionalProperties => 0,
  41         properties => {},
  42     },
  43     returns => {
  44         type => 'array',
  45         items => {
  46             type => "object",
  47             properties => {},
  48         },
  49         links => [ { rel => 'child', href => "{name}" } ],
  50     },
  51     code => sub {
  52         my ($param) = @_;
  53
  54         my $result = [
  55             { name => 'nodes' },
  56             { name => 'totem' },
  57             { name => 'join' },
  58             { name => 'qdevice' },
  59         ];
  60
  61         return $result;
  62     }});
  63
  64 __PACKAGE__->register_method ({
  65     name => 'create',
  66     path => '',
  67     method => 'POST',
  68     protected => 1,
  69     description => "Generate new cluster configuration.",
  70     parameters => {
  71         additionalProperties => 0,
  72         properties => {
  73             clustername => {
  74                 description => "The name of the cluster.",
  75                 type => 'string', format => 'pve-node',
  76                 maxLength => 15,
  77             },
  78             nodeid => get_standard_option('corosync-nodeid'),
  79             votes => {
  80                 type => 'integer',
  81                 description => "Number of votes for this node.",
  82                 minimum => 1,
  83                 optional => 1,
  84             },
  85             link0 => get_standard_option('corosync-link'),
  86             link1 => get_standard_option('corosync-link'),
  87         },
  88     },
  89     returns => { type => 'string' },
  90     code => sub {
  91         my ($param) = @_;
  92
  93         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
  94
  95         my $rpcenv = PVE::RPCEnvironment::get();
  96         my $authuser = $rpcenv->get_user();
  97
  98         my $code = sub {
  99             STDOUT->autoflush();
 100             PVE::Cluster::setup_sshd_config(1);
 101             PVE::Cluster::setup_rootsshconfig();
 102             PVE::Cluster::setup_ssh_keys();
 103
 104             PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
 105                 if !-f $authfile;
 106             die "no authentication key available\n" if -f !$authfile;
 107
 108             my $nodename = PVE::INotify::nodename();
 109
 110             # get the corosync basis config for the new cluster
 111             my $config = PVE::Corosync::create_conf($nodename, %$param);
 112
 113             print "Writing corosync config to /etc/pve/corosync.conf\n";
 114             PVE::Corosync::atomic_write_conf($config);
 115
 116             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 117             PVE::Cluster::ssh_merge_keys();
 118             PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
 119             PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 120
 121             print "Restart corosync and cluster filesystem\n";
 122             PVE::Tools::run_command('systemctl restart corosync pve-cluster');
 123         };
 124
 125         my $worker = sub {
 126             PVE::Tools::lock_file($local_cluster_lock, 10, $code);
 127             die $@ if $@;
 128         };
 129
 130         return $rpcenv->fork_worker('clustercreate', $param->{clustername},  $authuser, $worker);
 131 }});
 132
 133 __PACKAGE__->register_method({
 134     name => 'nodes',
 135     path => 'nodes',
 136     method => 'GET',
 137     description => "Corosync node list.",
 138     permissions => {
 139         check => ['perm', '/', [ 'Sys.Audit' ]],
 140     },
 141     parameters => {
 142         additionalProperties => 0,
 143         properties => {},
 144     },
 145     returns => {
 146         type => 'array',
 147         items => {
 148             type => "object",
 149             properties => {
 150                 node => { type => 'string' },
 151             },
 152         },
 153         links => [ { rel => 'child', href => "{node}" } ],
 154     },
 155     code => sub {
 156         my ($param) = @_;
 157
 158
 159         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 160         my $nodelist = PVE::Corosync::nodelist($conf);
 161
 162         return PVE::RESTHandler::hash_to_array($nodelist, 'node');
 163     }});
 164
 165 # lock method to ensure local and cluster wide atomicity
 166 # if we're a single node cluster just lock locally, we have no other cluster
 167 # node which we could contend with, else also acquire a cluster wide lock
 168 my $config_change_lock = sub {
 169     my ($code) = @_;
 170
 171     PVE::Tools::lock_file($local_cluster_lock, 10, sub {
 172         PVE::Cluster::cfs_update(1);
 173         my $members = PVE::Cluster::get_members();
 174         if (scalar(keys %$members) > 1) {
 175             return PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 176         } else {
 177             return $code->();
 178         }
 179     });
 180 };
 181
 182 __PACKAGE__->register_method ({
 183     name => 'addnode',
 184     path => 'nodes/{node}',
 185     method => 'POST',
 186     protected => 1,
 187     description => "Adds a node to the cluster configuration. This call is for internal use.",
 188     parameters => {
 189         additionalProperties => 0,
 190         properties => {
 191             node => get_standard_option('pve-node'),
 192             nodeid => get_standard_option('corosync-nodeid'),
 193             votes => {
 194                 type => 'integer',
 195                 description => "Number of votes for this node",
 196                 minimum => 0,
 197                 optional => 1,
 198             },
 199             force => {
 200                 type => 'boolean',
 201                 description => "Do not throw error if node already exists.",
 202                 optional => 1,
 203             },
 204             link0 => get_standard_option('corosync-link'),
 205             link1 => get_standard_option('corosync-link'),
 206         },
 207     },
 208     returns => {
 209         type => "object",
 210         properties => {
 211             corosync_authkey => {
 212                 type => 'string',
 213             },
 214             corosync_conf => {
 215                 type => 'string',
 216             }
 217         },
 218     },
 219     code => sub {
 220         my ($param) = @_;
 221
 222         PVE::Cluster::check_cfs_quorum();
 223
 224         my $code = sub {
 225             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 226             my $nodelist = PVE::Corosync::nodelist($conf);
 227             my $totem_cfg = PVE::Corosync::totem_config($conf);
 228
 229             my $name = $param->{node};
 230
 231             # ensure we do not reuse an address, that can crash the whole cluster!
 232             my $check_duplicate_addr = sub {
 233                 my $link = shift;
 234                 return if !defined($link) || !defined($link->{address});
 235                 my $addr = $link->{address};
 236
 237                 while (my ($k, $v) = each %$nodelist) {
 238                     next if $k eq $name; # allows re-adding a node if force is set
 239
 240                     for my $linknumber (0..1) {
 241                         my $id = "ring${linknumber}_addr";
 242                         next if !defined($v->{$id});
 243
 244                         die "corosync: address '$addr' already used on link $id by node '$k'\n"
 245                             if $v->{$id} eq $addr;
 246                     }
 247                 }
 248             };
 249
 250             my $link0 = PVE::Cluster::parse_corosync_link($param->{link0});
 251             my $link1 = PVE::Cluster::parse_corosync_link($param->{link1});
 252
 253             $check_duplicate_addr->($link0);
 254             $check_duplicate_addr->($link1);
 255
 256             # FIXME: handle all links (0-7), they're all independent now
 257             $link0->{address} //= $name if exists($totem_cfg->{interface}->{0});
 258
 259             die "corosync: using 'link1' parameter needs a interface with linknumber '1' configured!\n"
 260                 if $link1 && !defined($totem_cfg->{interface}->{1});
 261
 262             die "corosync: totem interface with linknumber 1 configured but 'link1' parameter not defined!\n"
 263                 if defined($totem_cfg->{interface}->{1}) && !defined($link1);
 264
 265             if (defined(my $res = $nodelist->{$name})) {
 266                 $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 267                 $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 268
 269                 if ($res->{quorum_votes} == $param->{votes} &&
 270                     $res->{nodeid} == $param->{nodeid} && $param->{force}) {
 271                     print "forcing overwrite of configured node '$name'\n";
 272                 } else {
 273                     die "can't add existing node '$name'\n";
 274                 }
 275             } elsif (!$param->{nodeid}) {
 276                 my $nodeid = 1;
 277
 278                 while(1) {
 279                     my $found = 0;
 280                     foreach my $v (values %$nodelist) {
 281                         if ($v->{nodeid} eq $nodeid) {
 282                             $found = 1;
 283                             $nodeid++;
 284                             last;
 285                         }
 286                     }
 287                     last if !$found;
 288                 };
 289
 290                 $param->{nodeid} = $nodeid;
 291             }
 292
 293             $param->{votes} = 1 if !defined($param->{votes});
 294
 295             PVE::Cluster::gen_local_dirs($name);
 296
 297             eval { PVE::Cluster::ssh_merge_keys(); };
 298             warn $@ if $@;
 299
 300             $nodelist->{$name} = {
 301                 ring0_addr => $link0->{address},
 302                 nodeid => $param->{nodeid},
 303                 name => $name,
 304             };
 305             $nodelist->{$name}->{ring1_addr} = $link1->{address} if defined($link1);
 306             $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 307
 308             PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");
 309
 310             PVE::Corosync::update_nodelist($conf, $nodelist);
 311         };
 312
 313         $config_change_lock->($code);
 314         die $@ if $@;
 315
 316         my $res = {
 317             corosync_authkey => PVE::Tools::file_get_contents($authfile),
 318             corosync_conf => PVE::Tools::file_get_contents($clusterconf),
 319         };
 320
 321         return $res;
 322     }});
 323
 324
 325 __PACKAGE__->register_method ({
 326     name => 'delnode',
 327     path => 'nodes/{node}',
 328     method => 'DELETE',
 329     protected => 1,
 330     description => "Removes a node from the cluster configuration.",
 331     parameters => {
 332         additionalProperties => 0,
 333         properties => {
 334             node => get_standard_option('pve-node'),
 335         },
 336     },
 337     returns => { type => 'null' },
 338     code => sub {
 339         my ($param) = @_;
 340
 341         my $local_node = PVE::INotify::nodename();
 342         die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;
 343
 344         PVE::Cluster::check_cfs_quorum();
 345
 346         my $code = sub {
 347             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 348             my $nodelist = PVE::Corosync::nodelist($conf);
 349
 350             my $node;
 351             my $nodeid;
 352
 353             foreach my $tmp_node (keys %$nodelist) {
 354                 my $d = $nodelist->{$tmp_node};
 355                 my $ring0_addr = $d->{ring0_addr};
 356                 my $ring1_addr = $d->{ring1_addr};
 357                 if (($tmp_node eq $param->{node}) ||
 358                     (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
 359                     (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
 360                     $node = $tmp_node;
 361                     $nodeid = $d->{nodeid};
 362                     last;
 363                 }
 364             }
 365
 366             die "Node/IP: $param->{node} is not a known host of the cluster.\n"
 367                 if !defined($node);
 368
 369             PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");
 370
 371             delete $nodelist->{$node};
 372
 373             PVE::Corosync::update_nodelist($conf, $nodelist);
 374
 375             PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
 376         };
 377
 378         $config_change_lock->($code);
 379         die $@ if $@;
 380
 381         return undef;
 382     }});
 383
 384 __PACKAGE__->register_method ({
 385     name => 'join_info',
 386     path => 'join',
 387     permissions => {
 388         check => ['perm', '/', [ 'Sys.Audit' ]],
 389     },
 390     method => 'GET',
 391     description => "Get information needed to join this cluster over the connected node.",
 392     parameters => {
 393         additionalProperties => 0,
 394         properties => {
 395             node => get_standard_option('pve-node', {
 396                 description => "The node for which the joinee gets the nodeinfo. ",
 397                 default => "current connected node",
 398                 optional => 1,
 399             }),
 400         },
 401     },
 402     returns => {
 403         type => 'object',
 404         additionalProperties => 0,
 405         properties => {
 406             nodelist => {
 407                 type => 'array',
 408                 items => {
 409                     type => "object",
 410                     additionalProperties => 1,
 411                     properties => {
 412                         name => get_standard_option('pve-node'),
 413                         nodeid => get_standard_option('corosync-nodeid'),
 414                         ring0_addr => get_standard_option('corosync-link'),
 415                         quorum_votes => { type => 'integer', minimum => 0 },
 416                         pve_addr => { type => 'string', format => 'ip' },
 417                         pve_fp => get_standard_option('fingerprint-sha256'),
 418                     },
 419                 },
 420             },
 421             preferred_node => get_standard_option('pve-node'),
 422             totem => { type => 'object' },
 423             config_digest => { type => 'string' },
 424         },
 425     },
 426     code => sub {
 427         my ($param) = @_;
 428
 429         my $nodename = $param->{node} // PVE::INotify::nodename();
 430
 431         PVE::Cluster::cfs_update(1);
 432         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 433
 434         die "node is not in a cluster, no join info available!\n"
 435             if !($conf && $conf->{main});
 436
 437         my $totem_cfg = $conf->{main}->{totem} // {};
 438         my $nodelist = $conf->{main}->{nodelist}->{node} // {};
 439         my $corosync_config_digest = $conf->{digest};
 440
 441         die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};
 442
 443         foreach my $name (keys %$nodelist) {
 444             my $node = $nodelist->{$name};
 445             $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
 446             $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
 447         }
 448
 449         my $res = {
 450             nodelist => [ values %$nodelist ],
 451             preferred_node => $nodename,
 452             totem => $totem_cfg,
 453             config_digest => $corosync_config_digest,
 454         };
 455
 456         return $res;
 457     }});
 458
 459 __PACKAGE__->register_method ({
 460     name => 'join',
 461     path => 'join',
 462     method => 'POST',
 463     protected => 1,
 464     description => "Joins this node into an existing cluster.",
 465     parameters => {
 466         additionalProperties => 0,
 467         properties => {
 468             hostname => {
 469                 type => 'string',
 470                 description => "Hostname (or IP) of an existing cluster member."
 471             },
 472             nodeid => get_standard_option('corosync-nodeid'),
 473             votes => {
 474                 type => 'integer',
 475                 description => "Number of votes for this node",
 476                 minimum => 0,
 477                 optional => 1,
 478             },
 479             force => {
 480                 type => 'boolean',
 481                 description => "Do not throw error if node already exists.",
 482                 optional => 1,
 483             },
 484             link0 => get_standard_option('corosync-link', {
 485                 default => "IP resolved by node's hostname",
 486             }),
 487             link1 => get_standard_option('corosync-link'),
 488             fingerprint => get_standard_option('fingerprint-sha256'),
 489             password => {
 490                 description => "Superuser (root) password of peer node.",
 491                 type => 'string',
 492                 maxLength => 128,
 493             },
 494         },
 495     },
 496     returns => { type => 'string' },
 497     code => sub {
 498         my ($param) = @_;
 499
 500         my $rpcenv = PVE::RPCEnvironment::get();
 501         my $authuser = $rpcenv->get_user();
 502
 503         my $worker = sub {
 504             STDOUT->autoflush();
 505             PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
 506             die $@ if $@;
 507         };
 508
 509         return $rpcenv->fork_worker('clusterjoin', undef,  $authuser, $worker);
 510     }});
 511
 512
 513 __PACKAGE__->register_method({
 514     name => 'totem',
 515     path => 'totem',
 516     method => 'GET',
 517     description => "Get corosync totem protocol settings.",
 518     permissions => {
 519         check => ['perm', '/', [ 'Sys.Audit' ]],
 520     },
 521     parameters => {
 522         additionalProperties => 0,
 523         properties => {},
 524     },
 525     returns => {
 526         type => "object",
 527         properties => {},
 528     },
 529     code => sub {
 530         my ($param) = @_;
 531
 532
 533         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 534
 535         my $totem_cfg = $conf->{main}->{totem};
 536
 537         return $totem_cfg;
 538     }});
 539
 540 __PACKAGE__->register_method ({
 541     name => 'status',
 542     path => 'qdevice',
 543     method => 'GET',
 544     description => 'Get QDevice status',
 545     permissions => {
 546         check => ['perm', '/', [ 'Sys.Audit' ]],
 547     },
 548     parameters => {
 549         additionalProperties => 0,
 550         properties => {},
 551     },
 552     returns => {
 553         type => "object",
 554     },
 555     code => sub {
 556         my ($param) = @_;
 557
 558         my $result = {};
 559         my $socket_path = "/var/run/corosync-qdevice/corosync-qdevice.sock";
 560         return $result if !-S $socket_path;
 561
 562         my $qdevice_socket = IO::Socket::UNIX->new(
 563             Type => SOCK_STREAM,
 564             Peer => $socket_path,
 565         );
 566
 567         print $qdevice_socket "status verbose\n";
 568         my $qdevice_keys = {
 569             "Algorithm" => 1,
 570             "Echo reply" => 1,
 571             "Last poll call" => 1,
 572             "Model" => 1,
 573             "QNetd host" => 1,
 574             "State" => 1,
 575             "Tie-breaker" => 1,
 576         };
 577         while (my $line = <$qdevice_socket>) {
 578             chomp $line;
 579             next if $line =~ /^\s/;
 580             if ($line =~ /^(.*?)\s*:\s*(.*)$/) {
 581                 $result->{$1} = $2 if $qdevice_keys->{$1};
 582             }
 583         }
 584
 585         return $result;
 586     }});
 587 #TODO: possibly add setup and remove methods
 588
 589
 590 1;