data/PVE/API2/ClusterConfig.pm

   1 package PVE::API2::ClusterConfig;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Tools;
   7 use PVE::SafeSyslog;
   8 use PVE::RESTHandler;
   9 use PVE::RPCEnvironment;
  10 use PVE::JSONSchema qw(get_standard_option);
  11 use PVE::Cluster;
  12 use PVE::APIClient::LWP;
  13 use PVE::Corosync;
  14 use PVE::Cluster::Setup;
  15
  16 use IO::Socket::UNIX;
  17
  18 use base qw(PVE::RESTHandler);
  19
  20 my $clusterconf = "/etc/pve/corosync.conf";
  21 my $authfile = "/etc/corosync/authkey";
  22 my $local_cluster_lock = "/var/lock/pvecm.lock";
  23
  24 my $nodeid_desc = {
  25     type => 'integer',
  26     description => "Node id for this node.",
  27     minimum => 1,
  28     optional => 1,
  29 };
  30 PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);
  31
  32 __PACKAGE__->register_method({
  33     name => 'index',
  34     path => '',
  35     method => 'GET',
  36     description => "Directory index.",
  37     permissions => {
  38         check => ['perm', '/', [ 'Sys.Audit' ]],
  39     },
  40     parameters => {
  41         additionalProperties => 0,
  42         properties => {},
  43     },
  44     returns => {
  45         type => 'array',
  46         items => {
  47             type => "object",
  48             properties => {},
  49         },
  50         links => [ { rel => 'child', href => "{name}" } ],
  51     },
  52     code => sub {
  53         my ($param) = @_;
  54
  55         my $result = [
  56             { name => 'nodes' },
  57             { name => 'totem' },
  58             { name => 'join' },
  59             { name => 'qdevice' },
  60         ];
  61
  62         return $result;
  63     }});
  64
  65 __PACKAGE__->register_method ({
  66     name => 'create',
  67     path => '',
  68     method => 'POST',
  69     protected => 1,
  70     description => "Generate new cluster configuration.",
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             clustername => {
  75                 description => "The name of the cluster.",
  76                 type => 'string', format => 'pve-node',
  77                 maxLength => 15,
  78             },
  79             nodeid => get_standard_option('corosync-nodeid'),
  80             votes => {
  81                 type => 'integer',
  82                 description => "Number of votes for this node.",
  83                 minimum => 1,
  84                 optional => 1,
  85             },
  86             link0 => get_standard_option('corosync-link'),
  87             link1 => get_standard_option('corosync-link'),
  88         },
  89     },
  90     returns => { type => 'string' },
  91     code => sub {
  92         my ($param) = @_;
  93
  94         -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
  95
  96         my $rpcenv = PVE::RPCEnvironment::get();
  97         my $authuser = $rpcenv->get_user();
  98
  99         my $code = sub {
 100             STDOUT->autoflush();
 101             PVE::Cluster::Setup::setup_sshd_config(1);
 102             PVE::Cluster::Setup::setup_rootsshconfig();
 103             PVE::Cluster::Setup::setup_ssh_keys();
 104
 105             PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
 106                 if !-f $authfile;
 107             die "no authentication key available\n" if -f !$authfile;
 108
 109             my $nodename = PVE::INotify::nodename();
 110
 111             # get the corosync basis config for the new cluster
 112             my $config = PVE::Corosync::create_conf($nodename, %$param);
 113
 114             print "Writing corosync config to /etc/pve/corosync.conf\n";
 115             PVE::Corosync::atomic_write_conf($config);
 116
 117             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 118             PVE::Cluster::Setup::ssh_merge_keys();
 119             PVE::Cluster::Setup::gen_pve_node_files($nodename, $local_ip_address);
 120             PVE::Cluster::Setup::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 121
 122             print "Restart corosync and cluster filesystem\n";
 123             PVE::Tools::run_command('systemctl restart corosync pve-cluster');
 124         };
 125
 126         my $worker = sub {
 127             PVE::Tools::lock_file($local_cluster_lock, 10, $code);
 128             die $@ if $@;
 129         };
 130
 131         return $rpcenv->fork_worker('clustercreate', $param->{clustername},  $authuser, $worker);
 132 }});
 133
 134 __PACKAGE__->register_method({
 135     name => 'nodes',
 136     path => 'nodes',
 137     method => 'GET',
 138     description => "Corosync node list.",
 139     permissions => {
 140         check => ['perm', '/', [ 'Sys.Audit' ]],
 141     },
 142     parameters => {
 143         additionalProperties => 0,
 144         properties => {},
 145     },
 146     returns => {
 147         type => 'array',
 148         items => {
 149             type => "object",
 150             properties => {
 151                 node => { type => 'string' },
 152             },
 153         },
 154         links => [ { rel => 'child', href => "{node}" } ],
 155     },
 156     code => sub {
 157         my ($param) = @_;
 158
 159
 160         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 161         my $nodelist = PVE::Corosync::nodelist($conf);
 162
 163         return PVE::RESTHandler::hash_to_array($nodelist, 'node');
 164     }});
 165
 166 # lock method to ensure local and cluster wide atomicity
 167 # if we're a single node cluster just lock locally, we have no other cluster
 168 # node which we could contend with, else also acquire a cluster wide lock
 169 my $config_change_lock = sub {
 170     my ($code) = @_;
 171
 172     PVE::Tools::lock_file($local_cluster_lock, 10, sub {
 173         PVE::Cluster::cfs_update(1);
 174         my $members = PVE::Cluster::get_members();
 175         if (scalar(keys %$members) > 1) {
 176             my $res = PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 177
 178             # cfs_lock_file only sets $@ but lock_file doesn't propagates $@ unless we die here
 179             die $@ if defined($@);
 180
 181             return $res;
 182         } else {
 183             return $code->();
 184         }
 185     });
 186 };
 187
 188 __PACKAGE__->register_method ({
 189     name => 'addnode',
 190     path => 'nodes/{node}',
 191     method => 'POST',
 192     protected => 1,
 193     description => "Adds a node to the cluster configuration. This call is for internal use.",
 194     parameters => {
 195         additionalProperties => 0,
 196         properties => {
 197             node => get_standard_option('pve-node'),
 198             nodeid => get_standard_option('corosync-nodeid'),
 199             votes => {
 200                 type => 'integer',
 201                 description => "Number of votes for this node",
 202                 minimum => 0,
 203                 optional => 1,
 204             },
 205             force => {
 206                 type => 'boolean',
 207                 description => "Do not throw error if node already exists.",
 208                 optional => 1,
 209             },
 210             link0 => get_standard_option('corosync-link'),
 211             link1 => get_standard_option('corosync-link'),
 212         },
 213     },
 214     returns => {
 215         type => "object",
 216         properties => {
 217             corosync_authkey => {
 218                 type => 'string',
 219             },
 220             corosync_conf => {
 221                 type => 'string',
 222             }
 223         },
 224     },
 225     code => sub {
 226         my ($param) = @_;
 227
 228         PVE::Cluster::check_cfs_quorum();
 229
 230         my $code = sub {
 231             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 232             my $nodelist = PVE::Corosync::nodelist($conf);
 233             my $totem_cfg = PVE::Corosync::totem_config($conf);
 234
 235             my $name = $param->{node};
 236
 237             # ensure we do not reuse an address, that can crash the whole cluster!
 238             my $check_duplicate_addr = sub {
 239                 my $link = shift;
 240                 return if !defined($link) || !defined($link->{address});
 241                 my $addr = $link->{address};
 242
 243                 while (my ($k, $v) = each %$nodelist) {
 244                     next if $k eq $name; # allows re-adding a node if force is set
 245
 246                     for my $linknumber (0..1) {
 247                         my $id = "ring${linknumber}_addr";
 248                         next if !defined($v->{$id});
 249
 250                         die "corosync: address '$addr' already used on link $id by node '$k'\n"
 251                             if $v->{$id} eq $addr;
 252                     }
 253                 }
 254             };
 255
 256             my $link0 = PVE::Corosync::parse_corosync_link($param->{link0});
 257             my $link1 = PVE::Corosync::parse_corosync_link($param->{link1});
 258
 259             $check_duplicate_addr->($link0);
 260             $check_duplicate_addr->($link1);
 261
 262             # FIXME: handle all links (0-7), they're all independent now
 263             $link0->{address} //= $name if exists($totem_cfg->{interface}->{0});
 264
 265             die "corosync: using 'link1' parameter needs a interface with linknumber '1' configured!\n"
 266                 if $link1 && !defined($totem_cfg->{interface}->{1});
 267
 268             die "corosync: totem interface with linknumber 1 configured but 'link1' parameter not defined!\n"
 269                 if defined($totem_cfg->{interface}->{1}) && !defined($link1);
 270
 271             if (defined(my $res = $nodelist->{$name})) {
 272                 $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 273                 $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 274
 275                 if ($res->{quorum_votes} == $param->{votes} &&
 276                     $res->{nodeid} == $param->{nodeid} && $param->{force}) {
 277                     print "forcing overwrite of configured node '$name'\n";
 278                 } else {
 279                     die "can't add existing node '$name'\n";
 280                 }
 281             } elsif (!$param->{nodeid}) {
 282                 my $nodeid = 1;
 283
 284                 while(1) {
 285                     my $found = 0;
 286                     foreach my $v (values %$nodelist) {
 287                         if ($v->{nodeid} eq $nodeid) {
 288                             $found = 1;
 289                             $nodeid++;
 290                             last;
 291                         }
 292                     }
 293                     last if !$found;
 294                 };
 295
 296                 $param->{nodeid} = $nodeid;
 297             }
 298
 299             $param->{votes} = 1 if !defined($param->{votes});
 300
 301             PVE::Cluster::Setup::gen_local_dirs($name);
 302
 303             eval { PVE::Cluster::Setup::ssh_merge_keys(); };
 304             warn $@ if $@;
 305
 306             $nodelist->{$name} = {
 307                 ring0_addr => $link0->{address},
 308                 nodeid => $param->{nodeid},
 309                 name => $name,
 310             };
 311             $nodelist->{$name}->{ring1_addr} = $link1->{address} if defined($link1);
 312             $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 313
 314             PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");
 315
 316             PVE::Corosync::update_nodelist($conf, $nodelist);
 317         };
 318
 319         $config_change_lock->($code);
 320         die $@ if $@;
 321
 322         my $res = {
 323             corosync_authkey => PVE::Tools::file_get_contents($authfile),
 324             corosync_conf => PVE::Tools::file_get_contents($clusterconf),
 325         };
 326
 327         return $res;
 328     }});
 329
 330
 331 __PACKAGE__->register_method ({
 332     name => 'delnode',
 333     path => 'nodes/{node}',
 334     method => 'DELETE',
 335     protected => 1,
 336     description => "Removes a node from the cluster configuration.",
 337     parameters => {
 338         additionalProperties => 0,
 339         properties => {
 340             node => get_standard_option('pve-node'),
 341         },
 342     },
 343     returns => { type => 'null' },
 344     code => sub {
 345         my ($param) = @_;
 346
 347         my $local_node = PVE::INotify::nodename();
 348         die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;
 349
 350         PVE::Cluster::check_cfs_quorum();
 351
 352         my $code = sub {
 353             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 354             my $nodelist = PVE::Corosync::nodelist($conf);
 355
 356             my $node;
 357             my $nodeid;
 358
 359             foreach my $tmp_node (keys %$nodelist) {
 360                 my $d = $nodelist->{$tmp_node};
 361                 my $ring0_addr = $d->{ring0_addr};
 362                 my $ring1_addr = $d->{ring1_addr};
 363                 if (($tmp_node eq $param->{node}) ||
 364                     (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
 365                     (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
 366                     $node = $tmp_node;
 367                     $nodeid = $d->{nodeid};
 368                     last;
 369                 }
 370             }
 371
 372             die "Node/IP: $param->{node} is not a known host of the cluster.\n"
 373                 if !defined($node);
 374
 375             PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");
 376
 377             delete $nodelist->{$node};
 378
 379             PVE::Corosync::update_nodelist($conf, $nodelist);
 380
 381             PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid]) if defined($nodeid);
 382         };
 383
 384         $config_change_lock->($code);
 385         die $@ if $@;
 386
 387         return undef;
 388     }});
 389
 390 __PACKAGE__->register_method ({
 391     name => 'join_info',
 392     path => 'join',
 393     permissions => {
 394         check => ['perm', '/', [ 'Sys.Audit' ]],
 395     },
 396     method => 'GET',
 397     description => "Get information needed to join this cluster over the connected node.",
 398     parameters => {
 399         additionalProperties => 0,
 400         properties => {
 401             node => get_standard_option('pve-node', {
 402                 description => "The node for which the joinee gets the nodeinfo. ",
 403                 default => "current connected node",
 404                 optional => 1,
 405             }),
 406         },
 407     },
 408     returns => {
 409         type => 'object',
 410         additionalProperties => 0,
 411         properties => {
 412             nodelist => {
 413                 type => 'array',
 414                 items => {
 415                     type => "object",
 416                     additionalProperties => 1,
 417                     properties => {
 418                         name => get_standard_option('pve-node'),
 419                         nodeid => get_standard_option('corosync-nodeid'),
 420                         ring0_addr => get_standard_option('corosync-link'),
 421                         quorum_votes => { type => 'integer', minimum => 0 },
 422                         pve_addr => { type => 'string', format => 'ip' },
 423                         pve_fp => get_standard_option('fingerprint-sha256'),
 424                     },
 425                 },
 426             },
 427             preferred_node => get_standard_option('pve-node'),
 428             totem => { type => 'object' },
 429             config_digest => { type => 'string' },
 430         },
 431     },
 432     code => sub {
 433         my ($param) = @_;
 434
 435         my $nodename = $param->{node} // PVE::INotify::nodename();
 436
 437         PVE::Cluster::cfs_update(1);
 438         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 439
 440         die "node is not in a cluster, no join info available!\n"
 441             if !($conf && $conf->{main});
 442
 443         my $totem_cfg = $conf->{main}->{totem} // {};
 444         my $nodelist = $conf->{main}->{nodelist}->{node} // {};
 445         my $corosync_config_digest = $conf->{digest};
 446
 447         die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};
 448
 449         foreach my $name (keys %$nodelist) {
 450             my $node = $nodelist->{$name};
 451             $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
 452             $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
 453         }
 454
 455         my $res = {
 456             nodelist => [ values %$nodelist ],
 457             preferred_node => $nodename,
 458             totem => $totem_cfg,
 459             config_digest => $corosync_config_digest,
 460         };
 461
 462         return $res;
 463     }});
 464
 465 __PACKAGE__->register_method ({
 466     name => 'join',
 467     path => 'join',
 468     method => 'POST',
 469     protected => 1,
 470     description => "Joins this node into an existing cluster.",
 471     parameters => {
 472         additionalProperties => 0,
 473         properties => {
 474             hostname => {
 475                 type => 'string',
 476                 description => "Hostname (or IP) of an existing cluster member."
 477             },
 478             nodeid => get_standard_option('corosync-nodeid'),
 479             votes => {
 480                 type => 'integer',
 481                 description => "Number of votes for this node",
 482                 minimum => 0,
 483                 optional => 1,
 484             },
 485             force => {
 486                 type => 'boolean',
 487                 description => "Do not throw error if node already exists.",
 488                 optional => 1,
 489             },
 490             link0 => get_standard_option('corosync-link', {
 491                 default => "IP resolved by node's hostname",
 492             }),
 493             link1 => get_standard_option('corosync-link'),
 494             fingerprint => get_standard_option('fingerprint-sha256'),
 495             password => {
 496                 description => "Superuser (root) password of peer node.",
 497                 type => 'string',
 498                 maxLength => 128,
 499             },
 500         },
 501     },
 502     returns => { type => 'string' },
 503     code => sub {
 504         my ($param) = @_;
 505
 506         my $rpcenv = PVE::RPCEnvironment::get();
 507         my $authuser = $rpcenv->get_user();
 508
 509         my $worker = sub {
 510             STDOUT->autoflush();
 511             PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param);
 512             die $@ if $@;
 513         };
 514
 515         return $rpcenv->fork_worker('clusterjoin', undef,  $authuser, $worker);
 516     }});
 517
 518
 519 __PACKAGE__->register_method({
 520     name => 'totem',
 521     path => 'totem',
 522     method => 'GET',
 523     description => "Get corosync totem protocol settings.",
 524     permissions => {
 525         check => ['perm', '/', [ 'Sys.Audit' ]],
 526     },
 527     parameters => {
 528         additionalProperties => 0,
 529         properties => {},
 530     },
 531     returns => {
 532         type => "object",
 533         properties => {},
 534     },
 535     code => sub {
 536         my ($param) = @_;
 537
 538
 539         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 540
 541         my $totem_cfg = $conf->{main}->{totem};
 542
 543         return $totem_cfg;
 544     }});
 545
 546 __PACKAGE__->register_method ({
 547     name => 'status',
 548     path => 'qdevice',
 549     method => 'GET',
 550     description => 'Get QDevice status',
 551     permissions => {
 552         check => ['perm', '/', [ 'Sys.Audit' ]],
 553     },
 554     parameters => {
 555         additionalProperties => 0,
 556         properties => {},
 557     },
 558     returns => {
 559         type => "object",
 560     },
 561     code => sub {
 562         my ($param) = @_;
 563
 564         my $result = {};
 565         my $socket_path = "/var/run/corosync-qdevice/corosync-qdevice.sock";
 566         return $result if !-S $socket_path;
 567
 568         my $qdevice_socket = IO::Socket::UNIX->new(
 569             Type => SOCK_STREAM,
 570             Peer => $socket_path,
 571         );
 572
 573         print $qdevice_socket "status verbose\n";
 574         my $qdevice_keys = {
 575             "Algorithm" => 1,
 576             "Echo reply" => 1,
 577             "Last poll call" => 1,
 578             "Model" => 1,
 579             "QNetd host" => 1,
 580             "State" => 1,
 581             "Tie-breaker" => 1,
 582         };
 583         while (my $line = <$qdevice_socket>) {
 584             chomp $line;
 585             next if $line =~ /^\s/;
 586             if ($line =~ /^(.*?)\s*:\s*(.*)$/) {
 587                 $result->{$1} = $2 if $qdevice_keys->{$1};
 588             }
 589         }
 590
 591         return $result;
 592     }});
 593 #TODO: possibly add setup and remove methods
 594
 595
 596 1;