data/PVE/API2/ClusterConfig.pm

   1 package PVE::API2::ClusterConfig;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Exception;
   7 use PVE::Tools;
   8 use PVE::SafeSyslog;
   9 use PVE::RESTHandler;
  10 use PVE::RPCEnvironment;
  11 use PVE::JSONSchema qw(get_standard_option);
  12 use PVE::Cluster;
  13 use PVE::APIClient::LWP;
  14 use PVE::Corosync;
  15 use PVE::Cluster::Setup;
  16
  17 use IO::Socket::UNIX;
  18
  19 use base qw(PVE::RESTHandler);
  20
  21 my $clusterconf = "/etc/pve/corosync.conf";
  22 my $authfile = "/etc/corosync/authkey";
  23 my $local_cluster_lock = "/var/lock/pvecm.lock";
  24
  25 my $nodeid_desc = {
  26     type => 'integer',
  27     description => "Node id for this node.",
  28     minimum => 1,
  29     optional => 1,
  30 };
  31 PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc);
  32
  33 __PACKAGE__->register_method({
  34     name => 'index',
  35     path => '',
  36     method => 'GET',
  37     description => "Directory index.",
  38     permissions => {
  39         check => ['perm', '/', [ 'Sys.Audit' ]],
  40     },
  41     parameters => {
  42         additionalProperties => 0,
  43         properties => {},
  44     },
  45     returns => {
  46         type => 'array',
  47         items => {
  48             type => "object",
  49             properties => {},
  50         },
  51         links => [ { rel => 'child', href => "{name}" } ],
  52     },
  53     code => sub {
  54         my ($param) = @_;
  55
  56         my $result = [
  57             { name => 'nodes' },
  58             { name => 'totem' },
  59             { name => 'join' },
  60             { name => 'qdevice' },
  61             { name => 'apiversion' },
  62         ];
  63
  64         return $result;
  65     }});
  66
  67 __PACKAGE__->register_method ({
  68     name => 'join_api_version',
  69     path => 'apiversion',
  70     method => 'GET',
  71     description => "Return the version of the cluster join API available on this node.",
  72     permissions => {
  73         check => ['perm', '/', [ 'Sys.Audit' ]],
  74     },
  75     parameters => {
  76         additionalProperties => 0,
  77         properties => {},
  78     },
  79     returns => {
  80         type => 'integer',
  81         minimum => 0,
  82         description => "Cluster Join API version, currently " . PVE::Cluster::Setup::JOIN_API_VERSION,
  83     },
  84     code => sub {
  85         return PVE::Cluster::Setup::JOIN_API_VERSION;
  86     }});
  87
  88 __PACKAGE__->register_method ({
  89     name => 'create',
  90     path => '',
  91     method => 'POST',
  92     protected => 1,
  93     description => "Generate new cluster configuration. If no links given,"
  94                  . " default to local IP address as link0.",
  95     parameters => {
  96         additionalProperties => 0,
  97         properties => PVE::Corosync::add_corosync_link_properties({
  98             clustername => {
  99                 description => "The name of the cluster.",
 100                 type => 'string', format => 'pve-node',
 101                 maxLength => 15,
 102             },
 103             nodeid => get_standard_option('corosync-nodeid'),
 104             votes => {
 105                 type => 'integer',
 106                 description => "Number of votes for this node.",
 107                 minimum => 1,
 108                 optional => 1,
 109             },
 110         }),
 111     },
 112     returns => { type => 'string' },
 113     code => sub {
 114         my ($param) = @_;
 115
 116         die "cluster config '$clusterconf' already exists\n" if -f $clusterconf;
 117
 118         my $rpcenv = PVE::RPCEnvironment::get();
 119         my $authuser = $rpcenv->get_user();
 120
 121         my $code = sub {
 122             STDOUT->autoflush();
 123             PVE::Cluster::Setup::setup_sshd_config(1);
 124             PVE::Cluster::Setup::setup_rootsshconfig();
 125             PVE::Cluster::Setup::setup_ssh_keys();
 126
 127             PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
 128                 if !-f $authfile;
 129             die "no authentication key available\n" if -f !$authfile;
 130
 131             my $nodename = PVE::INotify::nodename();
 132
 133             # get the corosync basis config for the new cluster
 134             my $config = PVE::Corosync::create_conf($nodename, $param);
 135
 136             print "Writing corosync config to /etc/pve/corosync.conf\n";
 137             PVE::Corosync::atomic_write_conf($config);
 138
 139             my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 140             PVE::Cluster::Setup::ssh_merge_keys();
 141             PVE::Cluster::Setup::gen_pve_node_files($nodename, $local_ip_address);
 142             PVE::Cluster::Setup::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 143
 144             print "Restart corosync and cluster filesystem\n";
 145             PVE::Tools::run_command('systemctl restart corosync pve-cluster');
 146         };
 147
 148         my $worker = sub {
 149             PVE::Tools::lock_file($local_cluster_lock, 10, $code);
 150             die $@ if $@;
 151         };
 152
 153         return $rpcenv->fork_worker('clustercreate', $param->{clustername},  $authuser, $worker);
 154 }});
 155
 156 __PACKAGE__->register_method({
 157     name => 'nodes',
 158     path => 'nodes',
 159     method => 'GET',
 160     description => "Corosync node list.",
 161     permissions => {
 162         check => ['perm', '/', [ 'Sys.Audit' ]],
 163     },
 164     parameters => {
 165         additionalProperties => 0,
 166         properties => {},
 167     },
 168     returns => {
 169         type => 'array',
 170         items => {
 171             type => "object",
 172             properties => {
 173                 node => { type => 'string' },
 174             },
 175         },
 176         links => [ { rel => 'child', href => "{node}" } ],
 177     },
 178     code => sub {
 179         my ($param) = @_;
 180
 181
 182         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 183         my $nodelist = PVE::Corosync::nodelist($conf);
 184
 185         return PVE::RESTHandler::hash_to_array($nodelist, 'node');
 186     }});
 187
 188 # lock method to ensure local and cluster wide atomicity
 189 # if we're a single node cluster just lock locally, we have no other cluster
 190 # node which we could contend with, else also acquire a cluster wide lock
 191 my $config_change_lock = sub {
 192     my ($code) = @_;
 193
 194     PVE::Tools::lock_file($local_cluster_lock, 10, sub {
 195         PVE::Cluster::cfs_update(1);
 196         my $members = PVE::Cluster::get_members();
 197         if (scalar(keys %$members) > 1) {
 198             my $res = PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
 199
 200             # cfs_lock_file only sets $@ but lock_file doesn't propagates $@ unless we die here
 201             die $@ if defined($@);
 202
 203             return $res;
 204         } else {
 205             return $code->();
 206         }
 207     });
 208 };
 209
 210 __PACKAGE__->register_method ({
 211     name => 'addnode',
 212     path => 'nodes/{node}',
 213     method => 'POST',
 214     protected => 1,
 215     description => "Adds a node to the cluster configuration. This call is for internal use.",
 216     parameters => {
 217         additionalProperties => 0,
 218         properties => PVE::Corosync::add_corosync_link_properties({
 219             node => get_standard_option('pve-node'),
 220             nodeid => get_standard_option('corosync-nodeid'),
 221             votes => {
 222                 type => 'integer',
 223                 description => "Number of votes for this node",
 224                 minimum => 0,
 225                 optional => 1,
 226             },
 227             force => {
 228                 type => 'boolean',
 229                 description => "Do not throw error if node already exists.",
 230                 optional => 1,
 231             },
 232             new_node_ip => {
 233                 type => 'string',
 234                 description => "IP Address of node to add. Used as fallback if no links are given.",
 235                 format => 'ip',
 236                 optional => 1,
 237             },
 238             apiversion => {
 239                 type => 'integer',
 240                 description => 'The JOIN_API_VERSION of the new node.',
 241                 optional => 1,
 242             },
 243         }),
 244     },
 245     returns => {
 246         type => "object",
 247         properties => {
 248             corosync_authkey => {
 249                 type => 'string',
 250             },
 251             corosync_conf => {
 252                 type => 'string',
 253             },
 254             warnings => {
 255                 type => 'array',
 256                 items => {
 257                     type => 'string',
 258                 },
 259             },
 260         },
 261     },
 262     code => sub {
 263         my ($param) = @_;
 264
 265         $param->{apiversion} //= 0;
 266         PVE::Cluster::Setup::assert_node_can_join_our_version($param->{apiversion});
 267
 268         PVE::Cluster::check_cfs_quorum();
 269
 270         my $vc_errors;
 271         my $vc_warnings;
 272
 273         my $code = sub {
 274             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 275             my $nodelist = PVE::Corosync::nodelist($conf);
 276             my $totem_cfg = PVE::Corosync::totem_config($conf);
 277
 278             ($vc_errors, $vc_warnings) = PVE::Corosync::verify_conf($conf);
 279             die if scalar(@$vc_errors);
 280
 281             my $name = $param->{node};
 282
 283             # ensure we do not reuse an address, that can crash the whole cluster!
 284             my $check_duplicate_addr = sub {
 285                 my $link = shift;
 286                 return if !defined($link) || !defined($link->{address});
 287                 my $addr = $link->{address};
 288
 289                 while (my ($k, $v) = each %$nodelist) {
 290                     next if $k eq $name; # allows re-adding a node if force is set
 291
 292                     for my $linknumber (0..PVE::Corosync::MAX_LINK_INDEX) {
 293                         my $id = "ring${linknumber}_addr";
 294                         next if !defined($v->{$id});
 295
 296                         die "corosync: address '$addr' already used on link $id by node '$k'\n"
 297                             if $v->{$id} eq $addr;
 298                     }
 299                 }
 300             };
 301
 302             my $links = PVE::Corosync::extract_corosync_link_args($param);
 303             foreach my $link (values %$links) {
 304                 $check_duplicate_addr->($link);
 305             }
 306
 307             # Make sure that the newly added node has links compatible with the
 308             # rest of the cluster. To start, extract all links that currently
 309             # exist. Check any node, all have the same links here (because of
 310             # verify_conf above).
 311             my $node_options = (values %$nodelist)[0];
 312             my $cluster_links = {};
 313             foreach my $opt (keys %$node_options) {
 314                 my ($linktype, $linkid) = PVE::Corosync::parse_link_entry($opt);
 315                 next if !defined($linktype);
 316                 $cluster_links->{$linkid} = $node_options->{$opt};
 317             }
 318
 319             # in case no fallback IP was passed, but instead only a single link,
 320             # treat it as fallback to allow link-IDs to be matched automatically
 321             # FIXME: remove in 8.0 or when joining an old node not supporting
 322             # new_node_ip becomes infeasible otherwise
 323             my $legacy_fallback = 0;
 324             if (!$param->{new_node_ip} && scalar(%$links) == 1 && $param->{apiversion} == 0) {
 325                 my $passed_link_id = (keys %$links)[0];
 326                 my $passed_link = delete $links->{$passed_link_id};
 327                 $param->{new_node_ip} = $passed_link->{address};
 328                 $legacy_fallback = 1;
 329             }
 330
 331             if (scalar(%$links)) {
 332                 # verify specified links exist and none are missing
 333                 for my $linknum (0..PVE::Corosync::MAX_LINK_INDEX) {
 334                     my $have_cluster_link = defined($cluster_links->{$linknum});
 335                     my $have_new_link = defined($links->{$linknum});
 336
 337                     die "corosync: link $linknum exists in cluster config but wasn't specified for new node\n"
 338                         if $have_cluster_link && !$have_new_link;
 339                     die "corosync: link $linknum specified for new node doesn't exist in cluster config\n"
 340                         if !$have_cluster_link && $have_new_link;
 341                 }
 342             } else {
 343                 # when called without any link parameters, fall back to
 344                 # new_node_ip, if all existing nodes only have a single link too
 345                 die "no links and no fallback ip (new_node_ip) given, cannot join cluster\n"
 346                     if !$param->{new_node_ip};
 347
 348                 my $cluster_link_count = scalar(%$cluster_links);
 349                 if ($cluster_link_count == 1) {
 350                     my $linknum = (keys %$cluster_links)[0];
 351                     $links->{$linknum} = { address => $param->{new_node_ip} };
 352                 } else {
 353                     die "cluster has $cluster_link_count links, but only 1 given"
 354                         if $legacy_fallback;
 355                     die "no links given but multiple links found on other nodes,"
 356                       . " fallback only supported on single-link clusters\n";
 357                 }
 358             }
 359
 360             if (defined(my $res = $nodelist->{$name})) {
 361                 $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
 362                 $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
 363
 364                 if ($res->{quorum_votes} == $param->{votes} &&
 365                     $res->{nodeid} == $param->{nodeid} && $param->{force}) {
 366                     print "forcing overwrite of configured node '$name'\n";
 367                 } else {
 368                     die "can't add existing node '$name'\n";
 369                 }
 370             } elsif (!$param->{nodeid}) {
 371                 my $nodeid = 1;
 372
 373                 while(1) {
 374                     my $found = 0;
 375                     foreach my $v (values %$nodelist) {
 376                         if ($v->{nodeid} eq $nodeid) {
 377                             $found = 1;
 378                             $nodeid++;
 379                             last;
 380                         }
 381                     }
 382                     last if !$found;
 383                 };
 384
 385                 $param->{nodeid} = $nodeid;
 386             }
 387
 388             $param->{votes} = 1 if !defined($param->{votes});
 389
 390             PVE::Cluster::Setup::gen_local_dirs($name);
 391
 392             eval { PVE::Cluster::Setup::ssh_merge_keys(); };
 393             warn $@ if $@;
 394
 395             $nodelist->{$name} = {
 396                 nodeid => $param->{nodeid},
 397                 name => $name,
 398             };
 399             $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
 400
 401             foreach my $link (keys %$links) {
 402                 $nodelist->{$name}->{"ring${link}_addr"} = $links->{$link}->{address};
 403             }
 404
 405             PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster");
 406
 407             PVE::Corosync::update_nodelist($conf, $nodelist);
 408         };
 409
 410         $config_change_lock->($code);
 411
 412         # If vc_errors is set, we died because of verify_conf.
 413         # Raise this error, since it contains more information than just a single-line string.
 414         if (defined($vc_errors) && scalar(@$vc_errors)) {
 415             my $err_hash = {};
 416             my $add_errs = sub {
 417                 my ($type, @arr) = @_;
 418                 return if !scalar(@arr);
 419                 $err_hash->{"${type}$_"} = $arr[$_] for 0..$#arr;
 420             };
 421
 422             $add_errs->("warning", @$vc_warnings);
 423             $add_errs->("error", @$vc_errors);
 424
 425             PVE::Exception::raise("invalid corosync.conf\n", errors => $err_hash);
 426         }
 427
 428         die $@ if $@;
 429
 430         my $res = {
 431             corosync_authkey => PVE::Tools::file_get_contents($authfile),
 432             corosync_conf => PVE::Tools::file_get_contents($clusterconf),
 433             warnings => $vc_warnings,
 434         };
 435
 436         return $res;
 437     }});
 438
 439
 440 __PACKAGE__->register_method ({
 441     name => 'delnode',
 442     path => 'nodes/{node}',
 443     method => 'DELETE',
 444     protected => 1,
 445     description => "Removes a node from the cluster configuration.",
 446     parameters => {
 447         additionalProperties => 0,
 448         properties => {
 449             node => get_standard_option('pve-node'),
 450         },
 451     },
 452     returns => { type => 'null' },
 453     code => sub {
 454         my ($param) = @_;
 455
 456         my $local_node = PVE::INotify::nodename();
 457         die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node;
 458
 459         PVE::Cluster::check_cfs_quorum();
 460
 461         my $code = sub {
 462             my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
 463             my $nodelist = PVE::Corosync::nodelist($conf);
 464
 465             my $node;
 466             my $nodeid;
 467
 468             foreach my $tmp_node (keys %$nodelist) {
 469                 my $d = $nodelist->{$tmp_node};
 470                 my $ring0_addr = $d->{ring0_addr};
 471                 my $ring1_addr = $d->{ring1_addr};
 472                 if (($tmp_node eq $param->{node}) ||
 473                     (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
 474                     (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
 475                     $node = $tmp_node;
 476                     $nodeid = $d->{nodeid};
 477                     last;
 478                 }
 479             }
 480
 481             die "Node/IP: $param->{node} is not a known host of the cluster.\n"
 482                 if !defined($node);
 483
 484             PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster");
 485
 486             delete $nodelist->{$node};
 487
 488             # allowed to fail when node is already shut down!
 489             eval {
 490                 PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid])
 491                     if defined($nodeid);
 492             };
 493
 494             PVE::Corosync::update_nodelist($conf, $nodelist);
 495         };
 496
 497         $config_change_lock->($code);
 498         die $@ if $@;
 499
 500         return undef;
 501     }});
 502
 503 __PACKAGE__->register_method ({
 504     name => 'join_info',
 505     path => 'join',
 506     permissions => {
 507         check => ['perm', '/', [ 'Sys.Audit' ]],
 508     },
 509     method => 'GET',
 510     description => "Get information needed to join this cluster over the connected node.",
 511     parameters => {
 512         additionalProperties => 0,
 513         properties => {
 514             node => get_standard_option('pve-node', {
 515                 description => "The node for which the joinee gets the nodeinfo. ",
 516                 default => "current connected node",
 517                 optional => 1,
 518             }),
 519         },
 520     },
 521     returns => {
 522         type => 'object',
 523         additionalProperties => 0,
 524         properties => {
 525             nodelist => {
 526                 type => 'array',
 527                 items => {
 528                     type => "object",
 529                     additionalProperties => 1,
 530                     properties => {
 531                         name => get_standard_option('pve-node'),
 532                         nodeid => get_standard_option('corosync-nodeid'),
 533                         ring0_addr => get_standard_option('corosync-link'),
 534                         quorum_votes => { type => 'integer', minimum => 0 },
 535                         pve_addr => { type => 'string', format => 'ip' },
 536                         pve_fp => get_standard_option('fingerprint-sha256'),
 537                     },
 538                 },
 539             },
 540             preferred_node => get_standard_option('pve-node'),
 541             totem => { type => 'object' },
 542             config_digest => { type => 'string' },
 543         },
 544     },
 545     code => sub {
 546         my ($param) = @_;
 547
 548         my $nodename = $param->{node} // PVE::INotify::nodename();
 549
 550         PVE::Cluster::cfs_update(1);
 551         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 552
 553         # FIXME: just return undef or empty object in PVE 8.0 (check if manager can handle it!)
 554         PVE::Exception::raise(
 555             'node is not in a cluster, no join info available!',
 556             code => HTTP::Status::HTTP_FAILED_DEPENDENCY,
 557         ) if !($conf && $conf->{main});
 558
 559         my $totem_cfg = $conf->{main}->{totem} // {};
 560         my $nodelist = $conf->{main}->{nodelist}->{node} // {};
 561         my $corosync_config_digest = $conf->{digest};
 562
 563         die "unknown node '$nodename'\n" if ! $nodelist->{$nodename};
 564
 565         foreach my $name (keys %$nodelist) {
 566             my $node = $nodelist->{$name};
 567             $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name);
 568             $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name));
 569         }
 570
 571         my $res = {
 572             nodelist => [ values %$nodelist ],
 573             preferred_node => $nodename,
 574             totem => $totem_cfg,
 575             config_digest => $corosync_config_digest,
 576         };
 577
 578         return $res;
 579     }});
 580
 581 __PACKAGE__->register_method ({
 582     name => 'join',
 583     path => 'join',
 584     method => 'POST',
 585     protected => 1,
 586     description => "Joins this node into an existing cluster. If no links are"
 587                  . " given, default to IP resolved by node's hostname on single"
 588                  . " link (fallback fails for clusters with multiple links).",
 589     parameters => {
 590         additionalProperties => 0,
 591         properties => PVE::Corosync::add_corosync_link_properties({
 592             hostname => {
 593                 type => 'string',
 594                 description => "Hostname (or IP) of an existing cluster member."
 595             },
 596             nodeid => get_standard_option('corosync-nodeid'),
 597             votes => {
 598                 type => 'integer',
 599                 description => "Number of votes for this node",
 600                 minimum => 0,
 601                 optional => 1,
 602             },
 603             force => {
 604                 type => 'boolean',
 605                 description => "Do not throw error if node already exists.",
 606                 optional => 1,
 607             },
 608             fingerprint => get_standard_option('fingerprint-sha256'),
 609             password => {
 610                 description => "Superuser (root) password of peer node.",
 611                 type => 'string',
 612                 maxLength => 128,
 613             },
 614         }),
 615     },
 616     returns => { type => 'string' },
 617     code => sub {
 618         my ($param) = @_;
 619
 620         my $rpcenv = PVE::RPCEnvironment::get();
 621         my $authuser = $rpcenv->get_user();
 622
 623         my $worker = sub {
 624             STDOUT->autoflush();
 625             PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param);
 626             die $@ if $@;
 627         };
 628
 629         return $rpcenv->fork_worker('clusterjoin', undef,  $authuser, $worker);
 630     }});
 631
 632
 633 __PACKAGE__->register_method({
 634     name => 'totem',
 635     path => 'totem',
 636     method => 'GET',
 637     description => "Get corosync totem protocol settings.",
 638     permissions => {
 639         check => ['perm', '/', [ 'Sys.Audit' ]],
 640     },
 641     parameters => {
 642         additionalProperties => 0,
 643         properties => {},
 644     },
 645     returns => {
 646         type => "object",
 647         properties => {},
 648     },
 649     code => sub {
 650         my ($param) = @_;
 651
 652
 653         my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 654
 655         my $totem_cfg = $conf->{main}->{totem};
 656
 657         return $totem_cfg;
 658     }});
 659
 660 __PACKAGE__->register_method ({
 661     name => 'status',
 662     path => 'qdevice',
 663     method => 'GET',
 664     protected => 1,
 665     description => 'Get QDevice status',
 666     permissions => {
 667         check => ['perm', '/', [ 'Sys.Audit' ]],
 668     },
 669     parameters => {
 670         additionalProperties => 0,
 671         properties => {},
 672     },
 673     returns => {
 674         type => "object",
 675     },
 676     code => sub {
 677         my ($param) = @_;
 678
 679         my $result = {};
 680         my $socket_path = "/var/run/corosync-qdevice/corosync-qdevice.sock";
 681         return $result if !-S $socket_path;
 682
 683         my $qdevice_socket = IO::Socket::UNIX->new(
 684             Type => SOCK_STREAM,
 685             Peer => $socket_path,
 686         );
 687
 688         print $qdevice_socket "status verbose\n";
 689         my $qdevice_keys = {
 690             "Algorithm" => 1,
 691             "Echo reply" => 1,
 692             "Last poll call" => 1,
 693             "Model" => 1,
 694             "QNetd host" => 1,
 695             "State" => 1,
 696             "Tie-breaker" => 1,
 697         };
 698         while (my $line = <$qdevice_socket>) {
 699             chomp $line;
 700             next if $line =~ /^\s/;
 701             if ($line =~ /^(.*?)\s*:\s*(.*)$/) {
 702                 $result->{$1} = $2 if $qdevice_keys->{$1};
 703             }
 704         }
 705
 706         return $result;
 707     }});
 708 #TODO: possibly add setup and remove methods
 709
 710
 711 1;