2 * Copyright (c) 2002-2005 MontaVista Software, Inc.
3 * Copyright (c) 2006-2013 Red Hat, Inc.
7 * Author: Steven Dake (sdake@redhat.com)
8 * Jan Friesse (jfriesse@redhat.com)
10 * This software licensed under BSD license, the text of which follows:
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions are met:
15 * - Redistributions of source code must retain the above copyright notice,
16 * this list of conditions and the following disclaimer.
17 * - Redistributions in binary form must reproduce the above copyright notice,
18 * this list of conditions and the following disclaimer in the documentation
19 * and/or other materials provided with the distribution.
20 * - Neither the name of the MontaVista Software, Inc. nor the names of its
21 * contributors may be used to endorse or promote products derived from this
22 * software without specific prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
34 * THE POSSIBILITY OF SUCH DAMAGE.
44 #include <sys/socket.h>
45 #include <sys/types.h>
48 #include <netinet/in.h>
49 #include <arpa/inet.h>
50 #include <sys/param.h>
52 #include <corosync/swab.h>
53 #include <qb/qblist.h>
54 #include <qb/qbdefs.h>
56 #include <corosync/totem/totem.h>
57 #include <corosync/config.h>
58 #include <corosync/logsys.h>
59 #include <corosync/icmap.h>
62 #include "totemconfig.h"
64 #define TOKEN_RETRANSMITS_BEFORE_LOSS_CONST 4
65 #define TOKEN_TIMEOUT 1000
66 #define TOKEN_COEFFICIENT 650
67 #define JOIN_TIMEOUT 50
68 #define MERGE_TIMEOUT 200
69 #define DOWNCHECK_TIMEOUT 1000
70 #define FAIL_TO_RECV_CONST 2500
71 #define SEQNO_UNCHANGED_CONST 30
72 #define MINIMUM_TIMEOUT (int)(1000/HZ)*3
73 #define MAX_NETWORK_DELAY 50
74 #define WINDOW_SIZE 50
75 #define MAX_MESSAGES 17
76 #define MISS_COUNT_CONST 5
78 /* These currently match the defaults in libknet.h */
79 #define KNET_PING_INTERVAL 1000
80 #define KNET_PING_TIMEOUT 2000
81 #define KNET_PING_PRECISION 2048
82 #define KNET_PONG_COUNT 2
83 #define KNET_PMTUD_INTERVAL 30
84 #define KNET_DEFAULT_TRANSPORT KNET_TRANSPORT_UDP
86 #define DEFAULT_PORT 5405
88 static char error_string_response
[512];
90 static void add_totem_config_notification(struct totem_config
*totem_config
);
92 static void *totem_get_param_by_name(struct totem_config
*totem_config
, const char *param_name
)
94 if (strcmp(param_name
, "totem.token") == 0)
95 return &totem_config
->token_timeout
;
96 if (strcmp(param_name
, "totem.token_retransmit") == 0)
97 return &totem_config
->token_retransmit_timeout
;
98 if (strcmp(param_name
, "totem.hold") == 0)
99 return &totem_config
->token_hold_timeout
;
100 if (strcmp(param_name
, "totem.token_retransmits_before_loss_const") == 0)
101 return &totem_config
->token_retransmits_before_loss_const
;
102 if (strcmp(param_name
, "totem.join") == 0)
103 return &totem_config
->join_timeout
;
104 if (strcmp(param_name
, "totem.send_join") == 0)
105 return &totem_config
->send_join_timeout
;
106 if (strcmp(param_name
, "totem.consensus") == 0)
107 return &totem_config
->consensus_timeout
;
108 if (strcmp(param_name
, "totem.merge") == 0)
109 return &totem_config
->merge_timeout
;
110 if (strcmp(param_name
, "totem.downcheck") == 0)
111 return &totem_config
->downcheck_timeout
;
112 if (strcmp(param_name
, "totem.fail_recv_const") == 0)
113 return &totem_config
->fail_to_recv_const
;
114 if (strcmp(param_name
, "totem.seqno_unchanged_const") == 0)
115 return &totem_config
->seqno_unchanged_const
;
116 if (strcmp(param_name
, "totem.heartbeat_failures_allowed") == 0)
117 return &totem_config
->heartbeat_failures_allowed
;
118 if (strcmp(param_name
, "totem.max_network_delay") == 0)
119 return &totem_config
->max_network_delay
;
120 if (strcmp(param_name
, "totem.window_size") == 0)
121 return &totem_config
->window_size
;
122 if (strcmp(param_name
, "totem.max_messages") == 0)
123 return &totem_config
->max_messages
;
124 if (strcmp(param_name
, "totem.miss_count_const") == 0)
125 return &totem_config
->miss_count_const
;
126 if (strcmp(param_name
, "totem.knet_pmtud_interval") == 0)
127 return &totem_config
->knet_pmtud_interval
;
128 if (strcmp(param_name
, "totem.knet_compression_threshold") == 0)
129 return &totem_config
->knet_compression_threshold
;
130 if (strcmp(param_name
, "totem.knet_compression_level") == 0)
131 return &totem_config
->knet_compression_level
;
132 if (strcmp(param_name
, "totem.knet_compression_model") == 0)
133 return &totem_config
->knet_compression_model
;
139 * Read key_name from icmap. If key is not found or key_name == delete_key or if allow_zero is false
140 * and readed value is zero, default value is used and stored into totem_config.
142 static void totem_volatile_config_set_uint32_value (struct totem_config
*totem_config
,
143 const char *key_name
, const char *deleted_key
, unsigned int default_value
,
144 int allow_zero_value
)
146 char runtime_key_name
[ICMAP_KEYNAME_MAXLEN
];
148 if (icmap_get_uint32(key_name
, totem_get_param_by_name(totem_config
, key_name
)) != CS_OK
||
149 (deleted_key
!= NULL
&& strcmp(deleted_key
, key_name
) == 0) ||
150 (!allow_zero_value
&& *(uint32_t *)totem_get_param_by_name(totem_config
, key_name
) == 0)) {
151 *(uint32_t *)totem_get_param_by_name(totem_config
, key_name
) = default_value
;
155 * Store totem_config value to cmap runtime section
157 if (strlen("runtime.config.") + strlen(key_name
) >= ICMAP_KEYNAME_MAXLEN
) {
159 * This shouldn't happen
164 strcpy(runtime_key_name
, "runtime.config.");
165 strcat(runtime_key_name
, key_name
);
167 icmap_set_uint32(runtime_key_name
, *(uint32_t *)totem_get_param_by_name(totem_config
, key_name
));
170 static void totem_volatile_config_set_int32_value (struct totem_config
*totem_config
,
171 const char *key_name
, const char *deleted_key
, int default_value
,
172 int allow_zero_value
)
174 char runtime_key_name
[ICMAP_KEYNAME_MAXLEN
];
176 if (icmap_get_int32(key_name
, totem_get_param_by_name(totem_config
, key_name
)) != CS_OK
||
177 (deleted_key
!= NULL
&& strcmp(deleted_key
, key_name
) == 0) ||
178 (!allow_zero_value
&& *(int32_t *)totem_get_param_by_name(totem_config
, key_name
) == 0)) {
179 *(int32_t *)totem_get_param_by_name(totem_config
, key_name
) = default_value
;
183 * Store totem_config value to cmap runtime section
185 if (strlen("runtime.config.") + strlen(key_name
) >= ICMAP_KEYNAME_MAXLEN
) {
187 * This shouldn't happen
192 strcpy(runtime_key_name
, "runtime.config.");
193 strcat(runtime_key_name
, key_name
);
195 icmap_set_int32(runtime_key_name
, *(int32_t *)totem_get_param_by_name(totem_config
, key_name
));
198 static void totem_volatile_config_set_string_value (struct totem_config
*totem_config
,
199 const char *key_name
, const char *deleted_key
, const char *default_value
)
201 char runtime_key_name
[ICMAP_KEYNAME_MAXLEN
];
203 void *old_config_ptr
;
205 config_value
= totem_get_param_by_name(totem_config
, key_name
);
206 old_config_ptr
= *config_value
;
207 if (icmap_get_string(key_name
, totem_get_param_by_name(totem_config
, key_name
)) != CS_OK
||
208 (deleted_key
!= NULL
&& strcmp(deleted_key
, key_name
) == 0)) {
210 /* Need to strdup() here so that the free() below works for a default and a configured value */
211 *config_value
= strdup(default_value
);
213 free(old_config_ptr
);
216 * Store totem_config value to cmap runtime section
218 if (strlen("runtime.config.") + strlen(key_name
) >= ICMAP_KEYNAME_MAXLEN
) {
220 * This shouldn't happen
225 strcpy(runtime_key_name
, "runtime.config.");
226 strcat(runtime_key_name
, key_name
);
228 icmap_set_string(runtime_key_name
, (char *)*config_value
);
233 * Read and validate config values from cmap and store them into totem_config. If key doesn't exists,
234 * default value is stored. deleted_key is name of key beeing processed by delete operation
235 * from cmap. It is considered as non existing even if it can be read. Can be NULL.
237 static void totem_volatile_config_read (struct totem_config
*totem_config
, const char *deleted_key
)
241 totem_volatile_config_set_uint32_value(totem_config
, "totem.token_retransmits_before_loss_const", deleted_key
,
242 TOKEN_RETRANSMITS_BEFORE_LOSS_CONST
, 0);
244 totem_volatile_config_set_uint32_value(totem_config
, "totem.token", deleted_key
, TOKEN_TIMEOUT
, 0);
246 if (totem_config
->interfaces
[0].member_count
> 2) {
247 u32
= TOKEN_COEFFICIENT
;
248 icmap_get_uint32("totem.token_coefficient", &u32
);
249 totem_config
->token_timeout
+= (totem_config
->interfaces
[0].member_count
- 2) * u32
;
252 * Store totem_config value to cmap runtime section
254 icmap_set_uint32("runtime.config.totem.token", totem_config
->token_timeout
);
257 totem_volatile_config_set_uint32_value(totem_config
, "totem.max_network_delay", deleted_key
, MAX_NETWORK_DELAY
, 0);
259 totem_volatile_config_set_uint32_value(totem_config
, "totem.window_size", deleted_key
, WINDOW_SIZE
, 0);
261 totem_volatile_config_set_uint32_value(totem_config
, "totem.max_messages", deleted_key
, MAX_MESSAGES
, 0);
263 totem_volatile_config_set_uint32_value(totem_config
, "totem.miss_count_const", deleted_key
, MISS_COUNT_CONST
, 0);
264 totem_volatile_config_set_uint32_value(totem_config
, "totem.knet_pmtud_interval", deleted_key
, KNET_PMTUD_INTERVAL
, 0);
266 totem_volatile_config_set_uint32_value(totem_config
, "totem.token_retransmit", deleted_key
,
267 (int)(totem_config
->token_timeout
/ (totem_config
->token_retransmits_before_loss_const
+ 0.2)), 0);
269 totem_volatile_config_set_uint32_value(totem_config
, "totem.hold", deleted_key
,
270 (int)(totem_config
->token_retransmit_timeout
* 0.8 - (1000/HZ
)), 0);
272 totem_volatile_config_set_uint32_value(totem_config
, "totem.join", deleted_key
, JOIN_TIMEOUT
, 0);
274 totem_volatile_config_set_uint32_value(totem_config
, "totem.consensus", deleted_key
,
275 (int)(float)(1.2 * totem_config
->token_timeout
), 0);
277 totem_volatile_config_set_uint32_value(totem_config
, "totem.merge", deleted_key
, MERGE_TIMEOUT
, 0);
279 totem_volatile_config_set_uint32_value(totem_config
, "totem.downcheck", deleted_key
, DOWNCHECK_TIMEOUT
, 0);
281 totem_volatile_config_set_uint32_value(totem_config
, "totem.fail_recv_const", deleted_key
, FAIL_TO_RECV_CONST
, 0);
283 totem_volatile_config_set_uint32_value(totem_config
, "totem.seqno_unchanged_const", deleted_key
,
284 SEQNO_UNCHANGED_CONST
, 0);
286 totem_volatile_config_set_uint32_value(totem_config
, "totem.send_join", deleted_key
, 0, 1);
288 totem_volatile_config_set_uint32_value(totem_config
, "totem.heartbeat_failures_allowed", deleted_key
, 0, 1);
290 totem_volatile_config_set_uint32_value(totem_config
, "totem.knet_compression_threshold", deleted_key
, 0, 1);
292 totem_volatile_config_set_int32_value(totem_config
, "totem.knet_compression_level", deleted_key
, 0, 1);
294 totem_volatile_config_set_string_value(totem_config
, "totem.knet_compression_model", deleted_key
, "none");
299 static int totem_volatile_config_validate (
300 struct totem_config
*totem_config
,
301 const char **error_string
)
303 static char local_error_reason
[512];
304 const char *error_reason
= local_error_reason
;
306 if (totem_config
->max_network_delay
< MINIMUM_TIMEOUT
) {
307 snprintf (local_error_reason
, sizeof(local_error_reason
),
308 "The max_network_delay parameter (%d ms) may not be less than (%d ms).",
309 totem_config
->max_network_delay
, MINIMUM_TIMEOUT
);
313 if (totem_config
->token_timeout
< MINIMUM_TIMEOUT
) {
314 snprintf (local_error_reason
, sizeof(local_error_reason
),
315 "The token timeout parameter (%d ms) may not be less than (%d ms).",
316 totem_config
->token_timeout
, MINIMUM_TIMEOUT
);
320 if (totem_config
->token_retransmit_timeout
< MINIMUM_TIMEOUT
) {
321 snprintf (local_error_reason
, sizeof(local_error_reason
),
322 "The token retransmit timeout parameter (%d ms) may not be less than (%d ms).",
323 totem_config
->token_retransmit_timeout
, MINIMUM_TIMEOUT
);
327 if (totem_config
->token_hold_timeout
< MINIMUM_TIMEOUT
) {
328 snprintf (local_error_reason
, sizeof(local_error_reason
),
329 "The token hold timeout parameter (%d ms) may not be less than (%d ms).",
330 totem_config
->token_hold_timeout
, MINIMUM_TIMEOUT
);
334 if (totem_config
->join_timeout
< MINIMUM_TIMEOUT
) {
335 snprintf (local_error_reason
, sizeof(local_error_reason
),
336 "The join timeout parameter (%d ms) may not be less than (%d ms).",
337 totem_config
->join_timeout
, MINIMUM_TIMEOUT
);
341 if (totem_config
->consensus_timeout
< MINIMUM_TIMEOUT
) {
342 snprintf (local_error_reason
, sizeof(local_error_reason
),
343 "The consensus timeout parameter (%d ms) may not be less than (%d ms).",
344 totem_config
->consensus_timeout
, MINIMUM_TIMEOUT
);
348 if (totem_config
->consensus_timeout
< totem_config
->join_timeout
) {
349 snprintf (local_error_reason
, sizeof(local_error_reason
),
350 "The consensus timeout parameter (%d ms) may not be less than join timeout (%d ms).",
351 totem_config
->consensus_timeout
, totem_config
->join_timeout
);
355 if (totem_config
->merge_timeout
< MINIMUM_TIMEOUT
) {
356 snprintf (local_error_reason
, sizeof(local_error_reason
),
357 "The merge timeout parameter (%d ms) may not be less than (%d ms).",
358 totem_config
->merge_timeout
, MINIMUM_TIMEOUT
);
362 if (totem_config
->downcheck_timeout
< MINIMUM_TIMEOUT
) {
363 snprintf (local_error_reason
, sizeof(local_error_reason
),
364 "The downcheck timeout parameter (%d ms) may not be less than (%d ms).",
365 totem_config
->downcheck_timeout
, MINIMUM_TIMEOUT
);
372 snprintf (error_string_response
, sizeof(error_string_response
),
373 "parse error in config: %s\n", error_reason
);
374 *error_string
= error_string_response
;
379 static int totem_get_crypto(struct totem_config
*totem_config
, const char **error_string
)
382 const char *tmp_cipher
;
383 const char *tmp_hash
;
384 const char *tmp_model
;
390 if (icmap_get_string("totem.crypto_model", &str
) == CS_OK
) {
391 if (strcmp(str
, "nss") == 0) {
394 if (strcmp(str
, "openssl") == 0) {
395 tmp_model
= "openssl";
402 if (icmap_get_string("totem.crypto_cipher", &str
) == CS_OK
) {
403 if (strcmp(str
, "none") == 0) {
406 if (strcmp(str
, "aes256") == 0) {
407 tmp_cipher
= "aes256";
409 if (strcmp(str
, "aes192") == 0) {
410 tmp_cipher
= "aes192";
412 if (strcmp(str
, "aes128") == 0) {
413 tmp_cipher
= "aes128";
415 if (strcmp(str
, "3des") == 0) {
421 if (icmap_get_string("totem.crypto_hash", &str
) == CS_OK
) {
422 if (strcmp(str
, "none") == 0) {
425 if (strcmp(str
, "md5") == 0) {
428 if (strcmp(str
, "sha1") == 0) {
431 if (strcmp(str
, "sha256") == 0) {
434 if (strcmp(str
, "sha384") == 0) {
437 if (strcmp(str
, "sha512") == 0) {
443 if ((strcmp(tmp_cipher
, "none") != 0) &&
444 (strcmp(tmp_hash
, "none") == 0)) {
445 *error_string
= "crypto_cipher requires crypto_hash with value other than none";
449 if (strcmp(tmp_model
, "none") == 0) {
450 *error_string
= "crypto_model should be 'nss' or 'openssl'";
454 free(totem_config
->crypto_cipher_type
);
455 free(totem_config
->crypto_hash_type
);
456 free(totem_config
->crypto_model
);
458 totem_config
->crypto_cipher_type
= strdup(tmp_cipher
);
459 totem_config
->crypto_hash_type
= strdup(tmp_hash
);
460 totem_config
->crypto_model
= strdup(tmp_model
);
465 static int totem_config_get_ip_version(struct totem_config
*totem_config
)
471 if (totem_config
->transport_number
== TOTEM_TRANSPORT_KNET
) {
474 if (icmap_get_string("totem.ip_version", &str
) == CS_OK
) {
475 if (strcmp(str
, "ipv4") == 0) {
478 if (strcmp(str
, "ipv6") == 0) {
488 static uint16_t generate_cluster_id (const char *cluster_name
)
493 for (i
= 0; i
< strlen(cluster_name
); i
++) {
495 value
+= cluster_name
[i
];
498 return (value
& 0xFFFF);
501 static int get_cluster_mcast_addr (
502 const char *cluster_name
,
503 unsigned int linknumber
,
505 struct totem_ip_address
*res
)
508 char addr
[INET6_ADDRSTRLEN
+ 1];
511 if (cluster_name
== NULL
) {
515 clusterid
= generate_cluster_id(cluster_name
) + linknumber
;
516 memset (res
, 0, sizeof(*res
));
518 switch (ip_version
) {
520 snprintf(addr
, sizeof(addr
), "239.192.%d.%d", clusterid
>> 8, clusterid
% 0xFF);
523 snprintf(addr
, sizeof(addr
), "ff15::%x", clusterid
);
532 err
= totemip_parse (res
, addr
, ip_version
);
537 static unsigned int generate_nodeid_for_duplicate_test(
538 struct totem_config
*totem_config
,
542 struct totem_ip_address totemip
;
544 /* AF_INET hard-coded here because auto-generated nodeids
546 if (totemip_parse(&totemip
, addr
, AF_INET
) != 0)
549 memcpy (&nodeid
, &totemip
.addr
, sizeof (unsigned int));
551 #if __BYTE_ORDER == __LITTLE_ENDIAN
552 nodeid
= swab32 (nodeid
);
555 if (totem_config
->clear_node_high_bit
) {
556 nodeid
&= 0x7FFFFFFF;
561 static int check_for_duplicate_nodeids(
562 struct totem_config
*totem_config
,
563 const char **error_string
)
566 icmap_iter_t subiter
;
567 const char *iter_key
;
570 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
571 char *ring0_addr
=NULL
;
572 char *ring0_addr1
=NULL
;
573 unsigned int node_pos
;
574 unsigned int node_pos1
;
576 unsigned int nodeid1
;
579 iter
= icmap_iter_init("nodelist.node.");
580 while ((iter_key
= icmap_iter_next(iter
, NULL
, NULL
)) != NULL
) {
581 res
= sscanf(iter_key
, "nodelist.node.%u.%s", &node_pos
, tmp_key
);
586 if (strcmp(tmp_key
, "ring0_addr") != 0) {
590 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.nodeid", node_pos
);
592 if (icmap_get_uint32(tmp_key
, &nodeid
) != CS_OK
) {
594 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.ring0_addr", node_pos
);
595 if (icmap_get_string(tmp_key
, &ring0_addr
) != CS_OK
) {
599 /* Generate nodeid so we can check that auto-generated nodeids don't clash either */
600 nodeid
= generate_nodeid_for_duplicate_test(totem_config
, ring0_addr
);
608 subiter
= icmap_iter_init("nodelist.node.");
609 while (((iter_key
= icmap_iter_next(subiter
, NULL
, NULL
)) != NULL
) && (node_pos1
< node_pos
)) {
610 res
= sscanf(iter_key
, "nodelist.node.%u.%s", &node_pos1
, tmp_key
);
611 if ((res
!= 2) || (node_pos1
>= node_pos
)) {
615 if (strcmp(tmp_key
, "ring0_addr") != 0) {
619 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.nodeid", node_pos1
);
620 if (icmap_get_uint32(tmp_key
, &nodeid1
) != CS_OK
) {
622 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.ring0_addr", node_pos1
);
623 if (icmap_get_string(tmp_key
, &ring0_addr1
) != CS_OK
) {
626 nodeid1
= generate_nodeid_for_duplicate_test(totem_config
, ring0_addr1
);
632 if (nodeid
== nodeid1
) {
634 snprintf (error_string_response
, sizeof(error_string_response
),
635 "Nodeid %u%s%s%s appears twice in corosync.conf", nodeid
,
636 autogenerated
?"(autogenerated from ":"",
637 autogenerated
?ring0_addr
:"",
638 autogenerated
?")":"");
639 log_printf (LOGSYS_LEVEL_ERROR
, error_string_response
);
640 *error_string
= error_string_response
;
644 icmap_iter_finalize(subiter
);
646 icmap_iter_finalize(iter
);
651 static int find_local_node_in_nodelist(struct totem_config
*totem_config
)
654 const char *iter_key
;
656 unsigned int node_pos
;
657 int local_node_pos
= -1;
658 struct totem_ip_address bind_addr
;
659 int interface_up
, interface_num
;
660 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
662 struct totem_ip_address node_addr
;
664 res
= totemip_iface_check(&totem_config
->interfaces
[0].bindnet
,
665 &bind_addr
, &interface_up
, &interface_num
,
666 totem_config
->clear_node_high_bit
);
671 iter
= icmap_iter_init("nodelist.node.");
672 while ((iter_key
= icmap_iter_next(iter
, NULL
, NULL
)) != NULL
) {
673 res
= sscanf(iter_key
, "nodelist.node.%u.%s", &node_pos
, tmp_key
);
678 if (strcmp(tmp_key
, "ring0_addr") != 0) {
682 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.ring0_addr", node_pos
);
683 if (icmap_get_string(tmp_key
, &node_addr_str
) != CS_OK
) {
687 res
= totemip_parse (&node_addr
, node_addr_str
, totem_config
->ip_version
);
693 if (totemip_equal(&bind_addr
, &node_addr
)) {
694 local_node_pos
= node_pos
;
697 icmap_iter_finalize(iter
);
699 return (local_node_pos
);
703 * This needs to be done last of all. It would be nice to do it when reading the
704 * interface params, but the totem params need to have them to be read first. We
705 * need both, so this is a way round that circular dependancy.
707 static void calc_knet_ping_timers(struct totem_config
*totem_config
)
709 char runtime_key_name
[ICMAP_KEYNAME_MAXLEN
];
712 for (interface
= 0; interface
< INTERFACE_MAX
; interface
++) {
714 if (totem_config
->interfaces
[interface
].configured
) {
715 if (!totem_config
->interfaces
[interface
].knet_ping_timeout
) {
716 totem_config
->interfaces
[interface
].knet_ping_timeout
=
717 totem_config
->token_timeout
/ totem_config
->interfaces
[interface
].knet_pong_count
;
719 snprintf(runtime_key_name
, sizeof(runtime_key_name
),
720 "runtime.config.totem.interface.%d.knet_ping_timeout", interface
);
721 icmap_set_uint32(runtime_key_name
, totem_config
->interfaces
[interface
].knet_ping_timeout
);
723 if (!totem_config
->interfaces
[interface
].knet_ping_interval
) {
724 totem_config
->interfaces
[interface
].knet_ping_interval
=
725 totem_config
->token_timeout
/ (totem_config
->interfaces
[interface
].knet_pong_count
* 2);
727 snprintf(runtime_key_name
, sizeof(runtime_key_name
),
728 "runtime.config.totem.interface.%d.knet_ping_interval", interface
);
729 icmap_set_uint32(runtime_key_name
, totem_config
->interfaces
[interface
].knet_ping_interval
);
735 * Compute difference between two set of totem interface arrays. set1 and set2
736 * are changed so for same ring, ip existing in both set1 and set2 are cleared
737 * (set to 0), and ips which are only in set1 or set2 remains untouched.
738 * totempg_node_add/remove is called.
740 static void compute_interfaces_diff(struct totem_interface
*set1
,
741 struct totem_interface
*set2
)
743 int ring_no
, set1_pos
, set2_pos
;
744 struct totem_ip_address empty_ip_address
;
746 memset(&empty_ip_address
, 0, sizeof(empty_ip_address
));
748 for (ring_no
= 0; ring_no
< INTERFACE_MAX
; ring_no
++) {
749 if (!set1
[ring_no
].configured
&& !set2
[ring_no
].configured
) {
753 for (set1_pos
= 0; set1_pos
< set1
[ring_no
].member_count
; set1_pos
++) {
754 for (set2_pos
= 0; set2_pos
< set2
[ring_no
].member_count
; set2_pos
++) {
756 * For current ring_no remove all set1 items existing
759 if (memcmp(&set1
[ring_no
].member_list
[set1_pos
],
760 &set2
[ring_no
].member_list
[set2_pos
],
761 sizeof(struct totem_ip_address
)) == 0) {
762 memset(&set1
[ring_no
].member_list
[set1_pos
], 0,
763 sizeof(struct totem_ip_address
));
764 memset(&set2
[ring_no
].member_list
[set2_pos
], 0,
765 sizeof(struct totem_ip_address
));
771 for (ring_no
= 0; ring_no
< INTERFACE_MAX
; ring_no
++) {
772 for (set1_pos
= 0; set1_pos
< set1
[ring_no
].member_count
; set1_pos
++) {
774 * All items which remained in set1 doesn't exists in set2 any longer so
775 * node has to be removed.
777 if (memcmp(&set1
[ring_no
].member_list
[set1_pos
], &empty_ip_address
, sizeof(empty_ip_address
)) != 0) {
778 log_printf(LOGSYS_LEVEL_DEBUG
,
779 "removing dynamic member %s for ring %u",
780 totemip_print(&set1
[ring_no
].member_list
[set1_pos
]),
783 totempg_member_remove(&set1
[ring_no
].member_list
[set1_pos
], ring_no
);
786 if (!set2
[ring_no
].configured
) {
789 for (set2_pos
= 0; set2_pos
< set2
[ring_no
].member_count
; set2_pos
++) {
791 * All items which remained in set2 doesn't existed in set1 so this is no node
792 * and has to be added.
794 if (memcmp(&set2
[ring_no
].member_list
[set2_pos
], &empty_ip_address
, sizeof(empty_ip_address
)) != 0) {
795 log_printf(LOGSYS_LEVEL_DEBUG
,
796 "adding dynamic member %s for ring %u",
797 totemip_print(&set2
[ring_no
].member_list
[set2_pos
]),
800 totempg_member_add(&set2
[ring_no
].member_list
[set2_pos
], ring_no
);
807 * Reconfigure links in totempg. Sets new local IP address and adds params for new links.
809 static void reconfigure_links(struct totem_config
*totem_config
)
812 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
814 struct totem_ip_address local_ip
;
816 unsigned int local_node_pos
= find_local_node_in_nodelist(totem_config
);
818 for (i
= 0; i
<INTERFACE_MAX
; i
++) {
819 if (!totem_config
->interfaces
[i
].configured
) {
823 log_printf(LOGSYS_LEVEL_INFO
, "Configuring link %d\n", i
);
825 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.ring%u_addr", local_node_pos
, i
);
826 if (icmap_get_string(tmp_key
, &addr_string
) != CS_OK
) {
830 err
= totemip_parse(&local_ip
, addr_string
, AF_UNSPEC
);
834 local_ip
.nodeid
= totem_config
->node_id
;
836 /* In case this is a new link, fill in the defaults if there was no interface{} section for it */
837 if (!totem_config
->interfaces
[i
].knet_link_priority
)
838 totem_config
->interfaces
[i
].knet_link_priority
= 1;
840 /* knet_ping_interval & knet_ping_timeout are set later once we know all the other params */
842 if (!totem_config
->interfaces
[i
].knet_ping_precision
)
843 totem_config
->interfaces
[i
].knet_ping_precision
= KNET_PING_PRECISION
;
844 if (!totem_config
->interfaces
[i
].knet_pong_count
)
845 totem_config
->interfaces
[i
].knet_pong_count
= KNET_PONG_COUNT
;
846 if (!totem_config
->interfaces
[i
].knet_transport
)
847 totem_config
->interfaces
[i
].knet_transport
= KNET_TRANSPORT_UDP
;
848 if (!totem_config
->interfaces
[i
].ip_port
)
849 totem_config
->interfaces
[i
].ip_port
= DEFAULT_PORT
+ i
;
851 totempg_iface_set(&local_ip
, totem_config
->interfaces
[i
].ip_port
, i
);
855 static void put_nodelist_members_to_config(struct totem_config
*totem_config
, int reload
)
857 icmap_iter_t iter
, iter2
;
858 const char *iter_key
, *iter_key2
;
860 unsigned int node_pos
;
861 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
862 char tmp_key2
[ICMAP_KEYNAME_MAXLEN
];
865 unsigned int linknumber
= 0;
867 struct totem_interface
*new_interfaces
= NULL
;
871 * We need to compute diff only for reload. Also for initial configuration
872 * not all totem structures are initialized so corosync will crash during
875 new_interfaces
= malloc (sizeof (struct totem_interface
) * INTERFACE_MAX
);
876 assert(new_interfaces
!= NULL
);
879 /* Clear out nodelist so we can put the new one in if needed */
880 for (i
= 0; i
< INTERFACE_MAX
; i
++) {
881 for (j
= 0; j
< PROCESSOR_COUNT_MAX
; j
++) {
882 memset(&totem_config
->interfaces
[i
].member_list
[j
], 0, sizeof(struct totem_ip_address
));
884 totem_config
->interfaces
[i
].member_count
= 0;
887 iter
= icmap_iter_init("nodelist.node.");
888 while ((iter_key
= icmap_iter_next(iter
, NULL
, NULL
)) != NULL
) {
889 res
= sscanf(iter_key
, "nodelist.node.%u.%s", &node_pos
, tmp_key
);
894 if (strcmp(tmp_key
, "ring0_addr") != 0) {
898 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.", node_pos
);
899 iter2
= icmap_iter_init(tmp_key
);
900 while ((iter_key2
= icmap_iter_next(iter2
, NULL
, NULL
)) != NULL
) {
903 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.nodeid", node_pos
);
904 if (icmap_get_uint32(tmp_key
, &nodeid
) != CS_OK
) {
907 res
= sscanf(iter_key2
, "nodelist.node.%u.ring%u%s", &node_pos
, &linknumber
, tmp_key2
);
908 if (res
!= 3 || strcmp(tmp_key2
, "_addr") != 0) {
912 if (icmap_get_string(iter_key2
, &node_addr_str
) != CS_OK
) {
916 member_count
= totem_config
->interfaces
[linknumber
].member_count
;
918 res
= totemip_parse(&totem_config
->interfaces
[linknumber
].member_list
[member_count
],
919 node_addr_str
, totem_config
->ip_version
);
921 totem_config
->interfaces
[linknumber
].member_list
[member_count
].nodeid
= nodeid
;
922 totem_config
->interfaces
[linknumber
].member_count
++;
925 totem_config
->interfaces
[linknumber
].configured
= 1;
929 icmap_iter_finalize(iter2
);
932 icmap_iter_finalize(iter
);
935 log_printf(LOGSYS_LEVEL_DEBUG
, "About to reconfigure links from nodelist.\n");
936 reconfigure_links(totem_config
);
938 memcpy(new_interfaces
, totem_config
->interfaces
, sizeof (struct totem_interface
) * INTERFACE_MAX
);
940 compute_interfaces_diff(totem_config
->orig_interfaces
, new_interfaces
);
942 free(new_interfaces
);
946 static void nodelist_dynamic_notify(
948 const char *key_name
,
949 struct icmap_notify_value new_val
,
950 struct icmap_notify_value old_val
,
954 unsigned int ring_no
;
955 unsigned int member_no
;
956 char tmp_str
[ICMAP_KEYNAME_MAXLEN
];
958 struct totem_config
*totem_config
= (struct totem_config
*)user_data
;
961 * If a full reload is in progress then don't do anything until it's done and
962 * can reconfigure it all atomically
964 if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading
) == CS_OK
&& reloading
) {
968 res
= sscanf(key_name
, "nodelist.node.%u.ring%u%s", &member_no
, &ring_no
, tmp_str
);
972 if (strcmp(tmp_str
, "_addr") != 0) {
976 put_nodelist_members_to_config(totem_config
, 1);
981 * Tries to find node (node_pos) in config nodelist which address matches any
982 * local interface. Address can be stored in ring0_addr or if ipaddr_key_prefix is not NULL
983 * key with prefix ipaddr_key is used (there can be multiuple of them)
984 * This function differs * from find_local_node_in_nodelist because it doesn't need bindnetaddr,
985 * but doesn't work when bind addr is network address (so IP must be exact
988 * Returns 1 on success (address was found, node_pos is then correctly set) or 0 on failure.
990 int totem_config_find_local_addr_in_nodelist(struct totem_config
*totem_config
, const char *ipaddr_key_prefix
, unsigned int *node_pos
)
992 struct qb_list_head addrs
;
993 struct totem_ip_if_address
*if_addr
;
994 icmap_iter_t iter
, iter2
;
995 const char *iter_key
, *iter_key2
;
996 struct qb_list_head
*list
;
997 const char *ipaddr_key
;
999 struct totem_ip_address node_addr
;
1000 char *node_addr_str
;
1003 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
1005 if (totemip_getifaddrs(&addrs
) == -1) {
1009 ip_version
= totem_config_get_ip_version(totem_config
);
1011 iter
= icmap_iter_init("nodelist.node.");
1013 while ((iter_key
= icmap_iter_next(iter
, NULL
, NULL
)) != NULL
) {
1014 res
= sscanf(iter_key
, "nodelist.node.%u.%s", node_pos
, tmp_key
);
1019 if (strcmp(tmp_key
, "ring0_addr") != 0) {
1023 if (icmap_get_string(iter_key
, &node_addr_str
) != CS_OK
) {
1027 free(node_addr_str
);
1030 * ring0_addr found -> let's iterate thru ipaddr_key_prefix
1032 snprintf(tmp_key
, sizeof(tmp_key
), "nodelist.node.%u.%s", *node_pos
,
1033 (ipaddr_key_prefix
!= NULL
? ipaddr_key_prefix
: "ring0_addr"));
1035 iter2
= icmap_iter_init(tmp_key
);
1036 while ((iter_key2
= icmap_iter_next(iter2
, NULL
, NULL
)) != NULL
) {
1038 * ring0_addr must be exact match, not prefix
1040 ipaddr_key
= (ipaddr_key_prefix
!= NULL
? iter_key2
: tmp_key
);
1041 if (icmap_get_string(ipaddr_key
, &node_addr_str
) != CS_OK
) {
1045 if (totemip_parse(&node_addr
, node_addr_str
, ip_version
) == -1) {
1046 free(node_addr_str
);
1049 free(node_addr_str
);
1052 * Try to match ip with if_addrs
1055 qb_list_for_each(list
, &(addrs
)) {
1056 if_addr
= qb_list_entry(list
, struct totem_ip_if_address
, list
);
1058 if (totemip_equal(&node_addr
, &if_addr
->ip_addr
)) {
1069 icmap_iter_finalize(iter2
);
1076 icmap_iter_finalize(iter
);
1077 totemip_freeifaddrs(&addrs
);
1079 return (node_found
);
1082 static void config_convert_nodelist_to_interface(struct totem_config
*totem_config
)
1085 unsigned int node_pos
;
1086 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
1087 char tmp_key2
[ICMAP_KEYNAME_MAXLEN
];
1088 char *node_addr_str
;
1089 unsigned int linknumber
= 0;
1091 const char *iter_key
;
1093 if (totem_config_find_local_addr_in_nodelist(totem_config
, NULL
, &node_pos
)) {
1095 * We found node, so create interface section
1097 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.", node_pos
);
1098 iter
= icmap_iter_init(tmp_key
);
1099 while ((iter_key
= icmap_iter_next(iter
, NULL
, NULL
)) != NULL
) {
1100 res
= sscanf(iter_key
, "nodelist.node.%u.ring%u%s", &node_pos
, &linknumber
, tmp_key2
);
1101 if (res
!= 3 || strcmp(tmp_key2
, "_addr") != 0) {
1105 if (icmap_get_string(iter_key
, &node_addr_str
) != CS_OK
) {
1109 snprintf(tmp_key2
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.bindnetaddr", linknumber
);
1110 icmap_set_string(tmp_key2
, node_addr_str
);
1111 free(node_addr_str
);
1113 icmap_iter_finalize(iter
);
1117 static int get_interface_params(struct totem_config
*totem_config
,
1118 const char **error_string
, uint64_t *warnings
,
1122 unsigned int linknumber
= 0;
1123 int member_count
= 0;
1125 icmap_iter_t iter
, member_iter
;
1126 const char *iter_key
;
1127 const char *member_iter_key
;
1128 char linknumber_key
[ICMAP_KEYNAME_MAXLEN
];
1129 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
1133 char *cluster_name
= NULL
;
1136 for (i
=0; i
<INTERFACE_MAX
; i
++) {
1138 * Set back to defaults things that might have been configured and
1139 * now have been taken out of corosync.conf. These won't be caught by the
1140 * code below which only looks at interface{} sections that actually exist.
1142 totem_config
->interfaces
[i
].configured
= 0;
1143 totem_config
->interfaces
[i
].knet_ping_timeout
= 0;
1144 totem_config
->interfaces
[i
].knet_ping_interval
= 0;
1145 totem_config
->interfaces
[i
].knet_ping_precision
= KNET_PING_PRECISION
;
1146 totem_config
->interfaces
[i
].knet_pong_count
= KNET_PONG_COUNT
;
1149 if (icmap_get_string("totem.cluster_name", &cluster_name
) != CS_OK
) {
1150 cluster_name
= NULL
;
1153 iter
= icmap_iter_init("totem.interface.");
1154 while ((iter_key
= icmap_iter_next(iter
, NULL
, NULL
)) != NULL
) {
1155 res
= sscanf(iter_key
, "totem.interface.%[^.].%s", linknumber_key
, tmp_key
);
1160 if (strcmp(tmp_key
, "bindnetaddr") != 0 && totem_config
->transport_number
== TOTEM_TRANSPORT_UDP
) {
1165 linknumber
= atoi(linknumber_key
);
1167 if (linknumber
>= INTERFACE_MAX
) {
1170 snprintf (error_string_response
, sizeof(error_string_response
),
1171 "parse error in config: interface ring number %u is bigger than allowed maximum %u\n",
1172 linknumber
, INTERFACE_MAX
- 1);
1174 *error_string
= error_string_response
;
1178 /* These things are only valid for the initial read */
1181 * Get the bind net address
1183 if (icmap_get_string(iter_key
, &str
) == CS_OK
) {
1184 res
= totemip_parse (&totem_config
->interfaces
[linknumber
].bindnet
, str
,
1185 totem_config
->ip_version
);
1190 * Get interface multicast address
1192 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.mcastaddr", linknumber
);
1193 if (icmap_get_string(tmp_key
, &str
) == CS_OK
) {
1194 res
= totemip_parse (&totem_config
->interfaces
[linknumber
].mcast_addr
, str
, totem_config
->ip_version
);
1198 * User not specified address -> autogenerate one from cluster_name key
1199 * (if available). Return code is intentionally ignored, because
1200 * udpu doesn't need mcastaddr and validity of mcastaddr for udp is
1201 * checked later anyway.
1203 (void)get_cluster_mcast_addr (cluster_name
,
1205 totem_config
->ip_version
,
1206 &totem_config
->interfaces
[linknumber
].mcast_addr
);
1209 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.broadcast", linknumber
);
1210 if (icmap_get_string(tmp_key
, &str
) == CS_OK
) {
1211 if (strcmp (str
, "yes") == 0) {
1212 totem_config
->broadcast_use
= 1;
1218 /* These things are only valid for the initial read OR a newly-defined link */
1219 if (!reload
|| (totem_config
->interfaces
[linknumber
].configured
== 0)) {
1224 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.mcastport", linknumber
);
1225 if (icmap_get_uint16(tmp_key
, &totem_config
->interfaces
[linknumber
].ip_port
) != CS_OK
) {
1226 if (totem_config
->broadcast_use
) {
1227 totem_config
->interfaces
[linknumber
].ip_port
= DEFAULT_PORT
+ (2 * linknumber
);
1229 totem_config
->interfaces
[linknumber
].ip_port
= DEFAULT_PORT
+ linknumber
;
1236 totem_config
->interfaces
[linknumber
].ttl
= 1;
1238 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.ttl", linknumber
);
1240 if (icmap_get_uint8(tmp_key
, &u8
) == CS_OK
) {
1241 totem_config
->interfaces
[linknumber
].ttl
= u8
;
1244 totem_config
->interfaces
[linknumber
].knet_transport
= KNET_DEFAULT_TRANSPORT
;
1245 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.knet_transport", linknumber
);
1246 if (icmap_get_string(tmp_key
, &str
) == CS_OK
) {
1247 if (strcmp(str
, "sctp") == 0) {
1248 totem_config
->interfaces
[linknumber
].knet_transport
= KNET_TRANSPORT_SCTP
;
1250 else if (strcmp(str
, "udp") == 0) {
1251 totem_config
->interfaces
[linknumber
].knet_transport
= KNET_TRANSPORT_UDP
;
1254 *error_string
= "Unrecognised knet_transport. expected 'udp' or 'sctp'";
1259 totem_config
->interfaces
[linknumber
].configured
= 1;
1262 * Get the knet link params
1264 totem_config
->interfaces
[linknumber
].knet_link_priority
= 1;
1265 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.knet_link_priority", linknumber
);
1267 if (icmap_get_uint8(tmp_key
, &u8
) == CS_OK
) {
1268 totem_config
->interfaces
[linknumber
].knet_link_priority
= u8
;
1271 totem_config
->interfaces
[linknumber
].knet_ping_interval
= 0; /* real default applied later */
1272 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.knet_ping_interval", linknumber
);
1273 if (icmap_get_uint32(tmp_key
, &u32
) == CS_OK
) {
1274 totem_config
->interfaces
[linknumber
].knet_ping_interval
= u32
;
1276 totem_config
->interfaces
[linknumber
].knet_ping_timeout
= 0; /* real default applied later */
1277 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.knet_ping_timeout", linknumber
);
1278 if (icmap_get_uint32(tmp_key
, &u32
) == CS_OK
) {
1279 totem_config
->interfaces
[linknumber
].knet_ping_timeout
= u32
;
1281 totem_config
->interfaces
[linknumber
].knet_ping_precision
= KNET_PING_PRECISION
;
1282 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.knet_ping_precision", linknumber
);
1283 if (icmap_get_uint32(tmp_key
, &u32
) == CS_OK
) {
1284 totem_config
->interfaces
[linknumber
].knet_ping_precision
= u32
;
1286 totem_config
->interfaces
[linknumber
].knet_pong_count
= KNET_PONG_COUNT
;
1287 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.knet_pong_count", linknumber
);
1288 if (icmap_get_uint32(tmp_key
, &u32
) == CS_OK
) {
1289 totem_config
->interfaces
[linknumber
].knet_pong_count
= u32
;
1292 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.member.", linknumber
);
1293 member_iter
= icmap_iter_init(tmp_key
);
1294 while ((member_iter_key
= icmap_iter_next(member_iter
, NULL
, NULL
)) != NULL
) {
1295 if (member_count
== 0) {
1296 if (icmap_get_string("nodelist.node.0.ring0_addr", &str
) == CS_OK
) {
1298 *warnings
|= TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
;
1301 *warnings
|= TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
;
1305 if (icmap_get_string(member_iter_key
, &str
) == CS_OK
) {
1306 res
= totemip_parse (&totem_config
->interfaces
[linknumber
].member_list
[member_count
++],
1307 str
, totem_config
->ip_version
);
1310 icmap_iter_finalize(member_iter
);
1312 totem_config
->interfaces
[linknumber
].member_count
= member_count
;
1315 icmap_iter_finalize(iter
);
1320 extern int totem_config_read (
1321 struct totem_config
*totem_config
,
1322 const char **error_string
,
1326 char *str
, *ring0_addr_str
;
1327 char tmp_key
[ICMAP_KEYNAME_MAXLEN
];
1335 memset (totem_config
, 0, sizeof (struct totem_config
));
1336 totem_config
->interfaces
= malloc (sizeof (struct totem_interface
) * INTERFACE_MAX
);
1337 if (totem_config
->interfaces
== 0) {
1338 *error_string
= "Out of memory trying to allocate ethernet interface storage area";
1342 totem_config
->transport_number
= TOTEM_TRANSPORT_KNET
;
1343 if (icmap_get_string("totem.transport", &str
) == CS_OK
) {
1344 if (strcmp (str
, "udpu") == 0) {
1345 totem_config
->transport_number
= TOTEM_TRANSPORT_UDPU
;
1348 if (strcmp (str
, "udp") == 0) {
1349 totem_config
->transport_number
= TOTEM_TRANSPORT_UDP
;
1352 if (strcmp (str
, "knet") == 0) {
1353 totem_config
->transport_number
= TOTEM_TRANSPORT_KNET
;
1359 memset (totem_config
->interfaces
, 0,
1360 sizeof (struct totem_interface
) * INTERFACE_MAX
);
1362 strcpy (totem_config
->link_mode
, "passive");
1364 icmap_get_uint32("totem.version", (uint32_t *)&totem_config
->version
);
1366 if (totem_get_crypto(totem_config
, error_string
) != 0) {
1370 if (icmap_get_string("totem.link_mode", &str
) == CS_OK
) {
1371 if (strlen(str
) >= TOTEM_LINK_MODE_BYTES
) {
1372 *error_string
= "totem.link_mode is too long";
1377 strcpy (totem_config
->link_mode
, str
);
1381 icmap_get_uint32("totem.nodeid", &totem_config
->node_id
);
1383 totem_config
->clear_node_high_bit
= 0;
1384 if (icmap_get_string("totem.clear_node_high_bit", &str
) == CS_OK
) {
1385 if (strcmp (str
, "yes") == 0) {
1386 totem_config
->clear_node_high_bit
= 1;
1391 icmap_get_uint32("totem.threads", &totem_config
->threads
);
1393 icmap_get_uint32("totem.netmtu", &totem_config
->net_mtu
);
1395 totem_config
->ip_version
= totem_config_get_ip_version(totem_config
);
1397 if (icmap_get_string("totem.interface.0.bindnetaddr", &str
) != CS_OK
) {
1399 * We were not able to find ring 0 bindnet addr. Try to use nodelist informations
1401 config_convert_nodelist_to_interface(totem_config
);
1403 if (icmap_get_string("nodelist.node.0.ring0_addr", &ring0_addr_str
) == CS_OK
) {
1405 * Both bindnetaddr and ring0_addr are set.
1406 * Log warning information, and use nodelist instead
1408 *warnings
|= TOTEM_CONFIG_BINDNETADDR_NODELIST_SET
;
1410 config_convert_nodelist_to_interface(totem_config
);
1412 free(ring0_addr_str
);
1419 * Broadcast option is global but set in interface section,
1420 * so reset before processing interfaces.
1422 totem_config
->broadcast_use
= 0;
1424 res
= get_interface_params(totem_config
, error_string
, warnings
, 0);
1430 * Use broadcast is global, so if set, make sure to fill mcast addr correctly
1431 * broadcast is only supported for UDP so just do interface 0;
1433 if (totem_config
->broadcast_use
) {
1434 totemip_parse (&totem_config
->interfaces
[0].mcast_addr
,
1435 "255.255.255.255", 0);
1440 * Store automatically generated items back to icmap only for UDP
1442 if (totem_config
->transport_number
== TOTEM_TRANSPORT_UDP
) {
1443 for (i
= 0; i
< INTERFACE_MAX
; i
++) {
1444 if (!totem_config
->interfaces
[i
].configured
) {
1447 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.mcastaddr", i
);
1448 if (icmap_get_string(tmp_key
, &str
) == CS_OK
) {
1451 str
= (char *)totemip_print(&totem_config
->interfaces
[i
].mcast_addr
);
1452 icmap_set_string(tmp_key
, str
);
1455 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "totem.interface.%u.mcastport", i
);
1456 if (icmap_get_uint16(tmp_key
, &u16
) != CS_OK
) {
1457 icmap_set_uint16(tmp_key
, totem_config
->interfaces
[i
].ip_port
);
1463 * Check existence of nodelist
1465 if (icmap_get_string("nodelist.node.0.ring0_addr", &str
) == CS_OK
) {
1470 local_node_pos
= find_local_node_in_nodelist(totem_config
);
1471 if (local_node_pos
!= -1) {
1472 icmap_set_uint32("nodelist.local_node_pos", local_node_pos
);
1474 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.nodeid", local_node_pos
);
1476 nodeid_set
= (totem_config
->node_id
!= 0);
1477 if (icmap_get_uint32(tmp_key
, &totem_config
->node_id
) == CS_OK
&& nodeid_set
) {
1478 *warnings
|= TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED
;
1480 if ((totem_config
->transport_number
== TOTEM_TRANSPORT_KNET
) && (!totem_config
->node_id
)) {
1481 *error_string
= "With knet, you must specify nodeid for current node";
1486 * Make localnode ring0_addr read only, so we can be sure that local
1487 * node never changes. If rebinding to other IP would be in future
1488 * supported, this must be changed and handled properly!
1490 snprintf(tmp_key
, ICMAP_KEYNAME_MAXLEN
, "nodelist.node.%u.ring0_addr", local_node_pos
);
1491 icmap_set_ro_access(tmp_key
, 0, 1);
1492 icmap_set_ro_access("nodelist.local_node_pos", 0, 1);
1495 put_nodelist_members_to_config(totem_config
, 0);
1499 * Get things that might change in the future (and can depend on totem_config->interfaces);
1501 totem_volatile_config_read(totem_config
, NULL
);
1503 calc_knet_ping_timers(totem_config
);
1505 icmap_set_uint8("config.totemconfig_reload_in_progress", 0);
1507 add_totem_config_notification(totem_config
);
1513 int totem_config_validate (
1514 struct totem_config
*totem_config
,
1515 const char **error_string
)
1517 static char local_error_reason
[512];
1518 char parse_error
[512];
1519 const char *error_reason
= local_error_reason
;
1522 int num_configured
= 0;
1523 unsigned int interface_max
= INTERFACE_MAX
;
1527 for (i
= 0; i
< INTERFACE_MAX
; i
++) {
1528 if (totem_config
->interfaces
[i
].configured
) {
1532 if (num_configured
== 0) {
1533 error_reason
= "No interfaces defined";
1537 /* Check we found a local node address */
1538 if (icmap_get_uint32("nodelist.local_node_pos", &u32
) != CS_OK
) {
1539 error_reason
= "No valid address found for local host";
1543 for (i
= 0; i
< INTERFACE_MAX
; i
++) {
1545 * Some error checking of parsed data to make sure its valid
1548 struct totem_ip_address null_addr
;
1550 if (!totem_config
->interfaces
[i
].configured
) {
1554 memset (&null_addr
, 0, sizeof (struct totem_ip_address
));
1556 if ((totem_config
->transport_number
== TOTEM_TRANSPORT_UDP
) &&
1557 memcmp (&totem_config
->interfaces
[i
].mcast_addr
, &null_addr
,
1558 sizeof (struct totem_ip_address
)) == 0) {
1559 error_reason
= "No multicast address specified";
1563 if (totem_config
->interfaces
[i
].ip_port
== 0) {
1564 error_reason
= "No multicast port specified";
1568 if (totem_config
->interfaces
[i
].ttl
> 255) {
1569 error_reason
= "Invalid TTL (should be 0..255)";
1572 if (totem_config
->transport_number
!= TOTEM_TRANSPORT_UDP
&&
1573 totem_config
->interfaces
[i
].ttl
!= 1) {
1574 error_reason
= "Can only set ttl on multicast transport types";
1577 if (totem_config
->interfaces
[i
].knet_link_priority
> 255) {
1578 error_reason
= "Invalid link priority (should be 0..255)";
1581 if (totem_config
->transport_number
!= TOTEM_TRANSPORT_KNET
&&
1582 totem_config
->interfaces
[i
].knet_link_priority
!= 1) {
1583 error_reason
= "Can only set link priority on knet transport type";
1587 if (totem_config
->interfaces
[i
].mcast_addr
.family
== AF_INET6
&&
1588 totem_config
->node_id
== 0) {
1590 error_reason
= "An IPV6 network requires that a node ID be specified.";
1594 if (totem_config
->broadcast_use
== 0 && totem_config
->transport_number
== TOTEM_TRANSPORT_UDP
) {
1595 if (totem_config
->interfaces
[i
].mcast_addr
.family
!= totem_config
->interfaces
[i
].bindnet
.family
) {
1596 error_reason
= "Multicast address family does not match bind address family";
1600 if (totemip_is_mcast (&totem_config
->interfaces
[i
].mcast_addr
) != 0) {
1601 error_reason
= "mcastaddr is not a correct multicast address.";
1605 /* Verify that all nodes on the same knet link have the same IP family */
1606 for (j
=1; j
<totem_config
->interfaces
[i
].member_count
; j
++) {
1607 if (totem_config
->interfaces
[i
].configured
) {
1608 if (totem_config
->interfaces
[i
].member_list
[j
].family
!=
1609 totem_config
->interfaces
[i
].member_list
[0].family
) {
1610 snprintf (local_error_reason
, sizeof(local_error_reason
),
1611 "Nodes for link %d have different IP families", i
);
1618 if (totem_config
->version
!= 2) {
1619 error_reason
= "This totem parser can only parse version 2 configurations.";
1623 if (totem_volatile_config_validate(totem_config
, error_string
) == -1) {
1627 if (check_for_duplicate_nodeids(totem_config
, error_string
) == -1) {
1632 * KNET Link values validation
1634 if (strcmp (totem_config
->link_mode
, "active") &&
1635 strcmp (totem_config
->link_mode
, "rr") &&
1636 strcmp (totem_config
->link_mode
, "passive")) {
1637 snprintf (local_error_reason
, sizeof(local_error_reason
),
1638 "The Knet link mode \"%s\" specified is invalid. It must be active, passive or rr.\n", totem_config
->link_mode
);
1642 /* Only Knet does multiple interfaces */
1643 if (totem_config
->transport_number
!= TOTEM_TRANSPORT_KNET
) {
1647 if (interface_max
< num_configured
) {
1648 snprintf (parse_error
, sizeof(parse_error
),
1649 "%d is too many configured interfaces for non-Knet transport.",
1651 error_reason
= parse_error
;
1655 /* Only knet allows crypto */
1656 if (totem_config
->transport_number
!= TOTEM_TRANSPORT_KNET
) {
1657 if ((strcmp(totem_config
->crypto_cipher_type
, "none") != 0) ||
1658 (strcmp(totem_config
->crypto_hash_type
, "none") != 0)) {
1660 snprintf (parse_error
, sizeof(parse_error
),
1661 "crypto_cipher & crypto_hash are only valid for the Knet transport.");
1662 error_reason
= parse_error
;
1667 if (totem_config
->net_mtu
== 0) {
1668 if (totem_config
->transport_number
== TOTEM_TRANSPORT_KNET
) {
1669 totem_config
->net_mtu
= KNET_MAX_PACKET_SIZE
;
1672 totem_config
->net_mtu
= 1500;
1679 snprintf (error_string_response
, sizeof(error_string_response
),
1680 "parse error in config: %s\n", error_reason
);
1681 *error_string
= error_string_response
;
1686 static int read_keyfile (
1687 const char *key_location
,
1688 struct totem_config
*totem_config
,
1689 const char **error_string
)
1694 char error_str
[100];
1695 const char *error_ptr
;
1697 fd
= open (key_location
, O_RDONLY
);
1699 error_ptr
= qb_strerror_r(errno
, error_str
, sizeof(error_str
));
1700 snprintf (error_string_response
, sizeof(error_string_response
),
1701 "Could not open %s: %s\n",
1702 key_location
, error_ptr
);
1706 res
= read (fd
, totem_config
->private_key
, TOTEM_PRIVATE_KEY_LEN_MAX
);
1707 saved_errno
= errno
;
1711 error_ptr
= qb_strerror_r (saved_errno
, error_str
, sizeof(error_str
));
1712 snprintf (error_string_response
, sizeof(error_string_response
),
1713 "Could not read %s: %s\n",
1714 key_location
, error_ptr
);
1718 if (res
< TOTEM_PRIVATE_KEY_LEN_MIN
) {
1719 snprintf (error_string_response
, sizeof(error_string_response
),
1720 "Could only read %d bits of minimum %u bits from %s.\n",
1721 res
* 8, TOTEM_PRIVATE_KEY_LEN_MIN
* 8, key_location
);
1725 totem_config
->private_key_len
= res
;
1730 *error_string
= error_string_response
;
1734 int totem_config_keyread (
1735 struct totem_config
*totem_config
,
1736 const char **error_string
)
1739 char *key_location
= NULL
;
1743 memset (totem_config
->private_key
, 0, sizeof(totem_config
->private_key
));
1744 totem_config
->private_key_len
= 0;
1746 if (strcmp(totem_config
->crypto_cipher_type
, "none") == 0 &&
1747 strcmp(totem_config
->crypto_hash_type
, "none") == 0) {
1751 /* cmap may store the location of the key file */
1752 if (icmap_get_string("totem.keyfile", &key_location
) == CS_OK
) {
1753 res
= read_keyfile(key_location
, totem_config
, error_string
);
1759 } else { /* Or the key itself may be in the cmap */
1760 if (icmap_get("totem.key", NULL
, &key_len
, NULL
) == CS_OK
) {
1761 if (key_len
> sizeof(totem_config
->private_key
)) {
1762 sprintf(error_string_response
, "key is too long");
1765 if (key_len
< TOTEM_PRIVATE_KEY_LEN_MIN
) {
1766 sprintf(error_string_response
, "key is too short");
1769 if (icmap_get("totem.key", totem_config
->private_key
, &key_len
, NULL
) == CS_OK
) {
1770 totem_config
->private_key_len
= key_len
;
1773 sprintf(error_string_response
, "can't load private key");
1779 /* In desperation we read the default filename */
1781 const char *filename
= getenv("COROSYNC_TOTEM_AUTHKEY_FILE");
1783 filename
= COROSYSCONFDIR
"/authkey";
1784 res
= read_keyfile(filename
, totem_config
, error_string
);
1793 *error_string
= error_string_response
;
1798 static void debug_dump_totem_config(const struct totem_config
*totem_config
)
1801 log_printf(LOGSYS_LEVEL_DEBUG
, "Token Timeout (%d ms) retransmit timeout (%d ms)",
1802 totem_config
->token_timeout
, totem_config
->token_retransmit_timeout
);
1803 log_printf(LOGSYS_LEVEL_DEBUG
, "token hold (%d ms) retransmits before loss (%d retrans)",
1804 totem_config
->token_hold_timeout
, totem_config
->token_retransmits_before_loss_const
);
1805 log_printf(LOGSYS_LEVEL_DEBUG
, "join (%d ms) send_join (%d ms) consensus (%d ms) merge (%d ms)",
1806 totem_config
->join_timeout
, totem_config
->send_join_timeout
, totem_config
->consensus_timeout
,
1807 totem_config
->merge_timeout
);
1808 log_printf(LOGSYS_LEVEL_DEBUG
, "downcheck (%d ms) fail to recv const (%d msgs)",
1809 totem_config
->downcheck_timeout
, totem_config
->fail_to_recv_const
);
1810 log_printf(LOGSYS_LEVEL_DEBUG
,
1811 "seqno unchanged const (%d rotations) Maximum network MTU %d",
1812 totem_config
->seqno_unchanged_const
, totem_config
->net_mtu
);
1813 log_printf(LOGSYS_LEVEL_DEBUG
,
1814 "window size per rotation (%d messages) maximum messages per rotation (%d messages)",
1815 totem_config
->window_size
, totem_config
->max_messages
);
1816 log_printf(LOGSYS_LEVEL_DEBUG
, "missed count const (%d messages)", totem_config
->miss_count_const
);
1817 log_printf(LOGSYS_LEVEL_DEBUG
, "heartbeat_failures_allowed (%d)",
1818 totem_config
->heartbeat_failures_allowed
);
1819 log_printf(LOGSYS_LEVEL_DEBUG
, "max_network_delay (%d ms)", totem_config
->max_network_delay
);
1822 static void totem_change_notify(
1824 const char *key_name
,
1825 struct icmap_notify_value new_val
,
1826 struct icmap_notify_value old_val
,
1829 struct totem_config
*totem_config
= (struct totem_config
*)user_data
;
1832 const char *deleted_key
= NULL
;
1833 const char *error_string
;
1836 * If a full reload is in progress then don't do anything until it's done and
1837 * can reconfigure it all atomically
1839 if (icmap_get_uint8("config.reload_in_progress", &reloading
) == CS_OK
&& reloading
)
1842 param
= totem_get_param_by_name((struct totem_config
*)user_data
, key_name
);
1844 * Process change only if changed key is found in totem_config (-> param is not NULL)
1845 * or for special key token_coefficient. token_coefficient key is not stored in
1846 * totem_config, but it is used for computation of token timeout.
1848 if (!param
&& strcmp(key_name
, "totem.token_coefficient") != 0)
1852 * Values other than UINT32 are not supported, or needed (yet)
1855 case ICMAP_TRACK_DELETE
:
1856 deleted_key
= key_name
;
1858 case ICMAP_TRACK_ADD
:
1859 case ICMAP_TRACK_MODIFY
:
1866 totem_volatile_config_read (totem_config
, deleted_key
);
1867 log_printf(LOGSYS_LEVEL_DEBUG
, "Totem related config key changed. Dumping actual totem config.");
1868 debug_dump_totem_config(totem_config
);
1869 if (totem_volatile_config_validate(totem_config
, &error_string
) == -1) {
1870 log_printf (LOGSYS_LEVEL_ERROR
, "%s", error_string
);
1872 * TODO: Consider corosync exit and/or load defaults for volatile
1873 * values. For now, log error seems to be enough
1878 static void totem_reload_notify(
1880 const char *key_name
,
1881 struct icmap_notify_value new_val
,
1882 struct icmap_notify_value old_val
,
1885 struct totem_config
*totem_config
= (struct totem_config
*)user_data
;
1886 uint32_t local_node_pos
;
1887 const char *error_string
;
1890 /* Reload has completed */
1891 if (*(uint8_t *)new_val
.data
== 0) {
1893 totem_config
->orig_interfaces
= malloc (sizeof (struct totem_interface
) * INTERFACE_MAX
);
1894 assert(totem_config
->orig_interfaces
!= NULL
);
1895 memcpy(totem_config
->orig_interfaces
, totem_config
->interfaces
, sizeof (struct totem_interface
) * INTERFACE_MAX
);
1897 get_interface_params(totem_config
, &error_string
, &warnings
, 1);
1898 put_nodelist_members_to_config (totem_config
, 1);
1899 totem_volatile_config_read (totem_config
, NULL
);
1901 calc_knet_ping_timers(totem_config
);
1903 log_printf(LOGSYS_LEVEL_DEBUG
, "Configuration reloaded. Dumping actual totem config.");
1904 debug_dump_totem_config(totem_config
);
1905 if (totem_volatile_config_validate(totem_config
, &error_string
) == -1) {
1906 log_printf (LOGSYS_LEVEL_ERROR
, "%s", error_string
);
1908 * TODO: Consider corosync exit and/or load defaults for volatile
1909 * values. For now, log error seems to be enough
1913 /* Reinstate the local_node_pos */
1914 local_node_pos
= find_local_node_in_nodelist(totem_config
);
1915 if (local_node_pos
!= -1) {
1916 icmap_set_uint32("nodelist.local_node_pos", local_node_pos
);
1919 /* Reconfigure network params as appropriate */
1920 totempg_reconfigure();
1922 free(totem_config
->orig_interfaces
);
1924 icmap_set_uint8("config.totemconfig_reload_in_progress", 0);
1926 icmap_set_uint8("config.totemconfig_reload_in_progress", 1);
1930 static void add_totem_config_notification(struct totem_config
*totem_config
)
1932 icmap_track_t icmap_track
;
1934 icmap_track_add("totem.",
1935 ICMAP_TRACK_ADD
| ICMAP_TRACK_DELETE
| ICMAP_TRACK_MODIFY
| ICMAP_TRACK_PREFIX
,
1936 totem_change_notify
,
1940 icmap_track_add("config.reload_in_progress",
1941 ICMAP_TRACK_ADD
| ICMAP_TRACK_MODIFY
,
1942 totem_reload_notify
,
1946 icmap_track_add("nodelist.node.",
1947 ICMAP_TRACK_ADD
| ICMAP_TRACK_DELETE
| ICMAP_TRACK_MODIFY
| ICMAP_TRACK_PREFIX
,
1948 nodelist_dynamic_notify
,
1949 (void *)totem_config
,