]> git.proxmox.com Git - mirror_corosync.git/blob - exec/totemknet.c
projects / mirror_corosync.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
configure: Modernize configure.ac a bit
[mirror_corosync.git] / exec / totemknet.c
1 /*
2 * Copyright (c) 2016-2020 Red Hat, Inc.
3 *
4 * All rights reserved.
5 *
6 * Author: Christine Caulfield (ccaulfie@redhat.com)
7
8 * This software licensed under BSD license, the text of which follows:
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * - Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * - Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 * - Neither the name of the MontaVista Software, Inc. nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <config.h>
36
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75
76 #include "main.h"
77 #include "util.h"
78
79 #include <libknet.h>
80 #include <corosync/totem/totemstats.h>
81
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92
93 struct totemknet_instance {
94 struct crypto_instance *crypto_inst;
95
96 qb_loop_t *poll_handle;
97
98 knet_handle_t knet_handle;
99
100 int link_mode;
101
102 void *context;
103
104 void (*totemknet_deliver_fn) (
105 void *context,
106 const void *msg,
107 unsigned int msg_len,
108 const struct sockaddr_storage *system_from);
109
110 void (*totemknet_iface_change_fn) (
111 void *context,
112 const struct totem_ip_address *iface_address,
113 unsigned int link_no);
114
115 void (*totemknet_mtu_changed) (
116 void *context,
117 int net_mtu);
118
119 void (*totemknet_target_set_completed) (void *context);
120
121 /*
122 * Function and data used to log messages
123 */
124 int totemknet_log_level_security;
125
126 int totemknet_log_level_error;
127
128 int totemknet_log_level_warning;
129
130 int totemknet_log_level_notice;
131
132 int totemknet_log_level_debug;
133
134 int totemknet_subsys_id;
135
136 int knet_subsys_id;
137
138 void (*totemknet_log_printf) (
139 int level,
140 int subsys,
141 const char *function,
142 const char *file,
143 int line,
144 const char *format,
145 ...)__attribute__((format(printf, 6, 7)));
146
147 void *knet_context;
148
149 char iov_buffer[KNET_MAX_PACKET_SIZE];
150
151 char *link_status[INTERFACE_MAX];
152
153 struct totem_ip_address my_ids[INTERFACE_MAX];
154
155 uint16_t ip_port[INTERFACE_MAX];
156
157 int our_nodeid;
158
159 int loopback_link;
160
161 struct totem_config *totem_config;
162
163 struct totem_ip_address token_target;
164
165 qb_loop_timer_handle timer_netif_check_timeout;
166
167 qb_loop_timer_handle timer_merge_detect_timeout;
168
169 int send_merge_detect_message;
170
171 unsigned int merge_detect_messages_sent_before_timeout;
172
173 int logpipes[2];
174 int knet_fd;
175
176 pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178 char *nozzle_name;
179 char *nozzle_ipaddr;
180 char *nozzle_prefix;
181 char *nozzle_macaddr;
182 nozzle_t nozzle_handle;
183 #endif
184 };
185
186 /* Awkward. But needed to get stats from knet */
187 struct totemknet_instance *global_instance;
188
189 struct work_item {
190 const void *msg;
191 unsigned int msg_len;
192 struct totemknet_instance *instance;
193 };
194
195 int totemknet_member_list_rebind_ip (
196 void *knet_context);
197
198
199 static int totemknet_configure_compression (
200 struct totemknet_instance *instance,
201 struct totem_config *totem_config);
202
203 static void totemknet_start_merge_detect_timeout(
204 void *knet_context);
205
206 static void totemknet_stop_merge_detect_timeout(
207 void *knet_context);
208
209 static void log_flush_messages (
210 void *knet_context);
211
212 static void totemknet_instance_initialize (struct totemknet_instance *instance)
213 {
214 int res;
215
216 memset (instance, 0, sizeof (struct totemknet_instance));
217 res = pthread_mutex_init(&instance->log_mutex, NULL);
218 /*
219 * There is not too much else what can be done.
220 */
221 assert(res == 0);
222 }
223
224 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
225 do { \
226 (void)pthread_mutex_lock(&instance->log_mutex); \
227 instance->totemknet_log_printf ( \
228 level, subsys, function, file, line, \
229 (const char *)format, ##args); \
230 (void)pthread_mutex_unlock(&instance->log_mutex); \
231 } while (0);
232
233 #define knet_log_printf(level, format, args...) \
234 do { \
235 knet_log_printf_lock ( \
236 level, instance->totemknet_subsys_id, \
237 __FUNCTION__, __FILE__, __LINE__, \
238 (const char *)format, ##args); \
239 } while (0);
240
241 #define libknet_log_printf(level, format, args...) \
242 do { \
243 knet_log_printf_lock ( \
244 level, instance->knet_subsys_id, \
245 __FUNCTION__, "libknet.h", __LINE__, \
246 (const char *)format, ##args); \
247 } while (0);
248
249 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
250 do { \
251 char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
252 const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
253 instance->totemknet_log_printf ( \
254 level, instance->totemknet_subsys_id, \
255 __FUNCTION__, __FILE__, __LINE__, \
256 fmt ": %s (%d)", ##args, _error_ptr, err_num); \
257 } while(0)
258
259
260 #ifdef HAVE_LIBNOZZLE
261 static inline int is_ether_addr_multicast(const uint8_t *addr)
262 {
263 return (addr[0] & 0x01);
264 }
265 static inline int is_ether_addr_zero(const uint8_t *addr)
266 {
267 return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
268 }
269
270 static int ether_host_filter_fn(void *private_data,
271 const unsigned char *outdata,
272 ssize_t outdata_len,
273 uint8_t tx_rx,
274 knet_node_id_t this_host_id,
275 knet_node_id_t src_host_id,
276 int8_t *channel,
277 knet_node_id_t *dst_host_ids,
278 size_t *dst_host_ids_entries)
279 {
280 struct ether_header *eth_h = (struct ether_header *)outdata;
281 uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
282 uint16_t dst_host_id;
283
284 if (is_ether_addr_zero(dst_mac))
285 return -1;
286
287 if (is_ether_addr_multicast(dst_mac)) {
288 return 1;
289 }
290
291 memmove(&dst_host_id, &dst_mac[4], 2);
292
293 dst_host_ids[0] = ntohs(dst_host_id);
294 *dst_host_ids_entries = 1;
295
296 return 0;
297 }
298 #endif
299
300 static int dst_host_filter_callback_fn(void *private_data,
301 const unsigned char *outdata,
302 ssize_t outdata_len,
303 uint8_t tx_rx,
304 knet_node_id_t this_host_id,
305 knet_node_id_t src_host_id,
306 int8_t *channel,
307 knet_node_id_t *dst_host_ids,
308 size_t *dst_host_ids_entries)
309 {
310 struct totem_message_header *header = (struct totem_message_header *)outdata;
311 int res;
312
313 #ifdef HAVE_LIBNOZZLE
314 if (*channel != 0) {
315 return ether_host_filter_fn(private_data,
316 outdata, outdata_len,
317 tx_rx,
318 this_host_id, src_host_id,
319 channel,
320 dst_host_ids,
321 dst_host_ids_entries);
322 }
323 #endif
324 if (header->target_nodeid) {
325 dst_host_ids[0] = header->target_nodeid;
326 *dst_host_ids_entries = 1;
327 res = 0; /* unicast message */
328 }
329 else {
330 *dst_host_ids_entries = 0;
331 res = 1; /* multicast message */
332 }
333 return res;
334 }
335
336 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
337 {
338 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
339
340 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
341 if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
342 knet_handle_remove_datafd(instance->knet_handle, datafd);
343 }
344 }
345
346 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
347 {
348 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
349
350 // TODO: what? if anything.
351 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
352 }
353
354 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
355 {
356 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
357 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
358
359 /* We don't need to tell corosync the actual knet MTU */
360 // instance->totemknet_mtu_changed(instance->context, data_mtu);
361 }
362
363 int totemknet_crypto_set (
364 void *knet_context,
365 const char *cipher_type,
366 const char *hash_type)
367 {
368 return (0);
369 }
370
371
372 static inline void ucast_sendmsg (
373 struct totemknet_instance *instance,
374 struct totem_ip_address *system_to,
375 const void *msg,
376 unsigned int msg_len)
377 {
378 int res = 0;
379 struct totem_message_header *header = (struct totem_message_header *)msg;
380 struct msghdr msg_ucast;
381 struct iovec iovec;
382
383 header->target_nodeid = system_to->nodeid;
384
385 iovec.iov_base = (void *)msg;
386 iovec.iov_len = msg_len;
387
388 /*
389 * Build unicast message
390 */
391 memset(&msg_ucast, 0, sizeof(msg_ucast));
392 msg_ucast.msg_iov = (void *)&iovec;
393 msg_ucast.msg_iovlen = 1;
394 #ifdef HAVE_MSGHDR_CONTROL
395 msg_ucast.msg_control = 0;
396 #endif
397 #ifdef HAVE_MSGHDR_CONTROLLEN
398 msg_ucast.msg_controllen = 0;
399 #endif
400 #ifdef HAVE_MSGHDR_FLAGS
401 msg_ucast.msg_flags = 0;
402 #endif
403 #ifdef HAVE_MSGHDR_ACCRIGHTS
404 msg_ucast.msg_accrights = NULL;
405 #endif
406 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
407 msg_ucast.msg_accrightslen = 0;
408 #endif
409
410 /*
411 * Transmit unicast message
412 * An error here is recovered by totemsrp
413 */
414
415 res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
416 if (res < 0) {
417 KNET_LOGSYS_PERROR (errno, instance->totemknet_log_level_debug,
418 "sendmsg(ucast) failed (non-critical)");
419 }
420 }
421
422 static inline void mcast_sendmsg (
423 struct totemknet_instance *instance,
424 const void *msg,
425 unsigned int msg_len,
426 int only_active)
427 {
428 int res;
429 struct totem_message_header *header = (struct totem_message_header *)msg;
430 struct msghdr msg_mcast;
431 struct iovec iovec;
432
433 iovec.iov_base = (void *)msg;
434 iovec.iov_len = msg_len;
435
436 header->target_nodeid = 0;
437
438 /*
439 * Build multicast message
440 */
441 memset(&msg_mcast, 0, sizeof(msg_mcast));
442 msg_mcast.msg_iov = (void *)&iovec;
443 msg_mcast.msg_iovlen = 1;
444 #ifdef HAVE_MSGHDR_CONTROL
445 msg_mcast.msg_control = 0;
446 #endif
447 #ifdef HAVE_MSGHDR_CONTROLLEN
448 msg_mcast.msg_controllen = 0;
449 #endif
450 #ifdef HAVE_MSGHDR_FLAGS
451 msg_mcast.msg_flags = 0;
452 #endif
453 #ifdef HAVE_MSGHDR_ACCRIGHTS
454 msg_mcast.msg_accrights = NULL;
455 #endif
456 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
457 msg_mcast.msg_accrightslen = 0;
458 #endif
459
460
461 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
462
463 res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
464 if (res < msg_len) {
465 knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
466 }
467
468 if (!only_active || instance->send_merge_detect_message) {
469 /*
470 * Current message was sent to all nodes
471 */
472 instance->merge_detect_messages_sent_before_timeout++;
473 instance->send_merge_detect_message = 0;
474 }
475 }
476
477 static int node_compare(const void *aptr, const void *bptr)
478 {
479 uint16_t a,b;
480
481 a = *(uint16_t *)aptr;
482 b = *(uint16_t *)bptr;
483
484 return a > b;
485 }
486
487 #ifndef OWN_INDEX_NONE
488 #define OWN_INDEX_NONE -1
489 #endif
490
491 int totemknet_nodestatus_get (
492 void *knet_context,
493 unsigned int nodeid,
494 struct totem_node_status *node_status)
495 {
496 int i;
497 int res = 0;
498 struct knet_link_status link_status;
499 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
500 struct knet_host_status knet_host_status;
501 uint8_t link_list[KNET_MAX_LINK];
502 size_t num_links;
503
504 if (!instance->knet_handle) {
505 return CS_ERR_NOT_EXIST; /* Not using knet */
506 }
507
508 if (!node_status) {
509 return CS_ERR_INVALID_PARAM;
510 }
511
512 res = knet_host_get_status(instance->knet_handle,
513 nodeid,
514 &knet_host_status);
515 if (res) {
516 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_host_status(%d) failed: %d", nodeid, res);
517 return (-1);
518 }
519 node_status->nodeid = nodeid;
520 node_status->reachable = knet_host_status.reachable;
521 node_status->remote = knet_host_status.remote;
522 node_status->external = knet_host_status.external;
523
524 #ifdef HAVE_KNET_ONWIRE_VER
525 res = knet_handle_get_onwire_ver(instance->knet_handle,
526 nodeid,
527 &node_status->onwire_min,
528 &node_status->onwire_max,
529 &node_status->onwire_ver);
530 if (res) {
531 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_onwire_ver(%d) failed: %d", nodeid, res);
532 return (-1);
533 }
534 #endif
535 /* Get link info */
536 res = knet_link_get_link_list(instance->knet_handle,
537 nodeid, link_list, &num_links);
538 if (res) {
539 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_list(%d) failed: %d", nodeid, res);
540 return (-1);
541 }
542
543 /* node_status[] has been zeroed for us in totempg.c */
544 for (i=0; i < num_links; i++) {
545 if (!instance->totem_config->interfaces[link_list[i]].configured) {
546 continue;
547 }
548 res = knet_link_get_status(instance->knet_handle,
549 nodeid,
550 link_list[i],
551 &link_status,
552 sizeof(link_status));
553 if (res == 0) {
554 node_status->link_status[link_list[i]].enabled = link_status.enabled;
555 node_status->link_status[link_list[i]].connected = link_status.connected;
556 node_status->link_status[link_list[i]].dynconnected = link_status.dynconnected;
557 node_status->link_status[link_list[i]].mtu = link_status.mtu;
558 memcpy(node_status->link_status[link_list[i]].src_ipaddr, link_status.src_ipaddr, KNET_MAX_HOST_LEN);
559 memcpy(node_status->link_status[link_list[i]].dst_ipaddr, link_status.dst_ipaddr, KNET_MAX_HOST_LEN);
560 } else {
561 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_status(%d, %d) failed: %d", nodeid, link_list[i], res);
562 }
563 }
564 return res;
565 }
566
567
568
569 int totemknet_ifaces_get (void *knet_context,
570 char ***status,
571 unsigned int *iface_count)
572 {
573 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
574 struct knet_link_status link_status;
575 knet_node_id_t host_list[KNET_MAX_HOST];
576 uint8_t link_list[KNET_MAX_LINK];
577 size_t num_hosts;
578 size_t num_links;
579 size_t link_idx;
580 int i,j;
581 char *ptr;
582 int res = 0;
583
584 /*
585 * Don't do the whole 'link_info' bit if the caller just wants
586 * a count of interfaces.
587 */
588 if (status) {
589 int own_idx = OWN_INDEX_NONE;
590
591 res = knet_host_get_host_list(instance->knet_handle,
592 host_list, &num_hosts);
593 if (res) {
594 return (-1);
595 }
596 qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
597
598 for (j=0; j<num_hosts; j++) {
599 if (host_list[j] == instance->our_nodeid) {
600 own_idx = j;
601 break;
602 }
603 }
604
605 for (i=0; i<INTERFACE_MAX; i++) {
606 memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
607 if (own_idx != OWN_INDEX_NONE) {
608 instance->link_status[i][own_idx] = 'n';
609 }
610 instance->link_status[i][num_hosts] = '\0';
611 }
612
613 /* This is all a bit "inside-out" because "status" is a set of strings per link
614 * and knet orders things by host
615 */
616 for (j=0; j<num_hosts; j++) {
617 if (own_idx != OWN_INDEX_NONE && j == own_idx) {
618 continue ;
619 }
620
621 res = knet_link_get_link_list(instance->knet_handle,
622 host_list[j], link_list, &num_links);
623 if (res) {
624 return (-1);
625 }
626
627 link_idx = 0;
628 for (i=0; i < num_links; i++) {
629 /*
630 * Skip over links that are unconfigured to corosync. This is basically
631 * link0 if corosync isn't using it for comms, as we will still
632 * have it set up for loopback.
633 */
634 if (!instance->totem_config->interfaces[link_list[i]].configured) {
635 continue;
636 }
637 ptr = instance->link_status[link_idx++];
638
639 res = knet_link_get_status(instance->knet_handle,
640 host_list[j],
641 link_list[i],
642 &link_status,
643 sizeof(link_status));
644 if (res == 0) {
645 ptr[j] = '0' + (link_status.enabled |
646 link_status.connected<<1 |
647 link_status.dynconnected<<2);
648 }
649 else {
650 knet_log_printf (LOGSYS_LEVEL_ERROR,
651 "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
652 ptr[j] = '?';
653 }
654 }
655 }
656 *status = instance->link_status;
657 }
658
659 *iface_count = INTERFACE_MAX;
660
661 return (res);
662 }
663
664 int totemknet_finalize (
665 void *knet_context)
666 {
667 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
668 int res = 0;
669 int i,j;
670 static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
671 uint8_t links[KNET_MAX_LINK];
672 size_t num_nodes;
673 size_t num_links;
674
675 knet_log_printf(LOG_DEBUG, "totemknet: finalize");
676
677 qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
678 qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
679
680 /*
681 * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
682 */
683 res = knet_handle_setfwd(instance->knet_handle, 0);
684 if (res) {
685 knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
686 }
687
688 res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
689 if (res) {
690 knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
691 /* Crash out anyway */
692 goto finalise_error;
693 }
694
695 /* Tidily shut down all nodes & links. */
696 for (i=0; i<num_nodes; i++) {
697
698 res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
699 if (res) {
700 knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
701 goto finalise_error;
702 }
703 for (j=0; j<num_links; j++) {
704 res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
705 if (res) {
706 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
707 }
708 res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
709 if (res) {
710 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
711 }
712 }
713 res = knet_host_remove(instance->knet_handle, nodes[i]);
714 if (res) {
715 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
716 }
717 }
718
719 finalise_error:
720 res = knet_handle_free(instance->knet_handle);
721 if (res) {
722 knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
723 }
724
725 totemknet_stop_merge_detect_timeout(instance);
726
727 log_flush_messages(instance);
728
729 /*
730 * Error is deliberately ignored
731 */
732 (void)pthread_mutex_destroy(&instance->log_mutex);
733
734 return (res);
735 }
736
737 static int log_deliver_fn (
738 int fd,
739 int revents,
740 void *data)
741 {
742 struct totemknet_instance *instance = (struct totemknet_instance *)data;
743 char buffer[sizeof(struct knet_log_msg)*4];
744 char *bufptr = buffer;
745 int done = 0;
746 int len;
747
748 len = read(fd, buffer, sizeof(buffer));
749 while (done < len) {
750 struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
751 switch (msg->msglevel) {
752 case KNET_LOG_ERR:
753 libknet_log_printf (LOGSYS_LEVEL_ERROR, "%s: %s",
754 knet_log_get_subsystem_name(msg->subsystem),
755 msg->msg);
756 break;
757 case KNET_LOG_WARN:
758 libknet_log_printf (LOGSYS_LEVEL_WARNING, "%s: %s",
759 knet_log_get_subsystem_name(msg->subsystem),
760 msg->msg);
761 break;
762 case KNET_LOG_INFO:
763 libknet_log_printf (LOGSYS_LEVEL_INFO, "%s: %s",
764 knet_log_get_subsystem_name(msg->subsystem),
765 msg->msg);
766 break;
767 case KNET_LOG_DEBUG:
768 libknet_log_printf (LOGSYS_LEVEL_DEBUG, "%s: %s",
769 knet_log_get_subsystem_name(msg->subsystem),
770 msg->msg);
771 break;
772 }
773 bufptr += sizeof(struct knet_log_msg);
774 done += sizeof(struct knet_log_msg);
775 }
776 return 0;
777 }
778
779 static int data_deliver_fn (
780 int fd,
781 int revents,
782 void *data)
783 {
784 struct totemknet_instance *instance = (struct totemknet_instance *)data;
785 struct msghdr msg_hdr;
786 struct iovec iov_recv;
787 struct sockaddr_storage system_from;
788 ssize_t msg_len;
789 int truncated_packet;
790
791 iov_recv.iov_base = instance->iov_buffer;
792 iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
793
794 msg_hdr.msg_name = &system_from;
795 msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
796 msg_hdr.msg_iov = &iov_recv;
797 msg_hdr.msg_iovlen = 1;
798 #ifdef HAVE_MSGHDR_CONTROL
799 msg_hdr.msg_control = 0;
800 #endif
801 #ifdef HAVE_MSGHDR_CONTROLLEN
802 msg_hdr.msg_controllen = 0;
803 #endif
804 #ifdef HAVE_MSGHDR_FLAGS
805 msg_hdr.msg_flags = 0;
806 #endif
807 #ifdef HAVE_MSGHDR_ACCRIGHTS
808 msg_hdr.msg_accrights = NULL;
809 #endif
810 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
811 msg_hdr.msg_accrightslen = 0;
812 #endif
813
814 msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
815 if (msg_len <= 0) {
816 return (0);
817 }
818
819 truncated_packet = 0;
820
821 #ifdef HAVE_MSGHDR_FLAGS
822 if (msg_hdr.msg_flags & MSG_TRUNC) {
823 truncated_packet = 1;
824 }
825 #else
826 /*
827 * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
828 * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
829 */
830 if (bytes_received == KNET_MAX_PACKET_SIZE) {
831 truncated_packet = 1;
832 }
833 #endif
834
835 if (truncated_packet) {
836 knet_log_printf(instance->totemknet_log_level_error,
837 "Received too big message. This may be because something bad is happening"
838 "on the network (attack?), or you tried join more nodes than corosync is"
839 "compiled with (%u) or bug in the code (bad estimation of "
840 "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
841 return (0);
842 }
843
844 /*
845 * Handle incoming message
846 */
847 instance->totemknet_deliver_fn (
848 instance->context,
849 instance->iov_buffer,
850 msg_len,
851 &system_from);
852
853 return (0);
854 }
855
856 static void timer_function_netif_check_timeout (
857 void *data)
858 {
859 struct totemknet_instance *instance = (struct totemknet_instance *)data;
860 int i;
861
862 for (i=0; i < INTERFACE_MAX; i++) {
863 if (!instance->totem_config->interfaces[i].configured) {
864 continue;
865 }
866 instance->totemknet_iface_change_fn (instance->context,
867 &instance->my_ids[i],
868 i);
869 }
870 }
871
872 static void knet_set_access_list_config(struct totemknet_instance *instance)
873 {
874 #ifdef HAVE_KNET_ACCESS_LIST
875 uint32_t value;
876 cs_error_t err;
877
878 value = instance->totem_config->block_unlisted_ips;
879 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
880
881 err = knet_handle_enable_access_lists(instance->knet_handle, value);
882 if (err) {
883 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
884 }
885 #endif
886 }
887
888 void totemknet_configure_log_level()
889 {
890 int logsys_log_mode;
891 int knet_log_mode = KNET_LOG_INFO;
892 uint8_t s;
893
894 if (!global_instance || !global_instance->knet_handle) {
895 return;
896 }
897
898 /* Reconfigure logging level */
899 logsys_log_mode = logsys_config_debug_get("KNET");
900
901 switch (logsys_log_mode) {
902 case LOGSYS_DEBUG_OFF:
903 knet_log_mode = KNET_LOG_INFO;
904 break;
905 case LOGSYS_DEBUG_ON:
906 knet_log_mode = KNET_LOG_DEBUG;
907 break;
908 case LOGSYS_DEBUG_TRACE:
909 knet_log_mode = KNET_LOG_DEBUG;
910 break;
911 }
912 log_printf (LOGSYS_LEVEL_DEBUG, "totemknet setting log level %s", knet_log_get_loglevel_name(knet_log_mode));
913 for (s = 0; s<KNET_MAX_SUBSYSTEMS; s++) {
914 knet_log_set_loglevel(global_instance->knet_handle, s, knet_log_mode);
915 }
916 }
917
918
919 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
920 static void totemknet_refresh_config(
921 int32_t event,
922 const char *key_name,
923 struct icmap_notify_value new_val,
924 struct icmap_notify_value old_val,
925 void *user_data)
926 {
927 uint8_t reloading;
928 uint32_t value;
929 uint32_t link_no;
930 size_t num_nodes;
931 knet_node_id_t host_ids[KNET_MAX_HOST];
932 int i;
933 int err;
934 struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
935
936 ENTER();
937
938 /*
939 * If a full reload is in progress then don't do anything until it's done and
940 * can reconfigure it all atomically
941 */
942 if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
943 return;
944 }
945
946 knet_set_access_list_config(instance);
947
948 if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
949
950 instance->totem_config->knet_pmtud_interval = value;
951 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
952 err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
953 if (err) {
954 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
955 }
956 }
957
958 /* Configure link parameters for each node */
959 err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
960 if (err != 0) {
961 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
962 }
963
964 for (i=0; i<num_nodes; i++) {
965 for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
966 if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
967 continue;
968 }
969
970 err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
971 instance->totem_config->interfaces[link_no].knet_ping_interval,
972 instance->totem_config->interfaces[link_no].knet_ping_timeout,
973 instance->totem_config->interfaces[link_no].knet_ping_precision);
974 if (err) {
975 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
976 }
977 err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
978 instance->totem_config->interfaces[link_no].knet_pong_count);
979 if (err) {
980 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
981 }
982 err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
983 instance->totem_config->interfaces[link_no].knet_link_priority);
984 if (err) {
985 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
986 }
987
988 }
989 }
990
991 /* Log levels get reconfigured from logconfig.c as that happens last in the reload */
992 LEAVE();
993 }
994
995 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
996 {
997 icmap_track_t icmap_track_totem = NULL;
998 icmap_track_t icmap_track_reload = NULL;
999
1000 ENTER();
1001
1002 icmap_track_add("totem.",
1003 ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX,
1004 totemknet_refresh_config,
1005 instance,
1006 &icmap_track_totem);
1007
1008 icmap_track_add("config.totemconfig_reload_in_progress",
1009 ICMAP_TRACK_ADD | ICMAP_TRACK_MODIFY,
1010 totemknet_refresh_config,
1011 instance,
1012 &icmap_track_reload);
1013
1014 LEAVE();
1015 }
1016
1017 static int totemknet_is_crypto_enabled(const struct totemknet_instance *instance)
1018 {
1019
1020 return (!(strcmp(instance->totem_config->crypto_cipher_type, "none") == 0 &&
1021 strcmp(instance->totem_config->crypto_hash_type, "none") == 0));
1022
1023 }
1024
1025 static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
1026 {
1027 struct knet_handle_crypto_cfg crypto_cfg;
1028 int res;
1029
1030 /* These have already been validated */
1031 memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
1032 memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
1033 memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
1034 memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1035 crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1036
1037 #ifdef HAVE_KNET_CRYPTO_RECONF
1038
1039 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
1040 crypto_cfg.crypto_model,
1041 crypto_cfg.crypto_cipher_type,
1042 crypto_cfg.crypto_hash_type,
1043 instance->totem_config->crypto_index
1044 );
1045
1046 /* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
1047 if (!totemknet_is_crypto_enabled(instance)) {
1048 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1049 if (res) {
1050 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
1051 }
1052 }
1053
1054 /* use_config will be called later when all nodes are synced */
1055 res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
1056 if (res == -1) {
1057 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
1058 goto exit_error;
1059 }
1060 if (res == -2) {
1061 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
1062 goto exit_error;
1063 }
1064 #else
1065 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
1066 crypto_cfg.crypto_model,
1067 crypto_cfg.crypto_cipher_type,
1068 crypto_cfg.crypto_hash_type
1069 );
1070
1071 res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1072 if (res == -1) {
1073 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1074 goto exit_error;
1075 }
1076 if (res == -2) {
1077 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1078 goto exit_error;
1079 }
1080 #endif
1081
1082
1083 exit_error:
1084 return res;
1085 }
1086
1087 /*
1088 * Create an instance
1089 */
1090 int totemknet_initialize (
1091 qb_loop_t *poll_handle,
1092 void **knet_context,
1093 struct totem_config *totem_config,
1094 totemsrp_stats_t *stats,
1095 void *context,
1096
1097 void (*deliver_fn) (
1098 void *context,
1099 const void *msg,
1100 unsigned int msg_len,
1101 const struct sockaddr_storage *system_from),
1102
1103 void (*iface_change_fn) (
1104 void *context,
1105 const struct totem_ip_address *iface_address,
1106 unsigned int link_no),
1107
1108 void (*mtu_changed) (
1109 void *context,
1110 int net_mtu),
1111
1112 void (*target_set_completed) (
1113 void *context))
1114 {
1115 struct totemknet_instance *instance;
1116 char *tmp_str;
1117 int8_t channel=0;
1118 int allow_knet_handle_fallback=0;
1119 int res;
1120 int i;
1121
1122 instance = malloc (sizeof (struct totemknet_instance));
1123 if (instance == NULL) {
1124 return (-1);
1125 }
1126
1127 totemknet_instance_initialize (instance);
1128
1129 instance->totem_config = totem_config;
1130
1131 /*
1132 * Configure logging
1133 */
1134 instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1135 instance->totemknet_log_level_error = totem_config->totem_logging_configuration.log_level_error;
1136 instance->totemknet_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
1137 instance->totemknet_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
1138 instance->totemknet_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
1139 instance->totemknet_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
1140 instance->totemknet_log_printf = totem_config->totem_logging_configuration.log_printf;
1141
1142 instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
1143
1144 /*
1145 * Initialize local variables for totemknet
1146 */
1147
1148 instance->our_nodeid = instance->totem_config->node_id;
1149
1150 for (i=0; i< INTERFACE_MAX; i++) {
1151 totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
1152 instance->my_ids[i].nodeid = instance->our_nodeid;
1153 instance->ip_port[i] = totem_config->interfaces[i].ip_port;
1154
1155 /* Needed for totemsrp */
1156 totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
1157 }
1158
1159 instance->poll_handle = poll_handle;
1160
1161 instance->context = context;
1162 instance->totemknet_deliver_fn = deliver_fn;
1163
1164 instance->totemknet_iface_change_fn = iface_change_fn;
1165
1166 instance->totemknet_mtu_changed = mtu_changed;
1167
1168 instance->totemknet_target_set_completed = target_set_completed;
1169
1170 instance->loopback_link = 0;
1171
1172 res = pipe(instance->logpipes);
1173 if (res == -1) {
1174 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
1175 goto exit_error;
1176 }
1177 if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
1178 fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
1179 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
1180 goto exit_error;
1181 }
1182
1183 if (icmap_get_string("system.allow_knet_handle_fallback", &tmp_str) == CS_OK) {
1184 if (strcmp(tmp_str, "yes") == 0) {
1185 allow_knet_handle_fallback = 1;
1186 }
1187 free(tmp_str);
1188 }
1189
1190 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
1191 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1192 #else
1193 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1194 #endif
1195
1196 if (allow_knet_handle_fallback && !instance->knet_handle && errno == ENAMETOOLONG) {
1197 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
1198 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
1199 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1200 #else
1201 instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1202 #endif
1203 }
1204
1205 if (!instance->knet_handle) {
1206 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1207 goto exit_error;
1208 }
1209
1210 knet_set_access_list_config(instance);
1211
1212 res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1213 if (res) {
1214 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1215 }
1216 res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1217 if (res) {
1218 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1219 }
1220 res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1221 if (res) {
1222 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1223 }
1224 res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1225 if (res) {
1226 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1227 }
1228 res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1229 if (res) {
1230 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1231 }
1232 global_instance = instance;
1233
1234 /* Setup knet logging level */
1235 totemknet_configure_log_level();
1236
1237 /* Get an fd into knet */
1238 instance->knet_fd = 0;
1239 res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1240 if (res) {
1241 knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1242 goto exit_error;
1243 }
1244
1245 /* Enable crypto if requested */
1246 #ifdef HAVE_KNET_CRYPTO_RECONF
1247 if (totemknet_is_crypto_enabled(instance)) {
1248 res = totemknet_set_knet_crypto(instance);
1249 if (res == 0) {
1250 res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
1251 if (res) {
1252 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
1253 goto exit_error;
1254 }
1255 } else {
1256 knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1257 goto exit_error;
1258 }
1259 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1260 if (res) {
1261 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
1262 goto exit_error;
1263 }
1264
1265 } else {
1266 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1267 if (res) {
1268 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
1269 goto exit_error;
1270 }
1271 }
1272 #else
1273 if (totemknet_is_crypto_enabled(instance)) {
1274 res = totemknet_set_knet_crypto(instance);
1275 if (res) {
1276 knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1277 goto exit_error;
1278 }
1279 }
1280 #endif
1281
1282 /* Set up compression */
1283 if (strcmp(totem_config->knet_compression_model, "none") != 0) {
1284 /* Not fatal, but will log */
1285 (void)totemknet_configure_compression(instance, totem_config);
1286 }
1287
1288 knet_handle_setfwd(instance->knet_handle, 1);
1289
1290 instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1291 if (strcmp(instance->totem_config->link_mode, "active")==0) {
1292 instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1293 }
1294 if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1295 instance->link_mode = KNET_LINK_POLICY_RR;
1296 }
1297
1298 for (i=0; i<INTERFACE_MAX; i++) {
1299 instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1300 if (!instance->link_status[i]) {
1301 goto exit_error;
1302 }
1303 }
1304
1305 qb_loop_poll_add (instance->poll_handle,
1306 QB_LOOP_MED,
1307 instance->logpipes[0],
1308 POLLIN, instance, log_deliver_fn);
1309
1310 qb_loop_poll_add (instance->poll_handle,
1311 QB_LOOP_HIGH,
1312 instance->knet_fd,
1313 POLLIN, instance, data_deliver_fn);
1314
1315 /*
1316 * Upper layer isn't ready to receive message because it hasn't
1317 * initialized yet. Add short timer to check the interfaces.
1318 */
1319 qb_loop_timer_add (instance->poll_handle,
1320 QB_LOOP_MED,
1321 100*QB_TIME_NS_IN_MSEC,
1322 (void *)instance,
1323 timer_function_netif_check_timeout,
1324 &instance->timer_netif_check_timeout);
1325
1326 totemknet_start_merge_detect_timeout(instance);
1327
1328 /* Start listening for config changes */
1329 totemknet_add_config_notifications(instance);
1330
1331 /* Add stats keys to icmap */
1332 stats_knet_add_handle();
1333
1334 knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1335 *knet_context = instance;
1336
1337 return (0);
1338
1339 exit_error:
1340 log_flush_messages(instance);
1341 free(instance);
1342 return (-1);
1343 }
1344
1345 void *totemknet_buffer_alloc (void)
1346 {
1347 /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1348 return malloc(KNET_MAX_PACKET_SIZE + 512);
1349 }
1350
1351 void totemknet_buffer_release (void *ptr)
1352 {
1353 return free (ptr);
1354 }
1355
1356 int totemknet_processor_count_set (
1357 void *knet_context,
1358 int processor_count)
1359 {
1360 return (0);
1361 }
1362
1363 int totemknet_recv_flush (void *knet_context)
1364 {
1365 return (0);
1366 }
1367
1368 int totemknet_send_flush (void *knet_context)
1369 {
1370 return (0);
1371 }
1372
1373 int totemknet_token_send (
1374 void *knet_context,
1375 const void *msg,
1376 unsigned int msg_len)
1377 {
1378 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1379 int res = 0;
1380
1381 ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1382
1383 return (res);
1384 }
1385 int totemknet_mcast_flush_send (
1386 void *knet_context,
1387 const void *msg,
1388 unsigned int msg_len)
1389 {
1390 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1391 int res = 0;
1392
1393 mcast_sendmsg (instance, msg, msg_len, 0);
1394
1395 return (res);
1396 }
1397
1398 int totemknet_mcast_noflush_send (
1399 void *knet_context,
1400 const void *msg,
1401 unsigned int msg_len)
1402 {
1403 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1404 int res = 0;
1405
1406 mcast_sendmsg (instance, msg, msg_len, 1);
1407
1408 return (res);
1409 }
1410
1411
1412 extern int totemknet_iface_check (void *knet_context)
1413 {
1414 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1415 int res = 0;
1416
1417 knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1418
1419 return (res);
1420 }
1421
1422 extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
1423 {
1424 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1425
1426 knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1427 }
1428
1429 int totemknet_token_target_set (
1430 void *knet_context,
1431 unsigned int nodeid)
1432 {
1433 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1434 int res = 0;
1435
1436 instance->token_target.nodeid = nodeid;
1437
1438 instance->totemknet_target_set_completed (instance->context);
1439
1440 return (res);
1441 }
1442
1443 extern int totemknet_recv_mcast_empty (
1444 void *knet_context)
1445 {
1446 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1447 unsigned int res;
1448 struct sockaddr_storage system_from;
1449 struct msghdr msg_hdr;
1450 struct iovec iov_recv;
1451 struct pollfd ufd;
1452 int nfds;
1453 int msg_processed = 0;
1454
1455 iov_recv.iov_base = instance->iov_buffer;
1456 iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1457
1458 msg_hdr.msg_name = &system_from;
1459 msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1460 msg_hdr.msg_iov = &iov_recv;
1461 msg_hdr.msg_iovlen = 1;
1462 #ifdef HAVE_MSGHDR_CONTROL
1463 msg_hdr.msg_control = 0;
1464 #endif
1465 #ifdef HAVE_MSGHDR_CONTROLLEN
1466 msg_hdr.msg_controllen = 0;
1467 #endif
1468 #ifdef HAVE_MSGHDR_FLAGS
1469 msg_hdr.msg_flags = 0;
1470 #endif
1471 #ifdef HAVE_MSGHDR_ACCRIGHTS
1472 msg_msg_hdr.msg_accrights = NULL;
1473 #endif
1474 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1475 msg_msg_hdr.msg_accrightslen = 0;
1476 #endif
1477
1478 do {
1479 ufd.fd = instance->knet_fd;
1480 ufd.events = POLLIN;
1481 nfds = poll (&ufd, 1, 0);
1482 if (nfds == 1 && ufd.revents & POLLIN) {
1483 res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1484 if (res != -1) {
1485 msg_processed = 1;
1486 } else {
1487 msg_processed = -1;
1488 }
1489 }
1490 } while (nfds == 1);
1491
1492 return (msg_processed);
1493 }
1494
1495 int totemknet_iface_set (void *knet_context,
1496 const struct totem_ip_address *local_addr,
1497 unsigned short ip_port,
1498 unsigned int iface_no)
1499 {
1500 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1501
1502 totemip_copy(&instance->my_ids[iface_no], local_addr);
1503
1504 knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1505
1506 instance->ip_port[iface_no] = ip_port;
1507
1508 return 0;
1509 }
1510
1511
1512 int totemknet_member_add (
1513 void *knet_context,
1514 const struct totem_ip_address *local,
1515 const struct totem_ip_address *member,
1516 int link_no)
1517 {
1518 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1519 int err;
1520 int port = instance->ip_port[link_no];
1521 struct sockaddr_storage remote_ss;
1522 struct sockaddr_storage local_ss;
1523 int addrlen;
1524 int i;
1525 int host_found = 0;
1526 knet_node_id_t host_ids[KNET_MAX_HOST];
1527 size_t num_host_ids;
1528
1529 /* Only create 1 loopback link and use link 0 */
1530 if (member->nodeid == instance->our_nodeid) {
1531 if (!instance->loopback_link) {
1532 link_no = 0;
1533 instance->loopback_link = 1;
1534 } else {
1535 /* Already done */
1536 return 0;
1537 }
1538 }
1539
1540 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1541 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1542
1543
1544 /* Only add the host if it doesn't already exist in knet */
1545 err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1546 if (err) {
1547 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1548 return -1;
1549 }
1550 for (i=0; i<num_host_ids; i++) {
1551 if (host_ids[i] == member->nodeid) {
1552 host_found = 1;
1553 }
1554 }
1555
1556 if (!host_found) {
1557 err = knet_host_add(instance->knet_handle, member->nodeid);
1558 if (err != 0 && errno != EEXIST) {
1559 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1560 return -1;
1561 }
1562 } else {
1563 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1564 }
1565
1566
1567 if (err == 0) {
1568 if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1569 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1570 return -1;
1571 }
1572 }
1573
1574 memset(&local_ss, 0, sizeof(local_ss));
1575 memset(&remote_ss, 0, sizeof(remote_ss));
1576 /* Casts to remove const */
1577 totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1578 totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1579
1580 if (member->nodeid == instance->our_nodeid) {
1581 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1582
1583 err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1584 KNET_TRANSPORT_LOOPBACK,
1585 &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1586 }
1587 else {
1588 err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1589 instance->totem_config->interfaces[link_no].knet_transport,
1590 &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1591 }
1592 if (err) {
1593 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1594 return -1;
1595 }
1596
1597 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1598 instance->totem_config->interfaces[link_no].knet_link_priority);
1599
1600 err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1601 instance->totem_config->interfaces[link_no].knet_link_priority);
1602 if (err) {
1603 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1604 }
1605
1606 /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1607 get done in totemknet_refresh_config */
1608 if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1609 err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1610 instance->totem_config->interfaces[link_no].knet_ping_interval,
1611 instance->totem_config->interfaces[link_no].knet_ping_timeout,
1612 instance->totem_config->interfaces[link_no].knet_ping_precision);
1613 if (err) {
1614 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1615 }
1616 err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1617 instance->totem_config->interfaces[link_no].knet_pong_count);
1618 if (err) {
1619 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1620 }
1621 }
1622
1623 err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1624 if (err) {
1625 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1626 return -1;
1627 }
1628
1629 /* register stats */
1630 stats_knet_add_member(member->nodeid, link_no);
1631 return (0);
1632 }
1633
1634 int totemknet_member_remove (
1635 void *knet_context,
1636 const struct totem_ip_address *token_target,
1637 int link_no)
1638 {
1639 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1640 int res;
1641 uint8_t link_list[KNET_MAX_LINK];
1642 size_t num_links;
1643
1644 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1645
1646 /* Don't remove the link with the loopback on it until we shut down */
1647 if (token_target->nodeid == instance->our_nodeid) {
1648 return 0;
1649 }
1650
1651 /* Tidy stats */
1652 stats_knet_del_member(token_target->nodeid, link_no);
1653
1654 /* Remove the link first */
1655 res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1656 if (res != 0) {
1657 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1658 return res;
1659 }
1660
1661 res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1662 if (res != 0) {
1663 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1664 return res;
1665 }
1666
1667 /* If this is the last link, then remove the node */
1668 res = knet_link_get_link_list(instance->knet_handle,
1669 token_target->nodeid, link_list, &num_links);
1670 if (res) {
1671 return (0); /* not really failure */
1672 }
1673
1674 if (num_links == 0) {
1675 res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1676 }
1677 return res;
1678 }
1679
1680 int totemknet_member_list_rebind_ip (
1681 void *knet_context)
1682 {
1683 return (0);
1684 }
1685
1686
1687 static int totemknet_configure_compression (
1688 struct totemknet_instance *instance,
1689 struct totem_config *totem_config)
1690 {
1691 struct knet_handle_compress_cfg compress_cfg;
1692 int res = 0;
1693
1694 assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1695 strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1696
1697 compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1698 compress_cfg.compress_level = totem_config->knet_compression_level;
1699
1700 res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1701 if (res) {
1702 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1703 }
1704 return res;
1705 }
1706
1707 int totemknet_reconfigure (
1708 void *knet_context,
1709 struct totem_config *totem_config)
1710 {
1711 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1712 int res = 0;
1713
1714 (void)totemknet_configure_compression(instance, totem_config);
1715
1716 #ifdef HAVE_LIBNOZZLE
1717 /* Set up nozzle device(s). Return code is ignored, because inability
1718 * configure nozzle is not fatal problem, errors are logged and
1719 * there is not much else we can do */
1720 (void)setup_nozzle(instance);
1721 #endif
1722
1723 if (totem_config->crypto_changed) {
1724 /* Flip crypto_index */
1725 totem_config->crypto_index = 3-totem_config->crypto_index;
1726 res = totemknet_set_knet_crypto(instance);
1727
1728 knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
1729 totem_config->crypto_model,
1730 totem_config->crypto_cipher_type,
1731 totem_config->crypto_hash_type);
1732 }
1733 return (res);
1734 }
1735
1736
1737 int totemknet_crypto_reconfigure_phase (
1738 void *knet_context,
1739 struct totem_config *totem_config,
1740 cfg_message_crypto_reconfig_phase_t phase)
1741 {
1742 #ifdef HAVE_KNET_CRYPTO_RECONF
1743 int res;
1744 int config_to_use;
1745 int config_to_clear;
1746 struct knet_handle_crypto_cfg crypto_cfg;
1747 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1748
1749 knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
1750
1751 switch (phase) {
1752 case CRYPTO_RECONFIG_PHASE_ACTIVATE:
1753 config_to_use = totem_config->crypto_index;
1754 if (!totemknet_is_crypto_enabled(instance)) {
1755 config_to_use = 0; /* we are clearing it */
1756 }
1757
1758 /* Enable the new config on this node */
1759 res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
1760 if (res == -1) {
1761 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
1762 }
1763 break;
1764
1765 case CRYPTO_RECONFIG_PHASE_CLEANUP:
1766 /*
1767 * All nodes should now have the new config. clear the old one out
1768 * OR disable crypto entirely if that's what the new config insists on.
1769 */
1770 config_to_clear = 3-totem_config->crypto_index;
1771 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
1772
1773 strcpy(crypto_cfg.crypto_model, "none");
1774 strcpy(crypto_cfg.crypto_cipher_type, "none");
1775 strcpy(crypto_cfg.crypto_hash_type, "none");
1776 res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
1777 if (res == -1) {
1778 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
1779 }
1780 if (res == -2) {
1781 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
1782 }
1783
1784 /* If crypto is enabled then disable all cleartext reception */
1785 if (totemknet_is_crypto_enabled(instance)) {
1786 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1787 if (res) {
1788 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
1789 }
1790 }
1791 }
1792 #endif
1793 return 0;
1794 }
1795
1796 void totemknet_stats_clear (
1797 void *knet_context)
1798 {
1799 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1800
1801 (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1802 }
1803
1804 /* For the stats module */
1805 int totemknet_link_get_status (
1806 knet_node_id_t node, uint8_t link_no,
1807 struct knet_link_status *status)
1808 {
1809 int res;
1810 int ret = CS_OK;
1811
1812 /* We are probably not using knet */
1813 if (!global_instance) {
1814 return CS_ERR_NOT_EXIST;
1815 }
1816
1817 if (link_no >= INTERFACE_MAX) {
1818 return CS_ERR_NOT_EXIST; /* Invalid link number */
1819 }
1820
1821 res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1822 if (res) {
1823 switch (errno) {
1824 case EINVAL:
1825 ret = CS_ERR_INVALID_PARAM;
1826 break;
1827 case EBUSY:
1828 ret = CS_ERR_BUSY;
1829 break;
1830 case EDEADLK:
1831 ret = CS_ERR_TRY_AGAIN;
1832 break;
1833 default:
1834 ret = CS_ERR_LIBRARY;
1835 break;
1836 }
1837 }
1838
1839 return (ret);
1840 }
1841
1842 int totemknet_handle_get_stats (
1843 struct knet_handle_stats *stats)
1844 {
1845 int res;
1846
1847 /* We are probably not using knet */
1848 if (!global_instance) {
1849 return CS_ERR_NOT_EXIST;
1850 }
1851
1852 res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1853 if (res != 0) {
1854 return (qb_to_cs_error(-errno));
1855 }
1856
1857 return CS_OK;
1858 }
1859
1860 static void timer_function_merge_detect_timeout (
1861 void *data)
1862 {
1863 struct totemknet_instance *instance = (struct totemknet_instance *)data;
1864
1865 if (instance->merge_detect_messages_sent_before_timeout == 0) {
1866 instance->send_merge_detect_message = 1;
1867 }
1868
1869 instance->merge_detect_messages_sent_before_timeout = 0;
1870
1871 totemknet_start_merge_detect_timeout(instance);
1872 }
1873
1874 static void totemknet_start_merge_detect_timeout(
1875 void *knet_context)
1876 {
1877 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1878
1879 qb_loop_timer_add(instance->poll_handle,
1880 QB_LOOP_MED,
1881 instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1882 (void *)instance,
1883 timer_function_merge_detect_timeout,
1884 &instance->timer_merge_detect_timeout);
1885
1886 }
1887
1888 static void totemknet_stop_merge_detect_timeout(
1889 void *knet_context)
1890 {
1891 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1892
1893 qb_loop_timer_del(instance->poll_handle,
1894 instance->timer_merge_detect_timeout);
1895 }
1896
1897 static void log_flush_messages (void *knet_context)
1898 {
1899 struct pollfd pfd;
1900 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1901 int cont;
1902
1903 cont = 1;
1904
1905 while (cont) {
1906 pfd.fd = instance->logpipes[0];
1907 pfd.events = POLLIN;
1908 pfd.revents = 0;
1909
1910 if ((poll(&pfd, 1, 0) > 0) &&
1911 (pfd.revents & POLLIN) &&
1912 (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1913 cont = 1;
1914 } else {
1915 cont = 0;
1916 }
1917 }
1918 }
1919
1920
1921 #ifdef HAVE_LIBNOZZLE
1922 #define NOZZLE_NAME "nozzle.name"
1923 #define NOZZLE_IPADDR "nozzle.ipaddr"
1924 #define NOZZLE_PREFIX "nozzle.ipprefix"
1925 #define NOZZLE_MACADDR "nozzle.macaddr"
1926
1927 #define NOZZLE_CHANNEL 1
1928
1929
1930 static char *get_nozzle_script_dir(void *knet_context)
1931 {
1932 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1933 char filename[PATH_MAX + FILENAME_MAX + 1];
1934 static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1935 int res;
1936 const char *dirname_res;
1937
1938 /*
1939 * Build script directory based on corosync.conf file location
1940 */
1941 res = snprintf(filename, sizeof(filename), "%s",
1942 corosync_get_config_file());
1943 if (res >= sizeof(filename)) {
1944 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1945 return NULL;
1946 }
1947
1948 dirname_res = dirname(filename);
1949
1950 res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1951 dirname_res, "updown.d");
1952 if (res >= sizeof(updown_dirname)) {
1953 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1954 return NULL;
1955 }
1956 return updown_dirname;
1957 }
1958
1959 /*
1960 * Deliberately doesn't return the status as caller doesn't care.
1961 * The result will be logged though
1962 */
1963 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1964 {
1965 int res;
1966 char *exec_string;
1967
1968 res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1969 if (res == -1 && errno != ENOENT) {
1970 knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1971 } else if (res == -2) {
1972 knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1973 knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1974 }
1975 }
1976
1977 /*
1978 * Reparse IP address to add in our node ID
1979 * IPv6 addresses must end in '::'
1980 * IPv4 addresses must just be valid
1981 * '/xx' lengths are optional for IPv6, mandatory for IPv4
1982 *
1983 * Returns the modified IP address as a string to pass into libnozzle
1984 */
1985 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1986 const char *input_addr,
1987 const char *prefix, int nodeid,
1988 char *output_addr, size_t output_len)
1989 {
1990 char *coloncolon;
1991 int bits;
1992 int max_prefix = 64;
1993 uint32_t nodeid_mask;
1994 uint32_t addr_mask;
1995 uint32_t masked_nodeid;
1996 struct in_addr *addr;
1997 struct totem_ip_address totemip;
1998
1999 coloncolon = strstr(input_addr, "::");
2000 if (!coloncolon) {
2001 max_prefix = 30;
2002 }
2003
2004 bits = atoi(prefix);
2005 if (bits < 8 || bits > max_prefix) {
2006 knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
2007 return -1;
2008 }
2009
2010 /* IPv6 is easy */
2011 if (coloncolon) {
2012 memcpy(output_addr, input_addr, coloncolon-input_addr);
2013 sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
2014 return 0;
2015 }
2016
2017 /* For IPv4 we need to parse the address into binary, mask off the required bits,
2018 * add in the masked_nodeid and 'print' it out again
2019 */
2020 nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
2021 addr_mask = UINT32_MAX ^ nodeid_mask;
2022 masked_nodeid = nodeid & nodeid_mask;
2023
2024 if (totemip_parse(&totemip, input_addr, AF_INET)) {
2025 knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
2026 return -1;
2027 }
2028 addr = (struct in_addr *)&totemip.addr;
2029 addr->s_addr &= htonl(addr_mask);
2030 addr->s_addr |= htonl(masked_nodeid);
2031
2032 inet_ntop(AF_INET, addr, output_addr, output_len);
2033 return 0;
2034 }
2035
2036 static int create_nozzle_device(void *knet_context, const char *name,
2037 const char *ipaddr, const char *prefix,
2038 const char *macaddr)
2039 {
2040 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2041 char device_name[IFNAMSIZ+1];
2042 size_t size = IFNAMSIZ;
2043 int8_t channel = NOZZLE_CHANNEL;
2044 nozzle_t nozzle_dev;
2045 int nozzle_fd;
2046 int res;
2047 char *updown_dir;
2048 char parsed_ipaddr[INET6_ADDRSTRLEN];
2049 char mac[19];
2050
2051 memset(device_name, 0, size);
2052 memset(&mac, 0, sizeof(mac));
2053 strncpy(device_name, name, size);
2054
2055 updown_dir = get_nozzle_script_dir(knet_context);
2056 knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
2057
2058 nozzle_dev = nozzle_open(device_name, size, updown_dir);
2059 if (!nozzle_dev) {
2060 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
2061 return -1;
2062 }
2063 instance->nozzle_handle = nozzle_dev;
2064
2065 if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
2066 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
2067 goto out_clean;
2068 }
2069
2070 if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
2071 /* Prints its own errors */
2072 goto out_clean;
2073 }
2074 knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
2075 if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
2076 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
2077 goto out_clean;
2078 }
2079
2080 nozzle_fd = nozzle_get_fd(nozzle_dev);
2081 knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
2082
2083 res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
2084 if (res != 0) {
2085 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
2086 goto out_clean;
2087 }
2088
2089 run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
2090
2091 res = nozzle_set_up(nozzle_dev);
2092 if (res != 0) {
2093 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
2094 goto out_clean;
2095 }
2096 run_nozzle_script(instance, NOZZLE_UP, "up");
2097
2098 return 0;
2099
2100 out_clean:
2101 nozzle_close(nozzle_dev);
2102 return -1;
2103 }
2104
2105 static int remove_nozzle_device(void *knet_context)
2106 {
2107 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2108 int res;
2109 int datafd;
2110
2111 res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
2112 if (res != 0) {
2113 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2114 return -1;
2115 }
2116
2117 res = knet_handle_remove_datafd(instance->knet_handle, datafd);
2118 if (res != 0) {
2119 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2120 return -1;
2121 }
2122
2123 run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
2124 res = nozzle_set_down(instance->nozzle_handle);
2125 if (res != 0) {
2126 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
2127 return -1;
2128 }
2129 run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
2130
2131 res = nozzle_close(instance->nozzle_handle);
2132 if (res != 0) {
2133 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
2134 return -1;
2135 }
2136 knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
2137 return 0;
2138 }
2139
2140 static void free_nozzle(struct totemknet_instance *instance)
2141 {
2142 free(instance->nozzle_name);
2143 free(instance->nozzle_ipaddr);
2144 free(instance->nozzle_prefix);
2145 free(instance->nozzle_macaddr);
2146
2147 instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
2148 instance->nozzle_macaddr = NULL;
2149 }
2150
2151 static int setup_nozzle(void *knet_context)
2152 {
2153 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2154 char *ipaddr_str = NULL;
2155 char *name_str = NULL;
2156 char *prefix_str = NULL;
2157 char *macaddr_str = NULL;
2158 char mac[32];
2159 int name_res;
2160 int macaddr_res;
2161 int res = -1;
2162
2163 /*
2164 * Return value ignored on purpose. icmap_get_string changes
2165 * ipaddr_str/prefix_str only on success.
2166 */
2167 (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
2168 (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
2169 macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
2170 name_res = icmap_get_string(NOZZLE_NAME, &name_str);
2171
2172 /* Is is being removed? */
2173 if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
2174 remove_nozzle_device(instance);
2175 free_nozzle(instance);
2176 goto out_free;
2177 }
2178
2179 if (!name_str) {
2180 /* no nozzle */
2181 goto out_free;
2182 }
2183
2184 if (!ipaddr_str) {
2185 knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
2186 goto out_free;
2187 }
2188
2189 if (!prefix_str) {
2190 knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
2191 goto out_free;
2192 }
2193
2194 if (macaddr_str && strlen(macaddr_str) != 17) {
2195 knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
2196 goto out_free;
2197 }
2198 if (!macaddr_str) {
2199 macaddr_str = (char*)"54:54:01:00:00:00";
2200 }
2201
2202 if (instance->nozzle_name &&
2203 (strcmp(name_str, instance->nozzle_name) == 0) &&
2204 (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
2205 (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
2206 (instance->nozzle_macaddr == NULL ||
2207 strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
2208 /* Nothing has changed */
2209 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
2210 goto out_free;
2211 }
2212
2213 /* Add nodeid into MAC address */
2214 memcpy(mac, macaddr_str, 12);
2215 snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
2216 instance->our_nodeid >> 8,
2217 instance->our_nodeid & 0xFF);
2218 knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
2219
2220 if (name_res == CS_OK && name_str) {
2221 /* Reconfigure */
2222 if (instance->nozzle_name) {
2223 remove_nozzle_device(instance);
2224 free_nozzle(instance);
2225 }
2226
2227 res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
2228 mac);
2229
2230 instance->nozzle_name = strdup(name_str);
2231 instance->nozzle_ipaddr = strdup(ipaddr_str);
2232 instance->nozzle_prefix = strdup(prefix_str);
2233 instance->nozzle_macaddr = strdup(macaddr_str);
2234 if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
2235 !instance->nozzle_prefix) {
2236 knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
2237 /*
2238 * This 'free' will cause a complete reconfigure of the device next time we reload
2239 * but will also let the the current device keep working until then.
2240 * remove_nozzle() only needs the, statically-allocated, nozzle_handle
2241 */
2242 free_nozzle(instance);
2243 }
2244 }
2245
2246 out_free:
2247 free(name_str);
2248 free(ipaddr_str);
2249 free(prefix_str);
2250 if (macaddr_res == CS_OK) {
2251 free(macaddr_str);
2252 }
2253
2254 return res;
2255 }
2256 #endif // HAVE_LIBNOZZLE
corosync mirror