2 * Copyright (c) 2015-2019 Red Hat, Inc.
6 * Author: Jan Friesse (jfriesse@redhat.com)
8 * This software licensed under BSD license, the text of which follows:
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * - Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * - Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 * - Neither the name of the Red Hat, Inc. nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
42 #include "qnet-config.h"
45 #include "dynar-str.h"
46 #include "dynar-getopt-lex.h"
49 #include "pr-poll-array.h"
50 #include "qnetd-advanced-settings.h"
51 #include "qnetd-algorithm.h"
52 #include "qnetd-instance.h"
53 #include "qnetd-ipc.h"
54 #include "qnetd-client-net.h"
55 #include "qnetd-client-msg-received.h"
56 #include "qnetd-poll-array-user-data.h"
60 #ifdef HAVE_LIBSYSTEMD
61 #include <systemd/sd-daemon.h>
65 * This is global variable used for comunication with main loop and signal (calls close)
67 struct qnetd_instance
*global_instance
;
69 enum tlv_decision_algorithm_type
70 qnetd_static_supported_decision_algorithms
[QNETD_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE
] = {
71 TLV_DECISION_ALGORITHM_TYPE_TEST
,
72 TLV_DECISION_ALGORITHM_TYPE_FFSPLIT
,
73 TLV_DECISION_ALGORITHM_TYPE_2NODELMS
,
74 TLV_DECISION_ALGORITHM_TYPE_LMS
,
81 log_nss(LOG_CRIT
, "NSS error");
90 log_nss(LOG_WARNING
, "NSS warning");
94 qnetd_pr_poll_array_create(struct qnetd_instance
*instance
)
96 struct pr_poll_array
*poll_array
;
97 const struct qnetd_client_list
*client_list
;
98 struct qnetd_client
*client
;
99 PRPollDesc
*poll_desc
;
100 struct qnetd_poll_array_user_data
*user_data
;
101 const struct unix_socket_client_list
*ipc_client_list
;
102 struct unix_socket_client
*ipc_client
;
104 poll_array
= &instance
->poll_array
;
105 client_list
= &instance
->clients
;
106 ipc_client_list
= &instance
->local_ipc
.clients
;
108 pr_poll_array_clean(poll_array
);
110 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
114 poll_desc
->fd
= instance
->server
.socket
;
115 poll_desc
->in_flags
= PR_POLL_READ
;
117 user_data
->type
= QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET
;
119 if (qnetd_ipc_is_closed(instance
)) {
120 log(LOG_DEBUG
, "Listening socket is closed");
125 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
129 poll_desc
->fd
= instance
->ipc_socket_poll_fd
;
130 poll_desc
->in_flags
= PR_POLL_READ
;
131 user_data
->type
= QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
;
133 TAILQ_FOREACH(client
, client_list
, entries
) {
134 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
137 poll_desc
->fd
= client
->socket
;
138 poll_desc
->in_flags
= PR_POLL_READ
;
140 if (!send_buffer_list_empty(&client
->send_buffer_list
)) {
141 poll_desc
->in_flags
|= PR_POLL_WRITE
;
144 user_data
->type
= QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT
;
145 user_data
->client
= client
;
148 TAILQ_FOREACH(ipc_client
, ipc_client_list
, entries
) {
149 if (!ipc_client
->reading_line
&& !ipc_client
->writing_buffer
) {
153 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
157 poll_desc
->fd
= ((struct qnetd_ipc_user_data
*)ipc_client
->user_data
)->nspr_poll_fd
;
158 if (ipc_client
->reading_line
) {
159 poll_desc
->in_flags
|= PR_POLL_READ
;
162 if (ipc_client
->writing_buffer
) {
163 poll_desc
->in_flags
|= PR_POLL_WRITE
;
166 user_data
->type
= QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
;
167 user_data
->ipc_client
= ipc_client
;
170 pr_poll_array_gc(poll_array
);
172 return (poll_array
->array
);
176 qnetd_poll(struct qnetd_instance
*instance
)
178 struct qnetd_client
*client
;
182 int client_disconnect
;
183 struct qnetd_poll_array_user_data
*user_data
;
184 struct unix_socket_client
*ipc_client
;
187 client_disconnect
= 0;
189 pfds
= qnetd_pr_poll_array_create(instance
);
194 if ((poll_res
= PR_Poll(pfds
, pr_poll_array_size(&instance
->poll_array
),
195 timer_list_time_to_expire(&instance
->main_timer_list
))) >= 0) {
196 timer_list_expire(&instance
->main_timer_list
);
199 * Walk thru pfds array and process events
201 for (i
= 0; i
< pr_poll_array_size(&instance
->poll_array
); i
++) {
202 user_data
= pr_poll_array_get_user_data(&instance
->poll_array
, i
);
206 client_disconnect
= 0;
208 switch (user_data
->type
) {
209 case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
211 case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT
:
212 client
= user_data
->client
;
213 client_disconnect
= client
->schedule_disconnect
;
215 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
217 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
218 ipc_client
= user_data
->ipc_client
;
219 client_disconnect
= ipc_client
->schedule_disconnect
;
222 if (!client_disconnect
&& poll_res
> 0 &&
223 pfds
[i
].out_flags
& PR_POLL_READ
) {
224 switch (user_data
->type
) {
225 case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
226 qnetd_client_net_accept(instance
);
228 case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT
:
229 if (qnetd_client_net_read(instance
, client
) == -1) {
230 client_disconnect
= 1;
233 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
234 qnetd_ipc_accept(instance
, &ipc_client
);
236 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
237 qnetd_ipc_io_read(instance
, ipc_client
);
242 if (!client_disconnect
&& poll_res
> 0 &&
243 pfds
[i
].out_flags
& PR_POLL_WRITE
) {
244 switch (user_data
->type
) {
245 case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
247 * Poll write on listen socket -> fatal error
249 log(LOG_CRIT
, "POLL_WRITE on listening socket");
253 case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT
:
254 if (qnetd_client_net_write(instance
, client
) == -1) {
255 client_disconnect
= 1;
258 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
259 log(LOG_CRIT
, "POLL_WRITE on listening IPC socket");
262 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
263 qnetd_ipc_io_write(instance
, ipc_client
);
268 if (!client_disconnect
&& poll_res
> 0 &&
269 (pfds
[i
].out_flags
& (PR_POLL_ERR
|PR_POLL_NVAL
|PR_POLL_HUP
|PR_POLL_EXCEPT
)) &&
270 !(pfds
[i
].out_flags
& (PR_POLL_READ
|PR_POLL_WRITE
))) {
271 switch (user_data
->type
) {
272 case QNETD_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
273 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
274 if (pfds
[i
].out_flags
!= PR_POLL_NVAL
) {
276 * Poll ERR on listening socket is fatal error.
277 * POLL_NVAL is used as a signal to quit poll loop.
279 log(LOG_CRIT
, "POLL_ERR (%u) on listening "
280 "socket", pfds
[i
].out_flags
);
282 log(LOG_DEBUG
, "Listening socket is closed");
287 case QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT
:
288 log(LOG_DEBUG
, "POLL_ERR (%u) on client socket. "
289 "Disconnecting.", pfds
[i
].out_flags
);
291 client_disconnect
= 1;
293 case QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
294 log(LOG_DEBUG
, "POLL_ERR (%u) on ipc client socket."
295 " Disconnecting.", pfds
[i
].out_flags
);
297 client_disconnect
= 1;
303 * If client is scheduled for disconnect, disconnect it
305 if (user_data
->type
== QNETD_POLL_ARRAY_USER_DATA_TYPE_CLIENT
&&
307 qnetd_instance_client_disconnect(instance
, client
, 0);
308 } else if (user_data
->type
== QNETD_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
&&
309 (client_disconnect
|| ipc_client
->schedule_disconnect
)) {
310 qnetd_ipc_client_disconnect(instance
, ipc_client
);
320 signal_int_handler(int sig
)
323 log(LOG_DEBUG
, "SIGINT received - closing server IPC socket");
325 qnetd_ipc_close(global_instance
);
329 signal_term_handler(int sig
)
332 log(LOG_DEBUG
, "SIGTERM received - closing server IPC socket");
334 qnetd_ipc_close(global_instance
);
338 signal_handlers_register(void)
340 struct sigaction act
;
342 act
.sa_handler
= signal_int_handler
;
343 sigemptyset(&act
.sa_mask
);
344 act
.sa_flags
= SA_RESTART
;
346 sigaction(SIGINT
, &act
, NULL
);
348 act
.sa_handler
= signal_term_handler
;
349 sigemptyset(&act
.sa_mask
);
350 act
.sa_flags
= SA_RESTART
;
352 sigaction(SIGTERM
, &act
, NULL
);
359 printf("usage: %s [-46dfhv] [-l listen_addr] [-p listen_port] [-s tls]\n", QNETD_PROGRAM_NAME
);
360 printf("%14s[-c client_cert_required] [-m max_clients] [-S option=value[,option2=value2,...]]\n", "");
364 display_version(void)
366 enum msg_type
*supported_messages
;
367 size_t no_supported_messages
;
370 msg_get_supported_messages(&supported_messages
, &no_supported_messages
);
371 printf("Corosync Qdevice Network Daemon, version '%s'\n\n", VERSION
);
372 printf("Supported algorithms: ");
373 for (zi
= 0; zi
< QNETD_STATIC_SUPPORTED_DECISION_ALGORITHMS_SIZE
; zi
++) {
378 tlv_decision_algorithm_type_to_str(qnetd_static_supported_decision_algorithms
[zi
]),
379 qnetd_static_supported_decision_algorithms
[zi
]);
382 printf("Supported message types: ");
383 for (zi
= 0; zi
< no_supported_messages
; zi
++) {
387 printf("%s (%u)", msg_type_to_str(supported_messages
[zi
]), supported_messages
[zi
]);
393 cli_parse_long_opt(struct qnetd_advanced_settings
*advanced_settings
, const char *long_opt
)
395 struct dynar_getopt_lex lex
;
396 struct dynar dynar_long_opt
;
401 dynar_init(&dynar_long_opt
, strlen(long_opt
) + 1);
402 if (dynar_str_cpy(&dynar_long_opt
, long_opt
) != 0) {
403 errx(1, "Can't alloc memory for long option");
406 dynar_getopt_lex_init(&lex
, &dynar_long_opt
);
408 while (dynar_getopt_lex_token_next(&lex
) == 0 && strcmp(dynar_data(&lex
.option
), "") != 0) {
409 opt
= dynar_data(&lex
.option
);
410 val
= dynar_data(&lex
.value
);
412 res
= qnetd_advanced_settings_set(advanced_settings
, opt
, val
);
415 errx(1, "Unknown option '%s'", opt
);
418 errx(1, "Invalid value '%s' for option '%s'", val
, opt
);
423 dynar_getopt_lex_destroy(&lex
);
424 dynar_destroy(&dynar_long_opt
);
428 cli_parse(int argc
, char * const argv
[], char **host_addr
, uint16_t *host_port
, int *foreground
,
429 int *debug_log
, int *bump_log_priority
, enum tlv_tls_supported
*tls_supported
,
430 int *client_cert_required
, size_t *max_clients
, PRIntn
*address_family
,
431 struct qnetd_advanced_settings
*advanced_settings
)
437 *host_port
= QNETD_DEFAULT_HOST_PORT
;
440 *bump_log_priority
= 0;
441 *tls_supported
= QNETD_DEFAULT_TLS_SUPPORTED
;
442 *client_cert_required
= QNETD_DEFAULT_TLS_CLIENT_CERT_REQUIRED
;
443 *max_clients
= QNETD_DEFAULT_MAX_CLIENTS
;
444 *address_family
= PR_AF_UNSPEC
;
446 while ((ch
= getopt(argc
, argv
, "46dfhvc:l:m:p:S:s:")) != -1) {
449 *address_family
= PR_AF_INET
;
452 *address_family
= PR_AF_INET6
;
459 *bump_log_priority
= 1;
464 if ((*client_cert_required
= utils_parse_bool_str(optarg
)) == -1) {
465 errx(1, "client_cert_required should be on/yes/1, off/no/0");
470 *host_addr
= strdup(optarg
);
471 if (*host_addr
== NULL
) {
472 errx(1, "Can't alloc memory for host addr string");
476 if (utils_strtonum(optarg
, 0, LLONG_MAX
, &tmpll
) == -1) {
477 errx(1, "max clients value %s is invalid", optarg
);
480 *max_clients
= (size_t)tmpll
;
483 if (utils_strtonum(optarg
, 1, UINT16_MAX
, &tmpll
) == -1) {
484 errx(1, "host port must be in range 1-%u", UINT16_MAX
);
490 cli_parse_long_opt(advanced_settings
, optarg
);
493 if (strcasecmp(optarg
, "on") == 0) {
494 *tls_supported
= QNETD_DEFAULT_TLS_SUPPORTED
;
495 } else if (strcasecmp(optarg
, "off") == 0) {
496 *tls_supported
= TLV_TLS_UNSUPPORTED
;
497 } else if (strcasecmp(optarg
, "req") == 0) {
498 *tls_supported
= TLV_TLS_REQUIRED
;
500 errx(1, "tls must be one of on, off, req");
517 main(int argc
, char * const argv
[])
519 struct qnetd_instance instance
;
520 struct qnetd_advanced_settings advanced_settings
;
525 int bump_log_priority
;
526 enum tlv_tls_supported tls_supported
;
527 int client_cert_required
;
529 PRIntn address_family
;
531 int another_instance_running
;
533 if (qnetd_advanced_settings_init(&advanced_settings
) != 0) {
534 errx(1, "Can't alloc memory for advanced settings");
537 cli_parse(argc
, argv
, &host_addr
, &host_port
, &foreground
, &debug_log
, &bump_log_priority
,
538 &tls_supported
, &client_cert_required
, &max_clients
, &address_family
, &advanced_settings
);
541 log_init(QNETD_PROGRAM_NAME
, LOG_TARGET_STDERR
);
543 log_init(QNETD_PROGRAM_NAME
, LOG_TARGET_SYSLOG
);
546 log_set_debug(debug_log
);
547 log_set_priority_bump(bump_log_priority
);
550 * Check that it's possible to open NSS dir if needed
552 if (nss_sock_check_db_dir((tls_supported
!= TLV_TLS_UNSUPPORTED
?
553 advanced_settings
.nss_db_dir
: NULL
)) != 0) {
554 log_err(LOG_ERR
, "Can't open NSS DB directory");
566 if ((lock_file
= utils_flock(advanced_settings
.lock_file
, getpid(),
567 &another_instance_running
)) == -1) {
568 if (another_instance_running
) {
569 log(LOG_ERR
, "Another instance is running");
571 log_err(LOG_ERR
, "Can't acquire lock");
577 log(LOG_DEBUG
, "Initializing nss");
578 if (nss_sock_init_nss((tls_supported
!= TLV_TLS_UNSUPPORTED
?
579 advanced_settings
.nss_db_dir
: NULL
)) != 0) {
583 if (SSL_ConfigServerSessionIDCache(0, 0, 0, NULL
) != SECSuccess
) {
587 if (qnetd_instance_init(&instance
, tls_supported
, client_cert_required
,
588 max_clients
, &advanced_settings
) == -1) {
589 log(LOG_ERR
, "Can't initialize qnetd");
592 instance
.host_addr
= host_addr
;
593 instance
.host_port
= host_port
;
595 if (tls_supported
!= TLV_TLS_UNSUPPORTED
&& qnetd_instance_init_certs(&instance
) == -1) {
599 log(LOG_DEBUG
, "Initializing local socket");
600 if (qnetd_ipc_init(&instance
) != 0) {
604 log(LOG_DEBUG
, "Creating listening socket");
605 instance
.server
.socket
= nss_sock_create_listen_socket(instance
.host_addr
,
606 instance
.host_port
, address_family
);
607 if (instance
.server
.socket
== NULL
) {
611 if (nss_sock_set_non_blocking(instance
.server
.socket
) != 0) {
615 if (PR_Listen(instance
.server
.socket
, instance
.advanced_settings
->listen_backlog
) !=
620 global_instance
= &instance
;
621 signal_handlers_register();
623 log(LOG_DEBUG
, "Registering algorithms");
624 if (qnetd_algorithm_register_all() != 0) {
628 log(LOG_DEBUG
, "QNetd ready to provide service");
630 #ifdef HAVE_LIBSYSTEMD
631 sd_notify(0, "READY=1");
637 while (qnetd_poll(&instance
) == 0) {
643 qnetd_ipc_destroy(&instance
);
645 if (PR_Close(instance
.server
.socket
) != PR_SUCCESS
) {
649 CERT_DestroyCertificate(instance
.server
.cert
);
650 SECKEY_DestroyPrivateKey(instance
.server
.private_key
);
652 SSL_ClearSessionCache();
654 SSL_ShutdownServerSessionIDCache();
656 qnetd_instance_destroy(&instance
);
658 qnetd_advanced_settings_destroy(&advanced_settings
);
660 if (NSS_Shutdown() != SECSuccess
) {
664 if (PR_Cleanup() != PR_SUCCESS
) {