2 * Copyright (c) 2015-2017 Red Hat, Inc.
6 * Author: Jan Friesse (jfriesse@redhat.com)
8 * This software licensed under BSD license, the text of which follows:
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * - Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * - Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 * - Neither the name of the Red Hat, Inc. nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
35 #include "qdevice-cmap.h"
36 #include "qdevice-net-poll.h"
37 #include "qdevice-log.h"
38 #include "qdevice-net-send.h"
39 #include "qdevice-net-socket.h"
40 #include "qdevice-votequorum.h"
41 #include "qdevice-ipc.h"
42 #include "qdevice-net-poll-array-user-data.h"
43 #include "qdevice-heuristics.h"
44 #include "qdevice-heuristics-cmd.h"
47 * Needed for creating nspr handle from unix fd
49 #include <private/pprio.h>
52 qdevice_net_poll_read_socket(struct qdevice_net_instance
*instance
)
55 if (qdevice_net_socket_read(instance
) == -1) {
56 instance
->schedule_disconnect
= 1;
61 qdevice_net_poll_read_votequorum(struct qdevice_net_instance
*instance
)
64 if (qdevice_votequorum_dispatch(instance
->qdevice_instance_ptr
) == -1) {
65 instance
->schedule_disconnect
= 1;
66 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED
;
71 qdevice_net_poll_read_cmap(struct qdevice_net_instance
*instance
)
74 if (qdevice_cmap_dispatch(instance
->qdevice_instance_ptr
) == -1) {
75 instance
->schedule_disconnect
= 1;
76 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED
;
81 qdevice_net_poll_write_socket(struct qdevice_net_instance
*instance
, const PRPollDesc
*pfd
)
85 if (instance
->state
== QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT
) {
86 res
= nss_sock_non_blocking_client_succeeded(pfd
);
89 * Connect failed -> try next
91 res
= nss_sock_non_blocking_client_try_next(&instance
->non_blocking_client
);
93 qdevice_log_nss(LOG_ERR
, "Can't connect to qnetd host.");
94 nss_sock_non_blocking_client_destroy(&instance
->non_blocking_client
);
96 } else if (res
== 0) {
100 } else if (res
== 1) {
104 instance
->socket
= instance
->non_blocking_client
.socket
;
105 nss_sock_non_blocking_client_destroy(&instance
->non_blocking_client
);
106 instance
->non_blocking_client
.socket
= NULL
;
108 instance
->state
= QDEVICE_NET_INSTANCE_STATE_SENDING_PREINIT_REPLY
;
110 qdevice_log(LOG_DEBUG
, "Sending preinit msg to qnetd");
111 if (qdevice_net_send_preinit(instance
) != 0) {
112 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER
;
113 instance
->schedule_disconnect
= 1;
116 qdevice_log(LOG_CRIT
, "Unhandled nss_sock_non_blocking_client_succeeded");
120 if (qdevice_net_socket_write(instance
) == -1) {
121 instance
->schedule_disconnect
= 1;
127 qdevice_net_poll_err_socket(struct qdevice_net_instance
*instance
, const PRPollDesc
*pfd
)
130 if (instance
->state
== QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT
) {
132 * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work
133 * only with poll, select works as expected!???).
134 * So test if client is still valid and if pollout was not already called (ensured
135 * by default because of order in PR_Poll).
136 * If both applies it's possible to emulate pollout set by calling poll_write.
138 if (!instance
->non_blocking_client
.destroyed
) {
139 qdevice_net_poll_write_socket(instance
, pfd
);
142 qdevice_log(LOG_ERR
, "POLL_ERR (%u) on main socket", pfd
->out_flags
);
144 instance
->schedule_disconnect
= 1;
145 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION
;
150 qdevice_net_poll_read_heuristics_log(struct qdevice_net_instance
*instance
)
154 res
= qdevice_heuristics_log_read_from_pipe(&instance
->qdevice_instance_ptr
->heuristics_instance
);
156 instance
->schedule_disconnect
= 1;
157 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED
;
162 qdevice_net_poll_read_heuristics_cmd(struct qdevice_net_instance
*instance
)
166 res
= qdevice_heuristics_cmd_read_from_pipe(&instance
->qdevice_instance_ptr
->heuristics_instance
);
168 instance
->schedule_disconnect
= 1;
169 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED
;
174 qdevice_net_poll_write_heuristics_cmd(struct qdevice_net_instance
*instance
)
178 res
= qdevice_heuristics_cmd_write(&instance
->qdevice_instance_ptr
->heuristics_instance
);
180 instance
->schedule_disconnect
= 1;
181 instance
->disconnect_reason
=
182 QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG
;
187 qdevice_net_poll_read_ipc_socket(struct qdevice_net_instance
*instance
)
189 struct unix_socket_client
*client
;
191 struct qdevice_ipc_user_data
*user_data
;
193 if (qdevice_ipc_accept(instance
->qdevice_instance_ptr
, &client
) != 0) {
197 prfd
= PR_CreateSocketPollFd(client
->socket
);
199 qdevice_log_nss(LOG_CRIT
, "Can't create NSPR poll fd for IPC client. "
200 "Disconnecting client");
201 qdevice_ipc_client_disconnect(instance
->qdevice_instance_ptr
, client
);
206 user_data
= (struct qdevice_ipc_user_data
*)client
->user_data
;
207 user_data
->model_data
= (void *)prfd
;
211 qdevice_net_pr_poll_array_create(struct qdevice_net_instance
*instance
)
213 struct pr_poll_array
*poll_array
;
214 PRPollDesc
*poll_desc
;
215 struct qdevice_net_poll_array_user_data
*user_data
;
216 struct unix_socket_client
*ipc_client
;
217 const struct unix_socket_client_list
*ipc_client_list
;
218 struct qdevice_ipc_user_data
*qdevice_ipc_user_data
;
220 poll_array
= &instance
->poll_array
;
221 ipc_client_list
= &instance
->qdevice_instance_ptr
->local_ipc
.clients
;
223 if (qdevice_ipc_is_closed(instance
->qdevice_instance_ptr
)) {
224 qdevice_log(LOG_DEBUG
, "Local socket is closed");
225 instance
->schedule_disconnect
= 1;
226 instance
->disconnect_reason
= QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED
;
231 pr_poll_array_clean(poll_array
);
233 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
236 poll_desc
->fd
= instance
->votequorum_poll_fd
;
237 poll_desc
->in_flags
= PR_POLL_READ
;
238 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM
;
240 if (!instance
->qdevice_instance_ptr
->sync_in_progress
) {
241 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
244 poll_desc
->fd
= instance
->cmap_poll_fd
;
245 poll_desc
->in_flags
= PR_POLL_READ
;
246 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP
;
249 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
252 poll_desc
->fd
= instance
->ipc_socket_poll_fd
;
253 poll_desc
->in_flags
= PR_POLL_READ
;
254 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
;
256 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
260 poll_desc
->fd
= instance
->heuristics_pipe_log_recv_poll_fd
;
261 poll_desc
->in_flags
= PR_POLL_READ
;
262 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV
;
264 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
268 poll_desc
->fd
= instance
->heuristics_pipe_cmd_recv_poll_fd
;
269 poll_desc
->in_flags
= PR_POLL_READ
;
270 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV
;
272 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
276 if (!send_buffer_list_empty(
277 &instance
->qdevice_instance_ptr
->heuristics_instance
.cmd_out_buffer_list
)) {
278 poll_desc
->fd
= instance
->heuristics_pipe_cmd_send_poll_fd
;
279 poll_desc
->in_flags
= PR_POLL_WRITE
;
280 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND
;
282 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
287 if (instance
->state
!= QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT
||
288 !instance
->non_blocking_client
.destroyed
) {
289 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
293 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET
;
295 if (instance
->state
== QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT
) {
296 poll_desc
->fd
= instance
->non_blocking_client
.socket
;
297 poll_desc
->in_flags
= PR_POLL_WRITE
| PR_POLL_EXCEPT
;
299 poll_desc
->fd
= instance
->socket
;
300 poll_desc
->in_flags
= PR_POLL_READ
;
302 if (!send_buffer_list_empty(&instance
->send_buffer_list
)) {
303 poll_desc
->in_flags
|= PR_POLL_WRITE
;
308 TAILQ_FOREACH(ipc_client
, ipc_client_list
, entries
) {
309 if (!ipc_client
->reading_line
&& !ipc_client
->writing_buffer
) {
313 if (pr_poll_array_add(poll_array
, &poll_desc
, (void **)&user_data
) < 0) {
317 qdevice_ipc_user_data
= (struct qdevice_ipc_user_data
*)ipc_client
->user_data
;
318 poll_desc
->fd
= (PRFileDesc
*)qdevice_ipc_user_data
->model_data
;
319 if (ipc_client
->reading_line
) {
320 poll_desc
->in_flags
|= PR_POLL_READ
;
323 if (ipc_client
->writing_buffer
) {
324 poll_desc
->in_flags
|= PR_POLL_WRITE
;
327 user_data
->type
= QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
;
328 user_data
->ipc_client
= ipc_client
;
331 pr_poll_array_gc(poll_array
);
333 return (poll_array
->array
);
337 qdevice_net_poll(struct qdevice_net_instance
*instance
)
343 struct qdevice_net_poll_array_user_data
*user_data
;
344 struct unix_socket_client
*ipc_client
;
345 struct qdevice_ipc_user_data
*qdevice_ipc_user_data
;
348 pfds
= qdevice_net_pr_poll_array_create(instance
);
353 instance
->schedule_disconnect
= 0;
355 if ((poll_res
= PR_Poll(pfds
, pr_poll_array_size(&instance
->poll_array
),
356 timer_list_time_to_expire(&instance
->main_timer_list
))) > 0) {
357 for (i
= 0; i
< pr_poll_array_size(&instance
->poll_array
); i
++) {
358 user_data
= pr_poll_array_get_user_data(&instance
->poll_array
, i
);
360 ipc_client
= user_data
->ipc_client
;
362 if (pfds
[i
].out_flags
& PR_POLL_READ
) {
365 switch (user_data
->type
) {
366 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
368 qdevice_net_poll_read_socket(instance
);
370 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM
:
372 qdevice_net_poll_read_votequorum(instance
);
374 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP
:
376 qdevice_net_poll_read_cmap(instance
);
378 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
380 qdevice_net_poll_read_ipc_socket(instance
);
382 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
384 qdevice_ipc_io_read(instance
->qdevice_instance_ptr
, ipc_client
);
386 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND
:
388 * Read on heuristics cmd send fd shouldn't happen
391 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV
:
393 qdevice_net_poll_read_heuristics_cmd(instance
);
395 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV
:
397 qdevice_net_poll_read_heuristics_log(instance
);
400 * Default is not defined intentionally. Compiler shows warning when
401 * new poll_array_user_data_type is added
405 if (!case_processed
) {
406 qdevice_log(LOG_CRIT
, "Unhandled read on poll descriptor %u", i
);
411 if (!instance
->schedule_disconnect
&& pfds
[i
].out_flags
& PR_POLL_WRITE
) {
414 switch (user_data
->type
) {
415 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
417 qdevice_net_poll_write_socket(instance
, &pfds
[i
]);
419 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
421 qdevice_ipc_io_write(instance
->qdevice_instance_ptr
, ipc_client
);
423 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND
:
425 qdevice_net_poll_write_heuristics_cmd(instance
);
427 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM
:
428 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP
:
429 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
430 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV
:
431 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV
:
433 * Write on votequorum, cmap, ipc socket and
434 * heuristics log shouldn't happen.
438 * Default is not defined intentionally. Compiler shows warning when
439 * new poll_array_user_data_type is added
443 if (!case_processed
) {
444 qdevice_log(LOG_CRIT
, "Unhandled write on poll descriptor %u", i
);
449 if (!instance
->schedule_disconnect
&&
450 (pfds
[i
].out_flags
& (PR_POLL_ERR
|PR_POLL_NVAL
|PR_POLL_HUP
|PR_POLL_EXCEPT
)) &&
451 !(pfds
[i
].out_flags
& (PR_POLL_READ
|PR_POLL_WRITE
))) {
454 switch (user_data
->type
) {
455 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET
:
457 qdevice_net_poll_err_socket(instance
, &pfds
[i
]);
459 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET
:
461 if (pfds
[i
].out_flags
!= PR_POLL_NVAL
) {
462 qdevice_log(LOG_CRIT
, "POLLERR (%u) on local socket",
466 qdevice_log(LOG_DEBUG
, "Local socket is closed");
467 instance
->schedule_disconnect
= 1;
468 instance
->disconnect_reason
=
469 QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED
;
472 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
:
474 qdevice_log(LOG_DEBUG
, "POLL_ERR (%u) on ipc client socket. "
475 "Disconnecting.", pfds
[i
].out_flags
);
476 ipc_client
->schedule_disconnect
= 1;
478 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM
:
479 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP
:
481 qdevice_log(LOG_DEBUG
, "POLL_ERR (%u) on corosync socket. "
482 "Disconnecting.", pfds
[i
].out_flags
);
484 instance
->schedule_disconnect
= 1;
485 instance
->disconnect_reason
=
486 QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED
;
488 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV
:
489 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV
:
490 case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND
:
494 * Closed pipe doesn't mean return of PR_POLL_READ. To display
495 * better log message, we call read log as if POLL_READ would
498 qdevice_net_poll_read_heuristics_log(instance
);
500 qdevice_log(LOG_DEBUG
, "POLL_ERR (%u) on heuristics pipe. "
501 "Disconnecting.", pfds
[i
].out_flags
);
503 instance
->schedule_disconnect
= 1;
504 instance
->disconnect_reason
=
505 QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED
;
508 * Default is not defined intentionally. Compiler shows warning when
509 * new poll_array_user_data_type is added
513 if (!case_processed
) {
514 qdevice_log(LOG_CRIT
, "Unhandled error on poll descriptor %u", i
);
519 if (user_data
->type
== QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT
&&
520 ipc_client
->schedule_disconnect
) {
521 qdevice_ipc_user_data
= (struct qdevice_ipc_user_data
*)ipc_client
->user_data
;
522 prfd
= (PRFileDesc
*)qdevice_ipc_user_data
->model_data
;
524 if (PR_DestroySocketPollFd(prfd
) != PR_SUCCESS
) {
525 qdevice_log_nss(LOG_WARNING
, "Unable to destroy client IPC poll socket fd");
528 qdevice_ipc_client_disconnect(instance
->qdevice_instance_ptr
, ipc_client
);
533 if (!instance
->schedule_disconnect
) {
534 timer_list_expire(&instance
->main_timer_list
);
537 if (instance
->schedule_disconnect
) {
539 * Schedule disconnect can be set by this function, by some timer_list callback
540 * or cmap/votequorum callbacks