4 * Copyright (C) 1997-1999 Kunihiro Ishiguro
5 * Copyright (C) 2015-2018 Cumulus Networks, Inc.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along
19 * with this program; see the file COPYING; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25 /* clang-format off */
26 #include <errno.h> /* for errno */
27 #include <netinet/in.h> /* for sockaddr_in */
28 #include <stdint.h> /* for uint8_t */
29 #include <stdio.h> /* for snprintf */
30 #include <sys/socket.h> /* for sockaddr_storage, AF_UNIX, accept... */
31 #include <sys/stat.h> /* for umask, mode_t */
32 #include <sys/un.h> /* for sockaddr_un */
33 #include <time.h> /* for NULL, tm, gmtime, time_t */
34 #include <unistd.h> /* for close, unlink, ssize_t */
36 #include "lib/buffer.h" /* for BUFFER_EMPTY, BUFFER_ERROR, BUFFE... */
37 #include "lib/command.h" /* for vty, install_element, CMD_SUCCESS... */
38 #include "lib/hook.h" /* for DEFINE_HOOK, DEFINE_KOOH, hook_call */
39 #include "lib/linklist.h" /* for ALL_LIST_ELEMENTS_RO, ALL_LIST_EL... */
40 #include "lib/libfrr.h" /* for frr_zclient_addr */
41 #include "lib/log.h" /* for zlog_warn, zlog_debug, safe_strerror */
42 #include "lib/memory.h" /* for MTYPE_TMP, XCALLOC, XFREE */
43 #include "lib/monotime.h" /* for monotime, ONE_DAY_SECOND, ONE_WEE... */
44 #include "lib/network.h" /* for set_nonblocking */
45 #include "lib/privs.h" /* for zebra_privs_t, ZPRIVS_LOWER, ZPRI... */
46 #include "lib/route_types.h" /* for ZEBRA_ROUTE_MAX */
47 #include "lib/sockopt.h" /* for setsockopt_so_recvbuf, setsockopt... */
48 #include "lib/sockunion.h" /* for sockopt_reuseaddr, sockopt_reuseport */
49 #include "lib/stream.h" /* for STREAM_SIZE, stream (ptr only), ... */
50 #include "lib/thread.h" /* for thread (ptr only), THREAD_ARG, ... */
51 #include "lib/vrf.h" /* for vrf_info_lookup, VRF_DEFAULT */
52 #include "lib/vty.h" /* for vty_out, vty (ptr only) */
53 #include "lib/zclient.h" /* for zmsghdr, ZEBRA_HEADER_SIZE, ZEBRA... */
54 #include "lib/frr_pthread.h" /* for frr_pthread_new, frr_pthread_stop... */
55 #include "lib/frratomic.h" /* for atomic_load_explicit, atomic_stor... */
56 #include "lib/lib_errors.h" /* for generic ferr ids */
57 #include "lib/printfrr.h" /* for string functions */
59 #include "zebra/debug.h" /* for various debugging macros */
60 #include "zebra/rib.h" /* for rib_score_proto */
61 #include "zebra/zapi_msg.h" /* for zserv_handle_commands */
62 #include "zebra/zebra_vrf.h" /* for zebra_vrf_lookup_by_id, zvrf */
63 #include "zebra/zserv.h" /* for zserv */
64 #include "zebra/zebra_router.h"
65 #include "zebra/zebra_errors.h" /* for error messages */
69 extern struct zebra_privs_t zserv_privs
;
71 /* The listener socket for clients connecting to us */
74 /* The lock that protects access to zapi client objects */
75 static pthread_mutex_t client_mutex
;
77 static struct zserv
*find_client_internal(uint8_t proto
,
78 unsigned short instance
,
81 /* Mem type for zclients. */
82 DEFINE_MTYPE_STATIC(ZEBRA
, ZSERV_CLIENT
, "ZClients");
85 * Client thread events.
87 * These are used almost exclusively by client threads to drive their own event
88 * loops. The only exception is in zserv_client_create(), which pushes an
89 * initial ZSERV_CLIENT_READ event to start the API handler loop.
91 enum zserv_client_event
{
92 /* Schedule a socket read */
94 /* Schedule a buffer write */
101 * These are used by client threads to notify the main thread about various
102 * events and to make processing requests.
105 /* Schedule listen job on Zebra API socket */
107 /* The calling client has packets on its input buffer */
108 ZSERV_PROCESS_MESSAGES
,
109 /* The calling client wishes to be killed */
110 ZSERV_HANDLE_CLIENT_FAIL
,
114 * Zebra server event driver for all client threads.
116 * This is essentially a wrapper around thread_add_event() that centralizes
117 * those scheduling calls into one place.
119 * All calls to this function schedule an event on the pthread running the
123 * the client in question, and thread target
126 * the event to notify them about
128 static void zserv_client_event(struct zserv
*client
,
129 enum zserv_client_event event
);
132 * Zebra server event driver for the main thread.
134 * This is essentially a wrapper around thread_add_event() that centralizes
135 * those scheduling calls into one place.
137 * All calls to this function schedule an event on Zebra's main pthread.
140 * the client in question
143 * the event to notify the main thread about
145 static void zserv_event(struct zserv
*client
, enum zserv_event event
);
148 /* Client thread lifecycle -------------------------------------------------- */
151 * Free a zserv client object.
153 void zserv_client_delete(struct zserv
*client
)
155 XFREE(MTYPE_ZSERV_CLIENT
, client
);
159 * Log zapi message to zlog.
170 void zserv_log_message(const char *errmsg
, struct stream
*msg
,
173 zlog_debug("Rx'd ZAPI message");
175 zlog_debug("%s", errmsg
);
177 zlog_debug(" Length: %d", hdr
->length
);
178 zlog_debug("Command: %s", zserv_command_string(hdr
->command
));
179 zlog_debug(" VRF: %u", hdr
->vrf_id
);
185 * Gracefuly shut down a client connection.
187 * Cancel any pending tasks for the client's thread. Then schedule a task on
188 * the main thread to shut down the calling thread.
190 * It is not safe to close the client socket in this function. The socket is
191 * owned by the main thread.
193 * Must be called from the client pthread, never the main thread.
195 static void zserv_client_fail(struct zserv
*client
)
197 flog_warn(EC_ZEBRA_CLIENT_IO_ERROR
,
198 "Client '%s' encountered an error and is shutting down.",
199 zebra_route_string(client
->proto
));
201 atomic_store_explicit(&client
->pthread
->running
, false,
202 memory_order_relaxed
);
204 THREAD_OFF(client
->t_read
);
205 THREAD_OFF(client
->t_write
);
206 zserv_event(client
, ZSERV_HANDLE_CLIENT_FAIL
);
210 * Write all pending messages to client socket.
212 * This function first attempts to flush any buffered data. If unsuccessful,
213 * the function reschedules itself and returns. If successful, it pops all
214 * available messages from the output queue and continues to write data
215 * directly to the socket until the socket would block. If the socket never
216 * blocks and all data is written, the function returns without rescheduling
217 * itself. If the socket ends up throwing EWOULDBLOCK, the remaining data is
218 * buffered and the function reschedules itself.
220 * The utility of the buffer is that it allows us to vastly reduce lock
221 * contention by allowing us to pop *all* messages off the output queue at once
222 * instead of locking and unlocking each time we want to pop a single message
223 * off the queue. The same thing could arguably be accomplished faster by
224 * allowing the main thread to write directly into the buffer instead of
225 * enqueuing packets onto an intermediary queue, but the intermediary queue
226 * allows us to expose information about input and output queues to the user in
227 * terms of number of packets rather than size of data.
229 static void zserv_write(struct thread
*thread
)
231 struct zserv
*client
= THREAD_ARG(thread
);
234 struct stream_fifo
*cache
;
236 /* If we have any data pending, try to flush it first */
237 switch (buffer_flush_all(client
->wb
, client
->sock
)) {
241 atomic_store_explicit(&client
->last_write_time
, monotime(NULL
),
242 memory_order_relaxed
);
243 zserv_client_event(client
, ZSERV_CLIENT_WRITE
);
249 cache
= stream_fifo_new();
251 frr_with_mutex (&client
->obuf_mtx
) {
252 while (stream_fifo_head(client
->obuf_fifo
))
253 stream_fifo_push(cache
,
254 stream_fifo_pop(client
->obuf_fifo
));
259 stream_set_getp(msg
, 0);
260 wcmd
= stream_getw_from(msg
, ZAPI_HEADER_CMD_LOCATION
);
263 while (stream_fifo_head(cache
)) {
264 msg
= stream_fifo_pop(cache
);
265 buffer_put(client
->wb
, STREAM_DATA(msg
), stream_get_endp(msg
));
269 stream_fifo_free(cache
);
271 /* If we have any data pending, try to flush it first */
272 switch (buffer_flush_all(client
->wb
, client
->sock
)) {
276 atomic_store_explicit(&client
->last_write_time
, monotime(NULL
),
277 memory_order_relaxed
);
278 zserv_client_event(client
, ZSERV_CLIENT_WRITE
);
284 atomic_store_explicit(&client
->last_write_cmd
, wcmd
,
285 memory_order_relaxed
);
287 atomic_store_explicit(&client
->last_write_time
, monotime(NULL
),
288 memory_order_relaxed
);
293 flog_warn(EC_ZEBRA_CLIENT_WRITE_FAILED
,
294 "%s: could not write to %s [fd = %d], closing.", __func__
,
295 zebra_route_string(client
->proto
), client
->sock
);
296 zserv_client_fail(client
);
300 * Read and process data from a client socket.
302 * The responsibilities here are to read raw data from the client socket,
303 * validate the header, encapsulate it into a single stream object, push it
304 * onto the input queue and then notify the main thread that there is new data
307 * This function first looks for any data in the client structure's working
308 * input buffer. If data is present, it is assumed that reading stopped in a
309 * previous invocation of this task and needs to be resumed to finish a message.
310 * Otherwise, the socket data stream is assumed to be at the beginning of a new
311 * ZAPI message (specifically at the header). The header is read and validated.
312 * If the header passed validation then the length field found in the header is
313 * used to compute the total length of the message. That much data is read (but
314 * not inspected), appended to the header, placed into a stream and pushed onto
315 * the client's input queue. A task is then scheduled on the main thread to
316 * process the client's input queue. Finally, if all of this was successful,
317 * this task reschedules itself.
319 * Any failure in any of these actions is handled by terminating the client.
321 static void zserv_read(struct thread
*thread
)
323 struct zserv
*client
= THREAD_ARG(thread
);
326 struct stream_fifo
*cache
;
332 p2p_orig
= atomic_load_explicit(&zrouter
.packets_to_process
,
333 memory_order_relaxed
);
334 cache
= stream_fifo_new();
336 sock
= THREAD_FD(thread
);
343 already
= stream_get_endp(client
->ibuf_work
);
345 /* Read length and command (if we don't have it already). */
346 if (already
< ZEBRA_HEADER_SIZE
) {
347 nb
= stream_read_try(client
->ibuf_work
, sock
,
348 ZEBRA_HEADER_SIZE
- already
);
349 if ((nb
== 0 || nb
== -1)) {
350 if (IS_ZEBRA_DEBUG_EVENT
)
351 zlog_debug("connection closed socket [%d]",
355 if (nb
!= (ssize_t
)(ZEBRA_HEADER_SIZE
- already
)) {
356 /* Try again later. */
359 already
= ZEBRA_HEADER_SIZE
;
362 /* Reset to read from the beginning of the incoming packet. */
363 stream_set_getp(client
->ibuf_work
, 0);
365 /* Fetch header values */
366 hdrvalid
= zapi_parse_header(client
->ibuf_work
, &hdr
);
369 snprintf(errmsg
, sizeof(errmsg
),
370 "%s: Message has corrupt header", __func__
);
371 zserv_log_message(errmsg
, client
->ibuf_work
, NULL
);
375 /* Validate header */
376 if (hdr
.marker
!= ZEBRA_HEADER_MARKER
377 || hdr
.version
!= ZSERV_VERSION
) {
379 errmsg
, sizeof(errmsg
),
380 "Message has corrupt header\n%s: socket %d version mismatch, marker %d, version %d",
381 __func__
, sock
, hdr
.marker
, hdr
.version
);
382 zserv_log_message(errmsg
, client
->ibuf_work
, &hdr
);
385 if (hdr
.length
< ZEBRA_HEADER_SIZE
) {
387 errmsg
, sizeof(errmsg
),
388 "Message has corrupt header\n%s: socket %d message length %u is less than header size %d",
389 __func__
, sock
, hdr
.length
, ZEBRA_HEADER_SIZE
);
390 zserv_log_message(errmsg
, client
->ibuf_work
, &hdr
);
393 if (hdr
.length
> STREAM_SIZE(client
->ibuf_work
)) {
395 errmsg
, sizeof(errmsg
),
396 "Message has corrupt header\n%s: socket %d message length %u exceeds buffer size %lu",
397 __func__
, sock
, hdr
.length
,
398 (unsigned long)STREAM_SIZE(client
->ibuf_work
));
399 zserv_log_message(errmsg
, client
->ibuf_work
, &hdr
);
403 /* Read rest of data. */
404 if (already
< hdr
.length
) {
405 nb
= stream_read_try(client
->ibuf_work
, sock
,
406 hdr
.length
- already
);
407 if ((nb
== 0 || nb
== -1)) {
408 if (IS_ZEBRA_DEBUG_EVENT
)
410 "connection closed [%d] when reading zebra data",
414 if (nb
!= (ssize_t
)(hdr
.length
- already
)) {
415 /* Try again later. */
420 /* Debug packet information. */
421 if (IS_ZEBRA_DEBUG_PACKET
)
422 zlog_debug("zebra message[%s:%u:%u] comes from socket [%d]",
423 zserv_command_string(hdr
.command
),
424 hdr
.vrf_id
, hdr
.length
,
427 stream_set_getp(client
->ibuf_work
, 0);
428 struct stream
*msg
= stream_dup(client
->ibuf_work
);
430 stream_fifo_push(cache
, msg
);
431 stream_reset(client
->ibuf_work
);
435 if (p2p
< p2p_orig
) {
436 /* update session statistics */
437 atomic_store_explicit(&client
->last_read_time
, monotime(NULL
),
438 memory_order_relaxed
);
439 atomic_store_explicit(&client
->last_read_cmd
, hdr
.command
,
440 memory_order_relaxed
);
442 /* publish read packets on client's input queue */
443 frr_with_mutex (&client
->ibuf_mtx
) {
445 stream_fifo_push(client
->ibuf_fifo
,
446 stream_fifo_pop(cache
));
449 /* Schedule job to process those packets */
450 zserv_event(client
, ZSERV_PROCESS_MESSAGES
);
454 if (IS_ZEBRA_DEBUG_PACKET
)
455 zlog_debug("Read %d packets from client: %s", p2p_orig
- p2p
,
456 zebra_route_string(client
->proto
));
458 /* Reschedule ourselves */
459 zserv_client_event(client
, ZSERV_CLIENT_READ
);
461 stream_fifo_free(cache
);
466 stream_fifo_free(cache
);
467 zserv_client_fail(client
);
470 static void zserv_client_event(struct zserv
*client
,
471 enum zserv_client_event event
)
474 case ZSERV_CLIENT_READ
:
475 thread_add_read(client
->pthread
->master
, zserv_read
, client
,
476 client
->sock
, &client
->t_read
);
478 case ZSERV_CLIENT_WRITE
:
479 thread_add_write(client
->pthread
->master
, zserv_write
, client
,
480 client
->sock
, &client
->t_write
);
485 /* Main thread lifecycle ---------------------------------------------------- */
488 * Read and process messages from a client.
490 * This task runs on the main pthread. It is scheduled by client pthreads when
491 * they have new messages available on their input queues. The client is passed
492 * as the task argument.
494 * Each message is popped off the client's input queue and the action associated
495 * with the message is executed. This proceeds until there are no more messages,
496 * an error occurs, or the processing limit is reached.
498 * The client's I/O thread can push at most zrouter.packets_to_process messages
499 * onto the input buffer before notifying us there are packets to read. As long
500 * as we always process zrouter.packets_to_process messages here, then we can
501 * rely on the read thread to handle queuing this task enough times to process
502 * everything on the input queue.
504 static void zserv_process_messages(struct thread
*thread
)
506 struct zserv
*client
= THREAD_ARG(thread
);
508 struct stream_fifo
*cache
= stream_fifo_new();
509 uint32_t p2p
= zrouter
.packets_to_process
;
510 bool need_resched
= false;
512 frr_with_mutex (&client
->ibuf_mtx
) {
514 for (i
= 0; i
< p2p
&& stream_fifo_head(client
->ibuf_fifo
);
516 msg
= stream_fifo_pop(client
->ibuf_fifo
);
517 stream_fifo_push(cache
, msg
);
522 /* Need to reschedule processing work if there are still
523 * packets in the fifo.
525 if (stream_fifo_head(client
->ibuf_fifo
))
529 /* Process the batch of messages */
530 if (stream_fifo_head(cache
))
531 zserv_handle_commands(client
, cache
);
533 stream_fifo_free(cache
);
535 /* Reschedule ourselves if necessary */
537 zserv_event(client
, ZSERV_PROCESS_MESSAGES
);
540 int zserv_send_message(struct zserv
*client
, struct stream
*msg
)
542 frr_with_mutex (&client
->obuf_mtx
) {
543 stream_fifo_push(client
->obuf_fifo
, msg
);
546 zserv_client_event(client
, ZSERV_CLIENT_WRITE
);
552 * Send a batch of messages to a connected Zebra API client.
554 int zserv_send_batch(struct zserv
*client
, struct stream_fifo
*fifo
)
558 frr_with_mutex (&client
->obuf_mtx
) {
559 msg
= stream_fifo_pop(fifo
);
561 stream_fifo_push(client
->obuf_fifo
, msg
);
562 msg
= stream_fifo_pop(fifo
);
566 zserv_client_event(client
, ZSERV_CLIENT_WRITE
);
571 /* Hooks for client connect / disconnect */
572 DEFINE_HOOK(zserv_client_connect
, (struct zserv
*client
), (client
));
573 DEFINE_KOOH(zserv_client_close
, (struct zserv
*client
), (client
));
576 * Deinitialize zebra client.
578 * - Deregister and deinitialize related internal resources
579 * - Gracefuly close socket
580 * - Free associated resources
581 * - Free client structure
583 * This does *not* take any action on the struct thread * fields. These are
584 * managed by the owning pthread and any tasks associated with them must have
585 * been stopped prior to invoking this function.
587 static void zserv_client_free(struct zserv
*client
)
592 hook_call(zserv_client_close
, client
);
594 /* Close file descriptor. */
596 unsigned long nroutes
;
601 if (DYNAMIC_CLIENT_GR_DISABLED(client
)) {
602 zebra_mpls_client_cleanup_vrf_label(client
->proto
);
604 nroutes
= rib_score_proto(client
->proto
,
607 "client %d disconnected %lu %s routes removed from the rib",
608 client
->sock
, nroutes
,
609 zebra_route_string(client
->proto
));
611 /* Not worrying about instance for now */
612 nnhgs
= zebra_nhg_score_proto(client
->proto
);
614 "client %d disconnected %lu %s nhgs removed from the rib",
616 zebra_route_string(client
->proto
));
621 /* Free stream buffers. */
622 if (client
->ibuf_work
)
623 stream_free(client
->ibuf_work
);
624 if (client
->obuf_work
)
625 stream_free(client
->obuf_work
);
626 if (client
->ibuf_fifo
)
627 stream_fifo_free(client
->ibuf_fifo
);
628 if (client
->obuf_fifo
)
629 stream_fifo_free(client
->obuf_fifo
);
631 buffer_free(client
->wb
);
633 /* Free buffer mutexes */
634 pthread_mutex_destroy(&client
->obuf_mtx
);
635 pthread_mutex_destroy(&client
->ibuf_mtx
);
638 for (afi_t afi
= AFI_IP
; afi
< AFI_MAX
; afi
++) {
639 for (int i
= 0; i
< ZEBRA_ROUTE_MAX
; i
++) {
640 vrf_bitmap_free(client
->redist
[afi
][i
]);
641 redist_del_all_instances(&client
->mi_redist
[afi
][i
]);
644 vrf_bitmap_free(client
->redist_default
[afi
]);
645 vrf_bitmap_free(client
->ridinfo
[afi
]);
646 vrf_bitmap_free(client
->nhrp_neighinfo
[afi
]);
650 * If any instance are graceful restart enabled,
651 * client is not deleted
653 if (DYNAMIC_CLIENT_GR_DISABLED(client
)) {
654 if (IS_ZEBRA_DEBUG_EVENT
)
655 zlog_debug("%s: Deleting client %s", __func__
,
656 zebra_route_string(client
->proto
));
657 zserv_client_delete(client
);
659 /* Handle cases where client has GR instance. */
660 if (IS_ZEBRA_DEBUG_EVENT
)
661 zlog_debug("%s: client %s restart enabled", __func__
,
662 zebra_route_string(client
->proto
));
663 if (zebra_gr_client_disconnect(client
) < 0)
665 "%s: GR enabled but could not handle disconnect event",
670 void zserv_close_client(struct zserv
*client
)
674 if (client
->pthread
) {
675 /* synchronously stop and join pthread */
676 frr_pthread_stop(client
->pthread
, NULL
);
678 if (IS_ZEBRA_DEBUG_EVENT
)
679 zlog_debug("Closing client '%s'",
680 zebra_route_string(client
->proto
));
682 thread_cancel_event(zrouter
.master
, client
);
683 THREAD_OFF(client
->t_cleanup
);
684 THREAD_OFF(client
->t_process
);
686 /* destroy pthread */
687 frr_pthread_destroy(client
->pthread
);
688 client
->pthread
= NULL
;
692 * Final check in case the client struct is in use in another
693 * pthread: if not in-use, continue and free the client
695 frr_with_mutex (&client_mutex
) {
696 if (client
->busy_count
<= 0) {
697 /* remove from client list */
698 listnode_delete(zrouter
.client_list
, client
);
701 * The client session object may be in use, although
702 * the associated pthread is gone. Defer final
705 client
->is_closed
= true;
712 zserv_client_free(client
);
716 * This task is scheduled by a ZAPI client pthread on the main pthread when it
717 * wants to stop itself. When this executes, the client connection should
718 * already have been closed and the thread will most likely have died, but its
719 * resources still need to be cleaned up.
721 static void zserv_handle_client_fail(struct thread
*thread
)
723 struct zserv
*client
= THREAD_ARG(thread
);
725 zserv_close_client(client
);
729 * Create a new client.
731 * This is called when a new connection is accept()'d on the ZAPI socket. It
732 * initializes new client structure, notifies any subscribers of the connection
733 * event and spawns the client's thread.
736 * client's socket file descriptor
738 static struct zserv
*zserv_client_create(int sock
)
740 struct zserv
*client
;
742 MAX(ZEBRA_MAX_PACKET_SIZ
, sizeof(struct zapi_route
));
746 client
= XCALLOC(MTYPE_ZSERV_CLIENT
, sizeof(struct zserv
));
748 /* Make client input/output buffer. */
750 client
->ibuf_fifo
= stream_fifo_new();
751 client
->obuf_fifo
= stream_fifo_new();
752 client
->ibuf_work
= stream_new(stream_size
);
753 client
->obuf_work
= stream_new(stream_size
);
754 pthread_mutex_init(&client
->ibuf_mtx
, NULL
);
755 pthread_mutex_init(&client
->obuf_mtx
, NULL
);
756 client
->wb
= buffer_new(0);
757 TAILQ_INIT(&(client
->gr_info_queue
));
759 atomic_store_explicit(&client
->connect_time
, monotime(NULL
),
760 memory_order_relaxed
);
762 /* Initialize flags */
763 for (afi
= AFI_IP
; afi
< AFI_MAX
; afi
++) {
764 for (i
= 0; i
< ZEBRA_ROUTE_MAX
; i
++)
765 client
->redist
[afi
][i
] = vrf_bitmap_init();
766 client
->redist_default
[afi
] = vrf_bitmap_init();
767 client
->ridinfo
[afi
] = vrf_bitmap_init();
768 client
->nhrp_neighinfo
[afi
] = vrf_bitmap_init();
771 /* Add this client to linked list. */
772 frr_with_mutex (&client_mutex
) {
773 listnode_add(zrouter
.client_list
, client
);
776 struct frr_pthread_attr zclient_pthr_attrs
= {
777 .start
= frr_pthread_attr_default
.start
,
778 .stop
= frr_pthread_attr_default
.stop
781 frr_pthread_new(&zclient_pthr_attrs
, "Zebra API client thread",
784 /* start read loop */
785 zserv_client_event(client
, ZSERV_CLIENT_READ
);
788 hook_call(zserv_client_connect
, client
);
791 frr_pthread_run(client
->pthread
, NULL
);
797 * Retrieve a client object by the complete tuple of
798 * {protocol, instance, session}. This version supports use
799 * from a different pthread: the object will be returned marked
800 * in-use. The caller *must* release the client object with the
801 * release_client() api, to ensure that the in-use marker is cleared properly.
803 struct zserv
*zserv_acquire_client(uint8_t proto
, unsigned short instance
,
806 struct zserv
*client
= NULL
;
808 frr_with_mutex (&client_mutex
) {
809 client
= find_client_internal(proto
, instance
, session_id
);
811 /* Don't return a dead/closed client object */
812 if (client
->is_closed
)
815 client
->busy_count
++;
823 * Release a client object that was acquired with the acquire_client() api.
824 * After this has been called, the caller must not use the client pointer -
825 * it may be freed if the client has closed.
827 void zserv_release_client(struct zserv
*client
)
830 * Once we've decremented the client object's refcount, it's possible
831 * for it to be deleted as soon as we release the lock, so we won't
832 * touch the object again.
834 frr_with_mutex (&client_mutex
) {
835 client
->busy_count
--;
837 if (client
->busy_count
<= 0) {
839 * No more users of the client object. If the client
840 * session is closed, schedule cleanup on the zebra
843 if (client
->is_closed
)
844 thread_add_event(zrouter
.master
,
845 zserv_handle_client_fail
,
846 client
, 0, &client
->t_cleanup
);
851 * Cleanup must take place on the zebra main pthread, so we've
852 * scheduled an event.
857 * Accept socket connection.
859 static void zserv_accept(struct thread
*thread
)
863 struct sockaddr_in client
;
866 accept_sock
= THREAD_FD(thread
);
868 /* Reregister myself. */
869 zserv_event(NULL
, ZSERV_ACCEPT
);
871 len
= sizeof(struct sockaddr_in
);
872 client_sock
= accept(accept_sock
, (struct sockaddr
*)&client
, &len
);
874 if (client_sock
< 0) {
875 flog_err_sys(EC_LIB_SOCKET
, "Can't accept zebra socket: %s",
876 safe_strerror(errno
));
880 /* Make client socket non-blocking. */
881 set_nonblocking(client_sock
);
883 /* Create new zebra client. */
884 zserv_client_create(client_sock
);
887 void zserv_close(void)
890 * On shutdown, let's close the socket down
891 * so that long running processes of killing the
892 * routing table doesn't leave us in a bad
893 * state where a client tries to reconnect
898 /* Free client list's mutex */
899 pthread_mutex_destroy(&client_mutex
);
902 void zserv_start(char *path
)
906 struct sockaddr_storage sa
;
909 if (!frr_zclient_addr(&sa
, &sa_len
, path
))
910 /* should be caught in zebra main() */
914 old_mask
= umask(0077);
916 /* Make UNIX domain socket. */
917 zsock
= socket(sa
.ss_family
, SOCK_STREAM
, 0);
919 flog_err_sys(EC_LIB_SOCKET
, "Can't create zserv socket: %s",
920 safe_strerror(errno
));
924 if (sa
.ss_family
!= AF_UNIX
) {
925 sockopt_reuseaddr(zsock
);
926 sockopt_reuseport(zsock
);
928 struct sockaddr_un
*suna
= (struct sockaddr_un
*)&sa
;
929 if (suna
->sun_path
[0])
930 unlink(suna
->sun_path
);
933 setsockopt_so_recvbuf(zsock
, 1048576);
934 setsockopt_so_sendbuf(zsock
, 1048576);
936 frr_with_privs((sa
.ss_family
!= AF_UNIX
) ? &zserv_privs
: NULL
) {
937 ret
= bind(zsock
, (struct sockaddr
*)&sa
, sa_len
);
940 flog_err_sys(EC_LIB_SOCKET
, "Can't bind zserv socket on %s: %s",
941 path
, safe_strerror(errno
));
947 ret
= listen(zsock
, 5);
949 flog_err_sys(EC_LIB_SOCKET
,
950 "Can't listen to zserv socket %s: %s", path
,
951 safe_strerror(errno
));
959 zserv_event(NULL
, ZSERV_ACCEPT
);
962 void zserv_event(struct zserv
*client
, enum zserv_event event
)
966 thread_add_read(zrouter
.master
, zserv_accept
, NULL
, zsock
,
969 case ZSERV_PROCESS_MESSAGES
:
970 thread_add_event(zrouter
.master
, zserv_process_messages
, client
,
971 0, &client
->t_process
);
973 case ZSERV_HANDLE_CLIENT_FAIL
:
974 thread_add_event(zrouter
.master
, zserv_handle_client_fail
,
975 client
, 0, &client
->t_cleanup
);
980 /* General purpose ---------------------------------------------------------- */
982 #define ZEBRA_TIME_BUF 32
983 static char *zserv_time_buf(time_t *time1
, char *buf
, int buflen
)
988 assert(buflen
>= ZEBRA_TIME_BUF
);
989 assert(time1
!= NULL
);
992 snprintf(buf
, buflen
, "never ");
996 now
= monotime(NULL
);
999 frrtime_to_interval(now
, buf
, buflen
);
1004 /* Display client info details */
1005 static void zebra_show_client_detail(struct vty
*vty
, struct zserv
*client
)
1007 char cbuf
[ZEBRA_TIME_BUF
], rbuf
[ZEBRA_TIME_BUF
];
1008 char wbuf
[ZEBRA_TIME_BUF
], nhbuf
[ZEBRA_TIME_BUF
], mbuf
[ZEBRA_TIME_BUF
];
1009 time_t connect_time
, last_read_time
, last_write_time
;
1010 uint32_t last_read_cmd
, last_write_cmd
;
1012 vty_out(vty
, "Client: %s", zebra_route_string(client
->proto
));
1013 if (client
->instance
)
1014 vty_out(vty
, " Instance: %u", client
->instance
);
1015 if (client
->session_id
)
1016 vty_out(vty
, " [%u]", client
->session_id
);
1019 vty_out(vty
, "------------------------ \n");
1020 vty_out(vty
, "FD: %d \n", client
->sock
);
1022 connect_time
= (time_t) atomic_load_explicit(&client
->connect_time
,
1023 memory_order_relaxed
);
1025 vty_out(vty
, "Connect Time: %s \n",
1026 zserv_time_buf(&connect_time
, cbuf
, ZEBRA_TIME_BUF
));
1027 if (client
->nh_reg_time
) {
1028 vty_out(vty
, "Nexthop Registry Time: %s \n",
1029 zserv_time_buf(&client
->nh_reg_time
, nhbuf
,
1031 if (client
->nh_last_upd_time
)
1032 vty_out(vty
, "Nexthop Last Update Time: %s \n",
1033 zserv_time_buf(&client
->nh_last_upd_time
, mbuf
,
1036 vty_out(vty
, "No Nexthop Update sent\n");
1038 vty_out(vty
, "Not registered for Nexthop Updates\n");
1040 vty_out(vty
, "Client will %sbe notified about it's routes status\n",
1041 client
->notify_owner
? "" : "Not ");
1043 last_read_time
= (time_t)atomic_load_explicit(&client
->last_read_time
,
1044 memory_order_relaxed
);
1045 last_write_time
= (time_t)atomic_load_explicit(&client
->last_write_time
,
1046 memory_order_relaxed
);
1048 last_read_cmd
= atomic_load_explicit(&client
->last_read_cmd
,
1049 memory_order_relaxed
);
1050 last_write_cmd
= atomic_load_explicit(&client
->last_write_cmd
,
1051 memory_order_relaxed
);
1053 vty_out(vty
, "Last Msg Rx Time: %s \n",
1054 zserv_time_buf(&last_read_time
, rbuf
, ZEBRA_TIME_BUF
));
1055 vty_out(vty
, "Last Msg Tx Time: %s \n",
1056 zserv_time_buf(&last_write_time
, wbuf
, ZEBRA_TIME_BUF
));
1058 vty_out(vty
, "Last Rcvd Cmd: %s \n",
1059 zserv_command_string(last_read_cmd
));
1061 vty_out(vty
, "Last Sent Cmd: %s \n",
1062 zserv_command_string(last_write_cmd
));
1065 vty_out(vty
, "Type Add Update Del \n");
1066 vty_out(vty
, "================================================== \n");
1067 vty_out(vty
, "IPv4 %-12u%-12u%-12u\n", client
->v4_route_add_cnt
,
1068 client
->v4_route_upd8_cnt
, client
->v4_route_del_cnt
);
1069 vty_out(vty
, "IPv6 %-12u%-12u%-12u\n", client
->v6_route_add_cnt
,
1070 client
->v6_route_upd8_cnt
, client
->v6_route_del_cnt
);
1071 vty_out(vty
, "Redist:v4 %-12u%-12u%-12u\n", client
->redist_v4_add_cnt
,
1072 0, client
->redist_v4_del_cnt
);
1073 vty_out(vty
, "Redist:v6 %-12u%-12u%-12u\n", client
->redist_v6_add_cnt
,
1074 0, client
->redist_v6_del_cnt
);
1075 vty_out(vty
, "VRF %-12u%-12u%-12u\n", client
->vrfadd_cnt
, 0,
1076 client
->vrfdel_cnt
);
1077 vty_out(vty
, "Connected %-12u%-12u%-12u\n", client
->ifadd_cnt
, 0,
1079 vty_out(vty
, "Interface %-12u%-12u%-12u\n", client
->ifup_cnt
, 0,
1080 client
->ifdown_cnt
);
1081 vty_out(vty
, "Intf Addr %-12u%-12u%-12u\n",
1082 client
->connected_rt_add_cnt
, 0, client
->connected_rt_del_cnt
);
1083 vty_out(vty
, "BFD peer %-12u%-12u%-12u\n", client
->bfd_peer_add_cnt
,
1084 client
->bfd_peer_upd8_cnt
, client
->bfd_peer_del_cnt
);
1085 vty_out(vty
, "NHT v4 %-12u%-12u%-12u\n",
1086 client
->v4_nh_watch_add_cnt
, 0, client
->v4_nh_watch_rem_cnt
);
1087 vty_out(vty
, "NHT v6 %-12u%-12u%-12u\n",
1088 client
->v6_nh_watch_add_cnt
, 0, client
->v6_nh_watch_rem_cnt
);
1089 vty_out(vty
, "VxLAN SG %-12u%-12u%-12u\n", client
->vxlan_sg_add_cnt
,
1090 0, client
->vxlan_sg_del_cnt
);
1091 vty_out(vty
, "VNI %-12u%-12u%-12u\n", client
->vniadd_cnt
, 0,
1092 client
->vnidel_cnt
);
1093 vty_out(vty
, "L3-VNI %-12u%-12u%-12u\n", client
->l3vniadd_cnt
, 0,
1094 client
->l3vnidel_cnt
);
1095 vty_out(vty
, "MAC-IP %-12u%-12u%-12u\n", client
->macipadd_cnt
, 0,
1096 client
->macipdel_cnt
);
1097 vty_out(vty
, "ES %-12u%-12u%-12u\n", client
->local_es_add_cnt
,
1098 0, client
->local_es_del_cnt
);
1099 vty_out(vty
, "ES-EVI %-12u%-12u%-12u\n",
1100 client
->local_es_evi_add_cnt
, 0, client
->local_es_evi_del_cnt
);
1101 vty_out(vty
, "Errors: %u\n", client
->error_cnt
);
1103 #if defined DEV_BUILD
1104 vty_out(vty
, "Input Fifo: %zu:%zu Output Fifo: %zu:%zu\n",
1105 client
->ibuf_fifo
->count
, client
->ibuf_fifo
->max_count
,
1106 client
->obuf_fifo
->count
, client
->obuf_fifo
->max_count
);
1111 /* Display stale client information */
1112 static void zebra_show_stale_client_detail(struct vty
*vty
,
1113 struct zserv
*client
)
1115 char buf
[PREFIX2STR_BUFFER
];
1117 struct client_gr_info
*info
= NULL
;
1118 struct zserv
*s
= NULL
;
1119 bool first_p
= true;
1121 TAILQ_FOREACH (info
, &client
->gr_info_queue
, gr_info
) {
1123 vty_out(vty
, "Stale Client Information\n");
1124 vty_out(vty
, "------------------------\n");
1126 if (client
->instance
)
1127 vty_out(vty
, " Instance: %u", client
->instance
);
1128 if (client
->session_id
)
1129 vty_out(vty
, " [%u]", client
->session_id
);
1134 vty_out(vty
, "VRF : %s\n", vrf_id_to_name(info
->vrf_id
));
1135 vty_out(vty
, "Capabilities : ");
1136 switch (info
->capabilities
) {
1137 case ZEBRA_CLIENT_GR_CAPABILITIES
:
1138 vty_out(vty
, "Graceful Restart(%u seconds)\n",
1139 info
->stale_removal_time
);
1141 case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE
:
1142 case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING
:
1143 case ZEBRA_CLIENT_GR_DISABLE
:
1144 case ZEBRA_CLIENT_RIB_STALE_TIME
:
1145 vty_out(vty
, "None\n");
1149 if (ZEBRA_CLIENT_GR_ENABLED(info
->capabilities
)) {
1150 if (info
->stale_client_ptr
) {
1151 s
= (struct zserv
*)(info
->stale_client_ptr
);
1152 uptime
= monotime(NULL
);
1153 uptime
-= s
->restart_time
;
1155 frrtime_to_interval(uptime
, buf
, sizeof(buf
));
1157 vty_out(vty
, "Last restart time : %s ago\n",
1160 vty_out(vty
, "Stalepath removal time: %d sec\n",
1161 info
->stale_removal_time
);
1162 if (info
->t_stale_removal
) {
1164 "Stale delete timer: %ld sec\n",
1165 thread_timer_remain_second(
1166 info
->t_stale_removal
));
1169 vty_out(vty
, "Current AFI : %d\n", info
->current_afi
);
1170 if (info
->current_prefix
)
1171 vty_out(vty
, "Current prefix : %pFX\n",
1172 info
->current_prefix
);
1179 static void zebra_show_client_brief(struct vty
*vty
, struct zserv
*client
)
1181 char client_string
[80];
1182 char cbuf
[ZEBRA_TIME_BUF
], rbuf
[ZEBRA_TIME_BUF
];
1183 char wbuf
[ZEBRA_TIME_BUF
];
1184 time_t connect_time
, last_read_time
, last_write_time
;
1186 connect_time
= (time_t)atomic_load_explicit(&client
->connect_time
,
1187 memory_order_relaxed
);
1188 last_read_time
= (time_t)atomic_load_explicit(&client
->last_read_time
,
1189 memory_order_relaxed
);
1190 last_write_time
= (time_t)atomic_load_explicit(&client
->last_write_time
,
1191 memory_order_relaxed
);
1193 if (client
->instance
|| client
->session_id
)
1194 snprintfrr(client_string
, sizeof(client_string
), "%s[%u:%u]",
1195 zebra_route_string(client
->proto
), client
->instance
,
1196 client
->session_id
);
1198 snprintfrr(client_string
, sizeof(client_string
), "%s",
1199 zebra_route_string(client
->proto
));
1201 vty_out(vty
, "%-10s%12s %12s%12s %10d/%-10d %10d/%-10d\n",
1203 zserv_time_buf(&connect_time
, cbuf
, ZEBRA_TIME_BUF
),
1204 zserv_time_buf(&last_read_time
, rbuf
, ZEBRA_TIME_BUF
),
1205 zserv_time_buf(&last_write_time
, wbuf
, ZEBRA_TIME_BUF
),
1206 client
->v4_route_add_cnt
+ client
->v4_route_upd8_cnt
,
1207 client
->v4_route_del_cnt
,
1208 client
->v6_route_add_cnt
+ client
->v6_route_upd8_cnt
,
1209 client
->v6_route_del_cnt
);
1213 * Common logic that searches the client list for a zapi client; this
1214 * MUST be called holding the client list mutex.
1216 static struct zserv
*find_client_internal(uint8_t proto
,
1217 unsigned short instance
,
1218 uint32_t session_id
)
1220 struct listnode
*node
, *nnode
;
1221 struct zserv
*client
= NULL
;
1223 for (ALL_LIST_ELEMENTS(zrouter
.client_list
, node
, nnode
, client
)) {
1224 if (client
->proto
== proto
&& client
->instance
== instance
&&
1225 client
->session_id
== session_id
)
1233 * Public api that searches for a client session; this version is
1234 * used from the zebra main pthread.
1236 struct zserv
*zserv_find_client(uint8_t proto
, unsigned short instance
)
1238 struct zserv
*client
;
1240 frr_with_mutex (&client_mutex
) {
1241 client
= find_client_internal(proto
, instance
, 0);
1248 * Retrieve a client by its protocol, instance number, and session id.
1250 struct zserv
*zserv_find_client_session(uint8_t proto
, unsigned short instance
,
1251 uint32_t session_id
)
1253 struct zserv
*client
;
1255 frr_with_mutex (&client_mutex
) {
1256 client
= find_client_internal(proto
, instance
, session_id
);
1263 /* This command is for debugging purpose. */
1264 DEFUN (show_zebra_client
,
1265 show_zebra_client_cmd
,
1266 "show zebra client",
1269 "Client information\n")
1271 struct listnode
*node
;
1272 struct zserv
*client
;
1274 for (ALL_LIST_ELEMENTS_RO(zrouter
.client_list
, node
, client
)) {
1275 zebra_show_client_detail(vty
, client
);
1276 /* Show GR info if present */
1277 zebra_show_stale_client_detail(vty
, client
);
1283 /* This command is for debugging purpose. */
1284 DEFUN (show_zebra_client_summary
,
1285 show_zebra_client_summary_cmd
,
1286 "show zebra client summary",
1289 "Client information brief\n"
1292 struct listnode
*node
;
1293 struct zserv
*client
;
1296 "Name Connect Time Last Read Last Write IPv4 Routes IPv6 Routes\n");
1298 "------------------------------------------------------------------------------------------\n");
1300 for (ALL_LIST_ELEMENTS_RO(zrouter
.client_list
, node
, client
))
1301 zebra_show_client_brief(vty
, client
);
1303 vty_out(vty
, "Routes column shows (added+updated)/deleted\n");
1307 static int zserv_client_close_cb(struct zserv
*closed_client
)
1309 struct listnode
*node
, *nnode
;
1310 struct zserv
*client
= NULL
;
1312 for (ALL_LIST_ELEMENTS(zrouter
.client_list
, node
, nnode
, client
)) {
1313 if (client
->proto
== closed_client
->proto
)
1316 zsend_client_close_notify(client
, closed_client
);
1322 void zserv_init(void)
1324 /* Client list init. */
1325 zrouter
.client_list
= list_new();
1326 zrouter
.stale_client_list
= list_new();
1330 pthread_mutex_init(&client_mutex
, NULL
);
1332 install_element(ENABLE_NODE
, &show_zebra_client_cmd
);
1333 install_element(ENABLE_NODE
, &show_zebra_client_summary_cmd
);
1335 hook_register(zserv_client_close
, zserv_client_close_cb
);