]> git.proxmox.com Git - mirror_frr.git/blob - zebra/zebra_mlag.c
projects / mirror_frr.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
doc: Add `show ipv6 rpf X:X::X:X` command to docs
[mirror_frr.git] / zebra / zebra_mlag.c
1 /* Zebra Mlag Code.
2 * Copyright (C) 2018 Cumulus Networks, Inc.
3 * Donald Sharp
4 *
5 * This file is part of FRR.
6 *
7 * FRR is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2, or (at your option) any
10 * later version.
11 *
12 * FRR is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with FRR; see the file COPYING. If not, write to the Free
19 * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
20 * 02111-1307, USA.
21 */
22 #include "zebra.h"
23
24 #include "command.h"
25 #include "hook.h"
26 #include "frr_pthread.h"
27 #include "mlag.h"
28
29 #include "zebra/zebra_mlag.h"
30 #include "zebra/zebra_mlag_vty.h"
31 #include "zebra/zebra_router.h"
32 #include "zebra/zapi_msg.h"
33 #include "zebra/debug.h"
34
35 #ifdef HAVE_PROTOBUF_VERSION_3
36 #include "mlag/mlag.pb-c.h"
37 #endif
38
39 DEFINE_HOOK(zebra_mlag_private_write_data,
40 (uint8_t *data, uint32_t len), (data, len));
41 DEFINE_HOOK(zebra_mlag_private_monitor_state, (), ());
42 DEFINE_HOOK(zebra_mlag_private_open_channel, (), ());
43 DEFINE_HOOK(zebra_mlag_private_close_channel, (), ());
44 DEFINE_HOOK(zebra_mlag_private_cleanup_data, (), ());
45
46 #define ZEBRA_MLAG_METADATA_LEN 4
47 #define ZEBRA_MLAG_MSG_BCAST 0xFFFFFFFF
48
49 uint8_t mlag_wr_buffer[ZEBRA_MLAG_BUF_LIMIT];
50 uint8_t mlag_rd_buffer[ZEBRA_MLAG_BUF_LIMIT];
51
52 static bool test_mlag_in_progress;
53
54 static int zebra_mlag_signal_write_thread(void);
55 static void zebra_mlag_terminate_pthread(struct thread *event);
56 static void zebra_mlag_post_data_from_main_thread(struct thread *thread);
57 static void zebra_mlag_publish_process_state(struct zserv *client,
58 zebra_message_types_t msg_type);
59
60 /**********************MLAG Interaction***************************************/
61
62 /*
63 * API to post the Registration to MLAGD
64 * MLAG will not process any messages with out the registration
65 */
66 void zebra_mlag_send_register(void)
67 {
68 struct stream *s = NULL;
69
70 s = stream_new(sizeof(struct mlag_msg));
71
72 stream_putl(s, MLAG_REGISTER);
73 stream_putw(s, MLAG_MSG_NULL_PAYLOAD);
74 stream_putw(s, MLAG_MSG_NO_BATCH);
75 stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s);
76 zebra_mlag_signal_write_thread();
77
78 if (IS_ZEBRA_DEBUG_MLAG)
79 zlog_debug("%s: Enqueued MLAG Register to MLAG Thread ",
80 __func__);
81 }
82
83 /*
84 * API to post the De-Registration to MLAGD
85 * MLAG will not process any messages after the de-registration
86 */
87 void zebra_mlag_send_deregister(void)
88 {
89 struct stream *s = NULL;
90
91 s = stream_new(sizeof(struct mlag_msg));
92
93 stream_putl(s, MLAG_DEREGISTER);
94 stream_putw(s, MLAG_MSG_NULL_PAYLOAD);
95 stream_putw(s, MLAG_MSG_NO_BATCH);
96 stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s);
97 zebra_mlag_signal_write_thread();
98
99 if (IS_ZEBRA_DEBUG_MLAG)
100 zlog_debug("%s: Enqueued MLAG De-Register to MLAG Thread ",
101 __func__);
102 }
103
104 /*
105 * API To handle MLAG Received data
106 * Decodes the data using protobuf and enqueue to main thread
107 * main thread publish this to clients based on client subscription
108 */
109 void zebra_mlag_process_mlag_data(uint8_t *data, uint32_t len)
110 {
111 struct stream *s = NULL;
112 int msg_type = 0;
113
114 s = stream_new(ZEBRA_MLAG_BUF_LIMIT);
115 /*
116 * Place holder we need the message type first
117 */
118 stream_putl(s, msg_type);
119 msg_type = zebra_mlag_protobuf_decode_message(s, data, len);
120
121 if (msg_type <= 0) {
122 /* Something went wrong in decoding */
123 stream_free(s);
124 zlog_err("%s: failed to process mlag data-%d, %u", __func__,
125 msg_type, len);
126 return;
127 }
128
129 /*
130 * additional four bytes are for message type
131 */
132 stream_putl_at(s, 0, msg_type);
133 thread_add_event(zrouter.master, zebra_mlag_post_data_from_main_thread,
134 s, 0, NULL);
135 }
136
137 /**********************End of MLAG Interaction********************************/
138
139 /************************MLAG Thread Processing*******************************/
140
141 /*
142 * after posting every 'ZEBRA_MLAG_POST_LIMIT' packets, MLAG Thread will be
143 * yielded to give CPU for other threads
144 */
145 #define ZEBRA_MLAG_POST_LIMIT 100
146
147 /*
148 * This thread reads the clients data from the Global queue and encodes with
149 * protobuf and pass on to the MLAG socket.
150 */
151 static void zebra_mlag_client_msg_handler(struct thread *event)
152 {
153 struct stream *s;
154 uint32_t wr_count = 0;
155 uint32_t msg_type = 0;
156 uint32_t max_count = 0;
157 int len = 0;
158
159 wr_count = stream_fifo_count_safe(zrouter.mlag_info.mlag_fifo);
160 if (IS_ZEBRA_DEBUG_MLAG)
161 zlog_debug(":%s: Processing MLAG write, %u messages in queue",
162 __func__, wr_count);
163
164 max_count = MIN(wr_count, ZEBRA_MLAG_POST_LIMIT);
165
166 for (wr_count = 0; wr_count < max_count; wr_count++) {
167 s = stream_fifo_pop_safe(zrouter.mlag_info.mlag_fifo);
168 if (!s) {
169 zlog_debug(":%s: Got a NULL Messages, some thing wrong",
170 __func__);
171 break;
172 }
173
174 /*
175 * Encode the data now
176 */
177 len = zebra_mlag_protobuf_encode_client_data(s, &msg_type);
178
179 /*
180 * write to MCLAGD
181 */
182 if (len > 0) {
183 hook_call(zebra_mlag_private_write_data,
184 mlag_wr_buffer, len);
185
186 /*
187 * If message type is De-register, send a signal to main
188 * thread, so that necessary cleanup will be done by
189 * main thread.
190 */
191 if (msg_type == MLAG_DEREGISTER) {
192 thread_add_event(zrouter.master,
193 zebra_mlag_terminate_pthread,
194 NULL, 0, NULL);
195 }
196 }
197
198 stream_free(s);
199 }
200
201 if (IS_ZEBRA_DEBUG_MLAG)
202 zlog_debug(":%s: Posted %d messages to MLAGD", __func__,
203 wr_count);
204 /*
205 * Currently there is only message write task is enqueued to this
206 * thread, yielding was added for future purpose, so that this thread
207 * can server other tasks also and in case FIFO is empty, this task will
208 * be schedule when main thread adds some messages
209 */
210 if (wr_count >= ZEBRA_MLAG_POST_LIMIT)
211 zebra_mlag_signal_write_thread();
212 }
213
214 /*
215 * API to handle the process state.
216 * In case of Down, Zebra keep monitoring the MLAG state.
217 * all the state Notifications will be published to clients
218 */
219 void zebra_mlag_handle_process_state(enum zebra_mlag_state state)
220 {
221 if (state == MLAG_UP) {
222 zrouter.mlag_info.connected = true;
223 zebra_mlag_publish_process_state(NULL, ZEBRA_MLAG_PROCESS_UP);
224 zebra_mlag_send_register();
225 } else if (state == MLAG_DOWN) {
226 zrouter.mlag_info.connected = false;
227 zebra_mlag_publish_process_state(NULL, ZEBRA_MLAG_PROCESS_DOWN);
228 hook_call(zebra_mlag_private_monitor_state);
229 }
230 }
231
232 /***********************End of MLAG Thread processing*************************/
233
234 /*************************Multi-entratnt Api's********************************/
235
236 /*
237 * Provider api to signal that work/events are available
238 * for the Zebra MLAG Write pthread.
239 * This API is called from 2 pthreads..
240 * 1) by main thread when client posts a MLAG Message
241 * 2) by MLAG Thread, in case of yield
242 * though this api, is called from two threads we don't need any locking
243 * because Thread task enqueue is thread safe means internally it had
244 * necessary protection
245 */
246 static int zebra_mlag_signal_write_thread(void)
247 {
248 if (IS_ZEBRA_DEBUG_MLAG)
249 zlog_debug(":%s: Scheduling MLAG write", __func__);
250 /*
251 * This api will be called from Both main & MLAG Threads.
252 * main thread writes, "zrouter.mlag_info.th_master" only
253 * during Zebra Init/after MLAG thread is destroyed.
254 * so it is safe to use without any locking
255 */
256 thread_add_event(zrouter.mlag_info.th_master,
257 zebra_mlag_client_msg_handler, NULL, 0,
258 &zrouter.mlag_info.t_write);
259 return 0;
260 }
261
262 /*
263 * API will be used to publish the MLAG state to interested clients
264 * In case client is passed, state is posted only for that client,
265 * otherwise to all interested clients
266 * this api can be called from two threads.
267 * 1) from main thread: when client is passed
268 * 2) from MLAG Thread: when client is NULL
269 *
270 * In second case, to avoid global data access data will be post to Main
271 * thread, so that actual posting to clients will happen from Main thread.
272 */
273 static void zebra_mlag_publish_process_state(struct zserv *client,
274 zebra_message_types_t msg_type)
275 {
276 struct stream *s;
277
278 if (IS_ZEBRA_DEBUG_MLAG)
279 zlog_debug("%s: Publishing MLAG process state:%s to %s Client",
280 __func__,
281 (msg_type == ZEBRA_MLAG_PROCESS_UP) ? "UP" : "DOWN",
282 (client) ? "one" : "all");
283
284 if (client) {
285 s = stream_new(ZEBRA_HEADER_SIZE);
286 zclient_create_header(s, msg_type, VRF_DEFAULT);
287 zserv_send_message(client, s);
288 return;
289 }
290
291
292 /*
293 * additional four bytes are for mesasge type
294 */
295 s = stream_new(ZEBRA_HEADER_SIZE + ZEBRA_MLAG_METADATA_LEN);
296 stream_putl(s, ZEBRA_MLAG_MSG_BCAST);
297 zclient_create_header(s, msg_type, VRF_DEFAULT);
298 thread_add_event(zrouter.master, zebra_mlag_post_data_from_main_thread,
299 s, 0, NULL);
300 }
301
302 /**************************End of Multi-entrant Apis**************************/
303
304 /***********************Zebra Main thread processing**************************/
305
306 /*
307 * To avoid data corruption, messages will be post to clients only from
308 * main thread, because for that access was needed for clients list.
309 * so instead of forcing the locks, messages will be posted from main thread.
310 */
311 static void zebra_mlag_post_data_from_main_thread(struct thread *thread)
312 {
313 struct stream *s = THREAD_ARG(thread);
314 struct stream *zebra_s = NULL;
315 struct listnode *node;
316 struct zserv *client;
317 uint32_t msg_type = 0;
318 uint32_t msg_len = 0;
319
320 if (!s)
321 return;
322
323 STREAM_GETL(s, msg_type);
324 if (IS_ZEBRA_DEBUG_MLAG)
325 zlog_debug(
326 "%s: Posting MLAG data for msg_type:0x%x to interested clients",
327 __func__, msg_type);
328
329 msg_len = s->endp - ZEBRA_MLAG_METADATA_LEN;
330 for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client)) {
331 if (client->mlag_updates_interested == true) {
332 if (msg_type != ZEBRA_MLAG_MSG_BCAST
333 && !CHECK_FLAG(client->mlag_reg_mask1,
334 (1 << msg_type))) {
335 continue;
336 }
337
338 if (IS_ZEBRA_DEBUG_MLAG)
339 zlog_debug(
340 "%s: Posting MLAG data of length-%d to client:%d ",
341 __func__, msg_len, client->proto);
342
343 zebra_s = stream_new(msg_len);
344 STREAM_GET(zebra_s->data, s, msg_len);
345 zebra_s->endp = msg_len;
346 stream_putw_at(zebra_s, 0, msg_len);
347
348 /*
349 * This stream will be enqueued to client_obuf, it will
350 * be freed after posting to client socket.
351 */
352 zserv_send_message(client, zebra_s);
353 zebra_s = NULL;
354 }
355 }
356
357 stream_free(s);
358 return;
359 stream_failure:
360 stream_free(s);
361 if (zebra_s)
362 stream_free(zebra_s);
363 }
364
365 /*
366 * Start the MLAG Thread, this will be used to write client data on to
367 * MLAG Process and to read the data from MLAG and post to clients.
368 * when all clients are un-registered, this Thread will be
369 * suspended.
370 */
371 static void zebra_mlag_spawn_pthread(void)
372 {
373 /* Start MLAG write pthread */
374
375 struct frr_pthread_attr pattr = {.start =
376 frr_pthread_attr_default.start,
377 .stop = frr_pthread_attr_default.stop};
378
379 zrouter.mlag_info.zebra_pth_mlag =
380 frr_pthread_new(&pattr, "Zebra MLAG thread", "Zebra MLAG");
381
382 zrouter.mlag_info.th_master = zrouter.mlag_info.zebra_pth_mlag->master;
383
384
385 /* Enqueue an initial event to the Newly spawn MLAG pthread */
386 zebra_mlag_signal_write_thread();
387
388 frr_pthread_run(zrouter.mlag_info.zebra_pth_mlag, NULL);
389 }
390
391 /*
392 * all clients are un-registered for MLAG Updates, terminate the
393 * MLAG write thread
394 */
395 static void zebra_mlag_terminate_pthread(struct thread *event)
396 {
397 if (IS_ZEBRA_DEBUG_MLAG)
398 zlog_debug("Zebra MLAG write thread terminate called");
399
400 if (zrouter.mlag_info.clients_interested_cnt) {
401 if (IS_ZEBRA_DEBUG_MLAG)
402 zlog_debug(
403 "Zebra MLAG: still some clients are interested");
404 return;
405 }
406
407 frr_pthread_stop(zrouter.mlag_info.zebra_pth_mlag, NULL);
408
409 /* Destroy pthread */
410 frr_pthread_destroy(zrouter.mlag_info.zebra_pth_mlag);
411 zrouter.mlag_info.zebra_pth_mlag = NULL;
412 zrouter.mlag_info.th_master = NULL;
413 zrouter.mlag_info.t_read = NULL;
414 zrouter.mlag_info.t_write = NULL;
415
416 /*
417 * Send Notification to clean private data
418 */
419 hook_call(zebra_mlag_private_cleanup_data);
420 }
421
422 /*
423 * API to register zebra client for MLAG Updates
424 */
425 void zebra_mlag_client_register(ZAPI_HANDLER_ARGS)
426 {
427 struct stream *s;
428 uint32_t reg_mask = 0;
429 int rc = 0;
430
431 if (IS_ZEBRA_DEBUG_MLAG)
432 zlog_debug("Received MLAG Registration from client-proto:%d",
433 client->proto);
434
435
436 /* Get input stream. */
437 s = msg;
438
439 /* Get data. */
440 STREAM_GETL(s, reg_mask);
441
442 if (client->mlag_updates_interested == true) {
443
444 if (IS_ZEBRA_DEBUG_MLAG)
445 zlog_debug(
446 "Client is registered, existing mask: 0x%x, new mask: 0x%x",
447 client->mlag_reg_mask1, reg_mask);
448 if (client->mlag_reg_mask1 != reg_mask)
449 client->mlag_reg_mask1 = reg_mask;
450 /*
451 * Client might missed MLAG-UP Notification, post-it again
452 */
453 zebra_mlag_publish_process_state(client, ZEBRA_MLAG_PROCESS_UP);
454 return;
455 }
456
457
458 client->mlag_updates_interested = true;
459 client->mlag_reg_mask1 = reg_mask;
460 if (IS_ZEBRA_DEBUG_MLAG)
461 zlog_debug("Registering for MLAG Updates with mask: 0x%x, ",
462 client->mlag_reg_mask1);
463
464 zrouter.mlag_info.clients_interested_cnt++;
465
466 if (zrouter.mlag_info.clients_interested_cnt == 1) {
467 /*
468 * First-client for MLAG Updates,open the communication channel
469 * with MLAG
470 */
471 if (IS_ZEBRA_DEBUG_MLAG)
472 zlog_debug(
473 "First client, opening the channel with MLAG");
474
475 zebra_mlag_spawn_pthread();
476 rc = hook_call(zebra_mlag_private_open_channel);
477 if (rc < 0) {
478 /*
479 * For some reason, zebra not able to open the
480 * comm-channel with MLAG, so post MLAG-DOWN to client.
481 * later when the channel is open, zebra will send
482 * MLAG-UP
483 */
484 if (IS_ZEBRA_DEBUG_MLAG)
485 zlog_debug(
486 "Fail to open channel with MLAG,rc:%d, post Proto-down",
487 rc);
488 zebra_mlag_publish_process_state(
489 client, ZEBRA_MLAG_PROCESS_DOWN);
490 }
491 }
492
493 if (IS_ZEBRA_DEBUG_MLAG)
494 zlog_debug("Client Registered successfully for MLAG Updates");
495
496 if (zrouter.mlag_info.connected == true)
497 zebra_mlag_publish_process_state(client, ZEBRA_MLAG_PROCESS_UP);
498 stream_failure:
499 return;
500 }
501
502 /*
503 * API to un-register for MLAG Updates
504 */
505 void zebra_mlag_client_unregister(ZAPI_HANDLER_ARGS)
506 {
507 if (IS_ZEBRA_DEBUG_MLAG)
508 zlog_debug("Received MLAG De-Registration from client-proto:%d",
509 client->proto);
510
511 if (client->mlag_updates_interested == false)
512 /* Unexpected */
513 return;
514
515 client->mlag_updates_interested = false;
516 client->mlag_reg_mask1 = 0;
517 zrouter.mlag_info.clients_interested_cnt--;
518
519 if (zrouter.mlag_info.clients_interested_cnt == 0) {
520 /*
521 * No-client is interested for MLAG Updates,close the
522 * communication channel with MLAG
523 */
524 if (IS_ZEBRA_DEBUG_MLAG)
525 zlog_debug("Last client for MLAG, close the channel ");
526
527 /*
528 * Clean up flow:
529 * =============
530 * 1) main thread calls socket close which posts De-register
531 * to MLAG write thread
532 * 2) after MLAG write thread posts De-register it sends a
533 * signal back to main thread to do the thread cleanup
534 * this was mainly to make sure De-register is posted to MCLAGD.
535 */
536 hook_call(zebra_mlag_private_close_channel);
537 }
538
539 if (IS_ZEBRA_DEBUG_MLAG)
540 zlog_debug(
541 "Client De-Registered successfully for MLAG Updates");
542 }
543
544 /*
545 * Does following things.
546 * 1) allocated new local stream, and copies the client data and enqueue
547 * to MLAG Thread
548 * 2) MLAG Thread after dequeing, encode the client data using protobuf
549 * and write on to MLAG
550 */
551 void zebra_mlag_forward_client_msg(ZAPI_HANDLER_ARGS)
552 {
553 struct stream *zebra_s;
554 struct stream *mlag_s;
555
556 if (IS_ZEBRA_DEBUG_MLAG)
557 zlog_debug("Received Client MLAG Data from client-proto:%d",
558 client->proto);
559
560 /* Get input stream. */
561 zebra_s = msg;
562 mlag_s = stream_new(zebra_s->endp);
563
564 /*
565 * Client data is | Zebra Header + MLAG Data |
566 * we need to enqueue only the MLAG data, skipping Zebra Header
567 */
568 stream_put(mlag_s, zebra_s->data + zebra_s->getp,
569 STREAM_READABLE(zebra_s));
570 stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, mlag_s);
571 zebra_mlag_signal_write_thread();
572
573 if (IS_ZEBRA_DEBUG_MLAG)
574 zlog_debug("%s: Enqueued Client:%d data to MLAG Thread ",
575 __func__, client->proto);
576 }
577
578 /***********************End of Zebra Main thread processing*************/
579
580 enum mlag_role zebra_mlag_get_role(void)
581 {
582 return zrouter.mlag_info.role;
583 }
584
585 int32_t zebra_mlag_test_mlag_internal(const char *none, const char *primary,
586 const char *secondary)
587 {
588 enum mlag_role orig = zrouter.mlag_info.role;
589 char buf1[MLAG_ROLE_STRSIZE], buf2[MLAG_ROLE_STRSIZE];
590
591 if (none)
592 zrouter.mlag_info.role = MLAG_ROLE_NONE;
593 if (primary)
594 zrouter.mlag_info.role = MLAG_ROLE_PRIMARY;
595 if (secondary)
596 zrouter.mlag_info.role = MLAG_ROLE_SECONDARY;
597
598 if (IS_ZEBRA_DEBUG_MLAG)
599 zlog_debug("Test: Changing role from %s to %s",
600 mlag_role2str(orig, buf1, sizeof(buf1)),
601 mlag_role2str(orig, buf2, sizeof(buf2)));
602
603 if (orig != zrouter.mlag_info.role) {
604 zsend_capabilities_all_clients();
605 if (zrouter.mlag_info.role != MLAG_ROLE_NONE) {
606 if (zrouter.mlag_info.clients_interested_cnt == 0
607 && !test_mlag_in_progress) {
608 if (zrouter.mlag_info.zebra_pth_mlag == NULL)
609 zebra_mlag_spawn_pthread();
610 zrouter.mlag_info.clients_interested_cnt++;
611 test_mlag_in_progress = true;
612 hook_call(zebra_mlag_private_open_channel);
613 }
614 } else {
615 if (test_mlag_in_progress) {
616 test_mlag_in_progress = false;
617 zrouter.mlag_info.clients_interested_cnt--;
618 hook_call(zebra_mlag_private_close_channel);
619 }
620 }
621 }
622
623 return CMD_SUCCESS;
624 }
625
626 void zebra_mlag_init(void)
627 {
628 zebra_mlag_vty_init();
629
630 /*
631 * Intialiaze the MLAG Global variables
632 * write thread will be created during actual registration with MCLAG
633 */
634 zrouter.mlag_info.clients_interested_cnt = 0;
635 zrouter.mlag_info.connected = false;
636 zrouter.mlag_info.timer_running = false;
637 zrouter.mlag_info.mlag_fifo = stream_fifo_new();
638 zrouter.mlag_info.zebra_pth_mlag = NULL;
639 zrouter.mlag_info.th_master = NULL;
640 zrouter.mlag_info.t_read = NULL;
641 zrouter.mlag_info.t_write = NULL;
642 test_mlag_in_progress = false;
643 zebra_mlag_reset_read_buffer();
644 }
645
646 void zebra_mlag_terminate(void)
647 {
648 }
649
650
651 /*
652 *
653 * ProtoBuf Encoding APIs
654 */
655
656 #ifdef HAVE_PROTOBUF_VERSION_3
657
658 DEFINE_MTYPE_STATIC(ZEBRA, MLAG_PBUF, "ZEBRA MLAG PROTOBUF");
659
660 int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type)
661 {
662 ZebraMlagHeader hdr = ZEBRA_MLAG__HEADER__INIT;
663 struct mlag_msg mlag_msg;
664 uint8_t tmp_buf[ZEBRA_MLAG_BUF_LIMIT];
665 int len = 0;
666 int n_len = 0;
667 int rc = 0;
668 char buf[ZLOG_FILTER_LENGTH_MAX];
669 size_t length;
670
671 if (IS_ZEBRA_DEBUG_MLAG)
672 zlog_debug("%s: Entering..", __func__);
673
674 rc = mlag_lib_decode_mlag_hdr(s, &mlag_msg, &length);
675 if (rc)
676 return rc;
677
678 memset(tmp_buf, 0, ZEBRA_MLAG_BUF_LIMIT);
679
680 if (IS_ZEBRA_DEBUG_MLAG)
681 zlog_debug("%s: Mlag ProtoBuf encoding of message:%s, len:%d",
682 __func__,
683 mlag_lib_msgid_to_str(mlag_msg.msg_type, buf,
684 sizeof(buf)),
685 mlag_msg.data_len);
686 *msg_type = mlag_msg.msg_type;
687 switch (mlag_msg.msg_type) {
688 case MLAG_MROUTE_ADD: {
689 struct mlag_mroute_add msg;
690 ZebraMlagMrouteAdd pay_load = ZEBRA_MLAG_MROUTE_ADD__INIT;
691 uint32_t vrf_name_len = 0;
692
693 rc = mlag_lib_decode_mroute_add(s, &msg, &length);
694 if (rc)
695 return rc;
696
697 vrf_name_len = strlen(msg.vrf_name) + 1;
698 pay_load.vrf_name = XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
699 strlcpy(pay_load.vrf_name, msg.vrf_name, vrf_name_len);
700 pay_load.source_ip = msg.source_ip;
701 pay_load.group_ip = msg.group_ip;
702 pay_load.cost_to_rp = msg.cost_to_rp;
703 pay_load.owner_id = msg.owner_id;
704 pay_load.am_i_dr = msg.am_i_dr;
705 pay_load.am_i_dual_active = msg.am_i_dual_active;
706 pay_load.vrf_id = msg.vrf_id;
707
708 if (msg.owner_id == MLAG_OWNER_INTERFACE) {
709 vrf_name_len = strlen(msg.intf_name) + 1;
710 pay_load.intf_name =
711 XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
712 strlcpy(pay_load.intf_name, msg.intf_name,
713 vrf_name_len);
714 }
715
716 len = zebra_mlag_mroute_add__pack(&pay_load, tmp_buf);
717 XFREE(MTYPE_MLAG_PBUF, pay_load.vrf_name);
718 if (msg.owner_id == MLAG_OWNER_INTERFACE)
719 XFREE(MTYPE_MLAG_PBUF, pay_load.intf_name);
720 } break;
721 case MLAG_MROUTE_DEL: {
722 struct mlag_mroute_del msg;
723 ZebraMlagMrouteDel pay_load = ZEBRA_MLAG_MROUTE_DEL__INIT;
724 uint32_t vrf_name_len = 0;
725
726 rc = mlag_lib_decode_mroute_del(s, &msg, &length);
727 if (rc)
728 return rc;
729 vrf_name_len = strlen(msg.vrf_name) + 1;
730 pay_load.vrf_name = XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
731 strlcpy(pay_load.vrf_name, msg.vrf_name, vrf_name_len);
732 pay_load.source_ip = msg.source_ip;
733 pay_load.group_ip = msg.group_ip;
734 pay_load.owner_id = msg.owner_id;
735 pay_load.vrf_id = msg.vrf_id;
736
737 if (msg.owner_id == MLAG_OWNER_INTERFACE) {
738 vrf_name_len = strlen(msg.intf_name) + 1;
739 pay_load.intf_name =
740 XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
741 strlcpy(pay_load.intf_name, msg.intf_name,
742 vrf_name_len);
743 }
744
745 len = zebra_mlag_mroute_del__pack(&pay_load, tmp_buf);
746 XFREE(MTYPE_MLAG_PBUF, pay_load.vrf_name);
747 if (msg.owner_id == MLAG_OWNER_INTERFACE)
748 XFREE(MTYPE_MLAG_PBUF, pay_load.intf_name);
749 } break;
750 case MLAG_MROUTE_ADD_BULK: {
751 struct mlag_mroute_add msg;
752 ZebraMlagMrouteAddBulk Bulk_msg =
753 ZEBRA_MLAG_MROUTE_ADD_BULK__INIT;
754 ZebraMlagMrouteAdd **pay_load = NULL;
755 bool cleanup = false;
756 uint32_t i, actual;
757
758 Bulk_msg.n_mroute_add = mlag_msg.msg_cnt;
759 pay_load = XMALLOC(MTYPE_MLAG_PBUF, sizeof(ZebraMlagMrouteAdd *)
760 * mlag_msg.msg_cnt);
761
762 for (i = 0, actual = 0; i < mlag_msg.msg_cnt; i++, actual++) {
763
764 uint32_t vrf_name_len = 0;
765
766 rc = mlag_lib_decode_mroute_add(s, &msg, &length);
767 if (rc) {
768 cleanup = true;
769 break;
770 }
771 pay_load[i] = XMALLOC(MTYPE_MLAG_PBUF,
772 sizeof(ZebraMlagMrouteAdd));
773 zebra_mlag_mroute_add__init(pay_load[i]);
774
775 vrf_name_len = strlen(msg.vrf_name) + 1;
776 pay_load[i]->vrf_name =
777 XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
778 strlcpy(pay_load[i]->vrf_name, msg.vrf_name,
779 vrf_name_len);
780 pay_load[i]->source_ip = msg.source_ip;
781 pay_load[i]->group_ip = msg.group_ip;
782 pay_load[i]->cost_to_rp = msg.cost_to_rp;
783 pay_load[i]->owner_id = msg.owner_id;
784 pay_load[i]->am_i_dr = msg.am_i_dr;
785 pay_load[i]->am_i_dual_active = msg.am_i_dual_active;
786 pay_load[i]->vrf_id = msg.vrf_id;
787 if (msg.owner_id == MLAG_OWNER_INTERFACE) {
788 vrf_name_len = strlen(msg.intf_name) + 1;
789 pay_load[i]->intf_name =
790 XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
791
792 strlcpy(pay_load[i]->intf_name, msg.intf_name,
793 vrf_name_len);
794 }
795 }
796 if (!cleanup) {
797 Bulk_msg.mroute_add = pay_load;
798 len = zebra_mlag_mroute_add_bulk__pack(&Bulk_msg,
799 tmp_buf);
800 }
801
802 for (i = 0; i < actual; i++) {
803 /*
804 * The mlag_lib_decode_mroute_add can
805 * fail to properly decode and cause nothing
806 * to be allocated. Prevent a crash
807 */
808 if (!pay_load[i])
809 continue;
810
811 XFREE(MTYPE_MLAG_PBUF, pay_load[i]->vrf_name);
812 if (pay_load[i]->owner_id == MLAG_OWNER_INTERFACE
813 && pay_load[i]->intf_name)
814 XFREE(MTYPE_MLAG_PBUF, pay_load[i]->intf_name);
815 XFREE(MTYPE_MLAG_PBUF, pay_load[i]);
816 }
817 XFREE(MTYPE_MLAG_PBUF, pay_load);
818 if (cleanup)
819 return -1;
820 } break;
821 case MLAG_MROUTE_DEL_BULK: {
822 struct mlag_mroute_del msg;
823 ZebraMlagMrouteDelBulk Bulk_msg =
824 ZEBRA_MLAG_MROUTE_DEL_BULK__INIT;
825 ZebraMlagMrouteDel **pay_load = NULL;
826 bool cleanup = false;
827 uint32_t i, actual;
828
829 Bulk_msg.n_mroute_del = mlag_msg.msg_cnt;
830 pay_load = XMALLOC(MTYPE_MLAG_PBUF, sizeof(ZebraMlagMrouteDel *)
831 * mlag_msg.msg_cnt);
832
833 for (i = 0, actual = 0; i < mlag_msg.msg_cnt; i++, actual++) {
834
835 uint32_t vrf_name_len = 0;
836
837 rc = mlag_lib_decode_mroute_del(s, &msg, &length);
838 if (rc) {
839 cleanup = true;
840 break;
841 }
842
843 pay_load[i] = XMALLOC(MTYPE_MLAG_PBUF,
844 sizeof(ZebraMlagMrouteDel));
845 zebra_mlag_mroute_del__init(pay_load[i]);
846
847 vrf_name_len = strlen(msg.vrf_name) + 1;
848 pay_load[i]->vrf_name =
849 XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
850
851 strlcpy(pay_load[i]->vrf_name, msg.vrf_name,
852 vrf_name_len);
853 pay_load[i]->source_ip = msg.source_ip;
854 pay_load[i]->group_ip = msg.group_ip;
855 pay_load[i]->owner_id = msg.owner_id;
856 pay_load[i]->vrf_id = msg.vrf_id;
857 if (msg.owner_id == MLAG_OWNER_INTERFACE) {
858 vrf_name_len = strlen(msg.intf_name) + 1;
859 pay_load[i]->intf_name =
860 XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len);
861
862 strlcpy(pay_load[i]->intf_name, msg.intf_name,
863 vrf_name_len);
864 }
865 }
866 if (!cleanup) {
867 Bulk_msg.mroute_del = pay_load;
868 len = zebra_mlag_mroute_del_bulk__pack(&Bulk_msg,
869 tmp_buf);
870 }
871
872 for (i = 0; i < actual; i++) {
873 /*
874 * The mlag_lib_decode_mroute_add can
875 * fail to properly decode and cause nothing
876 * to be allocated. Prevent a crash
877 */
878 if (!pay_load[i])
879 continue;
880
881 XFREE(MTYPE_MLAG_PBUF, pay_load[i]->vrf_name);
882 if (pay_load[i]->owner_id == MLAG_OWNER_INTERFACE
883 && pay_load[i]->intf_name)
884 XFREE(MTYPE_MLAG_PBUF, pay_load[i]->intf_name);
885 XFREE(MTYPE_MLAG_PBUF, pay_load[i]);
886 }
887 XFREE(MTYPE_MLAG_PBUF, pay_load);
888 if (cleanup)
889 return -1;
890 } break;
891 case MLAG_REGISTER:
892 case MLAG_DEREGISTER:
893 case MLAG_STATUS_UPDATE:
894 case MLAG_DUMP:
895 case MLAG_PIM_CFG_DUMP:
896 case MLAG_VXLAN_UPDATE:
897 case MLAG_PEER_FRR_STATUS:
898 case MLAG_MSG_NONE:
899 break;
900 }
901
902 if (IS_ZEBRA_DEBUG_MLAG)
903 zlog_debug("%s: length of Mlag ProtoBuf encoded message:%s, %d",
904 __func__,
905 mlag_lib_msgid_to_str(mlag_msg.msg_type, buf,
906 sizeof(buf)),
907 len);
908 hdr.type = (ZebraMlagHeader__MessageType)mlag_msg.msg_type;
909 if (len != 0) {
910 hdr.data.len = len;
911 hdr.data.data = XMALLOC(MTYPE_MLAG_PBUF, len);
912 memcpy(hdr.data.data, tmp_buf, len);
913 }
914
915 /*
916 * ProtoBuf Infra will not support to demarc the pointers whem multiple
917 * messages are posted inside a single Buffer.
918 * 2 -solutions exist to solve this
919 * 1. add Unenoced length at the beginning of every message, this will
920 * be used to point to next message in the buffer
921 * 2. another solution is defining all messages insides another message
922 * But this will permit only 32 messages. this can be extended with
923 * multiple levels.
924 * for simplicity we are going with solution-1.
925 */
926 len = zebra_mlag__header__pack(&hdr,
927 (mlag_wr_buffer + ZEBRA_MLAG_LEN_SIZE));
928 n_len = htonl(len);
929 memcpy(mlag_wr_buffer, &n_len, ZEBRA_MLAG_LEN_SIZE);
930 len += ZEBRA_MLAG_LEN_SIZE;
931
932 if (IS_ZEBRA_DEBUG_MLAG)
933 zlog_debug(
934 "%s: length of Mlag ProtoBuf message:%s with Header %d",
935 __func__,
936 mlag_lib_msgid_to_str(mlag_msg.msg_type, buf,
937 sizeof(buf)),
938 len);
939 XFREE(MTYPE_MLAG_PBUF, hdr.data.data);
940
941 return len;
942 }
943
944 static void zebra_fill_protobuf_msg(struct stream *s, char *name, int len)
945 {
946 int str_len = strlen(name) + 1;
947
948 stream_put(s, name, str_len);
949 /* Fill the rest with Null Character for aligning */
950 stream_put(s, NULL, len - str_len);
951 }
952
953 int zebra_mlag_protobuf_decode_message(struct stream *s, uint8_t *data,
954 uint32_t len)
955 {
956 uint32_t msg_type;
957 ZebraMlagHeader *hdr;
958 char buf[80];
959
960 hdr = zebra_mlag__header__unpack(NULL, len, data);
961 if (hdr == NULL)
962 return -1;
963
964 /*
965 * ADD The MLAG Header
966 */
967 zclient_create_header(s, ZEBRA_MLAG_FORWARD_MSG, VRF_DEFAULT);
968
969 msg_type = hdr->type;
970
971 if (IS_ZEBRA_DEBUG_MLAG)
972 zlog_debug("%s: Mlag ProtoBuf decoding of message:%s", __func__,
973 mlag_lib_msgid_to_str(msg_type, buf, 80));
974
975 /*
976 * Internal MLAG Message-types & MLAG.proto message types should
977 * always match, otherwise there can be decoding errors
978 * To avoid exposing clients with Protobuf flags, using internal
979 * message-types
980 */
981 stream_putl(s, hdr->type);
982
983 if (hdr->data.len == 0) {
984 /* NULL Payload */
985 stream_putw(s, MLAG_MSG_NULL_PAYLOAD);
986 /* No Batching */
987 stream_putw(s, MLAG_MSG_NO_BATCH);
988 } else {
989 switch (msg_type) {
990 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_STATUS_UPDATE: {
991 ZebraMlagStatusUpdate *msg = NULL;
992
993 msg = zebra_mlag_status_update__unpack(
994 NULL, hdr->data.len, hdr->data.data);
995 if (msg == NULL) {
996 zebra_mlag__header__free_unpacked(hdr, NULL);
997 return -1;
998 }
999 /* Payload len */
1000 stream_putw(s, sizeof(struct mlag_status));
1001 /* No Batching */
1002 stream_putw(s, MLAG_MSG_NO_BATCH);
1003 /* Actual Data */
1004 zebra_fill_protobuf_msg(s, msg->peerlink,
1005 INTERFACE_NAMSIZ);
1006 stream_putl(s, msg->my_role);
1007 stream_putl(s, msg->peer_state);
1008 zebra_mlag_status_update__free_unpacked(msg, NULL);
1009 } break;
1010 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_VXLAN_UPDATE: {
1011 ZebraMlagVxlanUpdate *msg = NULL;
1012
1013 msg = zebra_mlag_vxlan_update__unpack(
1014 NULL, hdr->data.len, hdr->data.data);
1015 if (msg == NULL) {
1016 zebra_mlag__header__free_unpacked(hdr, NULL);
1017 return -1;
1018 }
1019 /* Payload len */
1020 stream_putw(s, sizeof(struct mlag_vxlan));
1021 /* No Batching */
1022 stream_putw(s, MLAG_MSG_NO_BATCH);
1023 /* Actual Data */
1024 stream_putl(s, msg->anycast_ip);
1025 stream_putl(s, msg->local_ip);
1026 zebra_mlag_vxlan_update__free_unpacked(msg, NULL);
1027 } break;
1028 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_ADD: {
1029 ZebraMlagMrouteAdd *msg = NULL;
1030
1031 msg = zebra_mlag_mroute_add__unpack(NULL, hdr->data.len,
1032 hdr->data.data);
1033 if (msg == NULL) {
1034 zebra_mlag__header__free_unpacked(hdr, NULL);
1035 return -1;
1036 }
1037 /* Payload len */
1038 stream_putw(s, sizeof(struct mlag_mroute_add));
1039 /* No Batching */
1040 stream_putw(s, MLAG_MSG_NO_BATCH);
1041 /* Actual Data */
1042 zebra_fill_protobuf_msg(s, msg->vrf_name, VRF_NAMSIZ);
1043
1044 stream_putl(s, msg->source_ip);
1045 stream_putl(s, msg->group_ip);
1046 stream_putl(s, msg->cost_to_rp);
1047 stream_putl(s, msg->owner_id);
1048 stream_putc(s, msg->am_i_dr);
1049 stream_putc(s, msg->am_i_dual_active);
1050 stream_putl(s, msg->vrf_id);
1051 if (msg->owner_id == MLAG_OWNER_INTERFACE)
1052 zebra_fill_protobuf_msg(s, msg->intf_name,
1053 INTERFACE_NAMSIZ);
1054 else
1055 stream_put(s, NULL, INTERFACE_NAMSIZ);
1056 zebra_mlag_mroute_add__free_unpacked(msg, NULL);
1057 } break;
1058 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_DEL: {
1059 ZebraMlagMrouteDel *msg = NULL;
1060
1061 msg = zebra_mlag_mroute_del__unpack(NULL, hdr->data.len,
1062 hdr->data.data);
1063 if (msg == NULL) {
1064 zebra_mlag__header__free_unpacked(hdr, NULL);
1065 return -1;
1066 }
1067 /* Payload len */
1068 stream_putw(s, sizeof(struct mlag_mroute_del));
1069 /* No Batching */
1070 stream_putw(s, MLAG_MSG_NO_BATCH);
1071 /* Actual Data */
1072 zebra_fill_protobuf_msg(s, msg->vrf_name, VRF_NAMSIZ);
1073
1074 stream_putl(s, msg->source_ip);
1075 stream_putl(s, msg->group_ip);
1076 stream_putl(s, msg->owner_id);
1077 stream_putl(s, msg->vrf_id);
1078 if (msg->owner_id == MLAG_OWNER_INTERFACE)
1079 zebra_fill_protobuf_msg(s, msg->intf_name,
1080 INTERFACE_NAMSIZ);
1081 else
1082 stream_put(s, NULL, INTERFACE_NAMSIZ);
1083 zebra_mlag_mroute_del__free_unpacked(msg, NULL);
1084 } break;
1085 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_ADD_BULK: {
1086 ZebraMlagMrouteAddBulk *Bulk_msg = NULL;
1087 ZebraMlagMrouteAdd *msg = NULL;
1088 size_t i, length_spot;
1089
1090 Bulk_msg = zebra_mlag_mroute_add_bulk__unpack(
1091 NULL, hdr->data.len, hdr->data.data);
1092 if (Bulk_msg == NULL) {
1093 zebra_mlag__header__free_unpacked(hdr, NULL);
1094 return -1;
1095 }
1096 /* Payload len */
1097 stream_putw(s, (Bulk_msg->n_mroute_add
1098 * sizeof(struct mlag_mroute_add)));
1099 /* No. of msgs in Batch */
1100 length_spot = stream_putw(s, Bulk_msg->n_mroute_add);
1101
1102 /* Actual Data */
1103 for (i = 0; i < Bulk_msg->n_mroute_add; i++) {
1104 if (STREAM_SIZE(s)
1105 < VRF_NAMSIZ + 22 + INTERFACE_NAMSIZ) {
1106 zlog_warn(
1107 "We have received more messages than we can parse at this point in time: %zu",
1108 Bulk_msg->n_mroute_add);
1109 break;
1110 }
1111
1112 msg = Bulk_msg->mroute_add[i];
1113
1114 zebra_fill_protobuf_msg(s, msg->vrf_name,
1115 VRF_NAMSIZ);
1116 stream_putl(s, msg->source_ip);
1117 stream_putl(s, msg->group_ip);
1118 stream_putl(s, msg->cost_to_rp);
1119 stream_putl(s, msg->owner_id);
1120 stream_putc(s, msg->am_i_dr);
1121 stream_putc(s, msg->am_i_dual_active);
1122 stream_putl(s, msg->vrf_id);
1123 if (msg->owner_id == MLAG_OWNER_INTERFACE)
1124 zebra_fill_protobuf_msg(
1125 s, msg->intf_name,
1126 INTERFACE_NAMSIZ);
1127 else
1128 stream_put(s, NULL, INTERFACE_NAMSIZ);
1129 }
1130
1131 stream_putw_at(s, length_spot, i + 1);
1132
1133 zebra_mlag_mroute_add_bulk__free_unpacked(Bulk_msg,
1134 NULL);
1135 } break;
1136 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_DEL_BULK: {
1137 ZebraMlagMrouteDelBulk *Bulk_msg = NULL;
1138 ZebraMlagMrouteDel *msg = NULL;
1139 size_t i, length_spot;
1140
1141 Bulk_msg = zebra_mlag_mroute_del_bulk__unpack(
1142 NULL, hdr->data.len, hdr->data.data);
1143 if (Bulk_msg == NULL) {
1144 zebra_mlag__header__free_unpacked(hdr, NULL);
1145 return -1;
1146 }
1147 /* Payload len */
1148 stream_putw(s, (Bulk_msg->n_mroute_del
1149 * sizeof(struct mlag_mroute_del)));
1150 /* No. of msgs in Batch */
1151 length_spot = stream_putw(s, Bulk_msg->n_mroute_del);
1152
1153 /* Actual Data */
1154 for (i = 0; i < Bulk_msg->n_mroute_del; i++) {
1155 if (STREAM_SIZE(s)
1156 < VRF_NAMSIZ + 16 + INTERFACE_NAMSIZ) {
1157 zlog_warn(
1158 "We have received more messages than we can parse at this time");
1159 break;
1160 }
1161
1162 msg = Bulk_msg->mroute_del[i];
1163
1164 zebra_fill_protobuf_msg(s, msg->vrf_name,
1165 VRF_NAMSIZ);
1166 stream_putl(s, msg->source_ip);
1167 stream_putl(s, msg->group_ip);
1168 stream_putl(s, msg->owner_id);
1169 stream_putl(s, msg->vrf_id);
1170 if (msg->owner_id == MLAG_OWNER_INTERFACE)
1171 zebra_fill_protobuf_msg(
1172 s, msg->intf_name,
1173 INTERFACE_NAMSIZ);
1174 else
1175 stream_put(s, NULL, INTERFACE_NAMSIZ);
1176 }
1177
1178 stream_putw_at(s, length_spot, i + 1);
1179
1180 zebra_mlag_mroute_del_bulk__free_unpacked(Bulk_msg,
1181 NULL);
1182 } break;
1183 case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_ZEBRA_STATUS_UPDATE: {
1184 ZebraMlagZebraStatusUpdate *msg = NULL;
1185
1186 msg = zebra_mlag_zebra_status_update__unpack(
1187 NULL, hdr->data.len, hdr->data.data);
1188 if (msg == NULL) {
1189 zebra_mlag__header__free_unpacked(hdr, NULL);
1190 return -1;
1191 }
1192 /* Payload len */
1193 stream_putw(s, sizeof(struct mlag_frr_status));
1194 /* No Batching */
1195 stream_putw(s, MLAG_MSG_NO_BATCH);
1196 /* Actual Data */
1197 stream_putl(s, msg->peer_frrstate);
1198 zebra_mlag_zebra_status_update__free_unpacked(msg,
1199 NULL);
1200 } break;
1201 default:
1202 break;
1203 }
1204 }
1205 zebra_mlag__header__free_unpacked(hdr, NULL);
1206 return msg_type;
1207 }
1208
1209 #else
1210 int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type)
1211 {
1212 return 0;
1213 }
1214
1215 int zebra_mlag_protobuf_decode_message(struct stream *s, uint8_t *data,
1216 uint32_t len)
1217 {
1218 return 0;
1219 }
1220 #endif
FRRouting mirror