]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
Merge tag 'ntb-4.13-bugfixes' of git://github.com/jonmason/ntb
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / mellanox / mlxsw / spectrum_router.c
1 /*
2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
19 *
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/rhashtable.h>
40 #include <linux/bitops.h>
41 #include <linux/in6.h>
42 #include <linux/notifier.h>
43 #include <linux/inetdevice.h>
44 #include <linux/netdevice.h>
45 #include <linux/if_bridge.h>
46 #include <net/netevent.h>
47 #include <net/neighbour.h>
48 #include <net/arp.h>
49 #include <net/ip_fib.h>
50 #include <net/fib_rules.h>
51 #include <net/l3mdev.h>
52
53 #include "spectrum.h"
54 #include "core.h"
55 #include "reg.h"
56 #include "spectrum_cnt.h"
57 #include "spectrum_dpipe.h"
58 #include "spectrum_router.h"
59
60 struct mlxsw_sp_vr;
61 struct mlxsw_sp_lpm_tree;
62 struct mlxsw_sp_rif_ops;
63
64 struct mlxsw_sp_router {
65 struct mlxsw_sp *mlxsw_sp;
66 struct mlxsw_sp_rif **rifs;
67 struct mlxsw_sp_vr *vrs;
68 struct rhashtable neigh_ht;
69 struct rhashtable nexthop_group_ht;
70 struct rhashtable nexthop_ht;
71 struct {
72 struct mlxsw_sp_lpm_tree *trees;
73 unsigned int tree_count;
74 } lpm;
75 struct {
76 struct delayed_work dw;
77 unsigned long interval; /* ms */
78 } neighs_update;
79 struct delayed_work nexthop_probe_dw;
80 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
81 struct list_head nexthop_neighs_list;
82 bool aborted;
83 struct notifier_block fib_nb;
84 const struct mlxsw_sp_rif_ops **rif_ops_arr;
85 };
86
87 struct mlxsw_sp_rif {
88 struct list_head nexthop_list;
89 struct list_head neigh_list;
90 struct net_device *dev;
91 struct mlxsw_sp_fid *fid;
92 unsigned char addr[ETH_ALEN];
93 int mtu;
94 u16 rif_index;
95 u16 vr_id;
96 const struct mlxsw_sp_rif_ops *ops;
97 struct mlxsw_sp *mlxsw_sp;
98
99 unsigned int counter_ingress;
100 bool counter_ingress_valid;
101 unsigned int counter_egress;
102 bool counter_egress_valid;
103 };
104
105 struct mlxsw_sp_rif_params {
106 struct net_device *dev;
107 union {
108 u16 system_port;
109 u16 lag_id;
110 };
111 u16 vid;
112 bool lag;
113 };
114
115 struct mlxsw_sp_rif_subport {
116 struct mlxsw_sp_rif common;
117 union {
118 u16 system_port;
119 u16 lag_id;
120 };
121 u16 vid;
122 bool lag;
123 };
124
125 struct mlxsw_sp_rif_ops {
126 enum mlxsw_sp_rif_type type;
127 size_t rif_size;
128
129 void (*setup)(struct mlxsw_sp_rif *rif,
130 const struct mlxsw_sp_rif_params *params);
131 int (*configure)(struct mlxsw_sp_rif *rif);
132 void (*deconfigure)(struct mlxsw_sp_rif *rif);
133 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
134 };
135
136 static unsigned int *
137 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
138 enum mlxsw_sp_rif_counter_dir dir)
139 {
140 switch (dir) {
141 case MLXSW_SP_RIF_COUNTER_EGRESS:
142 return &rif->counter_egress;
143 case MLXSW_SP_RIF_COUNTER_INGRESS:
144 return &rif->counter_ingress;
145 }
146 return NULL;
147 }
148
149 static bool
150 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
151 enum mlxsw_sp_rif_counter_dir dir)
152 {
153 switch (dir) {
154 case MLXSW_SP_RIF_COUNTER_EGRESS:
155 return rif->counter_egress_valid;
156 case MLXSW_SP_RIF_COUNTER_INGRESS:
157 return rif->counter_ingress_valid;
158 }
159 return false;
160 }
161
162 static void
163 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
164 enum mlxsw_sp_rif_counter_dir dir,
165 bool valid)
166 {
167 switch (dir) {
168 case MLXSW_SP_RIF_COUNTER_EGRESS:
169 rif->counter_egress_valid = valid;
170 break;
171 case MLXSW_SP_RIF_COUNTER_INGRESS:
172 rif->counter_ingress_valid = valid;
173 break;
174 }
175 }
176
177 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
178 unsigned int counter_index, bool enable,
179 enum mlxsw_sp_rif_counter_dir dir)
180 {
181 char ritr_pl[MLXSW_REG_RITR_LEN];
182 bool is_egress = false;
183 int err;
184
185 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
186 is_egress = true;
187 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
188 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
189 if (err)
190 return err;
191
192 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
193 is_egress);
194 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
195 }
196
197 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
198 struct mlxsw_sp_rif *rif,
199 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
200 {
201 char ricnt_pl[MLXSW_REG_RICNT_LEN];
202 unsigned int *p_counter_index;
203 bool valid;
204 int err;
205
206 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
207 if (!valid)
208 return -EINVAL;
209
210 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
211 if (!p_counter_index)
212 return -EINVAL;
213 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
214 MLXSW_REG_RICNT_OPCODE_NOP);
215 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
216 if (err)
217 return err;
218 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
219 return 0;
220 }
221
222 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
223 unsigned int counter_index)
224 {
225 char ricnt_pl[MLXSW_REG_RICNT_LEN];
226
227 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
228 MLXSW_REG_RICNT_OPCODE_CLEAR);
229 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
230 }
231
232 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
233 struct mlxsw_sp_rif *rif,
234 enum mlxsw_sp_rif_counter_dir dir)
235 {
236 unsigned int *p_counter_index;
237 int err;
238
239 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
240 if (!p_counter_index)
241 return -EINVAL;
242 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
243 p_counter_index);
244 if (err)
245 return err;
246
247 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
248 if (err)
249 goto err_counter_clear;
250
251 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
252 *p_counter_index, true, dir);
253 if (err)
254 goto err_counter_edit;
255 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
256 return 0;
257
258 err_counter_edit:
259 err_counter_clear:
260 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
261 *p_counter_index);
262 return err;
263 }
264
265 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
266 struct mlxsw_sp_rif *rif,
267 enum mlxsw_sp_rif_counter_dir dir)
268 {
269 unsigned int *p_counter_index;
270
271 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
272 return;
273
274 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
275 if (WARN_ON(!p_counter_index))
276 return;
277 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
278 *p_counter_index, false, dir);
279 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
280 *p_counter_index);
281 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
282 }
283
284 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
285 {
286 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
287 struct devlink *devlink;
288
289 devlink = priv_to_devlink(mlxsw_sp->core);
290 if (!devlink_dpipe_table_counter_enabled(devlink,
291 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
292 return;
293 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
294 }
295
296 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
297 {
298 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
299
300 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
301 }
302
303 static struct mlxsw_sp_rif *
304 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
305 const struct net_device *dev);
306
307 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
308
309 struct mlxsw_sp_prefix_usage {
310 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
311 };
312
313 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
314 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
315
316 static bool
317 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
318 struct mlxsw_sp_prefix_usage *prefix_usage2)
319 {
320 unsigned char prefix;
321
322 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
323 if (!test_bit(prefix, prefix_usage2->b))
324 return false;
325 }
326 return true;
327 }
328
329 static bool
330 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
331 struct mlxsw_sp_prefix_usage *prefix_usage2)
332 {
333 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
334 }
335
336 static bool
337 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
338 {
339 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
340
341 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
342 }
343
344 static void
345 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
346 struct mlxsw_sp_prefix_usage *prefix_usage2)
347 {
348 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
349 }
350
351 static void
352 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
353 unsigned char prefix_len)
354 {
355 set_bit(prefix_len, prefix_usage->b);
356 }
357
358 static void
359 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
360 unsigned char prefix_len)
361 {
362 clear_bit(prefix_len, prefix_usage->b);
363 }
364
365 struct mlxsw_sp_fib_key {
366 unsigned char addr[sizeof(struct in6_addr)];
367 unsigned char prefix_len;
368 };
369
370 enum mlxsw_sp_fib_entry_type {
371 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
372 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
373 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
374 };
375
376 struct mlxsw_sp_nexthop_group;
377 struct mlxsw_sp_fib;
378
379 struct mlxsw_sp_fib_node {
380 struct list_head entry_list;
381 struct list_head list;
382 struct rhash_head ht_node;
383 struct mlxsw_sp_fib *fib;
384 struct mlxsw_sp_fib_key key;
385 };
386
387 struct mlxsw_sp_fib_entry_params {
388 u32 tb_id;
389 u32 prio;
390 u8 tos;
391 u8 type;
392 };
393
394 struct mlxsw_sp_fib_entry {
395 struct list_head list;
396 struct mlxsw_sp_fib_node *fib_node;
397 enum mlxsw_sp_fib_entry_type type;
398 struct list_head nexthop_group_node;
399 struct mlxsw_sp_nexthop_group *nh_group;
400 struct mlxsw_sp_fib_entry_params params;
401 bool offloaded;
402 };
403
404 enum mlxsw_sp_l3proto {
405 MLXSW_SP_L3_PROTO_IPV4,
406 MLXSW_SP_L3_PROTO_IPV6,
407 };
408
409 struct mlxsw_sp_lpm_tree {
410 u8 id; /* tree ID */
411 unsigned int ref_count;
412 enum mlxsw_sp_l3proto proto;
413 struct mlxsw_sp_prefix_usage prefix_usage;
414 };
415
416 struct mlxsw_sp_fib {
417 struct rhashtable ht;
418 struct list_head node_list;
419 struct mlxsw_sp_vr *vr;
420 struct mlxsw_sp_lpm_tree *lpm_tree;
421 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
422 struct mlxsw_sp_prefix_usage prefix_usage;
423 enum mlxsw_sp_l3proto proto;
424 };
425
426 struct mlxsw_sp_vr {
427 u16 id; /* virtual router ID */
428 u32 tb_id; /* kernel fib table id */
429 unsigned int rif_count;
430 struct mlxsw_sp_fib *fib4;
431 };
432
433 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
434
435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
436 enum mlxsw_sp_l3proto proto)
437 {
438 struct mlxsw_sp_fib *fib;
439 int err;
440
441 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
442 if (!fib)
443 return ERR_PTR(-ENOMEM);
444 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
445 if (err)
446 goto err_rhashtable_init;
447 INIT_LIST_HEAD(&fib->node_list);
448 fib->proto = proto;
449 fib->vr = vr;
450 return fib;
451
452 err_rhashtable_init:
453 kfree(fib);
454 return ERR_PTR(err);
455 }
456
457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
458 {
459 WARN_ON(!list_empty(&fib->node_list));
460 WARN_ON(fib->lpm_tree);
461 rhashtable_destroy(&fib->ht);
462 kfree(fib);
463 }
464
465 static struct mlxsw_sp_lpm_tree *
466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
467 {
468 static struct mlxsw_sp_lpm_tree *lpm_tree;
469 int i;
470
471 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
472 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
473 if (lpm_tree->ref_count == 0)
474 return lpm_tree;
475 }
476 return NULL;
477 }
478
479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
480 struct mlxsw_sp_lpm_tree *lpm_tree)
481 {
482 char ralta_pl[MLXSW_REG_RALTA_LEN];
483
484 mlxsw_reg_ralta_pack(ralta_pl, true,
485 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
486 lpm_tree->id);
487 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
488 }
489
490 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
491 struct mlxsw_sp_lpm_tree *lpm_tree)
492 {
493 char ralta_pl[MLXSW_REG_RALTA_LEN];
494
495 mlxsw_reg_ralta_pack(ralta_pl, false,
496 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
497 lpm_tree->id);
498 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
499 }
500
501 static int
502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
503 struct mlxsw_sp_prefix_usage *prefix_usage,
504 struct mlxsw_sp_lpm_tree *lpm_tree)
505 {
506 char ralst_pl[MLXSW_REG_RALST_LEN];
507 u8 root_bin = 0;
508 u8 prefix;
509 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
510
511 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
512 root_bin = prefix;
513
514 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
515 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
516 if (prefix == 0)
517 continue;
518 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
519 MLXSW_REG_RALST_BIN_NO_CHILD);
520 last_prefix = prefix;
521 }
522 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
523 }
524
525 static struct mlxsw_sp_lpm_tree *
526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
527 struct mlxsw_sp_prefix_usage *prefix_usage,
528 enum mlxsw_sp_l3proto proto)
529 {
530 struct mlxsw_sp_lpm_tree *lpm_tree;
531 int err;
532
533 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
534 if (!lpm_tree)
535 return ERR_PTR(-EBUSY);
536 lpm_tree->proto = proto;
537 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
538 if (err)
539 return ERR_PTR(err);
540
541 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
542 lpm_tree);
543 if (err)
544 goto err_left_struct_set;
545 memcpy(&lpm_tree->prefix_usage, prefix_usage,
546 sizeof(lpm_tree->prefix_usage));
547 return lpm_tree;
548
549 err_left_struct_set:
550 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
551 return ERR_PTR(err);
552 }
553
554 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
555 struct mlxsw_sp_lpm_tree *lpm_tree)
556 {
557 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
558 }
559
560 static struct mlxsw_sp_lpm_tree *
561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
562 struct mlxsw_sp_prefix_usage *prefix_usage,
563 enum mlxsw_sp_l3proto proto)
564 {
565 struct mlxsw_sp_lpm_tree *lpm_tree;
566 int i;
567
568 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
569 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
570 if (lpm_tree->ref_count != 0 &&
571 lpm_tree->proto == proto &&
572 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
573 prefix_usage))
574 goto inc_ref_count;
575 }
576 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
577 proto);
578 if (IS_ERR(lpm_tree))
579 return lpm_tree;
580
581 inc_ref_count:
582 lpm_tree->ref_count++;
583 return lpm_tree;
584 }
585
586 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
587 struct mlxsw_sp_lpm_tree *lpm_tree)
588 {
589 if (--lpm_tree->ref_count == 0)
590 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
591 return 0;
592 }
593
594 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
595
596 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
597 {
598 struct mlxsw_sp_lpm_tree *lpm_tree;
599 u64 max_trees;
600 int i;
601
602 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
603 return -EIO;
604
605 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
606 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
607 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
608 sizeof(struct mlxsw_sp_lpm_tree),
609 GFP_KERNEL);
610 if (!mlxsw_sp->router->lpm.trees)
611 return -ENOMEM;
612
613 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
614 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
615 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
616 }
617
618 return 0;
619 }
620
621 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
622 {
623 kfree(mlxsw_sp->router->lpm.trees);
624 }
625
626 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
627 {
628 return !!vr->fib4;
629 }
630
631 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
632 {
633 struct mlxsw_sp_vr *vr;
634 int i;
635
636 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
637 vr = &mlxsw_sp->router->vrs[i];
638 if (!mlxsw_sp_vr_is_used(vr))
639 return vr;
640 }
641 return NULL;
642 }
643
644 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
645 const struct mlxsw_sp_fib *fib)
646 {
647 char raltb_pl[MLXSW_REG_RALTB_LEN];
648
649 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
650 (enum mlxsw_reg_ralxx_protocol) fib->proto,
651 fib->lpm_tree->id);
652 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
653 }
654
655 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
656 const struct mlxsw_sp_fib *fib)
657 {
658 char raltb_pl[MLXSW_REG_RALTB_LEN];
659
660 /* Bind to tree 0 which is default */
661 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
662 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
663 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
664 }
665
666 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
667 {
668 /* For our purpose, squash main and local table into one */
669 if (tb_id == RT_TABLE_LOCAL)
670 tb_id = RT_TABLE_MAIN;
671 return tb_id;
672 }
673
674 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
675 u32 tb_id)
676 {
677 struct mlxsw_sp_vr *vr;
678 int i;
679
680 tb_id = mlxsw_sp_fix_tb_id(tb_id);
681
682 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
683 vr = &mlxsw_sp->router->vrs[i];
684 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
685 return vr;
686 }
687 return NULL;
688 }
689
690 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
691 enum mlxsw_sp_l3proto proto)
692 {
693 switch (proto) {
694 case MLXSW_SP_L3_PROTO_IPV4:
695 return vr->fib4;
696 case MLXSW_SP_L3_PROTO_IPV6:
697 BUG_ON(1);
698 }
699 return NULL;
700 }
701
702 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
703 u32 tb_id)
704 {
705 struct mlxsw_sp_vr *vr;
706
707 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
708 if (!vr)
709 return ERR_PTR(-EBUSY);
710 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
711 if (IS_ERR(vr->fib4))
712 return ERR_CAST(vr->fib4);
713 vr->tb_id = tb_id;
714 return vr;
715 }
716
717 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
718 {
719 mlxsw_sp_fib_destroy(vr->fib4);
720 vr->fib4 = NULL;
721 }
722
723 static int
724 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
725 struct mlxsw_sp_prefix_usage *req_prefix_usage)
726 {
727 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
728 struct mlxsw_sp_lpm_tree *new_tree;
729 int err;
730
731 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
732 return 0;
733
734 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
735 fib->proto);
736 if (IS_ERR(new_tree)) {
737 /* We failed to get a tree according to the required
738 * prefix usage. However, the current tree might be still good
739 * for us if our requirement is subset of the prefixes used
740 * in the tree.
741 */
742 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
743 &lpm_tree->prefix_usage))
744 return 0;
745 return PTR_ERR(new_tree);
746 }
747
748 /* Prevent packet loss by overwriting existing binding */
749 fib->lpm_tree = new_tree;
750 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
751 if (err)
752 goto err_tree_bind;
753 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
754
755 return 0;
756
757 err_tree_bind:
758 fib->lpm_tree = lpm_tree;
759 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
760 return err;
761 }
762
763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
764 {
765 struct mlxsw_sp_vr *vr;
766
767 tb_id = mlxsw_sp_fix_tb_id(tb_id);
768 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
769 if (!vr)
770 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
771 return vr;
772 }
773
774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
775 {
776 if (!vr->rif_count && list_empty(&vr->fib4->node_list))
777 mlxsw_sp_vr_destroy(vr);
778 }
779
780 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
781 {
782 struct mlxsw_sp_vr *vr;
783 u64 max_vrs;
784 int i;
785
786 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
787 return -EIO;
788
789 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
790 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
791 GFP_KERNEL);
792 if (!mlxsw_sp->router->vrs)
793 return -ENOMEM;
794
795 for (i = 0; i < max_vrs; i++) {
796 vr = &mlxsw_sp->router->vrs[i];
797 vr->id = i;
798 }
799
800 return 0;
801 }
802
803 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
804
805 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
806 {
807 /* At this stage we're guaranteed not to have new incoming
808 * FIB notifications and the work queue is free from FIBs
809 * sitting on top of mlxsw netdevs. However, we can still
810 * have other FIBs queued. Flush the queue before flushing
811 * the device's tables. No need for locks, as we're the only
812 * writer.
813 */
814 mlxsw_core_flush_owq();
815 mlxsw_sp_router_fib_flush(mlxsw_sp);
816 kfree(mlxsw_sp->router->vrs);
817 }
818
819 struct mlxsw_sp_neigh_key {
820 struct neighbour *n;
821 };
822
823 struct mlxsw_sp_neigh_entry {
824 struct list_head rif_list_node;
825 struct rhash_head ht_node;
826 struct mlxsw_sp_neigh_key key;
827 u16 rif;
828 bool connected;
829 unsigned char ha[ETH_ALEN];
830 struct list_head nexthop_list; /* list of nexthops using
831 * this neigh entry
832 */
833 struct list_head nexthop_neighs_list_node;
834 };
835
836 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
837 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
838 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
839 .key_len = sizeof(struct mlxsw_sp_neigh_key),
840 };
841
842 static struct mlxsw_sp_neigh_entry *
843 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
844 u16 rif)
845 {
846 struct mlxsw_sp_neigh_entry *neigh_entry;
847
848 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
849 if (!neigh_entry)
850 return NULL;
851
852 neigh_entry->key.n = n;
853 neigh_entry->rif = rif;
854 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
855
856 return neigh_entry;
857 }
858
859 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
860 {
861 kfree(neigh_entry);
862 }
863
864 static int
865 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
866 struct mlxsw_sp_neigh_entry *neigh_entry)
867 {
868 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
869 &neigh_entry->ht_node,
870 mlxsw_sp_neigh_ht_params);
871 }
872
873 static void
874 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
875 struct mlxsw_sp_neigh_entry *neigh_entry)
876 {
877 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
878 &neigh_entry->ht_node,
879 mlxsw_sp_neigh_ht_params);
880 }
881
882 static struct mlxsw_sp_neigh_entry *
883 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
884 {
885 struct mlxsw_sp_neigh_entry *neigh_entry;
886 struct mlxsw_sp_rif *rif;
887 int err;
888
889 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
890 if (!rif)
891 return ERR_PTR(-EINVAL);
892
893 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
894 if (!neigh_entry)
895 return ERR_PTR(-ENOMEM);
896
897 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
898 if (err)
899 goto err_neigh_entry_insert;
900
901 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
902
903 return neigh_entry;
904
905 err_neigh_entry_insert:
906 mlxsw_sp_neigh_entry_free(neigh_entry);
907 return ERR_PTR(err);
908 }
909
910 static void
911 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
912 struct mlxsw_sp_neigh_entry *neigh_entry)
913 {
914 list_del(&neigh_entry->rif_list_node);
915 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
916 mlxsw_sp_neigh_entry_free(neigh_entry);
917 }
918
919 static struct mlxsw_sp_neigh_entry *
920 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
921 {
922 struct mlxsw_sp_neigh_key key;
923
924 key.n = n;
925 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
926 &key, mlxsw_sp_neigh_ht_params);
927 }
928
929 static void
930 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
931 {
932 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
933
934 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
935 }
936
937 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
938 char *rauhtd_pl,
939 int ent_index)
940 {
941 struct net_device *dev;
942 struct neighbour *n;
943 __be32 dipn;
944 u32 dip;
945 u16 rif;
946
947 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
948
949 if (!mlxsw_sp->router->rifs[rif]) {
950 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
951 return;
952 }
953
954 dipn = htonl(dip);
955 dev = mlxsw_sp->router->rifs[rif]->dev;
956 n = neigh_lookup(&arp_tbl, &dipn, dev);
957 if (!n) {
958 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
959 &dip);
960 return;
961 }
962
963 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
964 neigh_event_send(n, NULL);
965 neigh_release(n);
966 }
967
968 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
969 char *rauhtd_pl,
970 int rec_index)
971 {
972 u8 num_entries;
973 int i;
974
975 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
976 rec_index);
977 /* Hardware starts counting at 0, so add 1. */
978 num_entries++;
979
980 /* Each record consists of several neighbour entries. */
981 for (i = 0; i < num_entries; i++) {
982 int ent_index;
983
984 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
985 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
986 ent_index);
987 }
988
989 }
990
991 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
992 char *rauhtd_pl, int rec_index)
993 {
994 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
995 case MLXSW_REG_RAUHTD_TYPE_IPV4:
996 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
997 rec_index);
998 break;
999 case MLXSW_REG_RAUHTD_TYPE_IPV6:
1000 WARN_ON_ONCE(1);
1001 break;
1002 }
1003 }
1004
1005 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1006 {
1007 u8 num_rec, last_rec_index, num_entries;
1008
1009 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1010 last_rec_index = num_rec - 1;
1011
1012 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1013 return false;
1014 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1015 MLXSW_REG_RAUHTD_TYPE_IPV6)
1016 return true;
1017
1018 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1019 last_rec_index);
1020 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1021 return true;
1022 return false;
1023 }
1024
1025 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1026 {
1027 char *rauhtd_pl;
1028 u8 num_rec;
1029 int i, err;
1030
1031 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1032 if (!rauhtd_pl)
1033 return -ENOMEM;
1034
1035 /* Make sure the neighbour's netdev isn't removed in the
1036 * process.
1037 */
1038 rtnl_lock();
1039 do {
1040 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
1041 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1042 rauhtd_pl);
1043 if (err) {
1044 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1045 break;
1046 }
1047 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1048 for (i = 0; i < num_rec; i++)
1049 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1050 i);
1051 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1052 rtnl_unlock();
1053
1054 kfree(rauhtd_pl);
1055 return err;
1056 }
1057
1058 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1059 {
1060 struct mlxsw_sp_neigh_entry *neigh_entry;
1061
1062 /* Take RTNL mutex here to prevent lists from changes */
1063 rtnl_lock();
1064 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1065 nexthop_neighs_list_node)
1066 /* If this neigh have nexthops, make the kernel think this neigh
1067 * is active regardless of the traffic.
1068 */
1069 neigh_event_send(neigh_entry->key.n, NULL);
1070 rtnl_unlock();
1071 }
1072
1073 static void
1074 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1075 {
1076 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1077
1078 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1079 msecs_to_jiffies(interval));
1080 }
1081
1082 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1083 {
1084 struct mlxsw_sp_router *router;
1085 int err;
1086
1087 router = container_of(work, struct mlxsw_sp_router,
1088 neighs_update.dw.work);
1089 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1090 if (err)
1091 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1092
1093 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1094
1095 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1096 }
1097
1098 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1099 {
1100 struct mlxsw_sp_neigh_entry *neigh_entry;
1101 struct mlxsw_sp_router *router;
1102
1103 router = container_of(work, struct mlxsw_sp_router,
1104 nexthop_probe_dw.work);
1105 /* Iterate over nexthop neighbours, find those who are unresolved and
1106 * send arp on them. This solves the chicken-egg problem when
1107 * the nexthop wouldn't get offloaded until the neighbor is resolved
1108 * but it wouldn't get resolved ever in case traffic is flowing in HW
1109 * using different nexthop.
1110 *
1111 * Take RTNL mutex here to prevent lists from changes.
1112 */
1113 rtnl_lock();
1114 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1115 nexthop_neighs_list_node)
1116 if (!neigh_entry->connected)
1117 neigh_event_send(neigh_entry->key.n, NULL);
1118 rtnl_unlock();
1119
1120 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1121 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1122 }
1123
1124 static void
1125 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1126 struct mlxsw_sp_neigh_entry *neigh_entry,
1127 bool removing);
1128
1129 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1130 {
1131 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1132 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1133 }
1134
1135 static void
1136 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1137 struct mlxsw_sp_neigh_entry *neigh_entry,
1138 enum mlxsw_reg_rauht_op op)
1139 {
1140 struct neighbour *n = neigh_entry->key.n;
1141 u32 dip = ntohl(*((__be32 *) n->primary_key));
1142 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1143
1144 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1145 dip);
1146 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1147 }
1148
1149 static void
1150 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1151 struct mlxsw_sp_neigh_entry *neigh_entry,
1152 bool adding)
1153 {
1154 if (!adding && !neigh_entry->connected)
1155 return;
1156 neigh_entry->connected = adding;
1157 if (neigh_entry->key.n->tbl == &arp_tbl)
1158 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1159 mlxsw_sp_rauht_op(adding));
1160 else
1161 WARN_ON_ONCE(1);
1162 }
1163
1164 struct mlxsw_sp_neigh_event_work {
1165 struct work_struct work;
1166 struct mlxsw_sp *mlxsw_sp;
1167 struct neighbour *n;
1168 };
1169
1170 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1171 {
1172 struct mlxsw_sp_neigh_event_work *neigh_work =
1173 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1174 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1175 struct mlxsw_sp_neigh_entry *neigh_entry;
1176 struct neighbour *n = neigh_work->n;
1177 unsigned char ha[ETH_ALEN];
1178 bool entry_connected;
1179 u8 nud_state, dead;
1180
1181 /* If these parameters are changed after we release the lock,
1182 * then we are guaranteed to receive another event letting us
1183 * know about it.
1184 */
1185 read_lock_bh(&n->lock);
1186 memcpy(ha, n->ha, ETH_ALEN);
1187 nud_state = n->nud_state;
1188 dead = n->dead;
1189 read_unlock_bh(&n->lock);
1190
1191 rtnl_lock();
1192 entry_connected = nud_state & NUD_VALID && !dead;
1193 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1194 if (!entry_connected && !neigh_entry)
1195 goto out;
1196 if (!neigh_entry) {
1197 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1198 if (IS_ERR(neigh_entry))
1199 goto out;
1200 }
1201
1202 memcpy(neigh_entry->ha, ha, ETH_ALEN);
1203 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1204 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1205
1206 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1207 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1208
1209 out:
1210 rtnl_unlock();
1211 neigh_release(n);
1212 kfree(neigh_work);
1213 }
1214
1215 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1216 unsigned long event, void *ptr)
1217 {
1218 struct mlxsw_sp_neigh_event_work *neigh_work;
1219 struct mlxsw_sp_port *mlxsw_sp_port;
1220 struct mlxsw_sp *mlxsw_sp;
1221 unsigned long interval;
1222 struct neigh_parms *p;
1223 struct neighbour *n;
1224
1225 switch (event) {
1226 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1227 p = ptr;
1228
1229 /* We don't care about changes in the default table. */
1230 if (!p->dev || p->tbl != &arp_tbl)
1231 return NOTIFY_DONE;
1232
1233 /* We are in atomic context and can't take RTNL mutex,
1234 * so use RCU variant to walk the device chain.
1235 */
1236 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1237 if (!mlxsw_sp_port)
1238 return NOTIFY_DONE;
1239
1240 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1241 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1242 mlxsw_sp->router->neighs_update.interval = interval;
1243
1244 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1245 break;
1246 case NETEVENT_NEIGH_UPDATE:
1247 n = ptr;
1248
1249 if (n->tbl != &arp_tbl)
1250 return NOTIFY_DONE;
1251
1252 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1253 if (!mlxsw_sp_port)
1254 return NOTIFY_DONE;
1255
1256 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1257 if (!neigh_work) {
1258 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1259 return NOTIFY_BAD;
1260 }
1261
1262 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1263 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1264 neigh_work->n = n;
1265
1266 /* Take a reference to ensure the neighbour won't be
1267 * destructed until we drop the reference in delayed
1268 * work.
1269 */
1270 neigh_clone(n);
1271 mlxsw_core_schedule_work(&neigh_work->work);
1272 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1273 break;
1274 }
1275
1276 return NOTIFY_DONE;
1277 }
1278
1279 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1280 {
1281 int err;
1282
1283 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1284 &mlxsw_sp_neigh_ht_params);
1285 if (err)
1286 return err;
1287
1288 /* Initialize the polling interval according to the default
1289 * table.
1290 */
1291 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1292
1293 /* Create the delayed works for the activity_update */
1294 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1295 mlxsw_sp_router_neighs_update_work);
1296 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1297 mlxsw_sp_router_probe_unresolved_nexthops);
1298 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1299 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1300 return 0;
1301 }
1302
1303 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1304 {
1305 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1306 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1307 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1308 }
1309
1310 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1311 const struct mlxsw_sp_rif *rif)
1312 {
1313 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1314
1315 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1316 rif->rif_index, rif->addr);
1317 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1318 }
1319
1320 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1321 struct mlxsw_sp_rif *rif)
1322 {
1323 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1324
1325 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1326 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1327 rif_list_node)
1328 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1329 }
1330
1331 struct mlxsw_sp_nexthop_key {
1332 struct fib_nh *fib_nh;
1333 };
1334
1335 struct mlxsw_sp_nexthop {
1336 struct list_head neigh_list_node; /* member of neigh entry list */
1337 struct list_head rif_list_node;
1338 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1339 * this belongs to
1340 */
1341 struct rhash_head ht_node;
1342 struct mlxsw_sp_nexthop_key key;
1343 struct mlxsw_sp_rif *rif;
1344 u8 should_offload:1, /* set indicates this neigh is connected and
1345 * should be put to KVD linear area of this group.
1346 */
1347 offloaded:1, /* set in case the neigh is actually put into
1348 * KVD linear area of this group.
1349 */
1350 update:1; /* set indicates that MAC of this neigh should be
1351 * updated in HW
1352 */
1353 struct mlxsw_sp_neigh_entry *neigh_entry;
1354 };
1355
1356 struct mlxsw_sp_nexthop_group_key {
1357 struct fib_info *fi;
1358 };
1359
1360 struct mlxsw_sp_nexthop_group {
1361 struct rhash_head ht_node;
1362 struct list_head fib_list; /* list of fib entries that use this group */
1363 struct mlxsw_sp_nexthop_group_key key;
1364 u8 adj_index_valid:1,
1365 gateway:1; /* routes using the group use a gateway */
1366 u32 adj_index;
1367 u16 ecmp_size;
1368 u16 count;
1369 struct mlxsw_sp_nexthop nexthops[0];
1370 #define nh_rif nexthops[0].rif
1371 };
1372
1373 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1374 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1375 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1376 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1377 };
1378
1379 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1380 struct mlxsw_sp_nexthop_group *nh_grp)
1381 {
1382 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1383 &nh_grp->ht_node,
1384 mlxsw_sp_nexthop_group_ht_params);
1385 }
1386
1387 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1388 struct mlxsw_sp_nexthop_group *nh_grp)
1389 {
1390 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1391 &nh_grp->ht_node,
1392 mlxsw_sp_nexthop_group_ht_params);
1393 }
1394
1395 static struct mlxsw_sp_nexthop_group *
1396 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1397 struct mlxsw_sp_nexthop_group_key key)
1398 {
1399 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
1400 mlxsw_sp_nexthop_group_ht_params);
1401 }
1402
1403 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1404 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1405 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1406 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1407 };
1408
1409 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1410 struct mlxsw_sp_nexthop *nh)
1411 {
1412 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1413 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1414 }
1415
1416 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1417 struct mlxsw_sp_nexthop *nh)
1418 {
1419 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1420 mlxsw_sp_nexthop_ht_params);
1421 }
1422
1423 static struct mlxsw_sp_nexthop *
1424 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1425 struct mlxsw_sp_nexthop_key key)
1426 {
1427 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1428 mlxsw_sp_nexthop_ht_params);
1429 }
1430
1431 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1432 const struct mlxsw_sp_fib *fib,
1433 u32 adj_index, u16 ecmp_size,
1434 u32 new_adj_index,
1435 u16 new_ecmp_size)
1436 {
1437 char raleu_pl[MLXSW_REG_RALEU_LEN];
1438
1439 mlxsw_reg_raleu_pack(raleu_pl,
1440 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1441 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1442 new_ecmp_size);
1443 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1444 }
1445
1446 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1447 struct mlxsw_sp_nexthop_group *nh_grp,
1448 u32 old_adj_index, u16 old_ecmp_size)
1449 {
1450 struct mlxsw_sp_fib_entry *fib_entry;
1451 struct mlxsw_sp_fib *fib = NULL;
1452 int err;
1453
1454 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1455 if (fib == fib_entry->fib_node->fib)
1456 continue;
1457 fib = fib_entry->fib_node->fib;
1458 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1459 old_adj_index,
1460 old_ecmp_size,
1461 nh_grp->adj_index,
1462 nh_grp->ecmp_size);
1463 if (err)
1464 return err;
1465 }
1466 return 0;
1467 }
1468
1469 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1470 struct mlxsw_sp_nexthop *nh)
1471 {
1472 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473 char ratr_pl[MLXSW_REG_RATR_LEN];
1474
1475 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1476 true, adj_index, neigh_entry->rif);
1477 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1478 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1479 }
1480
1481 static int
1482 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1483 struct mlxsw_sp_nexthop_group *nh_grp,
1484 bool reallocate)
1485 {
1486 u32 adj_index = nh_grp->adj_index; /* base */
1487 struct mlxsw_sp_nexthop *nh;
1488 int i;
1489 int err;
1490
1491 for (i = 0; i < nh_grp->count; i++) {
1492 nh = &nh_grp->nexthops[i];
1493
1494 if (!nh->should_offload) {
1495 nh->offloaded = 0;
1496 continue;
1497 }
1498
1499 if (nh->update || reallocate) {
1500 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1501 adj_index, nh);
1502 if (err)
1503 return err;
1504 nh->update = 0;
1505 nh->offloaded = 1;
1506 }
1507 adj_index++;
1508 }
1509 return 0;
1510 }
1511
1512 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1513 struct mlxsw_sp_fib_entry *fib_entry);
1514
1515 static bool
1516 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
1517 const struct mlxsw_sp_fib_entry *fib_entry);
1518
1519 static int
1520 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1521 struct mlxsw_sp_nexthop_group *nh_grp)
1522 {
1523 struct mlxsw_sp_fib_entry *fib_entry;
1524 int err;
1525
1526 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1527 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
1528 fib_entry))
1529 continue;
1530 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1531 if (err)
1532 return err;
1533 }
1534 return 0;
1535 }
1536
1537 static void
1538 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1539 struct mlxsw_sp_nexthop_group *nh_grp)
1540 {
1541 struct mlxsw_sp_nexthop *nh;
1542 bool offload_change = false;
1543 u32 adj_index;
1544 u16 ecmp_size = 0;
1545 bool old_adj_index_valid;
1546 u32 old_adj_index;
1547 u16 old_ecmp_size;
1548 int i;
1549 int err;
1550
1551 if (!nh_grp->gateway) {
1552 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1553 return;
1554 }
1555
1556 for (i = 0; i < nh_grp->count; i++) {
1557 nh = &nh_grp->nexthops[i];
1558
1559 if (nh->should_offload ^ nh->offloaded) {
1560 offload_change = true;
1561 if (nh->should_offload)
1562 nh->update = 1;
1563 }
1564 if (nh->should_offload)
1565 ecmp_size++;
1566 }
1567 if (!offload_change) {
1568 /* Nothing was added or removed, so no need to reallocate. Just
1569 * update MAC on existing adjacency indexes.
1570 */
1571 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1572 false);
1573 if (err) {
1574 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1575 goto set_trap;
1576 }
1577 return;
1578 }
1579 if (!ecmp_size)
1580 /* No neigh of this group is connected so we just set
1581 * the trap and let everthing flow through kernel.
1582 */
1583 goto set_trap;
1584
1585 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1586 if (err) {
1587 /* We ran out of KVD linear space, just set the
1588 * trap and let everything flow through kernel.
1589 */
1590 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1591 goto set_trap;
1592 }
1593 old_adj_index_valid = nh_grp->adj_index_valid;
1594 old_adj_index = nh_grp->adj_index;
1595 old_ecmp_size = nh_grp->ecmp_size;
1596 nh_grp->adj_index_valid = 1;
1597 nh_grp->adj_index = adj_index;
1598 nh_grp->ecmp_size = ecmp_size;
1599 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1600 if (err) {
1601 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1602 goto set_trap;
1603 }
1604
1605 if (!old_adj_index_valid) {
1606 /* The trap was set for fib entries, so we have to call
1607 * fib entry update to unset it and use adjacency index.
1608 */
1609 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1610 if (err) {
1611 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1612 goto set_trap;
1613 }
1614 return;
1615 }
1616
1617 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1618 old_adj_index, old_ecmp_size);
1619 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1620 if (err) {
1621 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1622 goto set_trap;
1623 }
1624 return;
1625
1626 set_trap:
1627 old_adj_index_valid = nh_grp->adj_index_valid;
1628 nh_grp->adj_index_valid = 0;
1629 for (i = 0; i < nh_grp->count; i++) {
1630 nh = &nh_grp->nexthops[i];
1631 nh->offloaded = 0;
1632 }
1633 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1634 if (err)
1635 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1636 if (old_adj_index_valid)
1637 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1638 }
1639
1640 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1641 bool removing)
1642 {
1643 if (!removing && !nh->should_offload)
1644 nh->should_offload = 1;
1645 else if (removing && nh->offloaded)
1646 nh->should_offload = 0;
1647 nh->update = 1;
1648 }
1649
1650 static void
1651 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1652 struct mlxsw_sp_neigh_entry *neigh_entry,
1653 bool removing)
1654 {
1655 struct mlxsw_sp_nexthop *nh;
1656
1657 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1658 neigh_list_node) {
1659 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1660 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1661 }
1662 }
1663
1664 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1665 struct mlxsw_sp_rif *rif)
1666 {
1667 if (nh->rif)
1668 return;
1669
1670 nh->rif = rif;
1671 list_add(&nh->rif_list_node, &rif->nexthop_list);
1672 }
1673
1674 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1675 {
1676 if (!nh->rif)
1677 return;
1678
1679 list_del(&nh->rif_list_node);
1680 nh->rif = NULL;
1681 }
1682
1683 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1684 struct mlxsw_sp_nexthop *nh)
1685 {
1686 struct mlxsw_sp_neigh_entry *neigh_entry;
1687 struct fib_nh *fib_nh = nh->key.fib_nh;
1688 struct neighbour *n;
1689 u8 nud_state, dead;
1690 int err;
1691
1692 if (!nh->nh_grp->gateway || nh->neigh_entry)
1693 return 0;
1694
1695 /* Take a reference of neigh here ensuring that neigh would
1696 * not be detructed before the nexthop entry is finished.
1697 * The reference is taken either in neigh_lookup() or
1698 * in neigh_create() in case n is not found.
1699 */
1700 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1701 if (!n) {
1702 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1703 if (IS_ERR(n))
1704 return PTR_ERR(n);
1705 neigh_event_send(n, NULL);
1706 }
1707 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1708 if (!neigh_entry) {
1709 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1710 if (IS_ERR(neigh_entry)) {
1711 err = -EINVAL;
1712 goto err_neigh_entry_create;
1713 }
1714 }
1715
1716 /* If that is the first nexthop connected to that neigh, add to
1717 * nexthop_neighs_list
1718 */
1719 if (list_empty(&neigh_entry->nexthop_list))
1720 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1721 &mlxsw_sp->router->nexthop_neighs_list);
1722
1723 nh->neigh_entry = neigh_entry;
1724 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1725 read_lock_bh(&n->lock);
1726 nud_state = n->nud_state;
1727 dead = n->dead;
1728 read_unlock_bh(&n->lock);
1729 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1730
1731 return 0;
1732
1733 err_neigh_entry_create:
1734 neigh_release(n);
1735 return err;
1736 }
1737
1738 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1739 struct mlxsw_sp_nexthop *nh)
1740 {
1741 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1742 struct neighbour *n;
1743
1744 if (!neigh_entry)
1745 return;
1746 n = neigh_entry->key.n;
1747
1748 __mlxsw_sp_nexthop_neigh_update(nh, true);
1749 list_del(&nh->neigh_list_node);
1750 nh->neigh_entry = NULL;
1751
1752 /* If that is the last nexthop connected to that neigh, remove from
1753 * nexthop_neighs_list
1754 */
1755 if (list_empty(&neigh_entry->nexthop_list))
1756 list_del(&neigh_entry->nexthop_neighs_list_node);
1757
1758 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1759 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1760
1761 neigh_release(n);
1762 }
1763
1764 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1765 struct mlxsw_sp_nexthop_group *nh_grp,
1766 struct mlxsw_sp_nexthop *nh,
1767 struct fib_nh *fib_nh)
1768 {
1769 struct net_device *dev = fib_nh->nh_dev;
1770 struct in_device *in_dev;
1771 struct mlxsw_sp_rif *rif;
1772 int err;
1773
1774 nh->nh_grp = nh_grp;
1775 nh->key.fib_nh = fib_nh;
1776 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1777 if (err)
1778 return err;
1779
1780 if (!dev)
1781 return 0;
1782
1783 in_dev = __in_dev_get_rtnl(dev);
1784 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1785 fib_nh->nh_flags & RTNH_F_LINKDOWN)
1786 return 0;
1787
1788 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1789 if (!rif)
1790 return 0;
1791 mlxsw_sp_nexthop_rif_init(nh, rif);
1792
1793 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1794 if (err)
1795 goto err_nexthop_neigh_init;
1796
1797 return 0;
1798
1799 err_nexthop_neigh_init:
1800 mlxsw_sp_nexthop_rif_fini(nh);
1801 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1802 return err;
1803 }
1804
1805 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1806 struct mlxsw_sp_nexthop *nh)
1807 {
1808 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1809 mlxsw_sp_nexthop_rif_fini(nh);
1810 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1811 }
1812
1813 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1814 unsigned long event, struct fib_nh *fib_nh)
1815 {
1816 struct mlxsw_sp_nexthop_key key;
1817 struct mlxsw_sp_nexthop *nh;
1818 struct mlxsw_sp_rif *rif;
1819
1820 if (mlxsw_sp->router->aborted)
1821 return;
1822
1823 key.fib_nh = fib_nh;
1824 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1825 if (WARN_ON_ONCE(!nh))
1826 return;
1827
1828 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1829 if (!rif)
1830 return;
1831
1832 switch (event) {
1833 case FIB_EVENT_NH_ADD:
1834 mlxsw_sp_nexthop_rif_init(nh, rif);
1835 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1836 break;
1837 case FIB_EVENT_NH_DEL:
1838 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1839 mlxsw_sp_nexthop_rif_fini(nh);
1840 break;
1841 }
1842
1843 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1844 }
1845
1846 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1847 struct mlxsw_sp_rif *rif)
1848 {
1849 struct mlxsw_sp_nexthop *nh, *tmp;
1850
1851 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1852 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1853 mlxsw_sp_nexthop_rif_fini(nh);
1854 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1855 }
1856 }
1857
1858 static struct mlxsw_sp_nexthop_group *
1859 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1860 {
1861 struct mlxsw_sp_nexthop_group *nh_grp;
1862 struct mlxsw_sp_nexthop *nh;
1863 struct fib_nh *fib_nh;
1864 size_t alloc_size;
1865 int i;
1866 int err;
1867
1868 alloc_size = sizeof(*nh_grp) +
1869 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1870 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1871 if (!nh_grp)
1872 return ERR_PTR(-ENOMEM);
1873 INIT_LIST_HEAD(&nh_grp->fib_list);
1874 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1875 nh_grp->count = fi->fib_nhs;
1876 nh_grp->key.fi = fi;
1877 fib_info_hold(fi);
1878 for (i = 0; i < nh_grp->count; i++) {
1879 nh = &nh_grp->nexthops[i];
1880 fib_nh = &fi->fib_nh[i];
1881 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1882 if (err)
1883 goto err_nexthop_init;
1884 }
1885 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1886 if (err)
1887 goto err_nexthop_group_insert;
1888 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1889 return nh_grp;
1890
1891 err_nexthop_group_insert:
1892 err_nexthop_init:
1893 for (i--; i >= 0; i--) {
1894 nh = &nh_grp->nexthops[i];
1895 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1896 }
1897 fib_info_put(nh_grp->key.fi);
1898 kfree(nh_grp);
1899 return ERR_PTR(err);
1900 }
1901
1902 static void
1903 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1904 struct mlxsw_sp_nexthop_group *nh_grp)
1905 {
1906 struct mlxsw_sp_nexthop *nh;
1907 int i;
1908
1909 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1910 for (i = 0; i < nh_grp->count; i++) {
1911 nh = &nh_grp->nexthops[i];
1912 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1913 }
1914 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1915 WARN_ON_ONCE(nh_grp->adj_index_valid);
1916 fib_info_put(nh_grp->key.fi);
1917 kfree(nh_grp);
1918 }
1919
1920 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1921 struct mlxsw_sp_fib_entry *fib_entry,
1922 struct fib_info *fi)
1923 {
1924 struct mlxsw_sp_nexthop_group_key key;
1925 struct mlxsw_sp_nexthop_group *nh_grp;
1926
1927 key.fi = fi;
1928 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1929 if (!nh_grp) {
1930 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1931 if (IS_ERR(nh_grp))
1932 return PTR_ERR(nh_grp);
1933 }
1934 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1935 fib_entry->nh_group = nh_grp;
1936 return 0;
1937 }
1938
1939 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1940 struct mlxsw_sp_fib_entry *fib_entry)
1941 {
1942 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1943
1944 list_del(&fib_entry->nexthop_group_node);
1945 if (!list_empty(&nh_grp->fib_list))
1946 return;
1947 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1948 }
1949
1950 static bool
1951 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1952 {
1953 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1954
1955 if (fib_entry->params.tos)
1956 return false;
1957
1958 switch (fib_entry->type) {
1959 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1960 return !!nh_group->adj_index_valid;
1961 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1962 return !!nh_group->nh_rif;
1963 default:
1964 return false;
1965 }
1966 }
1967
1968 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1969 {
1970 fib_entry->offloaded = true;
1971
1972 switch (fib_entry->fib_node->fib->proto) {
1973 case MLXSW_SP_L3_PROTO_IPV4:
1974 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1975 break;
1976 case MLXSW_SP_L3_PROTO_IPV6:
1977 WARN_ON_ONCE(1);
1978 }
1979 }
1980
1981 static void
1982 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1983 {
1984 switch (fib_entry->fib_node->fib->proto) {
1985 case MLXSW_SP_L3_PROTO_IPV4:
1986 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1987 break;
1988 case MLXSW_SP_L3_PROTO_IPV6:
1989 WARN_ON_ONCE(1);
1990 }
1991
1992 fib_entry->offloaded = false;
1993 }
1994
1995 static void
1996 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1997 enum mlxsw_reg_ralue_op op, int err)
1998 {
1999 switch (op) {
2000 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2001 if (!fib_entry->offloaded)
2002 return;
2003 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2004 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2005 if (err)
2006 return;
2007 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2008 !fib_entry->offloaded)
2009 mlxsw_sp_fib_entry_offload_set(fib_entry);
2010 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2011 fib_entry->offloaded)
2012 mlxsw_sp_fib_entry_offload_unset(fib_entry);
2013 return;
2014 default:
2015 return;
2016 }
2017 }
2018
2019 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
2020 struct mlxsw_sp_fib_entry *fib_entry,
2021 enum mlxsw_reg_ralue_op op)
2022 {
2023 char ralue_pl[MLXSW_REG_RALUE_LEN];
2024 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2025 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2026 enum mlxsw_reg_ralue_trap_action trap_action;
2027 u16 trap_id = 0;
2028 u32 adjacency_index = 0;
2029 u16 ecmp_size = 0;
2030
2031 /* In case the nexthop group adjacency index is valid, use it
2032 * with provided ECMP size. Otherwise, setup trap and pass
2033 * traffic to kernel.
2034 */
2035 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2036 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2037 adjacency_index = fib_entry->nh_group->adj_index;
2038 ecmp_size = fib_entry->nh_group->ecmp_size;
2039 } else {
2040 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2041 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2042 }
2043
2044 mlxsw_reg_ralue_pack4(ralue_pl,
2045 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2046 fib->vr->id, fib_entry->fib_node->key.prefix_len,
2047 *p_dip);
2048 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2049 adjacency_index, ecmp_size);
2050 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2051 }
2052
2053 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
2054 struct mlxsw_sp_fib_entry *fib_entry,
2055 enum mlxsw_reg_ralue_op op)
2056 {
2057 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2058 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2059 enum mlxsw_reg_ralue_trap_action trap_action;
2060 char ralue_pl[MLXSW_REG_RALUE_LEN];
2061 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2062 u16 trap_id = 0;
2063 u16 rif_index = 0;
2064
2065 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2066 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2067 rif_index = rif->rif_index;
2068 } else {
2069 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2070 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2071 }
2072
2073 mlxsw_reg_ralue_pack4(ralue_pl,
2074 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2075 fib->vr->id, fib_entry->fib_node->key.prefix_len,
2076 *p_dip);
2077 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2078 rif_index);
2079 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2080 }
2081
2082 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
2083 struct mlxsw_sp_fib_entry *fib_entry,
2084 enum mlxsw_reg_ralue_op op)
2085 {
2086 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2087 char ralue_pl[MLXSW_REG_RALUE_LEN];
2088 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2089
2090 mlxsw_reg_ralue_pack4(ralue_pl,
2091 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2092 fib->vr->id, fib_entry->fib_node->key.prefix_len,
2093 *p_dip);
2094 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2095 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2096 }
2097
2098 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
2099 struct mlxsw_sp_fib_entry *fib_entry,
2100 enum mlxsw_reg_ralue_op op)
2101 {
2102 switch (fib_entry->type) {
2103 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2104 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
2105 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2106 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
2107 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2108 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
2109 }
2110 return -EINVAL;
2111 }
2112
2113 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2114 struct mlxsw_sp_fib_entry *fib_entry,
2115 enum mlxsw_reg_ralue_op op)
2116 {
2117 int err = -EINVAL;
2118
2119 switch (fib_entry->fib_node->fib->proto) {
2120 case MLXSW_SP_L3_PROTO_IPV4:
2121 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
2122 break;
2123 case MLXSW_SP_L3_PROTO_IPV6:
2124 return err;
2125 }
2126 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2127 return err;
2128 }
2129
2130 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2131 struct mlxsw_sp_fib_entry *fib_entry)
2132 {
2133 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2134 MLXSW_REG_RALUE_OP_WRITE_WRITE);
2135 }
2136
2137 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2138 struct mlxsw_sp_fib_entry *fib_entry)
2139 {
2140 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2141 MLXSW_REG_RALUE_OP_WRITE_DELETE);
2142 }
2143
2144 static int
2145 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2146 const struct fib_entry_notifier_info *fen_info,
2147 struct mlxsw_sp_fib_entry *fib_entry)
2148 {
2149 struct fib_info *fi = fen_info->fi;
2150
2151 switch (fen_info->type) {
2152 case RTN_BROADCAST: /* fall through */
2153 case RTN_LOCAL:
2154 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2155 return 0;
2156 case RTN_UNREACHABLE: /* fall through */
2157 case RTN_BLACKHOLE: /* fall through */
2158 case RTN_PROHIBIT:
2159 /* Packets hitting these routes need to be trapped, but
2160 * can do so with a lower priority than packets directed
2161 * at the host, so use action type local instead of trap.
2162 */
2163 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2164 return 0;
2165 case RTN_UNICAST:
2166 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2167 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2168 else
2169 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2170 return 0;
2171 default:
2172 return -EINVAL;
2173 }
2174 }
2175
2176 static struct mlxsw_sp_fib_entry *
2177 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2178 struct mlxsw_sp_fib_node *fib_node,
2179 const struct fib_entry_notifier_info *fen_info)
2180 {
2181 struct mlxsw_sp_fib_entry *fib_entry;
2182 int err;
2183
2184 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
2185 if (!fib_entry) {
2186 err = -ENOMEM;
2187 goto err_fib_entry_alloc;
2188 }
2189
2190 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2191 if (err)
2192 goto err_fib4_entry_type_set;
2193
2194 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2195 if (err)
2196 goto err_nexthop_group_get;
2197
2198 fib_entry->params.prio = fen_info->fi->fib_priority;
2199 fib_entry->params.tb_id = fen_info->tb_id;
2200 fib_entry->params.type = fen_info->type;
2201 fib_entry->params.tos = fen_info->tos;
2202
2203 fib_entry->fib_node = fib_node;
2204
2205 return fib_entry;
2206
2207 err_nexthop_group_get:
2208 err_fib4_entry_type_set:
2209 kfree(fib_entry);
2210 err_fib_entry_alloc:
2211 return ERR_PTR(err);
2212 }
2213
2214 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2215 struct mlxsw_sp_fib_entry *fib_entry)
2216 {
2217 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
2218 kfree(fib_entry);
2219 }
2220
2221 static struct mlxsw_sp_fib_node *
2222 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2223 const struct fib_entry_notifier_info *fen_info);
2224
2225 static struct mlxsw_sp_fib_entry *
2226 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2227 const struct fib_entry_notifier_info *fen_info)
2228 {
2229 struct mlxsw_sp_fib_entry *fib_entry;
2230 struct mlxsw_sp_fib_node *fib_node;
2231
2232 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2233 if (IS_ERR(fib_node))
2234 return NULL;
2235
2236 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2237 if (fib_entry->params.tb_id == fen_info->tb_id &&
2238 fib_entry->params.tos == fen_info->tos &&
2239 fib_entry->params.type == fen_info->type &&
2240 fib_entry->nh_group->key.fi == fen_info->fi) {
2241 return fib_entry;
2242 }
2243 }
2244
2245 return NULL;
2246 }
2247
2248 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2249 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2250 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2251 .key_len = sizeof(struct mlxsw_sp_fib_key),
2252 .automatic_shrinking = true,
2253 };
2254
2255 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2256 struct mlxsw_sp_fib_node *fib_node)
2257 {
2258 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2259 mlxsw_sp_fib_ht_params);
2260 }
2261
2262 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2263 struct mlxsw_sp_fib_node *fib_node)
2264 {
2265 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2266 mlxsw_sp_fib_ht_params);
2267 }
2268
2269 static struct mlxsw_sp_fib_node *
2270 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2271 size_t addr_len, unsigned char prefix_len)
2272 {
2273 struct mlxsw_sp_fib_key key;
2274
2275 memset(&key, 0, sizeof(key));
2276 memcpy(key.addr, addr, addr_len);
2277 key.prefix_len = prefix_len;
2278 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2279 }
2280
2281 static struct mlxsw_sp_fib_node *
2282 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2283 size_t addr_len, unsigned char prefix_len)
2284 {
2285 struct mlxsw_sp_fib_node *fib_node;
2286
2287 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2288 if (!fib_node)
2289 return NULL;
2290
2291 INIT_LIST_HEAD(&fib_node->entry_list);
2292 list_add(&fib_node->list, &fib->node_list);
2293 memcpy(fib_node->key.addr, addr, addr_len);
2294 fib_node->key.prefix_len = prefix_len;
2295
2296 return fib_node;
2297 }
2298
2299 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2300 {
2301 list_del(&fib_node->list);
2302 WARN_ON(!list_empty(&fib_node->entry_list));
2303 kfree(fib_node);
2304 }
2305
2306 static bool
2307 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2308 const struct mlxsw_sp_fib_entry *fib_entry)
2309 {
2310 return list_first_entry(&fib_node->entry_list,
2311 struct mlxsw_sp_fib_entry, list) == fib_entry;
2312 }
2313
2314 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2315 {
2316 unsigned char prefix_len = fib_node->key.prefix_len;
2317 struct mlxsw_sp_fib *fib = fib_node->fib;
2318
2319 if (fib->prefix_ref_count[prefix_len]++ == 0)
2320 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2321 }
2322
2323 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2324 {
2325 unsigned char prefix_len = fib_node->key.prefix_len;
2326 struct mlxsw_sp_fib *fib = fib_node->fib;
2327
2328 if (--fib->prefix_ref_count[prefix_len] == 0)
2329 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2330 }
2331
2332 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2333 struct mlxsw_sp_fib_node *fib_node,
2334 struct mlxsw_sp_fib *fib)
2335 {
2336 struct mlxsw_sp_prefix_usage req_prefix_usage;
2337 struct mlxsw_sp_lpm_tree *lpm_tree;
2338 int err;
2339
2340 err = mlxsw_sp_fib_node_insert(fib, fib_node);
2341 if (err)
2342 return err;
2343 fib_node->fib = fib;
2344
2345 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2346 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2347
2348 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2349 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2350 &req_prefix_usage);
2351 if (err)
2352 goto err_tree_check;
2353 } else {
2354 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2355 fib->proto);
2356 if (IS_ERR(lpm_tree))
2357 return PTR_ERR(lpm_tree);
2358 fib->lpm_tree = lpm_tree;
2359 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2360 if (err)
2361 goto err_tree_bind;
2362 }
2363
2364 mlxsw_sp_fib_node_prefix_inc(fib_node);
2365
2366 return 0;
2367
2368 err_tree_bind:
2369 fib->lpm_tree = NULL;
2370 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2371 err_tree_check:
2372 fib_node->fib = NULL;
2373 mlxsw_sp_fib_node_remove(fib, fib_node);
2374 return err;
2375 }
2376
2377 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2378 struct mlxsw_sp_fib_node *fib_node)
2379 {
2380 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2381 struct mlxsw_sp_fib *fib = fib_node->fib;
2382
2383 mlxsw_sp_fib_node_prefix_dec(fib_node);
2384
2385 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2386 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2387 fib->lpm_tree = NULL;
2388 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2389 } else {
2390 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2391 }
2392
2393 fib_node->fib = NULL;
2394 mlxsw_sp_fib_node_remove(fib, fib_node);
2395 }
2396
2397 static struct mlxsw_sp_fib_node *
2398 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2399 const struct fib_entry_notifier_info *fen_info)
2400 {
2401 struct mlxsw_sp_fib_node *fib_node;
2402 struct mlxsw_sp_fib *fib;
2403 struct mlxsw_sp_vr *vr;
2404 int err;
2405
2406 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2407 if (IS_ERR(vr))
2408 return ERR_CAST(vr);
2409 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2410
2411 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2412 sizeof(fen_info->dst),
2413 fen_info->dst_len);
2414 if (fib_node)
2415 return fib_node;
2416
2417 fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2418 sizeof(fen_info->dst),
2419 fen_info->dst_len);
2420 if (!fib_node) {
2421 err = -ENOMEM;
2422 goto err_fib_node_create;
2423 }
2424
2425 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2426 if (err)
2427 goto err_fib_node_init;
2428
2429 return fib_node;
2430
2431 err_fib_node_init:
2432 mlxsw_sp_fib_node_destroy(fib_node);
2433 err_fib_node_create:
2434 mlxsw_sp_vr_put(vr);
2435 return ERR_PTR(err);
2436 }
2437
2438 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2439 struct mlxsw_sp_fib_node *fib_node)
2440 {
2441 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2442
2443 if (!list_empty(&fib_node->entry_list))
2444 return;
2445 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2446 mlxsw_sp_fib_node_destroy(fib_node);
2447 mlxsw_sp_vr_put(vr);
2448 }
2449
2450 static struct mlxsw_sp_fib_entry *
2451 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2452 const struct mlxsw_sp_fib_entry_params *params)
2453 {
2454 struct mlxsw_sp_fib_entry *fib_entry;
2455
2456 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2457 if (fib_entry->params.tb_id > params->tb_id)
2458 continue;
2459 if (fib_entry->params.tb_id != params->tb_id)
2460 break;
2461 if (fib_entry->params.tos > params->tos)
2462 continue;
2463 if (fib_entry->params.prio >= params->prio ||
2464 fib_entry->params.tos < params->tos)
2465 return fib_entry;
2466 }
2467
2468 return NULL;
2469 }
2470
2471 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2472 struct mlxsw_sp_fib_entry *new_entry)
2473 {
2474 struct mlxsw_sp_fib_node *fib_node;
2475
2476 if (WARN_ON(!fib_entry))
2477 return -EINVAL;
2478
2479 fib_node = fib_entry->fib_node;
2480 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2481 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2482 fib_entry->params.tos != new_entry->params.tos ||
2483 fib_entry->params.prio != new_entry->params.prio)
2484 break;
2485 }
2486
2487 list_add_tail(&new_entry->list, &fib_entry->list);
2488 return 0;
2489 }
2490
2491 static int
2492 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2493 struct mlxsw_sp_fib_entry *new_entry,
2494 bool replace, bool append)
2495 {
2496 struct mlxsw_sp_fib_entry *fib_entry;
2497
2498 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2499
2500 if (append)
2501 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2502 if (replace && WARN_ON(!fib_entry))
2503 return -EINVAL;
2504
2505 /* Insert new entry before replaced one, so that we can later
2506 * remove the second.
2507 */
2508 if (fib_entry) {
2509 list_add_tail(&new_entry->list, &fib_entry->list);
2510 } else {
2511 struct mlxsw_sp_fib_entry *last;
2512
2513 list_for_each_entry(last, &fib_node->entry_list, list) {
2514 if (new_entry->params.tb_id > last->params.tb_id)
2515 break;
2516 fib_entry = last;
2517 }
2518
2519 if (fib_entry)
2520 list_add(&new_entry->list, &fib_entry->list);
2521 else
2522 list_add(&new_entry->list, &fib_node->entry_list);
2523 }
2524
2525 return 0;
2526 }
2527
2528 static void
2529 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2530 {
2531 list_del(&fib_entry->list);
2532 }
2533
2534 static int
2535 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2536 const struct mlxsw_sp_fib_node *fib_node,
2537 struct mlxsw_sp_fib_entry *fib_entry)
2538 {
2539 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2540 return 0;
2541
2542 /* To prevent packet loss, overwrite the previously offloaded
2543 * entry.
2544 */
2545 if (!list_is_singular(&fib_node->entry_list)) {
2546 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2547 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2548
2549 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2550 }
2551
2552 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2553 }
2554
2555 static void
2556 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2557 const struct mlxsw_sp_fib_node *fib_node,
2558 struct mlxsw_sp_fib_entry *fib_entry)
2559 {
2560 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2561 return;
2562
2563 /* Promote the next entry by overwriting the deleted entry */
2564 if (!list_is_singular(&fib_node->entry_list)) {
2565 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2566 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2567
2568 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2569 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2570 return;
2571 }
2572
2573 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2574 }
2575
2576 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2577 struct mlxsw_sp_fib_entry *fib_entry,
2578 bool replace, bool append)
2579 {
2580 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2581 int err;
2582
2583 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2584 append);
2585 if (err)
2586 return err;
2587
2588 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2589 if (err)
2590 goto err_fib4_node_entry_add;
2591
2592 return 0;
2593
2594 err_fib4_node_entry_add:
2595 mlxsw_sp_fib4_node_list_remove(fib_entry);
2596 return err;
2597 }
2598
2599 static void
2600 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2601 struct mlxsw_sp_fib_entry *fib_entry)
2602 {
2603 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2604
2605 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2606 mlxsw_sp_fib4_node_list_remove(fib_entry);
2607 }
2608
2609 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2610 struct mlxsw_sp_fib_entry *fib_entry,
2611 bool replace)
2612 {
2613 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2614 struct mlxsw_sp_fib_entry *replaced;
2615
2616 if (!replace)
2617 return;
2618
2619 /* We inserted the new entry before replaced one */
2620 replaced = list_next_entry(fib_entry, list);
2621
2622 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2623 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2624 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2625 }
2626
2627 static int
2628 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2629 const struct fib_entry_notifier_info *fen_info,
2630 bool replace, bool append)
2631 {
2632 struct mlxsw_sp_fib_entry *fib_entry;
2633 struct mlxsw_sp_fib_node *fib_node;
2634 int err;
2635
2636 if (mlxsw_sp->router->aborted)
2637 return 0;
2638
2639 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2640 if (IS_ERR(fib_node)) {
2641 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2642 return PTR_ERR(fib_node);
2643 }
2644
2645 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2646 if (IS_ERR(fib_entry)) {
2647 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2648 err = PTR_ERR(fib_entry);
2649 goto err_fib4_entry_create;
2650 }
2651
2652 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2653 append);
2654 if (err) {
2655 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2656 goto err_fib4_node_entry_link;
2657 }
2658
2659 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2660
2661 return 0;
2662
2663 err_fib4_node_entry_link:
2664 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2665 err_fib4_entry_create:
2666 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2667 return err;
2668 }
2669
2670 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2671 struct fib_entry_notifier_info *fen_info)
2672 {
2673 struct mlxsw_sp_fib_entry *fib_entry;
2674 struct mlxsw_sp_fib_node *fib_node;
2675
2676 if (mlxsw_sp->router->aborted)
2677 return;
2678
2679 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2680 if (WARN_ON(!fib_entry))
2681 return;
2682 fib_node = fib_entry->fib_node;
2683
2684 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2685 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2686 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2687 }
2688
2689 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2690 {
2691 char ralta_pl[MLXSW_REG_RALTA_LEN];
2692 char ralst_pl[MLXSW_REG_RALST_LEN];
2693 int i, err;
2694
2695 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2696 MLXSW_SP_LPM_TREE_MIN);
2697 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2698 if (err)
2699 return err;
2700
2701 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2702 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2703 if (err)
2704 return err;
2705
2706 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2707 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2708 char raltb_pl[MLXSW_REG_RALTB_LEN];
2709 char ralue_pl[MLXSW_REG_RALUE_LEN];
2710
2711 if (!mlxsw_sp_vr_is_used(vr))
2712 continue;
2713
2714 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2715 MLXSW_REG_RALXX_PROTOCOL_IPV4,
2716 MLXSW_SP_LPM_TREE_MIN);
2717 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2718 raltb_pl);
2719 if (err)
2720 return err;
2721
2722 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2723 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2724 0);
2725 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2726 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2727 ralue_pl);
2728 if (err)
2729 return err;
2730 }
2731
2732 return 0;
2733 }
2734
2735 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2736 struct mlxsw_sp_fib_node *fib_node)
2737 {
2738 struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2739
2740 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2741 bool do_break = &tmp->list == &fib_node->entry_list;
2742
2743 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2744 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2745 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2746 /* Break when entry list is empty and node was freed.
2747 * Otherwise, we'll access freed memory in the next
2748 * iteration.
2749 */
2750 if (do_break)
2751 break;
2752 }
2753 }
2754
2755 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2756 struct mlxsw_sp_fib_node *fib_node)
2757 {
2758 switch (fib_node->fib->proto) {
2759 case MLXSW_SP_L3_PROTO_IPV4:
2760 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2761 break;
2762 case MLXSW_SP_L3_PROTO_IPV6:
2763 WARN_ON_ONCE(1);
2764 break;
2765 }
2766 }
2767
2768 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2769 struct mlxsw_sp_vr *vr,
2770 enum mlxsw_sp_l3proto proto)
2771 {
2772 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2773 struct mlxsw_sp_fib_node *fib_node, *tmp;
2774
2775 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2776 bool do_break = &tmp->list == &fib->node_list;
2777
2778 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2779 if (do_break)
2780 break;
2781 }
2782 }
2783
2784 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2785 {
2786 int i;
2787
2788 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2789 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2790
2791 if (!mlxsw_sp_vr_is_used(vr))
2792 continue;
2793 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2794 }
2795 }
2796
2797 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2798 {
2799 int err;
2800
2801 if (mlxsw_sp->router->aborted)
2802 return;
2803 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2804 mlxsw_sp_router_fib_flush(mlxsw_sp);
2805 mlxsw_sp->router->aborted = true;
2806 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2807 if (err)
2808 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2809 }
2810
2811 struct mlxsw_sp_fib_event_work {
2812 struct work_struct work;
2813 union {
2814 struct fib_entry_notifier_info fen_info;
2815 struct fib_rule_notifier_info fr_info;
2816 struct fib_nh_notifier_info fnh_info;
2817 };
2818 struct mlxsw_sp *mlxsw_sp;
2819 unsigned long event;
2820 };
2821
2822 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2823 {
2824 struct mlxsw_sp_fib_event_work *fib_work =
2825 container_of(work, struct mlxsw_sp_fib_event_work, work);
2826 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2827 struct fib_rule *rule;
2828 bool replace, append;
2829 int err;
2830
2831 /* Protect internal structures from changes */
2832 rtnl_lock();
2833 switch (fib_work->event) {
2834 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2835 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2836 case FIB_EVENT_ENTRY_ADD:
2837 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2838 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2839 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2840 replace, append);
2841 if (err)
2842 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2843 fib_info_put(fib_work->fen_info.fi);
2844 break;
2845 case FIB_EVENT_ENTRY_DEL:
2846 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2847 fib_info_put(fib_work->fen_info.fi);
2848 break;
2849 case FIB_EVENT_RULE_ADD: /* fall through */
2850 case FIB_EVENT_RULE_DEL:
2851 rule = fib_work->fr_info.rule;
2852 if (!fib4_rule_default(rule) && !rule->l3mdev)
2853 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2854 fib_rule_put(rule);
2855 break;
2856 case FIB_EVENT_NH_ADD: /* fall through */
2857 case FIB_EVENT_NH_DEL:
2858 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2859 fib_work->fnh_info.fib_nh);
2860 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2861 break;
2862 }
2863 rtnl_unlock();
2864 kfree(fib_work);
2865 }
2866
2867 /* Called with rcu_read_lock() */
2868 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2869 unsigned long event, void *ptr)
2870 {
2871 struct mlxsw_sp_fib_event_work *fib_work;
2872 struct fib_notifier_info *info = ptr;
2873 struct mlxsw_sp_router *router;
2874
2875 if (!net_eq(info->net, &init_net))
2876 return NOTIFY_DONE;
2877
2878 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2879 if (WARN_ON(!fib_work))
2880 return NOTIFY_BAD;
2881
2882 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2883 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
2884 fib_work->mlxsw_sp = router->mlxsw_sp;
2885 fib_work->event = event;
2886
2887 switch (event) {
2888 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2889 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2890 case FIB_EVENT_ENTRY_ADD: /* fall through */
2891 case FIB_EVENT_ENTRY_DEL:
2892 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2893 /* Take referece on fib_info to prevent it from being
2894 * freed while work is queued. Release it afterwards.
2895 */
2896 fib_info_hold(fib_work->fen_info.fi);
2897 break;
2898 case FIB_EVENT_RULE_ADD: /* fall through */
2899 case FIB_EVENT_RULE_DEL:
2900 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2901 fib_rule_get(fib_work->fr_info.rule);
2902 break;
2903 case FIB_EVENT_NH_ADD: /* fall through */
2904 case FIB_EVENT_NH_DEL:
2905 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2906 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2907 break;
2908 }
2909
2910 mlxsw_core_schedule_work(&fib_work->work);
2911
2912 return NOTIFY_DONE;
2913 }
2914
2915 static struct mlxsw_sp_rif *
2916 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2917 const struct net_device *dev)
2918 {
2919 int i;
2920
2921 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2922 if (mlxsw_sp->router->rifs[i] &&
2923 mlxsw_sp->router->rifs[i]->dev == dev)
2924 return mlxsw_sp->router->rifs[i];
2925
2926 return NULL;
2927 }
2928
2929 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2930 {
2931 char ritr_pl[MLXSW_REG_RITR_LEN];
2932 int err;
2933
2934 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2935 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2936 if (WARN_ON_ONCE(err))
2937 return err;
2938
2939 mlxsw_reg_ritr_enable_set(ritr_pl, false);
2940 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2941 }
2942
2943 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2944 struct mlxsw_sp_rif *rif)
2945 {
2946 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2947 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2948 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2949 }
2950
2951 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2952 const struct in_device *in_dev,
2953 unsigned long event)
2954 {
2955 switch (event) {
2956 case NETDEV_UP:
2957 if (!rif)
2958 return true;
2959 return false;
2960 case NETDEV_DOWN:
2961 if (rif && !in_dev->ifa_list &&
2962 !netif_is_l3_slave(rif->dev))
2963 return true;
2964 /* It is possible we already removed the RIF ourselves
2965 * if it was assigned to a netdev that is now a bridge
2966 * or LAG slave.
2967 */
2968 return false;
2969 }
2970
2971 return false;
2972 }
2973
2974 static enum mlxsw_sp_rif_type
2975 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
2976 const struct net_device *dev)
2977 {
2978 enum mlxsw_sp_fid_type type;
2979
2980 /* RIF type is derived from the type of the underlying FID */
2981 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
2982 type = MLXSW_SP_FID_TYPE_8021Q;
2983 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
2984 type = MLXSW_SP_FID_TYPE_8021Q;
2985 else if (netif_is_bridge_master(dev))
2986 type = MLXSW_SP_FID_TYPE_8021D;
2987 else
2988 type = MLXSW_SP_FID_TYPE_RFID;
2989
2990 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
2991 }
2992
2993 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
2994 {
2995 int i;
2996
2997 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
2998 if (!mlxsw_sp->router->rifs[i]) {
2999 *p_rif_index = i;
3000 return 0;
3001 }
3002 }
3003
3004 return -ENOBUFS;
3005 }
3006
3007 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
3008 u16 vr_id,
3009 struct net_device *l3_dev)
3010 {
3011 struct mlxsw_sp_rif *rif;
3012
3013 rif = kzalloc(rif_size, GFP_KERNEL);
3014 if (!rif)
3015 return NULL;
3016
3017 INIT_LIST_HEAD(&rif->nexthop_list);
3018 INIT_LIST_HEAD(&rif->neigh_list);
3019 ether_addr_copy(rif->addr, l3_dev->dev_addr);
3020 rif->mtu = l3_dev->mtu;
3021 rif->vr_id = vr_id;
3022 rif->dev = l3_dev;
3023 rif->rif_index = rif_index;
3024
3025 return rif;
3026 }
3027
3028 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
3029 u16 rif_index)
3030 {
3031 return mlxsw_sp->router->rifs[rif_index];
3032 }
3033
3034 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
3035 {
3036 return rif->rif_index;
3037 }
3038
3039 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
3040 {
3041 return rif->dev->ifindex;
3042 }
3043
3044 static struct mlxsw_sp_rif *
3045 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
3046 const struct mlxsw_sp_rif_params *params)
3047 {
3048 u32 tb_id = l3mdev_fib_table(params->dev);
3049 const struct mlxsw_sp_rif_ops *ops;
3050 enum mlxsw_sp_rif_type type;
3051 struct mlxsw_sp_rif *rif;
3052 struct mlxsw_sp_fid *fid;
3053 struct mlxsw_sp_vr *vr;
3054 u16 rif_index;
3055 int err;
3056
3057 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
3058 ops = mlxsw_sp->router->rif_ops_arr[type];
3059
3060 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3061 if (IS_ERR(vr))
3062 return ERR_CAST(vr);
3063
3064 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
3065 if (err)
3066 goto err_rif_index_alloc;
3067
3068 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
3069 if (!rif) {
3070 err = -ENOMEM;
3071 goto err_rif_alloc;
3072 }
3073 rif->mlxsw_sp = mlxsw_sp;
3074 rif->ops = ops;
3075
3076 fid = ops->fid_get(rif);
3077 if (IS_ERR(fid)) {
3078 err = PTR_ERR(fid);
3079 goto err_fid_get;
3080 }
3081 rif->fid = fid;
3082
3083 if (ops->setup)
3084 ops->setup(rif, params);
3085
3086 err = ops->configure(rif);
3087 if (err)
3088 goto err_configure;
3089
3090 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
3091 mlxsw_sp_fid_index(fid), true);
3092 if (err)
3093 goto err_rif_fdb_op;
3094
3095 mlxsw_sp_rif_counters_alloc(rif);
3096 mlxsw_sp_fid_rif_set(fid, rif);
3097 mlxsw_sp->router->rifs[rif_index] = rif;
3098 vr->rif_count++;
3099
3100 return rif;
3101
3102 err_rif_fdb_op:
3103 ops->deconfigure(rif);
3104 err_configure:
3105 mlxsw_sp_fid_put(fid);
3106 err_fid_get:
3107 kfree(rif);
3108 err_rif_alloc:
3109 err_rif_index_alloc:
3110 mlxsw_sp_vr_put(vr);
3111 return ERR_PTR(err);
3112 }
3113
3114 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
3115 {
3116 const struct mlxsw_sp_rif_ops *ops = rif->ops;
3117 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3118 struct mlxsw_sp_fid *fid = rif->fid;
3119 struct mlxsw_sp_vr *vr;
3120
3121 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3122 vr = &mlxsw_sp->router->vrs[rif->vr_id];
3123
3124 vr->rif_count--;
3125 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
3126 mlxsw_sp_fid_rif_set(fid, NULL);
3127 mlxsw_sp_rif_counters_free(rif);
3128 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
3129 mlxsw_sp_fid_index(fid), false);
3130 ops->deconfigure(rif);
3131 mlxsw_sp_fid_put(fid);
3132 kfree(rif);
3133 mlxsw_sp_vr_put(vr);
3134 }
3135
3136 static void
3137 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
3138 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3139 {
3140 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3141
3142 params->vid = mlxsw_sp_port_vlan->vid;
3143 params->lag = mlxsw_sp_port->lagged;
3144 if (params->lag)
3145 params->lag_id = mlxsw_sp_port->lag_id;
3146 else
3147 params->system_port = mlxsw_sp_port->local_port;
3148 }
3149
3150 static int
3151 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
3152 struct net_device *l3_dev)
3153 {
3154 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3155 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
3156 u16 vid = mlxsw_sp_port_vlan->vid;
3157 struct mlxsw_sp_rif *rif;
3158 struct mlxsw_sp_fid *fid;
3159 int err;
3160
3161 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3162 if (!rif) {
3163 struct mlxsw_sp_rif_params params = {
3164 .dev = l3_dev,
3165 };
3166
3167 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
3168 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3169 if (IS_ERR(rif))
3170 return PTR_ERR(rif);
3171 }
3172
3173 /* FID was already created, just take a reference */
3174 fid = rif->ops->fid_get(rif);
3175 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
3176 if (err)
3177 goto err_fid_port_vid_map;
3178
3179 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
3180 if (err)
3181 goto err_port_vid_learning_set;
3182
3183 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
3184 BR_STATE_FORWARDING);
3185 if (err)
3186 goto err_port_vid_stp_set;
3187
3188 mlxsw_sp_port_vlan->fid = fid;
3189
3190 return 0;
3191
3192 err_port_vid_stp_set:
3193 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3194 err_port_vid_learning_set:
3195 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3196 err_fid_port_vid_map:
3197 mlxsw_sp_fid_put(fid);
3198 return err;
3199 }
3200
3201 void
3202 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3203 {
3204 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3205 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
3206 u16 vid = mlxsw_sp_port_vlan->vid;
3207
3208 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
3209 return;
3210
3211 mlxsw_sp_port_vlan->fid = NULL;
3212 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
3213 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3214 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3215 /* If router port holds the last reference on the rFID, then the
3216 * associated Sub-port RIF will be destroyed.
3217 */
3218 mlxsw_sp_fid_put(fid);
3219 }
3220
3221 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
3222 struct net_device *port_dev,
3223 unsigned long event, u16 vid)
3224 {
3225 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
3226 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
3227
3228 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
3229 if (WARN_ON(!mlxsw_sp_port_vlan))
3230 return -EINVAL;
3231
3232 switch (event) {
3233 case NETDEV_UP:
3234 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
3235 l3_dev);
3236 case NETDEV_DOWN:
3237 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
3238 break;
3239 }
3240
3241 return 0;
3242 }
3243
3244 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
3245 unsigned long event)
3246 {
3247 if (netif_is_bridge_port(port_dev) ||
3248 netif_is_lag_port(port_dev) ||
3249 netif_is_ovs_port(port_dev))
3250 return 0;
3251
3252 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
3253 }
3254
3255 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
3256 struct net_device *lag_dev,
3257 unsigned long event, u16 vid)
3258 {
3259 struct net_device *port_dev;
3260 struct list_head *iter;
3261 int err;
3262
3263 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
3264 if (mlxsw_sp_port_dev_check(port_dev)) {
3265 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
3266 port_dev,
3267 event, vid);
3268 if (err)
3269 return err;
3270 }
3271 }
3272
3273 return 0;
3274 }
3275
3276 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
3277 unsigned long event)
3278 {
3279 if (netif_is_bridge_port(lag_dev))
3280 return 0;
3281
3282 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
3283 }
3284
3285 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3286 unsigned long event)
3287 {
3288 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3289 struct mlxsw_sp_rif_params params = {
3290 .dev = l3_dev,
3291 };
3292 struct mlxsw_sp_rif *rif;
3293
3294 switch (event) {
3295 case NETDEV_UP:
3296 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3297 if (IS_ERR(rif))
3298 return PTR_ERR(rif);
3299 break;
3300 case NETDEV_DOWN:
3301 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3302 mlxsw_sp_rif_destroy(rif);
3303 break;
3304 }
3305
3306 return 0;
3307 }
3308
3309 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3310 unsigned long event)
3311 {
3312 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3313 u16 vid = vlan_dev_vlan_id(vlan_dev);
3314
3315 if (netif_is_bridge_port(vlan_dev))
3316 return 0;
3317
3318 if (mlxsw_sp_port_dev_check(real_dev))
3319 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
3320 event, vid);
3321 else if (netif_is_lag_master(real_dev))
3322 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3323 vid);
3324 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
3325 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
3326
3327 return 0;
3328 }
3329
3330 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
3331 unsigned long event)
3332 {
3333 if (mlxsw_sp_port_dev_check(dev))
3334 return mlxsw_sp_inetaddr_port_event(dev, event);
3335 else if (netif_is_lag_master(dev))
3336 return mlxsw_sp_inetaddr_lag_event(dev, event);
3337 else if (netif_is_bridge_master(dev))
3338 return mlxsw_sp_inetaddr_bridge_event(dev, event);
3339 else if (is_vlan_dev(dev))
3340 return mlxsw_sp_inetaddr_vlan_event(dev, event);
3341 else
3342 return 0;
3343 }
3344
3345 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3346 unsigned long event, void *ptr)
3347 {
3348 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3349 struct net_device *dev = ifa->ifa_dev->dev;
3350 struct mlxsw_sp *mlxsw_sp;
3351 struct mlxsw_sp_rif *rif;
3352 int err = 0;
3353
3354 mlxsw_sp = mlxsw_sp_lower_get(dev);
3355 if (!mlxsw_sp)
3356 goto out;
3357
3358 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3359 if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3360 goto out;
3361
3362 err = __mlxsw_sp_inetaddr_event(dev, event);
3363 out:
3364 return notifier_from_errno(err);
3365 }
3366
3367 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3368 const char *mac, int mtu)
3369 {
3370 char ritr_pl[MLXSW_REG_RITR_LEN];
3371 int err;
3372
3373 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3374 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3375 if (err)
3376 return err;
3377
3378 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3379 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3380 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3381 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3382 }
3383
3384 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3385 {
3386 struct mlxsw_sp *mlxsw_sp;
3387 struct mlxsw_sp_rif *rif;
3388 u16 fid_index;
3389 int err;
3390
3391 mlxsw_sp = mlxsw_sp_lower_get(dev);
3392 if (!mlxsw_sp)
3393 return 0;
3394
3395 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3396 if (!rif)
3397 return 0;
3398 fid_index = mlxsw_sp_fid_index(rif->fid);
3399
3400 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
3401 if (err)
3402 return err;
3403
3404 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3405 dev->mtu);
3406 if (err)
3407 goto err_rif_edit;
3408
3409 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
3410 if (err)
3411 goto err_rif_fdb_op;
3412
3413 ether_addr_copy(rif->addr, dev->dev_addr);
3414 rif->mtu = dev->mtu;
3415
3416 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3417
3418 return 0;
3419
3420 err_rif_fdb_op:
3421 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3422 err_rif_edit:
3423 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
3424 return err;
3425 }
3426
3427 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
3428 struct net_device *l3_dev)
3429 {
3430 struct mlxsw_sp_rif *rif;
3431
3432 /* If netdev is already associated with a RIF, then we need to
3433 * destroy it and create a new one with the new virtual router ID.
3434 */
3435 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3436 if (rif)
3437 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3438
3439 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
3440 }
3441
3442 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3443 struct net_device *l3_dev)
3444 {
3445 struct mlxsw_sp_rif *rif;
3446
3447 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3448 if (!rif)
3449 return;
3450 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3451 }
3452
3453 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
3454 struct netdev_notifier_changeupper_info *info)
3455 {
3456 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3457 int err = 0;
3458
3459 if (!mlxsw_sp)
3460 return 0;
3461
3462 switch (event) {
3463 case NETDEV_PRECHANGEUPPER:
3464 return 0;
3465 case NETDEV_CHANGEUPPER:
3466 if (info->linking)
3467 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
3468 else
3469 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
3470 break;
3471 }
3472
3473 return err;
3474 }
3475
3476 static struct mlxsw_sp_rif_subport *
3477 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
3478 {
3479 return container_of(rif, struct mlxsw_sp_rif_subport, common);
3480 }
3481
3482 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
3483 const struct mlxsw_sp_rif_params *params)
3484 {
3485 struct mlxsw_sp_rif_subport *rif_subport;
3486
3487 rif_subport = mlxsw_sp_rif_subport_rif(rif);
3488 rif_subport->vid = params->vid;
3489 rif_subport->lag = params->lag;
3490 if (params->lag)
3491 rif_subport->lag_id = params->lag_id;
3492 else
3493 rif_subport->system_port = params->system_port;
3494 }
3495
3496 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
3497 {
3498 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3499 struct mlxsw_sp_rif_subport *rif_subport;
3500 char ritr_pl[MLXSW_REG_RITR_LEN];
3501
3502 rif_subport = mlxsw_sp_rif_subport_rif(rif);
3503 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
3504 rif->rif_index, rif->vr_id, rif->dev->mtu,
3505 rif->dev->dev_addr);
3506 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
3507 rif_subport->lag ? rif_subport->lag_id :
3508 rif_subport->system_port,
3509 rif_subport->vid);
3510
3511 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3512 }
3513
3514 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
3515 {
3516 return mlxsw_sp_rif_subport_op(rif, true);
3517 }
3518
3519 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
3520 {
3521 mlxsw_sp_rif_subport_op(rif, false);
3522 }
3523
3524 static struct mlxsw_sp_fid *
3525 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
3526 {
3527 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
3528 }
3529
3530 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
3531 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
3532 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
3533 .setup = mlxsw_sp_rif_subport_setup,
3534 .configure = mlxsw_sp_rif_subport_configure,
3535 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
3536 .fid_get = mlxsw_sp_rif_subport_fid_get,
3537 };
3538
3539 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
3540 enum mlxsw_reg_ritr_if_type type,
3541 u16 vid_fid, bool enable)
3542 {
3543 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3544 char ritr_pl[MLXSW_REG_RITR_LEN];
3545
3546 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
3547 rif->dev->mtu, rif->dev->dev_addr);
3548 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
3549
3550 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3551 }
3552
3553 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
3554 {
3555 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
3556 }
3557
3558 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
3559 {
3560 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3561 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3562 int err;
3563
3564 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
3565 if (err)
3566 return err;
3567
3568 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3569 mlxsw_sp_router_port(mlxsw_sp), true);
3570 if (err)
3571 goto err_fid_bc_flood_set;
3572
3573 return 0;
3574
3575 err_fid_bc_flood_set:
3576 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3577 return err;
3578 }
3579
3580 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
3581 {
3582 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3583 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3584
3585 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3586 mlxsw_sp_router_port(mlxsw_sp), false);
3587 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3588 }
3589
3590 static struct mlxsw_sp_fid *
3591 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
3592 {
3593 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
3594
3595 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
3596 }
3597
3598 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
3599 .type = MLXSW_SP_RIF_TYPE_VLAN,
3600 .rif_size = sizeof(struct mlxsw_sp_rif),
3601 .configure = mlxsw_sp_rif_vlan_configure,
3602 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
3603 .fid_get = mlxsw_sp_rif_vlan_fid_get,
3604 };
3605
3606 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
3607 {
3608 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3609 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3610 int err;
3611
3612 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
3613 true);
3614 if (err)
3615 return err;
3616
3617 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3618 mlxsw_sp_router_port(mlxsw_sp), true);
3619 if (err)
3620 goto err_fid_bc_flood_set;
3621
3622 return 0;
3623
3624 err_fid_bc_flood_set:
3625 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3626 return err;
3627 }
3628
3629 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
3630 {
3631 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3632 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3633
3634 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3635 mlxsw_sp_router_port(mlxsw_sp), false);
3636 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3637 }
3638
3639 static struct mlxsw_sp_fid *
3640 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
3641 {
3642 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
3643 }
3644
3645 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
3646 .type = MLXSW_SP_RIF_TYPE_FID,
3647 .rif_size = sizeof(struct mlxsw_sp_rif),
3648 .configure = mlxsw_sp_rif_fid_configure,
3649 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
3650 .fid_get = mlxsw_sp_rif_fid_fid_get,
3651 };
3652
3653 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
3654 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
3655 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
3656 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
3657 };
3658
3659 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
3660 {
3661 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3662
3663 mlxsw_sp->router->rifs = kcalloc(max_rifs,
3664 sizeof(struct mlxsw_sp_rif *),
3665 GFP_KERNEL);
3666 if (!mlxsw_sp->router->rifs)
3667 return -ENOMEM;
3668
3669 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
3670
3671 return 0;
3672 }
3673
3674 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
3675 {
3676 int i;
3677
3678 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3679 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
3680
3681 kfree(mlxsw_sp->router->rifs);
3682 }
3683
3684 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3685 {
3686 struct mlxsw_sp_router *router;
3687
3688 /* Flush pending FIB notifications and then flush the device's
3689 * table before requesting another dump. The FIB notification
3690 * block is unregistered, so no need to take RTNL.
3691 */
3692 mlxsw_core_flush_owq();
3693 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3694 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
3695 }
3696
3697 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3698 {
3699 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3700 u64 max_rifs;
3701 int err;
3702
3703 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3704 return -EIO;
3705 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3706
3707 mlxsw_reg_rgcr_pack(rgcr_pl, true);
3708 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3709 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3710 if (err)
3711 return err;
3712 return 0;
3713 }
3714
3715 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3716 {
3717 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3718
3719 mlxsw_reg_rgcr_pack(rgcr_pl, false);
3720 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3721 }
3722
3723 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3724 {
3725 struct mlxsw_sp_router *router;
3726 int err;
3727
3728 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
3729 if (!router)
3730 return -ENOMEM;
3731 mlxsw_sp->router = router;
3732 router->mlxsw_sp = mlxsw_sp;
3733
3734 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
3735 err = __mlxsw_sp_router_init(mlxsw_sp);
3736 if (err)
3737 goto err_router_init;
3738
3739 err = mlxsw_sp_rifs_init(mlxsw_sp);
3740 if (err)
3741 goto err_rifs_init;
3742
3743 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
3744 &mlxsw_sp_nexthop_ht_params);
3745 if (err)
3746 goto err_nexthop_ht_init;
3747
3748 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
3749 &mlxsw_sp_nexthop_group_ht_params);
3750 if (err)
3751 goto err_nexthop_group_ht_init;
3752
3753 err = mlxsw_sp_lpm_init(mlxsw_sp);
3754 if (err)
3755 goto err_lpm_init;
3756
3757 err = mlxsw_sp_vrs_init(mlxsw_sp);
3758 if (err)
3759 goto err_vrs_init;
3760
3761 err = mlxsw_sp_neigh_init(mlxsw_sp);
3762 if (err)
3763 goto err_neigh_init;
3764
3765 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3766 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
3767 mlxsw_sp_router_fib_dump_flush);
3768 if (err)
3769 goto err_register_fib_notifier;
3770
3771 return 0;
3772
3773 err_register_fib_notifier:
3774 mlxsw_sp_neigh_fini(mlxsw_sp);
3775 err_neigh_init:
3776 mlxsw_sp_vrs_fini(mlxsw_sp);
3777 err_vrs_init:
3778 mlxsw_sp_lpm_fini(mlxsw_sp);
3779 err_lpm_init:
3780 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3781 err_nexthop_group_ht_init:
3782 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3783 err_nexthop_ht_init:
3784 mlxsw_sp_rifs_fini(mlxsw_sp);
3785 err_rifs_init:
3786 __mlxsw_sp_router_fini(mlxsw_sp);
3787 err_router_init:
3788 kfree(mlxsw_sp->router);
3789 return err;
3790 }
3791
3792 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3793 {
3794 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
3795 mlxsw_sp_neigh_fini(mlxsw_sp);
3796 mlxsw_sp_vrs_fini(mlxsw_sp);
3797 mlxsw_sp_lpm_fini(mlxsw_sp);
3798 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3799 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3800 mlxsw_sp_rifs_fini(mlxsw_sp);
3801 __mlxsw_sp_router_fini(mlxsw_sp);
3802 kfree(mlxsw_sp->router);
3803 }