drivers/net/team/team_mode_loadbalance.c

   1 /*
   2  * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team
   3  * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/types.h>
  13 #include <linux/module.h>
  14 #include <linux/init.h>
  15 #include <linux/errno.h>
  16 #include <linux/netdevice.h>
  17 #include <linux/etherdevice.h>
  18 #include <linux/filter.h>
  19 #include <linux/if_team.h>
  20
  21 static rx_handler_result_t lb_receive(struct team *team, struct team_port *port,
  22                                       struct sk_buff *skb)
  23 {
  24         if (unlikely(skb->protocol == htons(ETH_P_SLOW))) {
  25                 /* LACPDU packets should go to exact delivery */
  26                 const unsigned char *dest = eth_hdr(skb)->h_dest;
  27
  28                 if (is_link_local_ether_addr(dest) && dest[5] == 0x02)
  29                         return RX_HANDLER_EXACT;
  30         }
  31         return RX_HANDLER_ANOTHER;
  32 }
  33
  34 struct lb_priv;
  35
  36 typedef struct team_port *lb_select_tx_port_func_t(struct team *,
  37                                                    struct lb_priv *,
  38                                                    struct sk_buff *,
  39                                                    unsigned char);
  40
  41 #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */
  42
  43 struct lb_stats {
  44         u64 tx_bytes;
  45 };
  46
  47 struct lb_pcpu_stats {
  48         struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE];
  49         struct u64_stats_sync syncp;
  50 };
  51
  52 struct lb_stats_info {
  53         struct lb_stats stats;
  54         struct lb_stats last_stats;
  55         struct team_option_inst_info *opt_inst_info;
  56 };
  57
  58 struct lb_port_mapping {
  59         struct team_port __rcu *port;
  60         struct team_option_inst_info *opt_inst_info;
  61 };
  62
  63 struct lb_priv_ex {
  64         struct team *team;
  65         struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE];
  66         struct sock_fprog_kern *orig_fprog;
  67         struct {
  68                 unsigned int refresh_interval; /* in tenths of second */
  69                 struct delayed_work refresh_dw;
  70                 struct lb_stats_info info[LB_TX_HASHTABLE_SIZE];
  71         } stats;
  72 };
  73
  74 struct lb_priv {
  75         struct bpf_prog __rcu *fp;
  76         lb_select_tx_port_func_t __rcu *select_tx_port_func;
  77         struct lb_pcpu_stats __percpu *pcpu_stats;
  78         struct lb_priv_ex *ex; /* priv extension */
  79 };
  80
  81 static struct lb_priv *get_lb_priv(struct team *team)
  82 {
  83         return (struct lb_priv *) &team->mode_priv;
  84 }
  85
  86 struct lb_port_priv {
  87         struct lb_stats __percpu *pcpu_stats;
  88         struct lb_stats_info stats_info;
  89 };
  90
  91 static struct lb_port_priv *get_lb_port_priv(struct team_port *port)
  92 {
  93         return (struct lb_port_priv *) &port->mode_priv;
  94 }
  95
  96 #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \
  97         (lb_priv)->ex->tx_hash_to_port_mapping[hash].port
  98
  99 #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \
 100         (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info
 101
 102 static void lb_tx_hash_to_port_mapping_null_port(struct team *team,
 103                                                  struct team_port *port)
 104 {
 105         struct lb_priv *lb_priv = get_lb_priv(team);
 106         bool changed = false;
 107         int i;
 108
 109         for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) {
 110                 struct lb_port_mapping *pm;
 111
 112                 pm = &lb_priv->ex->tx_hash_to_port_mapping[i];
 113                 if (rcu_access_pointer(pm->port) == port) {
 114                         RCU_INIT_POINTER(pm->port, NULL);
 115                         team_option_inst_set_change(pm->opt_inst_info);
 116                         changed = true;
 117                 }
 118         }
 119         if (changed)
 120                 team_options_change_check(team);
 121 }
 122
 123 /* Basic tx selection based solely by hash */
 124 static struct team_port *lb_hash_select_tx_port(struct team *team,
 125                                                 struct lb_priv *lb_priv,
 126                                                 struct sk_buff *skb,
 127                                                 unsigned char hash)
 128 {
 129         int port_index = team_num_to_port_index(team, hash);
 130
 131         return team_get_port_by_index_rcu(team, port_index);
 132 }
 133
 134 /* Hash to port mapping select tx port */
 135 static struct team_port *lb_htpm_select_tx_port(struct team *team,
 136                                                 struct lb_priv *lb_priv,
 137                                                 struct sk_buff *skb,
 138                                                 unsigned char hash)
 139 {
 140         struct team_port *port;
 141
 142         port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash));
 143         if (likely(port))
 144                 return port;
 145         /* If no valid port in the table, fall back to simple hash */
 146         return lb_hash_select_tx_port(team, lb_priv, skb, hash);
 147 }
 148
 149 struct lb_select_tx_port {
 150         char *name;
 151         lb_select_tx_port_func_t *func;
 152 };
 153
 154 static const struct lb_select_tx_port lb_select_tx_port_list[] = {
 155         {
 156                 .name = "hash",
 157                 .func = lb_hash_select_tx_port,
 158         },
 159         {
 160                 .name = "hash_to_port_mapping",
 161                 .func = lb_htpm_select_tx_port,
 162         },
 163 };
 164 #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list)
 165
 166 static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func)
 167 {
 168         int i;
 169
 170         for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
 171                 const struct lb_select_tx_port *item;
 172
 173                 item = &lb_select_tx_port_list[i];
 174                 if (item->func == func)
 175                         return item->name;
 176         }
 177         return NULL;
 178 }
 179
 180 static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name)
 181 {
 182         int i;
 183
 184         for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
 185                 const struct lb_select_tx_port *item;
 186
 187                 item = &lb_select_tx_port_list[i];
 188                 if (!strcmp(item->name, name))
 189                         return item->func;
 190         }
 191         return NULL;
 192 }
 193
 194 static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv,
 195                                     struct sk_buff *skb)
 196 {
 197         struct bpf_prog *fp;
 198         uint32_t lhash;
 199         unsigned char *c;
 200
 201         fp = rcu_dereference_bh(lb_priv->fp);
 202         if (unlikely(!fp))
 203                 return 0;
 204         lhash = BPF_PROG_RUN(fp, skb);
 205         c = (char *) &lhash;
 206         return c[0] ^ c[1] ^ c[2] ^ c[3];
 207 }
 208
 209 static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv,
 210                                struct lb_port_priv *lb_port_priv,
 211                                unsigned char hash)
 212 {
 213         struct lb_pcpu_stats *pcpu_stats;
 214         struct lb_stats *port_stats;
 215         struct lb_stats *hash_stats;
 216
 217         pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats);
 218         port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats);
 219         hash_stats = &pcpu_stats->hash_stats[hash];
 220         u64_stats_update_begin(&pcpu_stats->syncp);
 221         port_stats->tx_bytes += tx_bytes;
 222         hash_stats->tx_bytes += tx_bytes;
 223         u64_stats_update_end(&pcpu_stats->syncp);
 224 }
 225
 226 static bool lb_transmit(struct team *team, struct sk_buff *skb)
 227 {
 228         struct lb_priv *lb_priv = get_lb_priv(team);
 229         lb_select_tx_port_func_t *select_tx_port_func;
 230         struct team_port *port;
 231         unsigned char hash;
 232         unsigned int tx_bytes = skb->len;
 233
 234         hash = lb_get_skb_hash(lb_priv, skb);
 235         select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func);
 236         port = select_tx_port_func(team, lb_priv, skb, hash);
 237         if (unlikely(!port))
 238                 goto drop;
 239         if (team_dev_queue_xmit(team, port, skb))
 240                 return false;
 241         lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash);
 242         return true;
 243
 244 drop:
 245         dev_kfree_skb_any(skb);
 246         return false;
 247 }
 248
 249 static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx)
 250 {
 251         struct lb_priv *lb_priv = get_lb_priv(team);
 252
 253         if (!lb_priv->ex->orig_fprog) {
 254                 ctx->data.bin_val.len = 0;
 255                 ctx->data.bin_val.ptr = NULL;
 256                 return 0;
 257         }
 258         ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len *
 259                                 sizeof(struct sock_filter);
 260         ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter;
 261         return 0;
 262 }
 263
 264 static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len,
 265                           const void *data)
 266 {
 267         struct sock_fprog_kern *fprog;
 268         struct sock_filter *filter = (struct sock_filter *) data;
 269
 270         if (data_len % sizeof(struct sock_filter))
 271                 return -EINVAL;
 272         fprog = kmalloc(sizeof(*fprog), GFP_KERNEL);
 273         if (!fprog)
 274                 return -ENOMEM;
 275         fprog->filter = kmemdup(filter, data_len, GFP_KERNEL);
 276         if (!fprog->filter) {
 277                 kfree(fprog);
 278                 return -ENOMEM;
 279         }
 280         fprog->len = data_len / sizeof(struct sock_filter);
 281         *pfprog = fprog;
 282         return 0;
 283 }
 284
 285 static void __fprog_destroy(struct sock_fprog_kern *fprog)
 286 {
 287         kfree(fprog->filter);
 288         kfree(fprog);
 289 }
 290
 291 static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx)
 292 {
 293         struct lb_priv *lb_priv = get_lb_priv(team);
 294         struct bpf_prog *fp = NULL;
 295         struct bpf_prog *orig_fp = NULL;
 296         struct sock_fprog_kern *fprog = NULL;
 297         int err;
 298
 299         if (ctx->data.bin_val.len) {
 300                 err = __fprog_create(&fprog, ctx->data.bin_val.len,
 301                                      ctx->data.bin_val.ptr);
 302                 if (err)
 303                         return err;
 304                 err = bpf_prog_create(&fp, fprog);
 305                 if (err) {
 306                         __fprog_destroy(fprog);
 307                         return err;
 308                 }
 309         }
 310
 311         if (lb_priv->ex->orig_fprog) {
 312                 /* Clear old filter data */
 313                 __fprog_destroy(lb_priv->ex->orig_fprog);
 314                 orig_fp = rcu_dereference_protected(lb_priv->fp,
 315                                                 lockdep_is_held(&team->lock));
 316         }
 317
 318         rcu_assign_pointer(lb_priv->fp, fp);
 319         lb_priv->ex->orig_fprog = fprog;
 320
 321         if (orig_fp) {
 322                 synchronize_rcu();
 323                 bpf_prog_destroy(orig_fp);
 324         }
 325         return 0;
 326 }
 327
 328 static void lb_bpf_func_free(struct team *team)
 329 {
 330         struct lb_priv *lb_priv = get_lb_priv(team);
 331         struct bpf_prog *fp;
 332
 333         if (!lb_priv->ex->orig_fprog)
 334                 return;
 335
 336         __fprog_destroy(lb_priv->ex->orig_fprog);
 337         fp = rcu_dereference_protected(lb_priv->fp,
 338                                        lockdep_is_held(&team->lock));
 339         bpf_prog_destroy(fp);
 340 }
 341
 342 static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx)
 343 {
 344         struct lb_priv *lb_priv = get_lb_priv(team);
 345         lb_select_tx_port_func_t *func;
 346         char *name;
 347
 348         func = rcu_dereference_protected(lb_priv->select_tx_port_func,
 349                                          lockdep_is_held(&team->lock));
 350         name = lb_select_tx_port_get_name(func);
 351         BUG_ON(!name);
 352         ctx->data.str_val = name;
 353         return 0;
 354 }
 355
 356 static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx)
 357 {
 358         struct lb_priv *lb_priv = get_lb_priv(team);
 359         lb_select_tx_port_func_t *func;
 360
 361         func = lb_select_tx_port_get_func(ctx->data.str_val);
 362         if (!func)
 363                 return -EINVAL;
 364         rcu_assign_pointer(lb_priv->select_tx_port_func, func);
 365         return 0;
 366 }
 367
 368 static int lb_tx_hash_to_port_mapping_init(struct team *team,
 369                                            struct team_option_inst_info *info)
 370 {
 371         struct lb_priv *lb_priv = get_lb_priv(team);
 372         unsigned char hash = info->array_index;
 373
 374         LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info;
 375         return 0;
 376 }
 377
 378 static int lb_tx_hash_to_port_mapping_get(struct team *team,
 379                                           struct team_gsetter_ctx *ctx)
 380 {
 381         struct lb_priv *lb_priv = get_lb_priv(team);
 382         struct team_port *port;
 383         unsigned char hash = ctx->info->array_index;
 384
 385         port = LB_HTPM_PORT_BY_HASH(lb_priv, hash);
 386         ctx->data.u32_val = port ? port->dev->ifindex : 0;
 387         return 0;
 388 }
 389
 390 static int lb_tx_hash_to_port_mapping_set(struct team *team,
 391                                           struct team_gsetter_ctx *ctx)
 392 {
 393         struct lb_priv *lb_priv = get_lb_priv(team);
 394         struct team_port *port;
 395         unsigned char hash = ctx->info->array_index;
 396
 397         list_for_each_entry(port, &team->port_list, list) {
 398                 if (ctx->data.u32_val == port->dev->ifindex &&
 399                     team_port_enabled(port)) {
 400                         rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash),
 401                                            port);
 402                         return 0;
 403                 }
 404         }
 405         return -ENODEV;
 406 }
 407
 408 static int lb_hash_stats_init(struct team *team,
 409                               struct team_option_inst_info *info)
 410 {
 411         struct lb_priv *lb_priv = get_lb_priv(team);
 412         unsigned char hash = info->array_index;
 413
 414         lb_priv->ex->stats.info[hash].opt_inst_info = info;
 415         return 0;
 416 }
 417
 418 static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
 419 {
 420         struct lb_priv *lb_priv = get_lb_priv(team);
 421         unsigned char hash = ctx->info->array_index;
 422
 423         ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats;
 424         ctx->data.bin_val.len = sizeof(struct lb_stats);
 425         return 0;
 426 }
 427
 428 static int lb_port_stats_init(struct team *team,
 429                               struct team_option_inst_info *info)
 430 {
 431         struct team_port *port = info->port;
 432         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 433
 434         lb_port_priv->stats_info.opt_inst_info = info;
 435         return 0;
 436 }
 437
 438 static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
 439 {
 440         struct team_port *port = ctx->info->port;
 441         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 442
 443         ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats;
 444         ctx->data.bin_val.len = sizeof(struct lb_stats);
 445         return 0;
 446 }
 447
 448 static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info)
 449 {
 450         memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats));
 451         memset(&s_info->stats, 0, sizeof(struct lb_stats));
 452 }
 453
 454 static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info,
 455                                           struct team *team)
 456 {
 457         if (memcmp(&s_info->last_stats, &s_info->stats,
 458             sizeof(struct lb_stats))) {
 459                 team_option_inst_set_change(s_info->opt_inst_info);
 460                 return true;
 461         }
 462         return false;
 463 }
 464
 465 static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats,
 466                                    struct lb_stats *cpu_stats,
 467                                    struct u64_stats_sync *syncp)
 468 {
 469         unsigned int start;
 470         struct lb_stats tmp;
 471
 472         do {
 473                 start = u64_stats_fetch_begin_irq(syncp);
 474                 tmp.tx_bytes = cpu_stats->tx_bytes;
 475         } while (u64_stats_fetch_retry_irq(syncp, start));
 476         acc_stats->tx_bytes += tmp.tx_bytes;
 477 }
 478
 479 static void lb_stats_refresh(struct work_struct *work)
 480 {
 481         struct team *team;
 482         struct lb_priv *lb_priv;
 483         struct lb_priv_ex *lb_priv_ex;
 484         struct lb_pcpu_stats *pcpu_stats;
 485         struct lb_stats *stats;
 486         struct lb_stats_info *s_info;
 487         struct team_port *port;
 488         bool changed = false;
 489         int i;
 490         int j;
 491
 492         lb_priv_ex = container_of(work, struct lb_priv_ex,
 493                                   stats.refresh_dw.work);
 494
 495         team = lb_priv_ex->team;
 496         lb_priv = get_lb_priv(team);
 497
 498         if (!mutex_trylock(&team->lock)) {
 499                 schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0);
 500                 return;
 501         }
 502
 503         for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) {
 504                 s_info = &lb_priv->ex->stats.info[j];
 505                 __lb_stats_info_refresh_prepare(s_info);
 506                 for_each_possible_cpu(i) {
 507                         pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 508                         stats = &pcpu_stats->hash_stats[j];
 509                         __lb_one_cpu_stats_add(&s_info->stats, stats,
 510                                                &pcpu_stats->syncp);
 511                 }
 512                 changed |= __lb_stats_info_refresh_check(s_info, team);
 513         }
 514
 515         list_for_each_entry(port, &team->port_list, list) {
 516                 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 517
 518                 s_info = &lb_port_priv->stats_info;
 519                 __lb_stats_info_refresh_prepare(s_info);
 520                 for_each_possible_cpu(i) {
 521                         pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 522                         stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i);
 523                         __lb_one_cpu_stats_add(&s_info->stats, stats,
 524                                                &pcpu_stats->syncp);
 525                 }
 526                 changed |= __lb_stats_info_refresh_check(s_info, team);
 527         }
 528
 529         if (changed)
 530                 team_options_change_check(team);
 531
 532         schedule_delayed_work(&lb_priv_ex->stats.refresh_dw,
 533                               (lb_priv_ex->stats.refresh_interval * HZ) / 10);
 534
 535         mutex_unlock(&team->lock);
 536 }
 537
 538 static int lb_stats_refresh_interval_get(struct team *team,
 539                                          struct team_gsetter_ctx *ctx)
 540 {
 541         struct lb_priv *lb_priv = get_lb_priv(team);
 542
 543         ctx->data.u32_val = lb_priv->ex->stats.refresh_interval;
 544         return 0;
 545 }
 546
 547 static int lb_stats_refresh_interval_set(struct team *team,
 548                                          struct team_gsetter_ctx *ctx)
 549 {
 550         struct lb_priv *lb_priv = get_lb_priv(team);
 551         unsigned int interval;
 552
 553         interval = ctx->data.u32_val;
 554         if (lb_priv->ex->stats.refresh_interval == interval)
 555                 return 0;
 556         lb_priv->ex->stats.refresh_interval = interval;
 557         if (interval)
 558                 schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0);
 559         else
 560                 cancel_delayed_work(&lb_priv->ex->stats.refresh_dw);
 561         return 0;
 562 }
 563
 564 static const struct team_option lb_options[] = {
 565         {
 566                 .name = "bpf_hash_func",
 567                 .type = TEAM_OPTION_TYPE_BINARY,
 568                 .getter = lb_bpf_func_get,
 569                 .setter = lb_bpf_func_set,
 570         },
 571         {
 572                 .name = "lb_tx_method",
 573                 .type = TEAM_OPTION_TYPE_STRING,
 574                 .getter = lb_tx_method_get,
 575                 .setter = lb_tx_method_set,
 576         },
 577         {
 578                 .name = "lb_tx_hash_to_port_mapping",
 579                 .array_size = LB_TX_HASHTABLE_SIZE,
 580                 .type = TEAM_OPTION_TYPE_U32,
 581                 .init = lb_tx_hash_to_port_mapping_init,
 582                 .getter = lb_tx_hash_to_port_mapping_get,
 583                 .setter = lb_tx_hash_to_port_mapping_set,
 584         },
 585         {
 586                 .name = "lb_hash_stats",
 587                 .array_size = LB_TX_HASHTABLE_SIZE,
 588                 .type = TEAM_OPTION_TYPE_BINARY,
 589                 .init = lb_hash_stats_init,
 590                 .getter = lb_hash_stats_get,
 591         },
 592         {
 593                 .name = "lb_port_stats",
 594                 .per_port = true,
 595                 .type = TEAM_OPTION_TYPE_BINARY,
 596                 .init = lb_port_stats_init,
 597                 .getter = lb_port_stats_get,
 598         },
 599         {
 600                 .name = "lb_stats_refresh_interval",
 601                 .type = TEAM_OPTION_TYPE_U32,
 602                 .getter = lb_stats_refresh_interval_get,
 603                 .setter = lb_stats_refresh_interval_set,
 604         },
 605 };
 606
 607 static int lb_init(struct team *team)
 608 {
 609         struct lb_priv *lb_priv = get_lb_priv(team);
 610         lb_select_tx_port_func_t *func;
 611         int i, err;
 612
 613         /* set default tx port selector */
 614         func = lb_select_tx_port_get_func("hash");
 615         BUG_ON(!func);
 616         rcu_assign_pointer(lb_priv->select_tx_port_func, func);
 617
 618         lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL);
 619         if (!lb_priv->ex)
 620                 return -ENOMEM;
 621         lb_priv->ex->team = team;
 622
 623         lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats);
 624         if (!lb_priv->pcpu_stats) {
 625                 err = -ENOMEM;
 626                 goto err_alloc_pcpu_stats;
 627         }
 628
 629         for_each_possible_cpu(i) {
 630                 struct lb_pcpu_stats *team_lb_stats;
 631                 team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 632                 u64_stats_init(&team_lb_stats->syncp);
 633         }
 634
 635
 636         INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh);
 637
 638         err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options));
 639         if (err)
 640                 goto err_options_register;
 641         return 0;
 642
 643 err_options_register:
 644         free_percpu(lb_priv->pcpu_stats);
 645 err_alloc_pcpu_stats:
 646         kfree(lb_priv->ex);
 647         return err;
 648 }
 649
 650 static void lb_exit(struct team *team)
 651 {
 652         struct lb_priv *lb_priv = get_lb_priv(team);
 653
 654         team_options_unregister(team, lb_options,
 655                                 ARRAY_SIZE(lb_options));
 656         lb_bpf_func_free(team);
 657         cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw);
 658         free_percpu(lb_priv->pcpu_stats);
 659         kfree(lb_priv->ex);
 660 }
 661
 662 static int lb_port_enter(struct team *team, struct team_port *port)
 663 {
 664         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 665
 666         lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats);
 667         if (!lb_port_priv->pcpu_stats)
 668                 return -ENOMEM;
 669         return 0;
 670 }
 671
 672 static void lb_port_leave(struct team *team, struct team_port *port)
 673 {
 674         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 675
 676         free_percpu(lb_port_priv->pcpu_stats);
 677 }
 678
 679 static void lb_port_disabled(struct team *team, struct team_port *port)
 680 {
 681         lb_tx_hash_to_port_mapping_null_port(team, port);
 682 }
 683
 684 static const struct team_mode_ops lb_mode_ops = {
 685         .init                   = lb_init,
 686         .exit                   = lb_exit,
 687         .port_enter             = lb_port_enter,
 688         .port_leave             = lb_port_leave,
 689         .port_disabled          = lb_port_disabled,
 690         .receive                = lb_receive,
 691         .transmit               = lb_transmit,
 692 };
 693
 694 static const struct team_mode lb_mode = {
 695         .kind           = "loadbalance",
 696         .owner          = THIS_MODULE,
 697         .priv_size      = sizeof(struct lb_priv),
 698         .port_priv_size = sizeof(struct lb_port_priv),
 699         .ops            = &lb_mode_ops,
 700         .lag_tx_type    = NETDEV_LAG_TX_TYPE_HASH,
 701 };
 702
 703 static int __init lb_init_module(void)
 704 {
 705         return team_mode_register(&lb_mode);
 706 }
 707
 708 static void __exit lb_cleanup_module(void)
 709 {
 710         team_mode_unregister(&lb_mode);
 711 }
 712
 713 module_init(lb_init_module);
 714 module_exit(lb_cleanup_module);
 715
 716 MODULE_LICENSE("GPL v2");
 717 MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>");
 718 MODULE_DESCRIPTION("Load-balancing mode for team");
 719 MODULE_ALIAS_TEAM_MODE("loadbalance");