net/netfilter/nf_tables_offload.c

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/init.h>
   3 #include <linux/module.h>
   4 #include <linux/netfilter.h>
   5 #include <net/flow_offload.h>
   6 #include <net/netfilter/nf_tables.h>
   7 #include <net/netfilter/nf_tables_offload.h>
   8 #include <net/pkt_cls.h>
   9
  10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
  11 {
  12         struct nft_flow_rule *flow;
  13
  14         flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
  15         if (!flow)
  16                 return NULL;
  17
  18         flow->rule = flow_rule_alloc(num_actions);
  19         if (!flow->rule) {
  20                 kfree(flow);
  21                 return NULL;
  22         }
  23
  24         flow->rule->match.dissector     = &flow->match.dissector;
  25         flow->rule->match.mask          = &flow->match.mask;
  26         flow->rule->match.key           = &flow->match.key;
  27
  28         return flow;
  29 }
  30
  31 void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
  32                                  enum flow_dissector_key_id addr_type)
  33 {
  34         struct nft_flow_match *match = &flow->match;
  35         struct nft_flow_key *mask = &match->mask;
  36         struct nft_flow_key *key = &match->key;
  37
  38         if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL))
  39                 return;
  40
  41         key->control.addr_type = addr_type;
  42         mask->control.addr_type = 0xffff;
  43         match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL);
  44         match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
  45                 offsetof(struct nft_flow_key, control);
  46 }
  47
  48 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
  49                                            const struct nft_rule *rule)
  50 {
  51         struct nft_offload_ctx *ctx;
  52         struct nft_flow_rule *flow;
  53         int num_actions = 0, err;
  54         struct nft_expr *expr;
  55
  56         expr = nft_expr_first(rule);
  57         while (nft_expr_more(rule, expr)) {
  58                 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
  59                         num_actions++;
  60
  61                 expr = nft_expr_next(expr);
  62         }
  63
  64         if (num_actions == 0)
  65                 return ERR_PTR(-EOPNOTSUPP);
  66
  67         flow = nft_flow_rule_alloc(num_actions);
  68         if (!flow)
  69                 return ERR_PTR(-ENOMEM);
  70
  71         expr = nft_expr_first(rule);
  72
  73         ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
  74         if (!ctx) {
  75                 err = -ENOMEM;
  76                 goto err_out;
  77         }
  78         ctx->net = net;
  79         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
  80
  81         while (nft_expr_more(rule, expr)) {
  82                 if (!expr->ops->offload) {
  83                         err = -EOPNOTSUPP;
  84                         goto err_out;
  85                 }
  86                 err = expr->ops->offload(ctx, flow, expr);
  87                 if (err < 0)
  88                         goto err_out;
  89
  90                 expr = nft_expr_next(expr);
  91         }
  92         flow->proto = ctx->dep.l3num;
  93         kfree(ctx);
  94
  95         return flow;
  96 err_out:
  97         kfree(ctx);
  98         nft_flow_rule_destroy(flow);
  99
 100         return ERR_PTR(err);
 101 }
 102
 103 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
 104 {
 105         struct flow_action_entry *entry;
 106         int i;
 107
 108         flow_action_for_each(i, entry, &flow->rule->action) {
 109                 switch (entry->id) {
 110                 case FLOW_ACTION_REDIRECT:
 111                 case FLOW_ACTION_MIRRED:
 112                         dev_put(entry->dev);
 113                         break;
 114                 default:
 115                         break;
 116                 }
 117         }
 118         kfree(flow->rule);
 119         kfree(flow);
 120 }
 121
 122 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
 123                                 enum nft_offload_dep_type type)
 124 {
 125         ctx->dep.type = type;
 126 }
 127
 128 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
 129                                    const void *data, u32 len)
 130 {
 131         switch (ctx->dep.type) {
 132         case NFT_OFFLOAD_DEP_NETWORK:
 133                 WARN_ON(len != sizeof(__u16));
 134                 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
 135                 break;
 136         case NFT_OFFLOAD_DEP_TRANSPORT:
 137                 WARN_ON(len != sizeof(__u8));
 138                 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
 139                 break;
 140         default:
 141                 break;
 142         }
 143         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
 144 }
 145
 146 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
 147                                          __be16 proto, int priority,
 148                                          struct netlink_ext_ack *extack)
 149 {
 150         common->protocol = proto;
 151         common->prio = priority;
 152         common->extack = extack;
 153 }
 154
 155 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
 156                              struct list_head *cb_list)
 157 {
 158         struct flow_block_cb *block_cb;
 159         int err;
 160
 161         list_for_each_entry(block_cb, cb_list, list) {
 162                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
 163                 if (err < 0)
 164                         return err;
 165         }
 166         return 0;
 167 }
 168
 169 int nft_chain_offload_priority(struct nft_base_chain *basechain)
 170 {
 171         if (basechain->ops.priority <= 0 ||
 172             basechain->ops.priority > USHRT_MAX)
 173                 return -1;
 174
 175         return 0;
 176 }
 177
 178 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
 179                                        const struct nft_base_chain *basechain,
 180                                        const struct nft_rule *rule,
 181                                        const struct nft_flow_rule *flow,
 182                                        struct netlink_ext_ack *extack,
 183                                        enum flow_cls_command command)
 184 {
 185         __be16 proto = ETH_P_ALL;
 186
 187         memset(cls_flow, 0, sizeof(*cls_flow));
 188
 189         if (flow)
 190                 proto = flow->proto;
 191
 192         nft_flow_offload_common_init(&cls_flow->common, proto,
 193                                      basechain->ops.priority, extack);
 194         cls_flow->command = command;
 195         cls_flow->cookie = (unsigned long) rule;
 196         if (flow)
 197                 cls_flow->rule = flow->rule;
 198 }
 199
 200 static int nft_flow_offload_rule(struct nft_chain *chain,
 201                                  struct nft_rule *rule,
 202                                  struct nft_flow_rule *flow,
 203                                  enum flow_cls_command command)
 204 {
 205         struct netlink_ext_ack extack = {};
 206         struct flow_cls_offload cls_flow;
 207         struct nft_base_chain *basechain;
 208
 209         if (!nft_is_base_chain(chain))
 210                 return -EOPNOTSUPP;
 211
 212         basechain = nft_base_chain(chain);
 213         nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
 214                                    command);
 215
 216         return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
 217                                  &basechain->flow_block.cb_list);
 218 }
 219
 220 static int nft_flow_offload_bind(struct flow_block_offload *bo,
 221                                  struct nft_base_chain *basechain)
 222 {
 223         list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
 224         return 0;
 225 }
 226
 227 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
 228                                    struct nft_base_chain *basechain)
 229 {
 230         struct flow_block_cb *block_cb, *next;
 231         struct flow_cls_offload cls_flow;
 232         struct netlink_ext_ack extack;
 233         struct nft_chain *chain;
 234         struct nft_rule *rule;
 235
 236         chain = &basechain->chain;
 237         list_for_each_entry(rule, &chain->rules, list) {
 238                 memset(&extack, 0, sizeof(extack));
 239                 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
 240                                            &extack, FLOW_CLS_DESTROY);
 241                 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
 242         }
 243
 244         list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
 245                 list_del(&block_cb->list);
 246                 flow_block_cb_free(block_cb);
 247         }
 248
 249         return 0;
 250 }
 251
 252 static int nft_block_setup(struct nft_base_chain *basechain,
 253                            struct flow_block_offload *bo,
 254                            enum flow_block_command cmd)
 255 {
 256         int err;
 257
 258         switch (cmd) {
 259         case FLOW_BLOCK_BIND:
 260                 err = nft_flow_offload_bind(bo, basechain);
 261                 break;
 262         case FLOW_BLOCK_UNBIND:
 263                 err = nft_flow_offload_unbind(bo, basechain);
 264                 break;
 265         default:
 266                 WARN_ON_ONCE(1);
 267                 err = -EOPNOTSUPP;
 268         }
 269
 270         return err;
 271 }
 272
 273 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
 274                                         struct net *net,
 275                                         enum flow_block_command cmd,
 276                                         struct nft_base_chain *basechain,
 277                                         struct netlink_ext_ack *extack)
 278 {
 279         memset(bo, 0, sizeof(*bo));
 280         bo->net         = net;
 281         bo->block       = &basechain->flow_block;
 282         bo->command     = cmd;
 283         bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
 284         bo->extack      = extack;
 285         INIT_LIST_HEAD(&bo->cb_list);
 286 }
 287
 288 static int nft_block_offload_cmd(struct nft_base_chain *chain,
 289                                  struct net_device *dev,
 290                                  enum flow_block_command cmd)
 291 {
 292         struct netlink_ext_ack extack = {};
 293         struct flow_block_offload bo;
 294         int err;
 295
 296         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
 297
 298         err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 299         if (err < 0)
 300                 return err;
 301
 302         return nft_block_setup(chain, &bo, cmd);
 303 }
 304
 305 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
 306 {
 307         struct nft_base_chain *basechain = block_cb->indr.data;
 308         struct net_device *dev = block_cb->indr.dev;
 309         struct netlink_ext_ack extack = {};
 310         struct net *net = dev_net(dev);
 311         struct flow_block_offload bo;
 312
 313         nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
 314                                     basechain, &extack);
 315         mutex_lock(&net->nft.commit_mutex);
 316         list_del(&block_cb->driver_list);
 317         list_move(&block_cb->list, &bo.cb_list);
 318         nft_flow_offload_unbind(&bo, basechain);
 319         mutex_unlock(&net->nft.commit_mutex);
 320 }
 321
 322 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
 323                                       struct net_device *dev,
 324                                       enum flow_block_command cmd)
 325 {
 326         struct netlink_ext_ack extack = {};
 327         struct flow_block_offload bo;
 328         int err;
 329
 330         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
 331
 332         err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
 333                                           nft_indr_block_cleanup);
 334         if (err < 0)
 335                 return err;
 336
 337         if (list_empty(&bo.cb_list))
 338                 return -EOPNOTSUPP;
 339
 340         return nft_block_setup(basechain, &bo, cmd);
 341 }
 342
 343 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
 344                                  struct net_device *dev,
 345                                  enum flow_block_command cmd)
 346 {
 347         int err;
 348
 349         if (dev->netdev_ops->ndo_setup_tc)
 350                 err = nft_block_offload_cmd(basechain, dev, cmd);
 351         else
 352                 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
 353
 354         return err;
 355 }
 356
 357 static int nft_flow_block_chain(struct nft_base_chain *basechain,
 358                                 const struct net_device *this_dev,
 359                                 enum flow_block_command cmd)
 360 {
 361         struct net_device *dev;
 362         struct nft_hook *hook;
 363         int err, i = 0;
 364
 365         list_for_each_entry(hook, &basechain->hook_list, list) {
 366                 dev = hook->ops.dev;
 367                 if (this_dev && this_dev != dev)
 368                         continue;
 369
 370                 err = nft_chain_offload_cmd(basechain, dev, cmd);
 371                 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
 372                         if (!this_dev)
 373                                 goto err_flow_block;
 374
 375                         return err;
 376                 }
 377                 i++;
 378         }
 379
 380         return 0;
 381
 382 err_flow_block:
 383         list_for_each_entry(hook, &basechain->hook_list, list) {
 384                 if (i-- <= 0)
 385                         break;
 386
 387                 dev = hook->ops.dev;
 388                 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
 389         }
 390         return err;
 391 }
 392
 393 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
 394                                   enum flow_block_command cmd)
 395 {
 396         struct nft_base_chain *basechain;
 397         u8 policy;
 398
 399         if (!nft_is_base_chain(chain))
 400                 return -EOPNOTSUPP;
 401
 402         basechain = nft_base_chain(chain);
 403         policy = ppolicy ? *ppolicy : basechain->policy;
 404
 405         /* Only default policy to accept is supported for now. */
 406         if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
 407                 return -EOPNOTSUPP;
 408
 409         return nft_flow_block_chain(basechain, NULL, cmd);
 410 }
 411
 412 static void nft_flow_rule_offload_abort(struct net *net,
 413                                         struct nft_trans *trans)
 414 {
 415         int err = 0;
 416
 417         list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
 418                 if (trans->ctx.family != NFPROTO_NETDEV)
 419                         continue;
 420
 421                 switch (trans->msg_type) {
 422                 case NFT_MSG_NEWCHAIN:
 423                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 424                             nft_trans_chain_update(trans))
 425                                 continue;
 426
 427                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 428                                                      FLOW_BLOCK_UNBIND);
 429                         break;
 430                 case NFT_MSG_DELCHAIN:
 431                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 432                                 continue;
 433
 434                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 435                                                      FLOW_BLOCK_BIND);
 436                         break;
 437                 case NFT_MSG_NEWRULE:
 438                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 439                                 continue;
 440
 441                         err = nft_flow_offload_rule(trans->ctx.chain,
 442                                                     nft_trans_rule(trans),
 443                                                     NULL, FLOW_CLS_DESTROY);
 444                         break;
 445                 case NFT_MSG_DELRULE:
 446                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 447                                 continue;
 448
 449                         err = nft_flow_offload_rule(trans->ctx.chain,
 450                                                     nft_trans_rule(trans),
 451                                                     nft_trans_flow_rule(trans),
 452                                                     FLOW_CLS_REPLACE);
 453                         break;
 454                 }
 455
 456                 if (WARN_ON_ONCE(err))
 457                         break;
 458         }
 459 }
 460
 461 int nft_flow_rule_offload_commit(struct net *net)
 462 {
 463         struct nft_trans *trans;
 464         int err = 0;
 465         u8 policy;
 466
 467         list_for_each_entry(trans, &net->nft.commit_list, list) {
 468                 if (trans->ctx.family != NFPROTO_NETDEV)
 469                         continue;
 470
 471                 switch (trans->msg_type) {
 472                 case NFT_MSG_NEWCHAIN:
 473                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 474                             nft_trans_chain_update(trans))
 475                                 continue;
 476
 477                         policy = nft_trans_chain_policy(trans);
 478                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 479                                                      FLOW_BLOCK_BIND);
 480                         break;
 481                 case NFT_MSG_DELCHAIN:
 482                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 483                                 continue;
 484
 485                         policy = nft_trans_chain_policy(trans);
 486                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 487                                                      FLOW_BLOCK_UNBIND);
 488                         break;
 489                 case NFT_MSG_NEWRULE:
 490                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 491                                 continue;
 492
 493                         if (trans->ctx.flags & NLM_F_REPLACE ||
 494                             !(trans->ctx.flags & NLM_F_APPEND)) {
 495                                 err = -EOPNOTSUPP;
 496                                 break;
 497                         }
 498                         err = nft_flow_offload_rule(trans->ctx.chain,
 499                                                     nft_trans_rule(trans),
 500                                                     nft_trans_flow_rule(trans),
 501                                                     FLOW_CLS_REPLACE);
 502                         break;
 503                 case NFT_MSG_DELRULE:
 504                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 505                                 continue;
 506
 507                         err = nft_flow_offload_rule(trans->ctx.chain,
 508                                                     nft_trans_rule(trans),
 509                                                     NULL, FLOW_CLS_DESTROY);
 510                         break;
 511                 }
 512
 513                 if (err) {
 514                         nft_flow_rule_offload_abort(net, trans);
 515                         break;
 516                 }
 517         }
 518
 519         list_for_each_entry(trans, &net->nft.commit_list, list) {
 520                 if (trans->ctx.family != NFPROTO_NETDEV)
 521                         continue;
 522
 523                 switch (trans->msg_type) {
 524                 case NFT_MSG_NEWRULE:
 525                 case NFT_MSG_DELRULE:
 526                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 527                                 continue;
 528
 529                         nft_flow_rule_destroy(nft_trans_flow_rule(trans));
 530                         break;
 531                 default:
 532                         break;
 533                 }
 534         }
 535
 536         return err;
 537 }
 538
 539 static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
 540 {
 541         struct nft_base_chain *basechain;
 542         struct net *net = dev_net(dev);
 543         struct nft_hook *hook, *found;
 544         const struct nft_table *table;
 545         struct nft_chain *chain;
 546
 547         list_for_each_entry(table, &net->nft.tables, list) {
 548                 if (table->family != NFPROTO_NETDEV)
 549                         continue;
 550
 551                 list_for_each_entry(chain, &table->chains, list) {
 552                         if (!nft_is_base_chain(chain) ||
 553                             !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
 554                                 continue;
 555
 556                         found = NULL;
 557                         basechain = nft_base_chain(chain);
 558                         list_for_each_entry(hook, &basechain->hook_list, list) {
 559                                 if (hook->ops.dev != dev)
 560                                         continue;
 561
 562                                 found = hook;
 563                                 break;
 564                         }
 565                         if (!found)
 566                                 continue;
 567
 568                         return chain;
 569                 }
 570         }
 571
 572         return NULL;
 573 }
 574
 575 static int nft_offload_netdev_event(struct notifier_block *this,
 576                                     unsigned long event, void *ptr)
 577 {
 578         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 579         struct net *net = dev_net(dev);
 580         struct nft_chain *chain;
 581
 582         if (event != NETDEV_UNREGISTER)
 583                 return NOTIFY_DONE;
 584
 585         mutex_lock(&net->nft.commit_mutex);
 586         chain = __nft_offload_get_chain(dev);
 587         if (chain)
 588                 nft_flow_block_chain(nft_base_chain(chain), dev,
 589                                      FLOW_BLOCK_UNBIND);
 590
 591         mutex_unlock(&net->nft.commit_mutex);
 592
 593         return NOTIFY_DONE;
 594 }
 595
 596 static struct notifier_block nft_offload_netdev_notifier = {
 597         .notifier_call  = nft_offload_netdev_event,
 598 };
 599
 600 int nft_offload_init(void)
 601 {
 602         return register_netdevice_notifier(&nft_offload_netdev_notifier);
 603 }
 604
 605 void nft_offload_exit(void)
 606 {
 607         unregister_netdevice_notifier(&nft_offload_netdev_notifier);
 608 }