net/netfilter/nf_tables_core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/kernel.h>
   9 #include <linux/module.h>
  10 #include <linux/init.h>
  11 #include <linux/list.h>
  12 #include <linux/rculist.h>
  13 #include <linux/skbuff.h>
  14 #include <linux/netlink.h>
  15 #include <linux/netfilter.h>
  16 #include <linux/static_key.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_tables_core.h>
  20 #include <net/netfilter/nf_tables.h>
  21 #include <net/netfilter/nf_log.h>
  22 #include <net/netfilter/nft_meta.h>
  23
  24 static noinline void __nft_trace_packet(struct nft_traceinfo *info,
  25                                         const struct nft_chain *chain,
  26                                         enum nft_trace_types type)
  27 {
  28         const struct nft_pktinfo *pkt = info->pkt;
  29
  30         if (!info->trace || !pkt->skb->nf_trace)
  31                 return;
  32
  33         info->chain = chain;
  34         info->type = type;
  35
  36         nft_trace_notify(info);
  37 }
  38
  39 static inline void nft_trace_packet(struct nft_traceinfo *info,
  40                                     const struct nft_chain *chain,
  41                                     const struct nft_rule *rule,
  42                                     enum nft_trace_types type)
  43 {
  44         if (static_branch_unlikely(&nft_trace_enabled)) {
  45                 info->rule = rule;
  46                 __nft_trace_packet(info, chain, type);
  47         }
  48 }
  49
  50 static void nft_cmp_fast_eval(const struct nft_expr *expr,
  51                               struct nft_regs *regs)
  52 {
  53         const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
  54         u32 mask = nft_cmp_fast_mask(priv->len);
  55
  56         if ((regs->data[priv->sreg] & mask) == priv->data)
  57                 return;
  58         regs->verdict.code = NFT_BREAK;
  59 }
  60
  61 static bool nft_payload_fast_eval(const struct nft_expr *expr,
  62                                   struct nft_regs *regs,
  63                                   const struct nft_pktinfo *pkt)
  64 {
  65         const struct nft_payload *priv = nft_expr_priv(expr);
  66         const struct sk_buff *skb = pkt->skb;
  67         u32 *dest = &regs->data[priv->dreg];
  68         unsigned char *ptr;
  69
  70         if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
  71                 ptr = skb_network_header(skb);
  72         else {
  73                 if (!pkt->tprot_set)
  74                         return false;
  75                 ptr = skb_network_header(skb) + pkt->xt.thoff;
  76         }
  77
  78         ptr += priv->offset;
  79
  80         if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
  81                 return false;
  82
  83         *dest = 0;
  84         if (priv->len == 2)
  85                 *(u16 *)dest = *(u16 *)ptr;
  86         else if (priv->len == 4)
  87                 *(u32 *)dest = *(u32 *)ptr;
  88         else
  89                 *(u8 *)dest = *(u8 *)ptr;
  90         return true;
  91 }
  92
  93 DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
  94
  95 static noinline void nft_update_chain_stats(const struct nft_chain *chain,
  96                                             const struct nft_pktinfo *pkt)
  97 {
  98         struct nft_base_chain *base_chain;
  99         struct nft_stats __percpu *pstats;
 100         struct nft_stats *stats;
 101
 102         base_chain = nft_base_chain(chain);
 103
 104         rcu_read_lock();
 105         pstats = READ_ONCE(base_chain->stats);
 106         if (pstats) {
 107                 local_bh_disable();
 108                 stats = this_cpu_ptr(pstats);
 109                 u64_stats_update_begin(&stats->syncp);
 110                 stats->pkts++;
 111                 stats->bytes += pkt->skb->len;
 112                 u64_stats_update_end(&stats->syncp);
 113                 local_bh_enable();
 114         }
 115         rcu_read_unlock();
 116 }
 117
 118 struct nft_jumpstack {
 119         const struct nft_chain  *chain;
 120         struct nft_rule *const *rules;
 121 };
 122
 123 static void expr_call_ops_eval(const struct nft_expr *expr,
 124                                struct nft_regs *regs,
 125                                struct nft_pktinfo *pkt)
 126 {
 127 #ifdef CONFIG_RETPOLINE
 128         unsigned long e = (unsigned long)expr->ops->eval;
 129 #define X(e, fun) \
 130         do { if ((e) == (unsigned long)(fun)) \
 131                 return fun(expr, regs, pkt); } while (0)
 132
 133         X(e, nft_payload_eval);
 134         X(e, nft_cmp_eval);
 135         X(e, nft_meta_get_eval);
 136         X(e, nft_lookup_eval);
 137         X(e, nft_range_eval);
 138         X(e, nft_immediate_eval);
 139         X(e, nft_byteorder_eval);
 140         X(e, nft_dynset_eval);
 141         X(e, nft_rt_get_eval);
 142         X(e, nft_bitwise_eval);
 143 #undef  X
 144 #endif /* CONFIG_RETPOLINE */
 145         expr->ops->eval(expr, regs, pkt);
 146 }
 147
 148 unsigned int
 149 nft_do_chain(struct nft_pktinfo *pkt, void *priv)
 150 {
 151         const struct nft_chain *chain = priv, *basechain = chain;
 152         const struct net *net = nft_net(pkt);
 153         struct nft_rule *const *rules;
 154         const struct nft_rule *rule;
 155         const struct nft_expr *expr, *last;
 156         struct nft_regs regs;
 157         unsigned int stackptr = 0;
 158         struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
 159         bool genbit = READ_ONCE(net->nft.gencursor);
 160         struct nft_traceinfo info;
 161
 162         info.trace = false;
 163         if (static_branch_unlikely(&nft_trace_enabled))
 164                 nft_trace_init(&info, pkt, &regs.verdict, basechain);
 165 do_chain:
 166         if (genbit)
 167                 rules = rcu_dereference(chain->rules_gen_1);
 168         else
 169                 rules = rcu_dereference(chain->rules_gen_0);
 170
 171 next_rule:
 172         rule = *rules;
 173         regs.verdict.code = NFT_CONTINUE;
 174         for (; *rules ; rules++) {
 175                 rule = *rules;
 176                 nft_rule_for_each_expr(expr, last, rule) {
 177                         if (expr->ops == &nft_cmp_fast_ops)
 178                                 nft_cmp_fast_eval(expr, &regs);
 179                         else if (expr->ops != &nft_payload_fast_ops ||
 180                                  !nft_payload_fast_eval(expr, &regs, pkt))
 181                                 expr_call_ops_eval(expr, &regs, pkt);
 182
 183                         if (regs.verdict.code != NFT_CONTINUE)
 184                                 break;
 185                 }
 186
 187                 switch (regs.verdict.code) {
 188                 case NFT_BREAK:
 189                         regs.verdict.code = NFT_CONTINUE;
 190                         continue;
 191                 case NFT_CONTINUE:
 192                         nft_trace_packet(&info, chain, rule,
 193                                          NFT_TRACETYPE_RULE);
 194                         continue;
 195                 }
 196                 break;
 197         }
 198
 199         switch (regs.verdict.code & NF_VERDICT_MASK) {
 200         case NF_ACCEPT:
 201         case NF_DROP:
 202         case NF_QUEUE:
 203         case NF_STOLEN:
 204                 nft_trace_packet(&info, chain, rule,
 205                                  NFT_TRACETYPE_RULE);
 206                 return regs.verdict.code;
 207         }
 208
 209         switch (regs.verdict.code) {
 210         case NFT_JUMP:
 211                 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
 212                         return NF_DROP;
 213                 jumpstack[stackptr].chain = chain;
 214                 jumpstack[stackptr].rules = rules + 1;
 215                 stackptr++;
 216                 fallthrough;
 217         case NFT_GOTO:
 218                 nft_trace_packet(&info, chain, rule,
 219                                  NFT_TRACETYPE_RULE);
 220
 221                 chain = regs.verdict.chain;
 222                 goto do_chain;
 223         case NFT_CONTINUE:
 224         case NFT_RETURN:
 225                 nft_trace_packet(&info, chain, rule,
 226                                  NFT_TRACETYPE_RETURN);
 227                 break;
 228         default:
 229                 WARN_ON(1);
 230         }
 231
 232         if (stackptr > 0) {
 233                 stackptr--;
 234                 chain = jumpstack[stackptr].chain;
 235                 rules = jumpstack[stackptr].rules;
 236                 goto next_rule;
 237         }
 238
 239         nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY);
 240
 241         if (static_branch_unlikely(&nft_counters_enabled))
 242                 nft_update_chain_stats(basechain, pkt);
 243
 244         return nft_base_chain(basechain)->policy;
 245 }
 246 EXPORT_SYMBOL_GPL(nft_do_chain);
 247
 248 static struct nft_expr_type *nft_basic_types[] = {
 249         &nft_imm_type,
 250         &nft_cmp_type,
 251         &nft_lookup_type,
 252         &nft_bitwise_type,
 253         &nft_byteorder_type,
 254         &nft_payload_type,
 255         &nft_dynset_type,
 256         &nft_range_type,
 257         &nft_meta_type,
 258         &nft_rt_type,
 259         &nft_exthdr_type,
 260 };
 261
 262 static struct nft_object_type *nft_basic_objects[] = {
 263 #ifdef CONFIG_NETWORK_SECMARK
 264         &nft_secmark_obj_type,
 265 #endif
 266 };
 267
 268 int __init nf_tables_core_module_init(void)
 269 {
 270         int err, i, j = 0;
 271
 272         for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
 273                 err = nft_register_obj(nft_basic_objects[i]);
 274                 if (err)
 275                         goto err;
 276         }
 277
 278         for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
 279                 err = nft_register_expr(nft_basic_types[j]);
 280                 if (err)
 281                         goto err;
 282         }
 283
 284         return 0;
 285
 286 err:
 287         while (j-- > 0)
 288                 nft_unregister_expr(nft_basic_types[j]);
 289
 290         while (i-- > 0)
 291                 nft_unregister_obj(nft_basic_objects[i]);
 292
 293         return err;
 294 }
 295
 296 void nf_tables_core_module_exit(void)
 297 {
 298         int i;
 299
 300         i = ARRAY_SIZE(nft_basic_types);
 301         while (i-- > 0)
 302                 nft_unregister_expr(nft_basic_types[i]);
 303
 304         i = ARRAY_SIZE(nft_basic_objects);
 305         while (i-- > 0)
 306                 nft_unregister_obj(nft_basic_objects[i]);
 307 }