fib: use indirect call wrappers in the most common fib_rules_ops

This avoids another inderect call per RX packet which save us around
20-40 ns.

Changelog:

v1 -> v2:
- Move declaraions to fib_rules.h to remove warnings

Reported-by: kernel test robot <lkp@intel.com>
Signed-off-by: Brian Vazquez <brianvv@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h
index a259050f84afcda7076a894226ed268bc116a8f0..4b10676c69d1917e4c30e086bf8f00b1e0f37ed4 100644 (file)
--- a/include/net/fib_rules.h
+++ b/include/net/fib_rules.h
@@ -10,6 +10,7 @@
 #include <net/flow.h>
 #include <net/rtnetlink.h>
 #include <net/fib_notifier.h>
+#include <linux/indirect_call_wrapper.h>
 
 struct fib_kuid_range {
        kuid_t start;
@@ -203,4 +204,21 @@ int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh,
                   struct netlink_ext_ack *extack);
 int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh,
                   struct netlink_ext_ack *extack);
+
+INDIRECT_CALLABLE_DECLARE(int fib6_rule_match(struct fib_rule *rule,
+                                           struct flowi *fl, int flags));
+INDIRECT_CALLABLE_DECLARE(int fib4_rule_match(struct fib_rule *rule,
+                                           struct flowi *fl, int flags));
+
+INDIRECT_CALLABLE_DECLARE(int fib6_rule_action(struct fib_rule *rule,
+                           struct flowi *flp, int flags,
+                           struct fib_lookup_arg *arg));
+INDIRECT_CALLABLE_DECLARE(int fib4_rule_action(struct fib_rule *rule,
+                           struct flowi *flp, int flags,
+                           struct fib_lookup_arg *arg));
+
+INDIRECT_CALLABLE_DECLARE(bool fib6_rule_suppress(struct fib_rule *rule,
+                                               struct fib_lookup_arg *arg));
+INDIRECT_CALLABLE_DECLARE(bool fib4_rule_suppress(struct fib_rule *rule,
+                                               struct fib_lookup_arg *arg));
 #endif

diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index bd7eba9066f8d6b39d04be38611535f238bf5cd0..e7a8f87b0bb2b471265be7e6231b9fba40818679 100644 (file)
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -14,6 +14,7 @@
 #include <net/sock.h>
 #include <net/fib_rules.h>
 #include <net/ip_tunnels.h>
+#include <linux/indirect_call_wrapper.h>
 
 static const struct fib_kuid_range fib_kuid_range_unset = {
        KUIDT_INIT(0),
@@ -267,7 +268,10 @@ static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
            uid_gt(fl->flowi_uid, rule->uid_range.end))
                goto out;
 
-       ret = ops->match(rule, fl, flags);
+       ret = INDIRECT_CALL_INET(ops->match,
+                                fib6_rule_match,
+                                fib4_rule_match,
+                                rule, fl, flags);
 out:
        return (rule->flags & FIB_RULE_INVERT) ? !ret : ret;
 }
@@ -298,9 +302,15 @@ jumped:
                } else if (rule->action == FR_ACT_NOP)
                        continue;
                else
-                       err = ops->action(rule, fl, flags, arg);
-
-               if (!err && ops->suppress && ops->suppress(rule, arg))
+                       err = INDIRECT_CALL_INET(ops->action,
+                                                fib6_rule_action,
+                                                fib4_rule_action,
+                                                rule, fl, flags, arg);
+
+               if (!err && ops->suppress && INDIRECT_CALL_INET(ops->suppress,
+                                                               fib6_rule_suppress,
+                                                               fib4_rule_suppress,
+                                                               rule, arg))
                        continue;
 
                if (err != -EAGAIN) {

diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c
index f99e3bac5cab20951d150b30efc438a7c1e4d404..ce54a30c2ef1e8e79c8922be5eee35055fa51178 100644 (file)
--- a/net/ipv4/fib_rules.c
+++ b/net/ipv4/fib_rules.c
@@ -29,6 +29,7 @@
 #include <net/ip_fib.h>
 #include <net/nexthop.h>
 #include <net/fib_rules.h>
+#include <linux/indirect_call_wrapper.h>
 
 struct fib4_rule {
        struct fib_rule         common;
@@ -103,8 +104,9 @@ int __fib_lookup(struct net *net, struct flowi4 *flp,
 }
 EXPORT_SYMBOL_GPL(__fib_lookup);
 
-static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
-                           int flags, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule,
+                                            struct flowi *flp, int flags,
+                                            struct fib_lookup_arg *arg)
 {
        int err = -EAGAIN;
        struct fib_table *tbl;
@@ -138,7 +140,8 @@ static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
        return err;
 }
 
-static bool fib4_rule_suppress(struct fib_rule *rule, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule,
+                                               struct fib_lookup_arg *arg)
 {
        struct fib_result *result = (struct fib_result *) arg->result;
        struct net_device *dev = NULL;
@@ -169,7 +172,8 @@ suppress_route:
        return true;
 }
 
-static int fib4_rule_match(struct fib_rule *rule, struct flowi *fl, int flags)
+INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule,
+                                           struct flowi *fl, int flags)
 {
        struct fib4_rule *r = (struct fib4_rule *) rule;
        struct flowi4 *fl4 = &fl->u.ip4;

diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c
index 6053ef851555537ba90c987cb336619beaf48edb..8f9a83314de7d5cd8f66e2bb404d3be132c7c64f 100644 (file)
--- a/net/ipv6/fib6_rules.c
+++ b/net/ipv6/fib6_rules.c
@@ -13,6 +13,7 @@
 #include <linux/netdevice.h>
 #include <linux/notifier.h>
 #include <linux/export.h>
+#include <linux/indirect_call_wrapper.h>
 
 #include <net/fib_rules.h>
 #include <net/ipv6.h>
@@ -255,8 +256,9 @@ out:
        return err;
 }
 
-static int fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
-                           int flags, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE int fib6_rule_action(struct fib_rule *rule,
+                                            struct flowi *flp, int flags,
+                                            struct fib_lookup_arg *arg)
 {
        if (arg->lookup_ptr == fib6_table_lookup)
                return fib6_rule_action_alt(rule, flp, flags, arg);
@@ -264,7 +266,8 @@ static int fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
        return __fib6_rule_action(rule, flp, flags, arg);
 }
 
-static bool fib6_rule_suppress(struct fib_rule *rule, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
+                                               struct fib_lookup_arg *arg)
 {
        struct fib6_result *res = arg->result;
        struct rt6_info *rt = res->rt6;
@@ -296,7 +299,8 @@ suppress_route:
        return true;
 }
 
-static int fib6_rule_match(struct fib_rule *rule, struct flowi *fl, int flags)
+INDIRECT_CALLABLE_SCOPE int fib6_rule_match(struct fib_rule *rule,
+                                           struct flowi *fl, int flags)
 {
        struct fib6_rule *r = (struct fib6_rule *) rule;
        struct flowi6 *fl6 = &fl->u.ip6;