bpf: fix kernel page fault in lpm map trie_get_next_key

Commit b471f2f1de8b ("bpf: implement MAP_GET_NEXT_KEY command
for LPM_TRIE map") introduces a bug likes below:

    if (!rcu_dereference(trie->root))
        return -ENOENT;
    if (!key || key->prefixlen > trie->max_prefixlen) {
        root = &trie->root;
        goto find_leftmost;
    }
    ......
  find_leftmost:
    for (node = rcu_dereference(*root); node;) {

In the code after label find_leftmost, it is assumed
that *root should not be NULL, but it is not true as
it is possbile trie->root is changed to NULL by an
asynchronous delete operation.

The issue is reported by syzbot and Eric Dumazet with the
below error log:
  ......
  kasan: CONFIG_KASAN_INLINE enabled
  kasan: GPF could be caused by NULL-ptr deref or user memory access
  general protection fault: 0000 [#1] SMP KASAN
  Dumping ftrace buffer:
     (ftrace buffer empty)
  Modules linked in:
  CPU: 1 PID: 8033 Comm: syz-executor3 Not tainted 4.15.0-rc8+ #4
  Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
  RIP: 0010:trie_get_next_key+0x3c2/0xf10 kernel/bpf/lpm_trie.c:682
  ......

This patch fixed the issue by use local rcu_dereferenced
pointer instead of *(&trie->root) later on.

Fixes: b471f2f1de8b ("bpf: implement MAP_GET_NEXT_KEY command or LPM_TRIE map")
Reported-by: syzbot <syzkaller@googlegroups.com>
Reported-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index 8f083ea9d4b9c11880a742f8aaaa3cd9527146bb..7b469d10d0e93a3e6b37fdce57f1e78082c6c9e4 100644 (file)
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -593,11 +593,10 @@ unlock:
 
 static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
 {
+       struct lpm_trie_node *node, *next_node = NULL, *parent, *search_root;
        struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
        struct bpf_lpm_trie_key *key = _key, *next_key = _next_key;
-       struct lpm_trie_node *node, *next_node = NULL, *parent;
        struct lpm_trie_node **node_stack = NULL;
-       struct lpm_trie_node __rcu **root;
        int err = 0, stack_ptr = -1;
        unsigned int next_bit;
        size_t matchlen;
@@ -614,14 +613,13 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
         */
 
        /* Empty trie */
-       if (!rcu_dereference(trie->root))
+       search_root = rcu_dereference(trie->root);
+       if (!search_root)
                return -ENOENT;
 
        /* For invalid key, find the leftmost node in the trie */
-       if (!key || key->prefixlen > trie->max_prefixlen) {
-               root = &trie->root;
+       if (!key || key->prefixlen > trie->max_prefixlen)
                goto find_leftmost;
-       }
 
        node_stack = kmalloc(trie->max_prefixlen * sizeof(struct lpm_trie_node *),
                             GFP_ATOMIC | __GFP_NOWARN);
@@ -629,7 +627,7 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
                return -ENOMEM;
 
        /* Try to find the exact node for the given key */
-       for (node = rcu_dereference(trie->root); node;) {
+       for (node = search_root; node;) {
                node_stack[++stack_ptr] = node;
                matchlen = longest_prefix_match(trie, node, key);
                if (node->prefixlen != matchlen ||
@@ -640,10 +638,8 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
                node = rcu_dereference(node->child[next_bit]);
        }
        if (!node || node->prefixlen != key->prefixlen ||
-           (node->flags & LPM_TREE_NODE_FLAG_IM)) {
-               root = &trie->root;
+           (node->flags & LPM_TREE_NODE_FLAG_IM))
                goto find_leftmost;
-       }
 
        /* The node with the exactly-matching key has been found,
         * find the first node in postorder after the matched node.
@@ -651,10 +647,10 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
        node = node_stack[stack_ptr];
        while (stack_ptr > 0) {
                parent = node_stack[stack_ptr - 1];
-               if (rcu_dereference(parent->child[0]) == node &&
-                   rcu_dereference(parent->child[1])) {
-                       root = &parent->child[1];
-                       goto find_leftmost;
+               if (rcu_dereference(parent->child[0]) == node) {
+                       search_root = rcu_dereference(parent->child[1]);
+                       if (search_root)
+                               goto find_leftmost;
                }
                if (!(parent->flags & LPM_TREE_NODE_FLAG_IM)) {
                        next_node = parent;
@@ -673,7 +669,7 @@ find_leftmost:
        /* Find the leftmost non-intermediate node, all intermediate nodes
         * have exact two children, so this function will never return NULL.
         */
-       for (node = rcu_dereference(*root); node;) {
+       for (node = search_root; node;) {
                if (!(node->flags & LPM_TREE_NODE_FLAG_IM))
                        next_node = node;
                node = rcu_dereference(node->child[0]);