@@ -593,11 +593,10 @@ static void trie_free(struct bpf_map *map)
static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
{
+ struct lpm_trie_node *node, *next_node = NULL, *parent, *search_root;
struct lpm_trie *trie = container_of(map, struct lpm_trie, map);
struct bpf_lpm_trie_key *key = _key, *next_key = _next_key;
- struct lpm_trie_node *node, *next_node = NULL, *parent;
struct lpm_trie_node **node_stack = NULL;
- struct lpm_trie_node __rcu **root;
int err = 0, stack_ptr = -1;
unsigned int next_bit;
size_t matchlen;
@@ -614,14 +613,13 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
*/
/* Empty trie */
- if (!rcu_dereference(trie->root))
+ search_root = rcu_dereference(trie->root);
+ if (!search_root)
return -ENOENT;
/* For invalid key, find the leftmost node in the trie */
- if (!key || key->prefixlen > trie->max_prefixlen) {
- root = &trie->root;
+ if (!key || key->prefixlen > trie->max_prefixlen)
goto find_leftmost;
- }
node_stack = kmalloc(trie->max_prefixlen * sizeof(struct lpm_trie_node *),
GFP_ATOMIC | __GFP_NOWARN);
@@ -629,7 +627,7 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
return -ENOMEM;
/* Try to find the exact node for the given key */
- for (node = rcu_dereference(trie->root); node;) {
+ for (node = search_root; node;) {
node_stack[++stack_ptr] = node;
matchlen = longest_prefix_match(trie, node, key);
if (node->prefixlen != matchlen ||
@@ -640,10 +638,8 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
node = rcu_dereference(node->child[next_bit]);
}
if (!node || node->prefixlen != key->prefixlen ||
- (node->flags & LPM_TREE_NODE_FLAG_IM)) {
- root = &trie->root;
+ (node->flags & LPM_TREE_NODE_FLAG_IM))
goto find_leftmost;
- }
/* The node with the exactly-matching key has been found,
* find the first node in postorder after the matched node.
@@ -651,10 +647,10 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
node = node_stack[stack_ptr];
while (stack_ptr > 0) {
parent = node_stack[stack_ptr - 1];
- if (rcu_dereference(parent->child[0]) == node &&
- rcu_dereference(parent->child[1])) {
- root = &parent->child[1];
- goto find_leftmost;
+ if (rcu_dereference(parent->child[0]) == node) {
+ search_root = rcu_dereference(parent->child[1]);
+ if (search_root)
+ goto find_leftmost;
}
if (!(parent->flags & LPM_TREE_NODE_FLAG_IM)) {
next_node = parent;
@@ -673,7 +669,7 @@ static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
/* Find the leftmost non-intermediate node, all intermediate nodes
* have exact two children, so this function will never return NULL.
*/
- for (node = rcu_dereference(*root); node;) {
+ for (node = search_root; node;) {
if (!(node->flags & LPM_TREE_NODE_FLAG_IM))
next_node = node;
node = rcu_dereference(node->child[0]);
Commit b471f2f1de8b ("bpf: implement MAP_GET_NEXT_KEY command for LPM_TRIE map") introduces a bug likes below: if (!rcu_dereference(trie->root)) return -ENOENT; if (!key || key->prefixlen > trie->max_prefixlen) { root = &trie->root; goto find_leftmost; } ...... find_leftmost: for (node = rcu_dereference(*root); node;) { In the code after label find_leftmost, it is assumed that *root should not be NULL, but it is not true as it is possbile trie->root is changed to NULL by an asynchronous delete operation. The issue is reported by syzbot and Eric Dumazet with the below error log: ...... kasan: CONFIG_KASAN_INLINE enabled kasan: GPF could be caused by NULL-ptr deref or user memory access general protection fault: 0000 [#1] SMP KASAN Dumping ftrace buffer: (ftrace buffer empty) Modules linked in: CPU: 1 PID: 8033 Comm: syz-executor3 Not tainted 4.15.0-rc8+ #4 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:trie_get_next_key+0x3c2/0xf10 kernel/bpf/lpm_trie.c:682 ...... This patch fixed the issue by use local rcu_dereferenced pointer instead of *(&trie->root) later on. Fixes: b471f2f1de8b ("bpf: implement MAP_GET_NEXT_KEY command or LPM_TRIE map") Reported-by: syzbot <syzkaller@googlegroups.com> Reported-by: Eric Dumazet <edumazet@google.com> Signed-off-by: Yonghong Song <yhs@fb.com> --- kernel/bpf/lpm_trie.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-)