[mirror_ubuntu-bionic-kernel.git] / kernel / bpf / bpf_lru_list.h

/* Copyright (c) 2016 Facebook
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 */
#ifndef __BPF_LRU_LIST_H_
#define __BPF_LRU_LIST_H_

#include <linux/list.h>
#include <linux/spinlock_types.h>

#define NR_BPF_LRU_LIST_T	(3)
#define NR_BPF_LRU_LIST_COUNT	(2)
#define NR_BPF_LRU_LOCAL_LIST_T (2)
#define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T

enum bpf_lru_list_type {
	BPF_LRU_LIST_T_ACTIVE,
	BPF_LRU_LIST_T_INACTIVE,
	BPF_LRU_LIST_T_FREE,
	BPF_LRU_LOCAL_LIST_T_FREE,
	BPF_LRU_LOCAL_LIST_T_PENDING,
};

struct bpf_lru_node {
	struct list_head list;
	u16 cpu;
	u8 type;
	u8 ref;
};

struct bpf_lru_list {
	struct list_head lists[NR_BPF_LRU_LIST_T];
	unsigned int counts[NR_BPF_LRU_LIST_COUNT];
	/* The next inacitve list rotation starts from here */
	struct list_head *next_inactive_rotation;

	raw_spinlock_t lock ____cacheline_aligned_in_smp;
};

struct bpf_lru_locallist {
	struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T];
	u16 next_steal;
	raw_spinlock_t lock;
};

struct bpf_common_lru {
	struct bpf_lru_list lru_list;
	struct bpf_lru_locallist __percpu *local_list;
};

typedef bool (*del_from_htab_func)(void *arg, struct bpf_lru_node *node);

struct bpf_lru {
	union {
		struct bpf_common_lru common_lru;
		struct bpf_lru_list __percpu *percpu_lru;
	};
	del_from_htab_func del_from_htab;
	void *del_arg;
	unsigned int hash_offset;
	unsigned int nr_scans;
	bool percpu;
};

static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node)
{
	/* ref is an approximation on access frequency.  It does not
	 * have to be very accurate.  Hence, no protection is used.
	 */
	if (!node->ref)
		node->ref = 1;
}

int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset,
		 del_from_htab_func del_from_htab, void *delete_arg);
void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset,
		      u32 elem_size, u32 nr_elems);
void bpf_lru_destroy(struct bpf_lru *lru);
struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash);
void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node);
void bpf_lru_promote(struct bpf_lru *lru, struct bpf_lru_node *node);

#endif
Commit	Line	Data
3a08c2fd MKL	1	/* Copyright (c) 2016 Facebook
	2	*
	3	* This program is free software; you can redistribute it and/or
	4	* modify it under the terms of version 2 of the GNU General Public
	5	* License as published by the Free Software Foundation.
	6	*/
	7	#ifndef __BPF_LRU_LIST_H_
	8	#define __BPF_LRU_LIST_H_
	9
	10	#include <linux/list.h>
	11	#include <linux/spinlock_types.h>
	12
	13	#define NR_BPF_LRU_LIST_T (3)
	14	#define NR_BPF_LRU_LIST_COUNT (2)
	15	#define NR_BPF_LRU_LOCAL_LIST_T (2)
	16	#define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T
	17
	18	enum bpf_lru_list_type {
	19	BPF_LRU_LIST_T_ACTIVE,
	20	BPF_LRU_LIST_T_INACTIVE,
	21	BPF_LRU_LIST_T_FREE,
	22	BPF_LRU_LOCAL_LIST_T_FREE,
	23	BPF_LRU_LOCAL_LIST_T_PENDING,
	24	};
	25
	26	struct bpf_lru_node {
	27	struct list_head list;
	28	u16 cpu;
	29	u8 type;
	30	u8 ref;
	31	};
	32
	33	struct bpf_lru_list {
	34	struct list_head lists[NR_BPF_LRU_LIST_T];
	35	unsigned int counts[NR_BPF_LRU_LIST_COUNT];
	36	/* The next inacitve list rotation starts from here */
	37	struct list_head *next_inactive_rotation;
	38
	39	raw_spinlock_t lock ____cacheline_aligned_in_smp;
	40	};
	41
	42	struct bpf_lru_locallist {
	43	struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T];
	44	u16 next_steal;
	45	raw_spinlock_t lock;
	46	};
	47
	48	struct bpf_common_lru {
	49	struct bpf_lru_list lru_list;
	50	struct bpf_lru_locallist __percpu *local_list;
	51	};
	52
	53	typedef bool (del_from_htab_func)(void arg, struct bpf_lru_node *node);
	54
	55	struct bpf_lru {
961578b6 MKL	56	union {
	57	struct bpf_common_lru common_lru;
	58	struct bpf_lru_list __percpu *percpu_lru;
	59	};
3a08c2fd MKL	60	del_from_htab_func del_from_htab;
	61	void *del_arg;
	62	unsigned int hash_offset;
	63	unsigned int nr_scans;
961578b6	64	bool percpu;
3a08c2fd MKL	65	};
	66
	67	static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node)
	68	{
	69	/* ref is an approximation on access frequency. It does not
	70	* have to be very accurate. Hence, no protection is used.
	71	*/
bb9b9f88 MKL	72	if (!node->ref)
bb9b9f88 MKL	73	node->ref = 1;
3a08c2fd MKL	74	}
3a08c2fd MKL	75
961578b6	76	int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset,
3a08c2fd MKL	77	del_from_htab_func del_from_htab, void *delete_arg);
	78	void bpf_lru_populate(struct bpf_lru lru, void buf, u32 node_offset,
	79	u32 elem_size, u32 nr_elems);
	80	void bpf_lru_destroy(struct bpf_lru *lru);
	81	struct bpf_lru_node bpf_lru_pop_free(struct bpf_lru lru, u32 hash);
	82	void bpf_lru_push_free(struct bpf_lru lru, struct bpf_lru_node node);
	83	void bpf_lru_promote(struct bpf_lru lru, struct bpf_lru_node node);
	84
	85	#endif