[mirror_ubuntu-bionic-kernel.git] / include / linux / vmstat.h

#ifndef _LINUX_VMSTAT_H
#define _LINUX_VMSTAT_H

#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <asm/atomic.h>

#ifdef CONFIG_ZONE_DMA
#define DMA_ZONE(xx) xx##_DMA,
#else
#define DMA_ZONE(xx)
#endif

#ifdef CONFIG_ZONE_DMA32
#define DMA32_ZONE(xx) xx##_DMA32,
#else
#define DMA32_ZONE(xx)
#endif

#ifdef CONFIG_HIGHMEM
#define HIGHMEM_ZONE(xx) , xx##_HIGH
#else
#define HIGHMEM_ZONE(xx)
#endif

#define FOR_ALL_ZONES(xx) DMA_ZONE(xx) DMA32_ZONE(xx) xx##_NORMAL HIGHMEM_ZONE(xx) , xx##_MOVABLE

enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
		FOR_ALL_ZONES(PGALLOC),
		PGFREE, PGACTIVATE, PGDEACTIVATE,
		PGFAULT, PGMAJFAULT,
		FOR_ALL_ZONES(PGREFILL),
		FOR_ALL_ZONES(PGSTEAL),
		FOR_ALL_ZONES(PGSCAN_KSWAPD),
		FOR_ALL_ZONES(PGSCAN_DIRECT),
		PGINODESTEAL, SLABS_SCANNED, KSWAPD_STEAL, KSWAPD_INODESTEAL,
		PAGEOUTRUN, ALLOCSTALL, PGROTATED,
		NR_VM_EVENT_ITEMS
};

#ifdef CONFIG_VM_EVENT_COUNTERS
/*
 * Light weight per cpu counter implementation.
 *
 * Counters should only be incremented and no critical kernel component
 * should rely on the counter values.
 *
 * Counters are handled completely inline. On many platforms the code
 * generated will simply be the increment of a global address.
 */

struct vm_event_state {
	unsigned long event[NR_VM_EVENT_ITEMS];
};

DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

static inline void __count_vm_event(enum vm_event_item item)
{
	__get_cpu_var(vm_event_states).event[item]++;
}

static inline void count_vm_event(enum vm_event_item item)
{
	get_cpu_var(vm_event_states).event[item]++;
	put_cpu();
}

static inline void __count_vm_events(enum vm_event_item item, long delta)
{
	__get_cpu_var(vm_event_states).event[item] += delta;
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
	get_cpu_var(vm_event_states).event[item] += delta;
	put_cpu();
}

extern void all_vm_events(unsigned long *);
#ifdef CONFIG_HOTPLUG
extern void vm_events_fold_cpu(int cpu);
#else
static inline void vm_events_fold_cpu(int cpu)
{
}
#endif

#else

/* Disable counters */
static inline void count_vm_event(enum vm_event_item item)
{
}
static inline void count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void __count_vm_event(enum vm_event_item item)
{
}
static inline void __count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void all_vm_events(unsigned long *ret)
{
}
static inline void vm_events_fold_cpu(int cpu)
{
}

#endif /* CONFIG_VM_EVENT_COUNTERS */

#define __count_zone_vm_events(item, zone, delta) \
		__count_vm_events(item##_NORMAL - ZONE_NORMAL + \
		zone_idx(zone), delta)

/*
 * Zone based page accounting with per cpu differentials.
 */
extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];

static inline void zone_page_state_add(long x, struct zone *zone,
				 enum zone_stat_item item)
{
	atomic_long_add(x, &zone->vm_stat[item]);
	atomic_long_add(x, &vm_stat[item]);
}

static inline unsigned long global_page_state(enum zone_stat_item item)
{
	long x = atomic_long_read(&vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long zone_page_state(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

#ifdef CONFIG_NUMA
/*
 * Determine the per node value of a stat item. This function
 * is called frequently in a NUMA machine, so try to be as
 * frugal as possible.
 */
static inline unsigned long node_page_state(int node,
				 enum zone_stat_item item)
{
	struct zone *zones = NODE_DATA(node)->node_zones;

	return
#ifdef CONFIG_ZONE_DMA
		zone_page_state(&zones[ZONE_DMA], item) +
#endif
#ifdef CONFIG_ZONE_DMA32
		zone_page_state(&zones[ZONE_DMA32], item) +
#endif
#ifdef CONFIG_HIGHMEM
		zone_page_state(&zones[ZONE_HIGHMEM], item) +
#endif
		zone_page_state(&zones[ZONE_NORMAL], item) +
		zone_page_state(&zones[ZONE_MOVABLE], item);
}

extern void zone_statistics(struct zonelist *, struct zone *);

#else

#define node_page_state(node, item) global_page_state(item)
#define zone_statistics(_zl,_z) do { } while (0)

#endif /* CONFIG_NUMA */

#define __add_zone_page_state(__z, __i, __d)	\
		__mod_zone_page_state(__z, __i, __d)
#define __sub_zone_page_state(__z, __i, __d)	\
		__mod_zone_page_state(__z, __i,-(__d))

#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))

static inline void zap_zone_vm_stats(struct zone *zone)
{
	memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
}

extern void inc_zone_state(struct zone *, enum zone_stat_item);

#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
void __dec_zone_page_state(struct page *, enum zone_stat_item);

void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
void inc_zone_page_state(struct page *, enum zone_stat_item);
void dec_zone_page_state(struct page *, enum zone_stat_item);

extern void inc_zone_state(struct zone *, enum zone_stat_item);
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
extern void dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_zone_state(struct zone *, enum zone_stat_item);

void refresh_cpu_vm_stats(int);
#else /* CONFIG_SMP */

/*
 * We do not maintain differentials in a single processor configuration.
 * The functions directly modify the zone and global counters.
 */
static inline void __mod_zone_page_state(struct zone *zone,
			enum zone_stat_item item, int delta)
{
	zone_page_state_add(delta, zone, item);
}

static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_inc(&zone->vm_stat[item]);
	atomic_long_inc(&vm_stat[item]);
}

static inline void __inc_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__inc_zone_state(page_zone(page), item);
}

static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_dec(&zone->vm_stat[item]);
	atomic_long_dec(&vm_stat[item]);
}

static inline void __dec_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	atomic_long_dec(&page_zone(page)->vm_stat[item]);
	atomic_long_dec(&vm_stat[item]);
}

/*
 * We only use atomic operations to update counters. So there is no need to
 * disable interrupts.
 */
#define inc_zone_page_state __inc_zone_page_state
#define dec_zone_page_state __dec_zone_page_state
#define mod_zone_page_state __mod_zone_page_state

static inline void refresh_cpu_vm_stats(int cpu) { }
#endif

#endif /* _LINUX_VMSTAT_H */
Commit	Line	Data
f6ac2354 CL	1	#ifndef _LINUX_VMSTAT_H
	2	#define _LINUX_VMSTAT_H
	3
	4	#include <linux/types.h>
	5	#include <linux/percpu.h>
96177299	6	#include <linux/mm.h>
2244b95a CL	7	#include <linux/mmzone.h>
2244b95a CL	8	#include <asm/atomic.h>
f6ac2354	9
4b51d669 CL	10	#ifdef CONFIG_ZONE_DMA
	11	#define DMA_ZONE(xx) xx##_DMA,
	12	#else
	13	#define DMA_ZONE(xx)
	14	#endif
	15
27bf71c2 CL	16	#ifdef CONFIG_ZONE_DMA32
	17	#define DMA32_ZONE(xx) xx##_DMA32,
	18	#else
	19	#define DMA32_ZONE(xx)
	20	#endif
	21
	22	#ifdef CONFIG_HIGHMEM
	23	#define HIGHMEM_ZONE(xx) , xx##_HIGH
	24	#else
	25	#define HIGHMEM_ZONE(xx)
	26	#endif
	27
2a1e274a	28	#define FOR_ALL_ZONES(xx) DMA_ZONE(xx) DMA32_ZONE(xx) xx##_NORMAL HIGHMEM_ZONE(xx) , xx##_MOVABLE
f8891e5e CL	29
	30	enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
	31	FOR_ALL_ZONES(PGALLOC),
	32	PGFREE, PGACTIVATE, PGDEACTIVATE,
	33	PGFAULT, PGMAJFAULT,
	34	FOR_ALL_ZONES(PGREFILL),
	35	FOR_ALL_ZONES(PGSTEAL),
	36	FOR_ALL_ZONES(PGSCAN_KSWAPD),
	37	FOR_ALL_ZONES(PGSCAN_DIRECT),
	38	PGINODESTEAL, SLABS_SCANNED, KSWAPD_STEAL, KSWAPD_INODESTEAL,
	39	PAGEOUTRUN, ALLOCSTALL, PGROTATED,
	40	NR_VM_EVENT_ITEMS
	41	};
	42
780a0656 AM	43	#ifdef CONFIG_VM_EVENT_COUNTERS
	44	/*
	45	* Light weight per cpu counter implementation.
	46	*
	47	* Counters should only be incremented and no critical kernel component
	48	* should rely on the counter values.
	49	*
	50	* Counters are handled completely inline. On many platforms the code
	51	* generated will simply be the increment of a global address.
	52	*/
	53
f8891e5e CL	54	struct vm_event_state {
f8891e5e CL	55	unsigned long event[NR_VM_EVENT_ITEMS];
f6ac2354 CL	56	};
f6ac2354 CL	57
f8891e5e CL	58	DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
	59
	60	static inline void __count_vm_event(enum vm_event_item item)
	61	{
38cbcdc0	62	__get_cpu_var(vm_event_states).event[item]++;
f8891e5e CL	63	}
	64
	65	static inline void count_vm_event(enum vm_event_item item)
	66	{
38cbcdc0	67	get_cpu_var(vm_event_states).event[item]++;
f8891e5e CL	68	put_cpu();
	69	}
	70
	71	static inline void __count_vm_events(enum vm_event_item item, long delta)
	72	{
38cbcdc0	73	__get_cpu_var(vm_event_states).event[item] += delta;
f8891e5e CL	74	}
	75
	76	static inline void count_vm_events(enum vm_event_item item, long delta)
	77	{
38cbcdc0	78	get_cpu_var(vm_event_states).event[item] += delta;
f8891e5e CL	79	put_cpu();
	80	}
	81
	82	extern void all_vm_events(unsigned long *);
e903387f	83	#ifdef CONFIG_HOTPLUG
f8891e5e	84	extern void vm_events_fold_cpu(int cpu);
e903387f MD	85	#else
	86	static inline void vm_events_fold_cpu(int cpu)
	87	{
	88	}
	89	#endif
f8891e5e CL	90
	91	#else
	92
	93	/* Disable counters */
780a0656 AM	94	static inline void count_vm_event(enum vm_event_item item)
	95	{
	96	}
	97	static inline void count_vm_events(enum vm_event_item item, long delta)
	98	{
	99	}
	100	static inline void __count_vm_event(enum vm_event_item item)
	101	{
	102	}
	103	static inline void __count_vm_events(enum vm_event_item item, long delta)
	104	{
	105	}
	106	static inline void all_vm_events(unsigned long *ret)
	107	{
	108	}
	109	static inline void vm_events_fold_cpu(int cpu)
	110	{
	111	}
f8891e5e CL	112
	113	#endif /* CONFIG_VM_EVENT_COUNTERS */
	114
	115	#define __count_zone_vm_events(item, zone, delta) \
4b51d669 CL	116	__count_vm_events(item##_NORMAL - ZONE_NORMAL + \
4b51d669 CL	117	zone_idx(zone), delta)
f6ac2354	118
2244b95a CL	119	/*
	120	* Zone based page accounting with per cpu differentials.
	121	*/
	122	extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
	123
	124	static inline void zone_page_state_add(long x, struct zone *zone,
	125	enum zone_stat_item item)
	126	{
	127	atomic_long_add(x, &zone->vm_stat[item]);
	128	atomic_long_add(x, &vm_stat[item]);
	129	}
	130
	131	static inline unsigned long global_page_state(enum zone_stat_item item)
	132	{
	133	long x = atomic_long_read(&vm_stat[item]);
	134	#ifdef CONFIG_SMP
	135	if (x < 0)
	136	x = 0;
	137	#endif
	138	return x;
	139	}
	140
	141	static inline unsigned long zone_page_state(struct zone *zone,
	142	enum zone_stat_item item)
	143	{
	144	long x = atomic_long_read(&zone->vm_stat[item]);
	145	#ifdef CONFIG_SMP
	146	if (x < 0)
	147	x = 0;
	148	#endif
	149	return x;
	150	}
	151
	152	#ifdef CONFIG_NUMA
	153	/*
	154	* Determine the per node value of a stat item. This function
	155	* is called frequently in a NUMA machine, so try to be as
	156	* frugal as possible.
	157	*/
	158	static inline unsigned long node_page_state(int node,
	159	enum zone_stat_item item)
	160	{
	161	struct zone *zones = NODE_DATA(node)->node_zones;
	162
	163	return
4b51d669 CL	164	#ifdef CONFIG_ZONE_DMA
	165	zone_page_state(&zones[ZONE_DMA], item) +
	166	#endif
fb0e7942	167	#ifdef CONFIG_ZONE_DMA32
2244b95a CL	168	zone_page_state(&zones[ZONE_DMA32], item) +
2244b95a CL	169	#endif
2244b95a CL	170	#ifdef CONFIG_HIGHMEM
	171	zone_page_state(&zones[ZONE_HIGHMEM], item) +
	172	#endif
2a1e274a MG	173	zone_page_state(&zones[ZONE_NORMAL], item) +
2a1e274a MG	174	zone_page_state(&zones[ZONE_MOVABLE], item);
2244b95a	175	}
ca889e6c CL	176
	177	extern void zone_statistics(struct zonelist , struct zone );
	178
2244b95a	179	#else
ca889e6c	180
2244b95a	181	#define node_page_state(node, item) global_page_state(item)
ca889e6c CL	182	#define zone_statistics(_zl,_z) do { } while (0)
	183
	184	#endif /* CONFIG_NUMA */
2244b95a CL	185
	186	#define __add_zone_page_state(__z, __i, __d) \
	187	__mod_zone_page_state(__z, __i, __d)
	188	#define __sub_zone_page_state(__z, __i, __d) \
	189	__mod_zone_page_state(__z, __i,-(__d))
	190
	191	#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
	192	#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
	193
	194	static inline void zap_zone_vm_stats(struct zone *zone)
	195	{
	196	memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
	197	}
	198
ca889e6c CL	199	extern void inc_zone_state(struct zone *, enum zone_stat_item);
ca889e6c CL	200
2244b95a CL	201	#ifdef CONFIG_SMP
	202	void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
	203	void __inc_zone_page_state(struct page *, enum zone_stat_item);
	204	void __dec_zone_page_state(struct page *, enum zone_stat_item);
f6ac2354	205
2244b95a CL	206	void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
	207	void inc_zone_page_state(struct page *, enum zone_stat_item);
	208	void dec_zone_page_state(struct page *, enum zone_stat_item);
	209
	210	extern void inc_zone_state(struct zone *, enum zone_stat_item);
c8785385 CL	211	extern void __inc_zone_state(struct zone *, enum zone_stat_item);
	212	extern void dec_zone_state(struct zone *, enum zone_stat_item);
	213	extern void __dec_zone_state(struct zone *, enum zone_stat_item);
2244b95a CL	214
2244b95a CL	215	void refresh_cpu_vm_stats(int);
2244b95a CL	216	#else /* CONFIG_SMP */
	217
	218	/*
	219	* We do not maintain differentials in a single processor configuration.
	220	* The functions directly modify the zone and global counters.
	221	*/
	222	static inline void __mod_zone_page_state(struct zone *zone,
	223	enum zone_stat_item item, int delta)
	224	{
	225	zone_page_state_add(delta, zone, item);
	226	}
	227
7f4599e9 CL	228	static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
	229	{
	230	atomic_long_inc(&zone->vm_stat[item]);
	231	atomic_long_inc(&vm_stat[item]);
	232	}
	233
2244b95a CL	234	static inline void __inc_zone_page_state(struct page *page,
	235	enum zone_stat_item item)
	236	{
7f4599e9	237	__inc_zone_state(page_zone(page), item);
2244b95a CL	238	}
2244b95a CL	239
c8785385 CL	240	static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
	241	{
	242	atomic_long_dec(&zone->vm_stat[item]);
	243	atomic_long_dec(&vm_stat[item]);
	244	}
	245
2244b95a CL	246	static inline void __dec_zone_page_state(struct page *page,
	247	enum zone_stat_item item)
	248	{
	249	atomic_long_dec(&page_zone(page)->vm_stat[item]);
	250	atomic_long_dec(&vm_stat[item]);
	251	}
	252
	253	/*
	254	* We only use atomic operations to update counters. So there is no need to
	255	* disable interrupts.
	256	*/
	257	#define inc_zone_page_state __inc_zone_page_state
	258	#define dec_zone_page_state __dec_zone_page_state
	259	#define mod_zone_page_state __mod_zone_page_state
	260
	261	static inline void refresh_cpu_vm_stats(int cpu) { }
2244b95a CL	262	#endif
	263
	264	#endif /* _LINUX_VMSTAT_H */