[mirror_ubuntu-bionic-kernel.git] / include / linux / vmstat.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_VMSTAT_H
#define _LINUX_VMSTAT_H

#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/mmzone.h>
#include <linux/vm_event_item.h>
#include <linux/atomic.h>
#include <linux/static_key.h>

extern int sysctl_stat_interval;

#ifdef CONFIG_NUMA
#define ENABLE_NUMA_STAT   1
#define DISABLE_NUMA_STAT   0
extern int sysctl_vm_numa_stat;
DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
		int write, void __user *buffer, size_t *length, loff_t *ppos);
#endif

#ifdef CONFIG_VM_EVENT_COUNTERS
/*
 * Light weight per cpu counter implementation.
 *
 * Counters should only be incremented and no critical kernel component
 * should rely on the counter values.
 *
 * Counters are handled completely inline. On many platforms the code
 * generated will simply be the increment of a global address.
 */

struct vm_event_state {
	unsigned long event[NR_VM_EVENT_ITEMS];
};

DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

/*
 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
 * local_irq_disable overhead.
 */
static inline void __count_vm_event(enum vm_event_item item)
{
	raw_cpu_inc(vm_event_states.event[item]);
}

static inline void count_vm_event(enum vm_event_item item)
{
	this_cpu_inc(vm_event_states.event[item]);
}

static inline void __count_vm_events(enum vm_event_item item, long delta)
{
	raw_cpu_add(vm_event_states.event[item], delta);
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
	this_cpu_add(vm_event_states.event[item], delta);
}

extern void all_vm_events(unsigned long *);

extern void vm_events_fold_cpu(int cpu);

#else

/* Disable counters */
static inline void count_vm_event(enum vm_event_item item)
{
}
static inline void count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void __count_vm_event(enum vm_event_item item)
{
}
static inline void __count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void all_vm_events(unsigned long *ret)
{
}
static inline void vm_events_fold_cpu(int cpu)
{
}

#endif /* CONFIG_VM_EVENT_COUNTERS */

#ifdef CONFIG_NUMA_BALANCING
#define count_vm_numa_event(x)     count_vm_event(x)
#define count_vm_numa_events(x, y) count_vm_events(x, y)
#else
#define count_vm_numa_event(x) do {} while (0)
#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
#endif /* CONFIG_NUMA_BALANCING */

#ifdef CONFIG_DEBUG_TLBFLUSH
#define count_vm_tlb_event(x)	   count_vm_event(x)
#define count_vm_tlb_events(x, y)  count_vm_events(x, y)
#else
#define count_vm_tlb_event(x)     do {} while (0)
#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
#endif

#ifdef CONFIG_DEBUG_VM_VMACACHE
#define count_vm_vmacache_event(x) count_vm_event(x)
#else
#define count_vm_vmacache_event(x) do {} while (0)
#endif

#define __count_zid_vm_events(item, zid, delta) \
	__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)

/*
 * Zone and node-based page accounting with per cpu differentials.
 */
extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];

#ifdef CONFIG_NUMA
static inline void zone_numa_state_add(long x, struct zone *zone,
				 enum numa_stat_item item)
{
	atomic_long_add(x, &zone->vm_numa_stat[item]);
	atomic_long_add(x, &vm_numa_stat[item]);
}

static inline unsigned long global_numa_state(enum numa_stat_item item)
{
	long x = atomic_long_read(&vm_numa_stat[item]);

	return x;
}

static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
					enum numa_stat_item item)
{
	long x = atomic_long_read(&zone->vm_numa_stat[item]);
	int cpu;

	for_each_online_cpu(cpu)
		x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];

	return x;
}
#endif /* CONFIG_NUMA */

static inline void zone_page_state_add(long x, struct zone *zone,
				 enum zone_stat_item item)
{
	atomic_long_add(x, &zone->vm_stat[item]);
	atomic_long_add(x, &vm_zone_stat[item]);
}

static inline void node_page_state_add(long x, struct pglist_data *pgdat,
				 enum node_stat_item item)
{
	atomic_long_add(x, &pgdat->vm_stat[item]);
	atomic_long_add(x, &vm_node_stat[item]);
}

static inline unsigned long global_zone_page_state(enum zone_stat_item item)
{
	long x = atomic_long_read(&vm_zone_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long global_node_page_state(enum node_stat_item item)
{
	long x = atomic_long_read(&vm_node_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long zone_page_state(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

/*
 * More accurate version that also considers the currently pending
 * deltas. For that we need to loop over all cpus to find the current
 * deltas. There is no synchronization so the result cannot be
 * exactly accurate either.
 */
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);

#ifdef CONFIG_SMP
	int cpu;
	for_each_online_cpu(cpu)
		x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];

	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long node_page_state_snapshot(pg_data_t *pgdat,
					enum node_stat_item item)
{
	long x = atomic_long_read(&pgdat->vm_stat[item]);

#ifdef CONFIG_SMP
	int cpu;
	for_each_online_cpu(cpu)
		x += per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->vm_node_stat_diff[item];

	if (x < 0)
		x = 0;
#endif
	return x;
}


#ifdef CONFIG_NUMA
extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
extern unsigned long sum_zone_node_page_state(int node,
					      enum zone_stat_item item);
extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
extern unsigned long node_page_state(struct pglist_data *pgdat,
						enum node_stat_item item);
#else
#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
#define node_page_state(node, item) global_node_page_state(item)
#endif /* CONFIG_NUMA */

#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
#define add_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, __d)
#define sub_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, -(__d))

#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
void __dec_zone_page_state(struct page *, enum zone_stat_item);

void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
void __inc_node_page_state(struct page *, enum node_stat_item);
void __dec_node_page_state(struct page *, enum node_stat_item);

void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
void inc_zone_page_state(struct page *, enum zone_stat_item);
void dec_zone_page_state(struct page *, enum zone_stat_item);

void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
void inc_node_page_state(struct page *, enum node_stat_item);
void dec_node_page_state(struct page *, enum node_stat_item);

extern void inc_node_state(struct pglist_data *, enum node_stat_item);
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
extern void dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_node_state(struct pglist_data *, enum node_stat_item);

void quiet_vmstat(void);
void cpu_vm_stats_fold(int cpu);
void refresh_zone_stat_thresholds(void);

struct ctl_table;
int vmstat_refresh(struct ctl_table *, int write,
		   void __user *buffer, size_t *lenp, loff_t *ppos);

void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);

int calculate_pressure_threshold(struct zone *zone);
int calculate_normal_threshold(struct zone *zone);
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
				int (*calculate_pressure)(struct zone *));
#else /* CONFIG_SMP */

/*
 * We do not maintain differentials in a single processor configuration.
 * The functions directly modify the zone and global counters.
 */
static inline void __mod_zone_page_state(struct zone *zone,
			enum zone_stat_item item, long delta)
{
	zone_page_state_add(delta, zone, item);
}

static inline void __mod_node_page_state(struct pglist_data *pgdat,
			enum node_stat_item item, int delta)
{
	node_page_state_add(delta, pgdat, item);
}

static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_inc(&zone->vm_stat[item]);
	atomic_long_inc(&vm_zone_stat[item]);
}

static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
	atomic_long_inc(&pgdat->vm_stat[item]);
	atomic_long_inc(&vm_node_stat[item]);
}

static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_dec(&zone->vm_stat[item]);
	atomic_long_dec(&vm_zone_stat[item]);
}

static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
	atomic_long_dec(&pgdat->vm_stat[item]);
	atomic_long_dec(&vm_node_stat[item]);
}

static inline void __inc_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__inc_zone_state(page_zone(page), item);
}

static inline void __inc_node_page_state(struct page *page,
			enum node_stat_item item)
{
	__inc_node_state(page_pgdat(page), item);
}


static inline void __dec_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
	__dec_zone_state(page_zone(page), item);
}

static inline void __dec_node_page_state(struct page *page,
			enum node_stat_item item)
{
	__dec_node_state(page_pgdat(page), item);
}


/*
 * We only use atomic operations to update counters. So there is no need to
 * disable interrupts.
 */
#define inc_zone_page_state __inc_zone_page_state
#define dec_zone_page_state __dec_zone_page_state
#define mod_zone_page_state __mod_zone_page_state

#define inc_node_page_state __inc_node_page_state
#define dec_node_page_state __dec_node_page_state
#define mod_node_page_state __mod_node_page_state

#define inc_zone_state __inc_zone_state
#define inc_node_state __inc_node_state
#define dec_zone_state __dec_zone_state

#define set_pgdat_percpu_threshold(pgdat, callback) { }

static inline void refresh_zone_stat_thresholds(void) { }
static inline void cpu_vm_stats_fold(int cpu) { }
static inline void quiet_vmstat(void) { }

static inline void drain_zonestat(struct zone *zone,
			struct per_cpu_pageset *pset) { }
#endif		/* CONFIG_SMP */

static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
					     int migratetype)
{
	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
	if (is_migrate_cma(migratetype))
		__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
}

extern const char * const vmstat_text[];

#endif /* _LINUX_VMSTAT_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
f6ac2354 CL	2	#ifndef _LINUX_VMSTAT_H
	3	#define _LINUX_VMSTAT_H
	4
	5	#include <linux/types.h>
	6	#include <linux/percpu.h>
2244b95a	7	#include <linux/mmzone.h>
f042e707	8	#include <linux/vm_event_item.h>
60063497	9	#include <linux/atomic.h>
4518085e	10	#include <linux/static_key.h>
f6ac2354	11
c748e134 AB	12	extern int sysctl_stat_interval;
c748e134 AB	13
4518085e KW	14	#ifdef CONFIG_NUMA
	15	#define ENABLE_NUMA_STAT 1
	16	#define DISABLE_NUMA_STAT 0
	17	extern int sysctl_vm_numa_stat;
	18	DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
	19	extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
	20	int write, void __user buffer, size_t length, loff_t *ppos);
	21	#endif
	22
780a0656 AM	23	#ifdef CONFIG_VM_EVENT_COUNTERS
	24	/*
	25	* Light weight per cpu counter implementation.
	26	*
	27	* Counters should only be incremented and no critical kernel component
	28	* should rely on the counter values.
	29	*
	30	* Counters are handled completely inline. On many platforms the code
	31	* generated will simply be the increment of a global address.
	32	*/
	33
f8891e5e CL	34	struct vm_event_state {
f8891e5e CL	35	unsigned long event[NR_VM_EVENT_ITEMS];
f6ac2354 CL	36	};
f6ac2354 CL	37
f8891e5e CL	38	DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
f8891e5e CL	39
293b6a4c CL	40	/*
	41	* vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
	42	* local_irq_disable overhead.
	43	*/
f8891e5e CL	44	static inline void __count_vm_event(enum vm_event_item item)
f8891e5e CL	45	{
293b6a4c	46	raw_cpu_inc(vm_event_states.event[item]);
f8891e5e CL	47	}
	48
	49	static inline void count_vm_event(enum vm_event_item item)
	50	{
dd17c8f7	51	this_cpu_inc(vm_event_states.event[item]);
f8891e5e CL	52	}
	53
	54	static inline void __count_vm_events(enum vm_event_item item, long delta)
	55	{
293b6a4c	56	raw_cpu_add(vm_event_states.event[item], delta);
f8891e5e CL	57	}
	58
	59	static inline void count_vm_events(enum vm_event_item item, long delta)
	60	{
dd17c8f7	61	this_cpu_add(vm_event_states.event[item], delta);
f8891e5e CL	62	}
	63
	64	extern void all_vm_events(unsigned long *);
f1cb0879	65
f8891e5e CL	66	extern void vm_events_fold_cpu(int cpu);
	67
	68	#else
	69
	70	/* Disable counters */
780a0656 AM	71	static inline void count_vm_event(enum vm_event_item item)
	72	{
	73	}
	74	static inline void count_vm_events(enum vm_event_item item, long delta)
	75	{
	76	}
	77	static inline void __count_vm_event(enum vm_event_item item)
	78	{
	79	}
	80	static inline void __count_vm_events(enum vm_event_item item, long delta)
	81	{
	82	}
	83	static inline void all_vm_events(unsigned long *ret)
	84	{
	85	}
	86	static inline void vm_events_fold_cpu(int cpu)
	87	{
	88	}
f8891e5e CL	89
	90	#endif /* CONFIG_VM_EVENT_COUNTERS */
	91
03c5a6e1 MG	92	#ifdef CONFIG_NUMA_BALANCING
	93	#define count_vm_numa_event(x) count_vm_event(x)
	94	#define count_vm_numa_events(x, y) count_vm_events(x, y)
	95	#else
	96	#define count_vm_numa_event(x) do {} while (0)
3c0ff468	97	#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
03c5a6e1 MG	98	#endif /* CONFIG_NUMA_BALANCING */
03c5a6e1 MG	99
ec659934 MG	100	#ifdef CONFIG_DEBUG_TLBFLUSH
	101	#define count_vm_tlb_event(x) count_vm_event(x)
	102	#define count_vm_tlb_events(x, y) count_vm_events(x, y)
	103	#else
	104	#define count_vm_tlb_event(x) do {} while (0)
	105	#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
	106	#endif
	107
4f115147 DB	108	#ifdef CONFIG_DEBUG_VM_VMACACHE
	109	#define count_vm_vmacache_event(x) count_vm_event(x)
	110	#else
	111	#define count_vm_vmacache_event(x) do {} while (0)
	112	#endif
	113
16709d1d MG	114	#define __count_zid_vm_events(item, zid, delta) \
16709d1d MG	115	__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
f6ac2354	116
2244b95a	117	/*
75ef7184	118	* Zone and node-based page accounting with per cpu differentials.
2244b95a	119	*/
75ef7184	120	extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
3a321d2a	121	extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
75ef7184	122	extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
2244b95a	123
3a321d2a KW	124	#ifdef CONFIG_NUMA
	125	static inline void zone_numa_state_add(long x, struct zone *zone,
	126	enum numa_stat_item item)
	127	{
	128	atomic_long_add(x, &zone->vm_numa_stat[item]);
	129	atomic_long_add(x, &vm_numa_stat[item]);
	130	}
	131
	132	static inline unsigned long global_numa_state(enum numa_stat_item item)
	133	{
	134	long x = atomic_long_read(&vm_numa_stat[item]);
	135
	136	return x;
	137	}
	138
63803222	139	static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
3a321d2a KW	140	enum numa_stat_item item)
	141	{
	142	long x = atomic_long_read(&zone->vm_numa_stat[item]);
63803222 KW	143	int cpu;
	144
	145	for_each_online_cpu(cpu)
	146	x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
3a321d2a KW	147
	148	return x;
	149	}
	150	#endif /* CONFIG_NUMA */
	151
2244b95a CL	152	static inline void zone_page_state_add(long x, struct zone *zone,
	153	enum zone_stat_item item)
	154	{
	155	atomic_long_add(x, &zone->vm_stat[item]);
75ef7184 MG	156	atomic_long_add(x, &vm_zone_stat[item]);
	157	}
	158
	159	static inline void node_page_state_add(long x, struct pglist_data *pgdat,
	160	enum node_stat_item item)
	161	{
	162	atomic_long_add(x, &pgdat->vm_stat[item]);
	163	atomic_long_add(x, &vm_node_stat[item]);
2244b95a CL	164	}
2244b95a CL	165
c41f012a	166	static inline unsigned long global_zone_page_state(enum zone_stat_item item)
2244b95a	167	{
75ef7184 MG	168	long x = atomic_long_read(&vm_zone_stat[item]);
	169	#ifdef CONFIG_SMP
	170	if (x < 0)
	171	x = 0;
	172	#endif
	173	return x;
	174	}
	175
	176	static inline unsigned long global_node_page_state(enum node_stat_item item)
	177	{
	178	long x = atomic_long_read(&vm_node_stat[item]);
2244b95a CL	179	#ifdef CONFIG_SMP
	180	if (x < 0)
	181	x = 0;
	182	#endif
	183	return x;
	184	}
	185
	186	static inline unsigned long zone_page_state(struct zone *zone,
	187	enum zone_stat_item item)
	188	{
	189	long x = atomic_long_read(&zone->vm_stat[item]);
	190	#ifdef CONFIG_SMP
	191	if (x < 0)
	192	x = 0;
	193	#endif
	194	return x;
	195	}
	196
aa454840 CL	197	/*
	198	* More accurate version that also considers the currently pending
	199	* deltas. For that we need to loop over all cpus to find the current
	200	* deltas. There is no synchronization so the result cannot be
	201	* exactly accurate either.
	202	*/
	203	static inline unsigned long zone_page_state_snapshot(struct zone *zone,
	204	enum zone_stat_item item)
	205	{
	206	long x = atomic_long_read(&zone->vm_stat[item]);
	207
	208	#ifdef CONFIG_SMP
	209	int cpu;
	210	for_each_online_cpu(cpu)
	211	x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
	212
	213	if (x < 0)
	214	x = 0;
	215	#endif
	216	return x;
	217	}
	218
599d0c95 MG	219	static inline unsigned long node_page_state_snapshot(pg_data_t *pgdat,
	220	enum node_stat_item item)
	221	{
	222	long x = atomic_long_read(&pgdat->vm_stat[item]);
	223
	224	#ifdef CONFIG_SMP
	225	int cpu;
	226	for_each_online_cpu(cpu)
	227	x += per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->vm_node_stat_diff[item];
	228
	229	if (x < 0)
	230	x = 0;
	231	#endif
	232	return x;
	233	}
	234
	235
2244b95a	236	#ifdef CONFIG_NUMA
3a321d2a	237	extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
75ef7184	238	extern unsigned long sum_zone_node_page_state(int node,
3a321d2a KW	239	enum zone_stat_item item);
3a321d2a KW	240	extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
75ef7184 MG	241	extern unsigned long node_page_state(struct pglist_data *pgdat,
75ef7184 MG	242	enum node_stat_item item);
2244b95a	243	#else
c41f012a	244	#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
75ef7184	245	#define node_page_state(node, item) global_node_page_state(item)
ca889e6c	246	#endif /* CONFIG_NUMA */
2244b95a	247
2244b95a CL	248	#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
2244b95a CL	249	#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
75ef7184 MG	250	#define add_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, __d)
75ef7184 MG	251	#define sub_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, -(__d))
2244b95a	252
2244b95a	253	#ifdef CONFIG_SMP
6cdb18ad	254	void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
2244b95a CL	255	void __inc_zone_page_state(struct page *, enum zone_stat_item);
2244b95a CL	256	void __dec_zone_page_state(struct page *, enum zone_stat_item);
f6ac2354	257
75ef7184 MG	258	void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
	259	void __inc_node_page_state(struct page *, enum node_stat_item);
	260	void __dec_node_page_state(struct page *, enum node_stat_item);
	261
6cdb18ad	262	void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
2244b95a CL	263	void inc_zone_page_state(struct page *, enum zone_stat_item);
	264	void dec_zone_page_state(struct page *, enum zone_stat_item);
	265
75ef7184 MG	266	void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
	267	void inc_node_page_state(struct page *, enum node_stat_item);
	268	void dec_node_page_state(struct page *, enum node_stat_item);
	269
75ef7184	270	extern void inc_node_state(struct pglist_data *, enum node_stat_item);
c8785385	271	extern void __inc_zone_state(struct zone *, enum zone_stat_item);
75ef7184	272	extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
c8785385 CL	273	extern void dec_zone_state(struct zone *, enum zone_stat_item);
c8785385 CL	274	extern void __dec_zone_state(struct zone *, enum zone_stat_item);
75ef7184	275	extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
2244b95a	276
0eb77e98	277	void quiet_vmstat(void);
2bb921e5	278	void cpu_vm_stats_fold(int cpu);
a6cccdc3	279	void refresh_zone_stat_thresholds(void);
b44129b3	280
52b6f46b HD	281	struct ctl_table;
	282	int vmstat_refresh(struct ctl_table *, int write,
	283	void __user buffer, size_t lenp, loff_t *ppos);
	284
5a883813 MK	285	void drain_zonestat(struct zone zone, struct per_cpu_pageset );
5a883813 MK	286
b44129b3 MG	287	int calculate_pressure_threshold(struct zone *zone);
	288	int calculate_normal_threshold(struct zone *zone);
	289	void set_pgdat_percpu_threshold(pg_data_t *pgdat,
	290	int (calculate_pressure)(struct zone ));
2244b95a CL	291	#else /* CONFIG_SMP */
	292
	293	/*
	294	* We do not maintain differentials in a single processor configuration.
	295	* The functions directly modify the zone and global counters.
	296	*/
	297	static inline void __mod_zone_page_state(struct zone *zone,
6cdb18ad	298	enum zone_stat_item item, long delta)
2244b95a CL	299	{
	300	zone_page_state_add(delta, zone, item);
	301	}
	302
75ef7184 MG	303	static inline void __mod_node_page_state(struct pglist_data *pgdat,
	304	enum node_stat_item item, int delta)
	305	{
	306	node_page_state_add(delta, pgdat, item);
	307	}
	308
7f4599e9 CL	309	static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
	310	{
	311	atomic_long_inc(&zone->vm_stat[item]);
75ef7184 MG	312	atomic_long_inc(&vm_zone_stat[item]);
	313	}
	314
	315	static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
	316	{
	317	atomic_long_inc(&pgdat->vm_stat[item]);
	318	atomic_long_inc(&vm_node_stat[item]);
7f4599e9 CL	319	}
7f4599e9 CL	320
c8785385 CL	321	static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
	322	{
	323	atomic_long_dec(&zone->vm_stat[item]);
75ef7184 MG	324	atomic_long_dec(&vm_zone_stat[item]);
	325	}
	326
	327	static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
	328	{
	329	atomic_long_dec(&pgdat->vm_stat[item]);
	330	atomic_long_dec(&vm_node_stat[item]);
c8785385 CL	331	}
c8785385 CL	332
6a3ed212 JW	333	static inline void __inc_zone_page_state(struct page *page,
	334	enum zone_stat_item item)
	335	{
	336	__inc_zone_state(page_zone(page), item);
	337	}
	338
75ef7184 MG	339	static inline void __inc_node_page_state(struct page *page,
	340	enum node_stat_item item)
	341	{
	342	__inc_node_state(page_pgdat(page), item);
	343	}
	344
	345
2244b95a CL	346	static inline void __dec_zone_page_state(struct page *page,
	347	enum zone_stat_item item)
	348	{
57ce36fe	349	__dec_zone_state(page_zone(page), item);
2244b95a CL	350	}
2244b95a CL	351
75ef7184 MG	352	static inline void __dec_node_page_state(struct page *page,
	353	enum node_stat_item item)
	354	{
	355	__dec_node_state(page_pgdat(page), item);
	356	}
	357
	358
2244b95a CL	359	/*
	360	* We only use atomic operations to update counters. So there is no need to
	361	* disable interrupts.
	362	*/
	363	#define inc_zone_page_state __inc_zone_page_state
	364	#define dec_zone_page_state __dec_zone_page_state
	365	#define mod_zone_page_state __mod_zone_page_state
	366
75ef7184 MG	367	#define inc_node_page_state __inc_node_page_state
	368	#define dec_node_page_state __dec_node_page_state
	369	#define mod_node_page_state __mod_node_page_state
	370
6a3ed212	371	#define inc_zone_state __inc_zone_state
75ef7184	372	#define inc_node_state __inc_node_state
6a3ed212 JW	373	#define dec_zone_state __dec_zone_state
6a3ed212 JW	374
b44129b3	375	#define set_pgdat_percpu_threshold(pgdat, callback) { }
88f5acf8	376
a6cccdc3	377	static inline void refresh_zone_stat_thresholds(void) { }
2bb921e5	378	static inline void cpu_vm_stats_fold(int cpu) { }
0eb77e98	379	static inline void quiet_vmstat(void) { }
a6cccdc3	380
5a883813 MK	381	static inline void drain_zonestat(struct zone *zone,
5a883813 MK	382	struct per_cpu_pageset *pset) { }
fa25c503 KM	383	#endif /* CONFIG_SMP */
fa25c503 KM	384
d1ce749a BZ	385	static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
	386	int migratetype)
	387	{
	388	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
	389	if (is_migrate_cma(migratetype))
	390	__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
	391	}
	392
fa25c503	393	extern const char * const vmstat_text[];
2244b95a CL	394
2244b95a CL	395	#endif /* _LINUX_VMSTAT_H */