include/linux/vmstat.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_VMSTAT_H
   3 #define _LINUX_VMSTAT_H
   4
   5 #include <linux/types.h>
   6 #include <linux/percpu.h>
   7 #include <linux/mmzone.h>
   8 #include <linux/vm_event_item.h>
   9 #include <linux/atomic.h>
  10 #include <linux/static_key.h>
  11
  12 extern int sysctl_stat_interval;
  13
  14 #ifdef CONFIG_NUMA
  15 #define ENABLE_NUMA_STAT   1
  16 #define DISABLE_NUMA_STAT   0
  17 extern int sysctl_vm_numa_stat;
  18 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
  19 extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
  20                 int write, void __user *buffer, size_t *length, loff_t *ppos);
  21 #endif
  22
  23 #ifdef CONFIG_VM_EVENT_COUNTERS
  24 /*
  25  * Light weight per cpu counter implementation.
  26  *
  27  * Counters should only be incremented and no critical kernel component
  28  * should rely on the counter values.
  29  *
  30  * Counters are handled completely inline. On many platforms the code
  31  * generated will simply be the increment of a global address.
  32  */
  33
  34 struct vm_event_state {
  35         unsigned long event[NR_VM_EVENT_ITEMS];
  36 };
  37
  38 DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
  39
  40 /*
  41  * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
  42  * local_irq_disable overhead.
  43  */
  44 static inline void __count_vm_event(enum vm_event_item item)
  45 {
  46         raw_cpu_inc(vm_event_states.event[item]);
  47 }
  48
  49 static inline void count_vm_event(enum vm_event_item item)
  50 {
  51         this_cpu_inc(vm_event_states.event[item]);
  52 }
  53
  54 static inline void __count_vm_events(enum vm_event_item item, long delta)
  55 {
  56         raw_cpu_add(vm_event_states.event[item], delta);
  57 }
  58
  59 static inline void count_vm_events(enum vm_event_item item, long delta)
  60 {
  61         this_cpu_add(vm_event_states.event[item], delta);
  62 }
  63
  64 extern void all_vm_events(unsigned long *);
  65
  66 extern void vm_events_fold_cpu(int cpu);
  67
  68 #else
  69
  70 /* Disable counters */
  71 static inline void count_vm_event(enum vm_event_item item)
  72 {
  73 }
  74 static inline void count_vm_events(enum vm_event_item item, long delta)
  75 {
  76 }
  77 static inline void __count_vm_event(enum vm_event_item item)
  78 {
  79 }
  80 static inline void __count_vm_events(enum vm_event_item item, long delta)
  81 {
  82 }
  83 static inline void all_vm_events(unsigned long *ret)
  84 {
  85 }
  86 static inline void vm_events_fold_cpu(int cpu)
  87 {
  88 }
  89
  90 #endif /* CONFIG_VM_EVENT_COUNTERS */
  91
  92 #ifdef CONFIG_NUMA_BALANCING
  93 #define count_vm_numa_event(x)     count_vm_event(x)
  94 #define count_vm_numa_events(x, y) count_vm_events(x, y)
  95 #else
  96 #define count_vm_numa_event(x) do {} while (0)
  97 #define count_vm_numa_events(x, y) do { (void)(y); } while (0)
  98 #endif /* CONFIG_NUMA_BALANCING */
  99
 100 #ifdef CONFIG_DEBUG_TLBFLUSH
 101 #define count_vm_tlb_event(x)      count_vm_event(x)
 102 #define count_vm_tlb_events(x, y)  count_vm_events(x, y)
 103 #else
 104 #define count_vm_tlb_event(x)     do {} while (0)
 105 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
 106 #endif
 107
 108 #ifdef CONFIG_DEBUG_VM_VMACACHE
 109 #define count_vm_vmacache_event(x) count_vm_event(x)
 110 #else
 111 #define count_vm_vmacache_event(x) do {} while (0)
 112 #endif
 113
 114 #define __count_zid_vm_events(item, zid, delta) \
 115         __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
 116
 117 /*
 118  * Zone and node-based page accounting with per cpu differentials.
 119  */
 120 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
 121 extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
 122 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
 123
 124 #ifdef CONFIG_NUMA
 125 static inline void zone_numa_state_add(long x, struct zone *zone,
 126                                  enum numa_stat_item item)
 127 {
 128         atomic_long_add(x, &zone->vm_numa_stat[item]);
 129         atomic_long_add(x, &vm_numa_stat[item]);
 130 }
 131
 132 static inline unsigned long global_numa_state(enum numa_stat_item item)
 133 {
 134         long x = atomic_long_read(&vm_numa_stat[item]);
 135
 136         return x;
 137 }
 138
 139 static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
 140                                         enum numa_stat_item item)
 141 {
 142         long x = atomic_long_read(&zone->vm_numa_stat[item]);
 143         int cpu;
 144
 145         for_each_online_cpu(cpu)
 146                 x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
 147
 148         return x;
 149 }
 150 #endif /* CONFIG_NUMA */
 151
 152 static inline void zone_page_state_add(long x, struct zone *zone,
 153                                  enum zone_stat_item item)
 154 {
 155         atomic_long_add(x, &zone->vm_stat[item]);
 156         atomic_long_add(x, &vm_zone_stat[item]);
 157 }
 158
 159 static inline void node_page_state_add(long x, struct pglist_data *pgdat,
 160                                  enum node_stat_item item)
 161 {
 162         atomic_long_add(x, &pgdat->vm_stat[item]);
 163         atomic_long_add(x, &vm_node_stat[item]);
 164 }
 165
 166 static inline unsigned long global_zone_page_state(enum zone_stat_item item)
 167 {
 168         long x = atomic_long_read(&vm_zone_stat[item]);
 169 #ifdef CONFIG_SMP
 170         if (x < 0)
 171                 x = 0;
 172 #endif
 173         return x;
 174 }
 175
 176 static inline unsigned long global_node_page_state(enum node_stat_item item)
 177 {
 178         long x = atomic_long_read(&vm_node_stat[item]);
 179 #ifdef CONFIG_SMP
 180         if (x < 0)
 181                 x = 0;
 182 #endif
 183         return x;
 184 }
 185
 186 static inline unsigned long zone_page_state(struct zone *zone,
 187                                         enum zone_stat_item item)
 188 {
 189         long x = atomic_long_read(&zone->vm_stat[item]);
 190 #ifdef CONFIG_SMP
 191         if (x < 0)
 192                 x = 0;
 193 #endif
 194         return x;
 195 }
 196
 197 /*
 198  * More accurate version that also considers the currently pending
 199  * deltas. For that we need to loop over all cpus to find the current
 200  * deltas. There is no synchronization so the result cannot be
 201  * exactly accurate either.
 202  */
 203 static inline unsigned long zone_page_state_snapshot(struct zone *zone,
 204                                         enum zone_stat_item item)
 205 {
 206         long x = atomic_long_read(&zone->vm_stat[item]);
 207
 208 #ifdef CONFIG_SMP
 209         int cpu;
 210         for_each_online_cpu(cpu)
 211                 x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
 212
 213         if (x < 0)
 214                 x = 0;
 215 #endif
 216         return x;
 217 }
 218
 219 static inline unsigned long node_page_state_snapshot(pg_data_t *pgdat,
 220                                         enum node_stat_item item)
 221 {
 222         long x = atomic_long_read(&pgdat->vm_stat[item]);
 223
 224 #ifdef CONFIG_SMP
 225         int cpu;
 226         for_each_online_cpu(cpu)
 227                 x += per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->vm_node_stat_diff[item];
 228
 229         if (x < 0)
 230                 x = 0;
 231 #endif
 232         return x;
 233 }
 234
 235
 236 #ifdef CONFIG_NUMA
 237 extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
 238 extern unsigned long sum_zone_node_page_state(int node,
 239                                               enum zone_stat_item item);
 240 extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
 241 extern unsigned long node_page_state(struct pglist_data *pgdat,
 242                                                 enum node_stat_item item);
 243 #else
 244 #define sum_zone_node_page_state(node, item) global_zone_page_state(item)
 245 #define node_page_state(node, item) global_node_page_state(item)
 246 #endif /* CONFIG_NUMA */
 247
 248 #define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
 249 #define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
 250 #define add_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, __d)
 251 #define sub_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, -(__d))
 252
 253 #ifdef CONFIG_SMP
 254 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
 255 void __inc_zone_page_state(struct page *, enum zone_stat_item);
 256 void __dec_zone_page_state(struct page *, enum zone_stat_item);
 257
 258 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
 259 void __inc_node_page_state(struct page *, enum node_stat_item);
 260 void __dec_node_page_state(struct page *, enum node_stat_item);
 261
 262 void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
 263 void inc_zone_page_state(struct page *, enum zone_stat_item);
 264 void dec_zone_page_state(struct page *, enum zone_stat_item);
 265
 266 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
 267 void inc_node_page_state(struct page *, enum node_stat_item);
 268 void dec_node_page_state(struct page *, enum node_stat_item);
 269
 270 extern void inc_node_state(struct pglist_data *, enum node_stat_item);
 271 extern void __inc_zone_state(struct zone *, enum zone_stat_item);
 272 extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
 273 extern void dec_zone_state(struct zone *, enum zone_stat_item);
 274 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
 275 extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
 276
 277 void quiet_vmstat(void);
 278 void cpu_vm_stats_fold(int cpu);
 279 void refresh_zone_stat_thresholds(void);
 280
 281 struct ctl_table;
 282 int vmstat_refresh(struct ctl_table *, int write,
 283                    void __user *buffer, size_t *lenp, loff_t *ppos);
 284
 285 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
 286
 287 int calculate_pressure_threshold(struct zone *zone);
 288 int calculate_normal_threshold(struct zone *zone);
 289 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
 290                                 int (*calculate_pressure)(struct zone *));
 291 #else /* CONFIG_SMP */
 292
 293 /*
 294  * We do not maintain differentials in a single processor configuration.
 295  * The functions directly modify the zone and global counters.
 296  */
 297 static inline void __mod_zone_page_state(struct zone *zone,
 298                         enum zone_stat_item item, long delta)
 299 {
 300         zone_page_state_add(delta, zone, item);
 301 }
 302
 303 static inline void __mod_node_page_state(struct pglist_data *pgdat,
 304                         enum node_stat_item item, int delta)
 305 {
 306         node_page_state_add(delta, pgdat, item);
 307 }
 308
 309 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
 310 {
 311         atomic_long_inc(&zone->vm_stat[item]);
 312         atomic_long_inc(&vm_zone_stat[item]);
 313 }
 314
 315 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
 316 {
 317         atomic_long_inc(&pgdat->vm_stat[item]);
 318         atomic_long_inc(&vm_node_stat[item]);
 319 }
 320
 321 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
 322 {
 323         atomic_long_dec(&zone->vm_stat[item]);
 324         atomic_long_dec(&vm_zone_stat[item]);
 325 }
 326
 327 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
 328 {
 329         atomic_long_dec(&pgdat->vm_stat[item]);
 330         atomic_long_dec(&vm_node_stat[item]);
 331 }
 332
 333 static inline void __inc_zone_page_state(struct page *page,
 334                         enum zone_stat_item item)
 335 {
 336         __inc_zone_state(page_zone(page), item);
 337 }
 338
 339 static inline void __inc_node_page_state(struct page *page,
 340                         enum node_stat_item item)
 341 {
 342         __inc_node_state(page_pgdat(page), item);
 343 }
 344
 345
 346 static inline void __dec_zone_page_state(struct page *page,
 347                         enum zone_stat_item item)
 348 {
 349         __dec_zone_state(page_zone(page), item);
 350 }
 351
 352 static inline void __dec_node_page_state(struct page *page,
 353                         enum node_stat_item item)
 354 {
 355         __dec_node_state(page_pgdat(page), item);
 356 }
 357
 358
 359 /*
 360  * We only use atomic operations to update counters. So there is no need to
 361  * disable interrupts.
 362  */
 363 #define inc_zone_page_state __inc_zone_page_state
 364 #define dec_zone_page_state __dec_zone_page_state
 365 #define mod_zone_page_state __mod_zone_page_state
 366
 367 #define inc_node_page_state __inc_node_page_state
 368 #define dec_node_page_state __dec_node_page_state
 369 #define mod_node_page_state __mod_node_page_state
 370
 371 #define inc_zone_state __inc_zone_state
 372 #define inc_node_state __inc_node_state
 373 #define dec_zone_state __dec_zone_state
 374
 375 #define set_pgdat_percpu_threshold(pgdat, callback) { }
 376
 377 static inline void refresh_zone_stat_thresholds(void) { }
 378 static inline void cpu_vm_stats_fold(int cpu) { }
 379 static inline void quiet_vmstat(void) { }
 380
 381 static inline void drain_zonestat(struct zone *zone,
 382                         struct per_cpu_pageset *pset) { }
 383 #endif          /* CONFIG_SMP */
 384
 385 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
 386                                              int migratetype)
 387 {
 388         __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
 389         if (is_migrate_cma(migratetype))
 390                 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
 391 }
 392
 393 extern const char * const vmstat_text[];
 394
 395 #endif /* _LINUX_VMSTAT_H */