]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - mm/zsmalloc.c
netfilter: use nf_conntrack_helpers_register when possible
[mirror_ubuntu-artful-kernel.git] / mm / zsmalloc.c
CommitLineData
61989a80
NG
1/*
2 * zsmalloc memory allocator
3 *
4 * Copyright (C) 2011 Nitin Gupta
31fc00bb 5 * Copyright (C) 2012, 2013 Minchan Kim
61989a80
NG
6 *
7 * This code is released using a dual license strategy: BSD/GPL
8 * You can choose the license that better fits your requirements.
9 *
10 * Released under the terms of 3-clause BSD License
11 * Released under the terms of GNU General Public License Version 2.0
12 */
13
2db51dae 14/*
2db51dae
NG
15 * Following is how we use various fields and flags of underlying
16 * struct page(s) to form a zspage.
17 *
18 * Usage of struct page fields:
3783689a 19 * page->private: points to zspage
48b4800a
MK
20 * page->freelist(index): links together all component pages of a zspage
21 * For the huge page, this is always 0, so we use this field
22 * to store handle.
fd854463 23 * page->units: first object offset in a subpage of zspage
2db51dae
NG
24 *
25 * Usage of struct page flags:
26 * PG_private: identifies the first component page
399d8eeb 27 * PG_owner_priv_1: identifies the huge component page
2db51dae
NG
28 *
29 */
30
4abaac9b
DS
31#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32
61989a80
NG
33#include <linux/module.h>
34#include <linux/kernel.h>
312fcae2 35#include <linux/sched.h>
50d34394 36#include <linux/magic.h>
61989a80
NG
37#include <linux/bitops.h>
38#include <linux/errno.h>
39#include <linux/highmem.h>
61989a80
NG
40#include <linux/string.h>
41#include <linux/slab.h>
42#include <asm/tlbflush.h>
43#include <asm/pgtable.h>
44#include <linux/cpumask.h>
45#include <linux/cpu.h>
0cbb613f 46#include <linux/vmalloc.h>
759b26b2 47#include <linux/preempt.h>
0959c63f
SJ
48#include <linux/spinlock.h>
49#include <linux/types.h>
0f050d99 50#include <linux/debugfs.h>
bcf1647d 51#include <linux/zsmalloc.h>
c795779d 52#include <linux/zpool.h>
48b4800a 53#include <linux/mount.h>
dd4123f3 54#include <linux/migrate.h>
48b4800a
MK
55#include <linux/pagemap.h>
56
57#define ZSPAGE_MAGIC 0x58
0959c63f
SJ
58
59/*
60 * This must be power of 2 and greater than of equal to sizeof(link_free).
61 * These two conditions ensure that any 'struct link_free' itself doesn't
62 * span more than 1 page which avoids complex case of mapping 2 pages simply
63 * to restore link_free pointer values.
64 */
65#define ZS_ALIGN 8
66
67/*
68 * A single 'zspage' is composed of up to 2^N discontiguous 0-order (single)
69 * pages. ZS_MAX_ZSPAGE_ORDER defines upper limit on N.
70 */
71#define ZS_MAX_ZSPAGE_ORDER 2
72#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
73
2e40e163
MK
74#define ZS_HANDLE_SIZE (sizeof(unsigned long))
75
0959c63f
SJ
76/*
77 * Object location (<PFN>, <obj_idx>) is encoded as
c3e3e88a 78 * as single (unsigned long) handle value.
0959c63f 79 *
bfd093f5 80 * Note that object index <obj_idx> starts from 0.
0959c63f
SJ
81 *
82 * This is made more complicated by various memory models and PAE.
83 */
84
85#ifndef MAX_PHYSMEM_BITS
86#ifdef CONFIG_HIGHMEM64G
87#define MAX_PHYSMEM_BITS 36
88#else /* !CONFIG_HIGHMEM64G */
89/*
90 * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
91 * be PAGE_SHIFT
92 */
93#define MAX_PHYSMEM_BITS BITS_PER_LONG
94#endif
95#endif
96#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
312fcae2
MK
97
98/*
99 * Memory for allocating for handle keeps object position by
100 * encoding <page, obj_idx> and the encoded value has a room
101 * in least bit(ie, look at obj_to_location).
102 * We use the bit to synchronize between object access by
103 * user and migration.
104 */
105#define HANDLE_PIN_BIT 0
106
107/*
108 * Head in allocated object should have OBJ_ALLOCATED_TAG
109 * to identify the object was allocated or not.
110 * It's okay to add the status bit in the least bit because
111 * header keeps handle which is 4byte-aligned address so we
112 * have room for two bit at least.
113 */
114#define OBJ_ALLOCATED_TAG 1
115#define OBJ_TAG_BITS 1
116#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
0959c63f
SJ
117#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
118
119#define MAX(a, b) ((a) >= (b) ? (a) : (b))
120/* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
121#define ZS_MIN_ALLOC_SIZE \
122 MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
2e40e163 123/* each chunk includes extra space to keep handle */
7b60a685 124#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
0959c63f
SJ
125
126/*
7eb52512 127 * On systems with 4K page size, this gives 255 size classes! There is a
0959c63f
SJ
128 * trader-off here:
129 * - Large number of size classes is potentially wasteful as free page are
130 * spread across these classes
131 * - Small number of size classes causes large internal fragmentation
132 * - Probably its better to use specific size classes (empirically
133 * determined). NOTE: all those class sizes must be set as multiple of
134 * ZS_ALIGN to make sure link_free itself never has to span 2 pages.
135 *
136 * ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
137 * (reason above)
138 */
3783689a 139#define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS)
0959c63f 140
0959c63f 141enum fullness_group {
0959c63f 142 ZS_EMPTY,
48b4800a
MK
143 ZS_ALMOST_EMPTY,
144 ZS_ALMOST_FULL,
145 ZS_FULL,
146 NR_ZS_FULLNESS,
0959c63f
SJ
147};
148
0f050d99 149enum zs_stat_type {
48b4800a
MK
150 CLASS_EMPTY,
151 CLASS_ALMOST_EMPTY,
152 CLASS_ALMOST_FULL,
153 CLASS_FULL,
0f050d99
GM
154 OBJ_ALLOCATED,
155 OBJ_USED,
48b4800a 156 NR_ZS_STAT_TYPE,
0f050d99
GM
157};
158
0f050d99
GM
159struct zs_size_stat {
160 unsigned long objs[NR_ZS_STAT_TYPE];
161};
162
57244594
SS
163#ifdef CONFIG_ZSMALLOC_STAT
164static struct dentry *zs_stat_root;
0f050d99
GM
165#endif
166
48b4800a
MK
167#ifdef CONFIG_COMPACTION
168static struct vfsmount *zsmalloc_mnt;
169#endif
170
40f9fb8c
MG
171/*
172 * number of size_classes
173 */
174static int zs_size_classes;
175
0959c63f
SJ
176/*
177 * We assign a page to ZS_ALMOST_EMPTY fullness group when:
178 * n <= N / f, where
179 * n = number of allocated objects
180 * N = total number of objects zspage can store
6dd9737e 181 * f = fullness_threshold_frac
0959c63f
SJ
182 *
183 * Similarly, we assign zspage to:
184 * ZS_ALMOST_FULL when n > N / f
185 * ZS_EMPTY when n == 0
186 * ZS_FULL when n == N
187 *
188 * (see: fix_fullness_group())
189 */
190static const int fullness_threshold_frac = 4;
191
192struct size_class {
57244594 193 spinlock_t lock;
48b4800a 194 struct list_head fullness_list[NR_ZS_FULLNESS];
0959c63f
SJ
195 /*
196 * Size of objects stored in this class. Must be multiple
197 * of ZS_ALIGN.
198 */
199 int size;
1fc6e27d 200 int objs_per_zspage;
7dfa4612
WY
201 /* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */
202 int pages_per_zspage;
48b4800a
MK
203
204 unsigned int index;
205 struct zs_size_stat stats;
0959c63f
SJ
206};
207
48b4800a
MK
208/* huge object: pages_per_zspage == 1 && maxobj_per_zspage == 1 */
209static void SetPageHugeObject(struct page *page)
210{
211 SetPageOwnerPriv1(page);
212}
213
214static void ClearPageHugeObject(struct page *page)
215{
216 ClearPageOwnerPriv1(page);
217}
218
219static int PageHugeObject(struct page *page)
220{
221 return PageOwnerPriv1(page);
222}
223
0959c63f
SJ
224/*
225 * Placed within free objects to form a singly linked list.
3783689a 226 * For every zspage, zspage->freeobj gives head of this list.
0959c63f
SJ
227 *
228 * This must be power of 2 and less than or equal to ZS_ALIGN
229 */
230struct link_free {
2e40e163
MK
231 union {
232 /*
bfd093f5 233 * Free object index;
2e40e163
MK
234 * It's valid for non-allocated object
235 */
bfd093f5 236 unsigned long next;
2e40e163
MK
237 /*
238 * Handle of allocated object.
239 */
240 unsigned long handle;
241 };
0959c63f
SJ
242};
243
244struct zs_pool {
6f3526d6 245 const char *name;
0f050d99 246
40f9fb8c 247 struct size_class **size_class;
2e40e163 248 struct kmem_cache *handle_cachep;
3783689a 249 struct kmem_cache *zspage_cachep;
0959c63f 250
13de8933 251 atomic_long_t pages_allocated;
0f050d99 252
7d3f3938 253 struct zs_pool_stats stats;
ab9d306d
SS
254
255 /* Compact classes */
256 struct shrinker shrinker;
257 /*
258 * To signify that register_shrinker() was successful
259 * and unregister_shrinker() will not Oops.
260 */
261 bool shrinker_enabled;
0f050d99
GM
262#ifdef CONFIG_ZSMALLOC_STAT
263 struct dentry *stat_dentry;
264#endif
48b4800a
MK
265#ifdef CONFIG_COMPACTION
266 struct inode *inode;
267 struct work_struct free_work;
268#endif
0959c63f 269};
61989a80 270
3783689a
MK
271#define FULLNESS_BITS 2
272#define CLASS_BITS 8
48b4800a
MK
273#define ISOLATED_BITS 3
274#define MAGIC_VAL_BITS 8
4f42047b 275
3783689a
MK
276struct zspage {
277 struct {
278 unsigned int fullness:FULLNESS_BITS;
85d492f2 279 unsigned int class:CLASS_BITS + 1;
48b4800a
MK
280 unsigned int isolated:ISOLATED_BITS;
281 unsigned int magic:MAGIC_VAL_BITS;
3783689a
MK
282 };
283 unsigned int inuse;
bfd093f5 284 unsigned int freeobj;
3783689a
MK
285 struct page *first_page;
286 struct list_head list; /* fullness list */
48b4800a
MK
287#ifdef CONFIG_COMPACTION
288 rwlock_t lock;
289#endif
3783689a 290};
61989a80 291
f553646a 292struct mapping_area {
1b945aee 293#ifdef CONFIG_PGTABLE_MAPPING
f553646a
SJ
294 struct vm_struct *vm; /* vm area for mapping object that span pages */
295#else
296 char *vm_buf; /* copy buffer for objects that span pages */
297#endif
298 char *vm_addr; /* address of kmap_atomic()'ed pages */
299 enum zs_mapmode vm_mm; /* mapping mode */
300};
301
48b4800a
MK
302#ifdef CONFIG_COMPACTION
303static int zs_register_migration(struct zs_pool *pool);
304static void zs_unregister_migration(struct zs_pool *pool);
305static void migrate_lock_init(struct zspage *zspage);
306static void migrate_read_lock(struct zspage *zspage);
307static void migrate_read_unlock(struct zspage *zspage);
308static void kick_deferred_free(struct zs_pool *pool);
309static void init_deferred_free(struct zs_pool *pool);
310static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
311#else
312static int zsmalloc_mount(void) { return 0; }
313static void zsmalloc_unmount(void) {}
314static int zs_register_migration(struct zs_pool *pool) { return 0; }
315static void zs_unregister_migration(struct zs_pool *pool) {}
316static void migrate_lock_init(struct zspage *zspage) {}
317static void migrate_read_lock(struct zspage *zspage) {}
318static void migrate_read_unlock(struct zspage *zspage) {}
319static void kick_deferred_free(struct zs_pool *pool) {}
320static void init_deferred_free(struct zs_pool *pool) {}
321static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}
322#endif
323
3783689a 324static int create_cache(struct zs_pool *pool)
2e40e163
MK
325{
326 pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
327 0, 0, NULL);
3783689a
MK
328 if (!pool->handle_cachep)
329 return 1;
330
331 pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),
332 0, 0, NULL);
333 if (!pool->zspage_cachep) {
334 kmem_cache_destroy(pool->handle_cachep);
335 pool->handle_cachep = NULL;
336 return 1;
337 }
338
339 return 0;
2e40e163
MK
340}
341
3783689a 342static void destroy_cache(struct zs_pool *pool)
2e40e163 343{
cd10add0 344 kmem_cache_destroy(pool->handle_cachep);
3783689a 345 kmem_cache_destroy(pool->zspage_cachep);
2e40e163
MK
346}
347
3783689a 348static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
2e40e163
MK
349{
350 return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
48b4800a 351 gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
2e40e163
MK
352}
353
3783689a 354static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
2e40e163
MK
355{
356 kmem_cache_free(pool->handle_cachep, (void *)handle);
357}
358
3783689a
MK
359static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)
360{
48b4800a
MK
361 return kmem_cache_alloc(pool->zspage_cachep,
362 flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
399d8eeb 363}
3783689a
MK
364
365static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
366{
367 kmem_cache_free(pool->zspage_cachep, zspage);
368}
369
2e40e163
MK
370static void record_obj(unsigned long handle, unsigned long obj)
371{
c102f07c
JL
372 /*
373 * lsb of @obj represents handle lock while other bits
374 * represent object value the handle is pointing so
375 * updating shouldn't do store tearing.
376 */
377 WRITE_ONCE(*(unsigned long *)handle, obj);
2e40e163
MK
378}
379
c795779d
DS
380/* zpool driver */
381
382#ifdef CONFIG_ZPOOL
383
6f3526d6 384static void *zs_zpool_create(const char *name, gfp_t gfp,
78672779 385 const struct zpool_ops *zpool_ops,
479305fd 386 struct zpool *zpool)
c795779d 387{
d0d8da2d
SS
388 /*
389 * Ignore global gfp flags: zs_malloc() may be invoked from
390 * different contexts and its caller must provide a valid
391 * gfp mask.
392 */
393 return zs_create_pool(name);
c795779d
DS
394}
395
396static void zs_zpool_destroy(void *pool)
397{
398 zs_destroy_pool(pool);
399}
400
401static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,
402 unsigned long *handle)
403{
d0d8da2d 404 *handle = zs_malloc(pool, size, gfp);
c795779d
DS
405 return *handle ? 0 : -1;
406}
407static void zs_zpool_free(void *pool, unsigned long handle)
408{
409 zs_free(pool, handle);
410}
411
412static int zs_zpool_shrink(void *pool, unsigned int pages,
413 unsigned int *reclaimed)
414{
415 return -EINVAL;
416}
417
418static void *zs_zpool_map(void *pool, unsigned long handle,
419 enum zpool_mapmode mm)
420{
421 enum zs_mapmode zs_mm;
422
423 switch (mm) {
424 case ZPOOL_MM_RO:
425 zs_mm = ZS_MM_RO;
426 break;
427 case ZPOOL_MM_WO:
428 zs_mm = ZS_MM_WO;
429 break;
430 case ZPOOL_MM_RW: /* fallthru */
431 default:
432 zs_mm = ZS_MM_RW;
433 break;
434 }
435
436 return zs_map_object(pool, handle, zs_mm);
437}
438static void zs_zpool_unmap(void *pool, unsigned long handle)
439{
440 zs_unmap_object(pool, handle);
441}
442
443static u64 zs_zpool_total_size(void *pool)
444{
722cdc17 445 return zs_get_total_pages(pool) << PAGE_SHIFT;
c795779d
DS
446}
447
448static struct zpool_driver zs_zpool_driver = {
449 .type = "zsmalloc",
450 .owner = THIS_MODULE,
451 .create = zs_zpool_create,
452 .destroy = zs_zpool_destroy,
453 .malloc = zs_zpool_malloc,
454 .free = zs_zpool_free,
455 .shrink = zs_zpool_shrink,
456 .map = zs_zpool_map,
457 .unmap = zs_zpool_unmap,
458 .total_size = zs_zpool_total_size,
459};
460
137f8cff 461MODULE_ALIAS("zpool-zsmalloc");
c795779d
DS
462#endif /* CONFIG_ZPOOL */
463
61989a80
NG
464/* per-cpu VM mapping areas for zspage accesses that cross page boundaries */
465static DEFINE_PER_CPU(struct mapping_area, zs_map_area);
466
48b4800a
MK
467static bool is_zspage_isolated(struct zspage *zspage)
468{
469 return zspage->isolated;
470}
471
61989a80
NG
472static int is_first_page(struct page *page)
473{
a27545bf 474 return PagePrivate(page);
61989a80
NG
475}
476
48b4800a 477/* Protected by class->lock */
3783689a 478static inline int get_zspage_inuse(struct zspage *zspage)
4f42047b 479{
3783689a 480 return zspage->inuse;
4f42047b
MK
481}
482
3783689a 483static inline void set_zspage_inuse(struct zspage *zspage, int val)
4f42047b 484{
3783689a 485 zspage->inuse = val;
4f42047b
MK
486}
487
3783689a 488static inline void mod_zspage_inuse(struct zspage *zspage, int val)
4f42047b 489{
3783689a 490 zspage->inuse += val;
4f42047b
MK
491}
492
48b4800a 493static inline struct page *get_first_page(struct zspage *zspage)
4f42047b 494{
48b4800a 495 struct page *first_page = zspage->first_page;
3783689a 496
48b4800a
MK
497 VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
498 return first_page;
4f42047b
MK
499}
500
48b4800a 501static inline int get_first_obj_offset(struct page *page)
4f42047b 502{
48b4800a
MK
503 return page->units;
504}
3783689a 505
48b4800a
MK
506static inline void set_first_obj_offset(struct page *page, int offset)
507{
508 page->units = offset;
4f42047b
MK
509}
510
bfd093f5 511static inline unsigned int get_freeobj(struct zspage *zspage)
4f42047b 512{
bfd093f5 513 return zspage->freeobj;
4f42047b
MK
514}
515
bfd093f5 516static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
4f42047b 517{
bfd093f5 518 zspage->freeobj = obj;
4f42047b
MK
519}
520
3783689a 521static void get_zspage_mapping(struct zspage *zspage,
a4209467 522 unsigned int *class_idx,
61989a80
NG
523 enum fullness_group *fullness)
524{
48b4800a
MK
525 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
526
3783689a
MK
527 *fullness = zspage->fullness;
528 *class_idx = zspage->class;
61989a80
NG
529}
530
3783689a 531static void set_zspage_mapping(struct zspage *zspage,
a4209467 532 unsigned int class_idx,
61989a80
NG
533 enum fullness_group fullness)
534{
3783689a
MK
535 zspage->class = class_idx;
536 zspage->fullness = fullness;
61989a80
NG
537}
538
c3e3e88a
NC
539/*
540 * zsmalloc divides the pool into various size classes where each
541 * class maintains a list of zspages where each zspage is divided
542 * into equal sized chunks. Each allocation falls into one of these
543 * classes depending on its size. This function returns index of the
544 * size class which has chunk size big enough to hold the give size.
545 */
61989a80
NG
546static int get_size_class_index(int size)
547{
548 int idx = 0;
549
550 if (likely(size > ZS_MIN_ALLOC_SIZE))
551 idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE,
552 ZS_SIZE_CLASS_DELTA);
553
7b60a685 554 return min(zs_size_classes - 1, idx);
61989a80
NG
555}
556
248ca1b0
MK
557static inline void zs_stat_inc(struct size_class *class,
558 enum zs_stat_type type, unsigned long cnt)
559{
48b4800a 560 class->stats.objs[type] += cnt;
248ca1b0
MK
561}
562
563static inline void zs_stat_dec(struct size_class *class,
564 enum zs_stat_type type, unsigned long cnt)
565{
48b4800a 566 class->stats.objs[type] -= cnt;
248ca1b0
MK
567}
568
569static inline unsigned long zs_stat_get(struct size_class *class,
570 enum zs_stat_type type)
571{
48b4800a 572 return class->stats.objs[type];
248ca1b0
MK
573}
574
57244594
SS
575#ifdef CONFIG_ZSMALLOC_STAT
576
4abaac9b 577static void __init zs_stat_init(void)
248ca1b0 578{
4abaac9b
DS
579 if (!debugfs_initialized()) {
580 pr_warn("debugfs not available, stat dir not created\n");
581 return;
582 }
248ca1b0
MK
583
584 zs_stat_root = debugfs_create_dir("zsmalloc", NULL);
585 if (!zs_stat_root)
4abaac9b 586 pr_warn("debugfs 'zsmalloc' stat dir creation failed\n");
248ca1b0
MK
587}
588
589static void __exit zs_stat_exit(void)
590{
591 debugfs_remove_recursive(zs_stat_root);
592}
593
1120ed54
SS
594static unsigned long zs_can_compact(struct size_class *class);
595
248ca1b0
MK
596static int zs_stats_size_show(struct seq_file *s, void *v)
597{
598 int i;
599 struct zs_pool *pool = s->private;
600 struct size_class *class;
601 int objs_per_zspage;
602 unsigned long class_almost_full, class_almost_empty;
1120ed54 603 unsigned long obj_allocated, obj_used, pages_used, freeable;
248ca1b0
MK
604 unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
605 unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
1120ed54 606 unsigned long total_freeable = 0;
248ca1b0 607
1120ed54 608 seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
248ca1b0
MK
609 "class", "size", "almost_full", "almost_empty",
610 "obj_allocated", "obj_used", "pages_used",
1120ed54 611 "pages_per_zspage", "freeable");
248ca1b0
MK
612
613 for (i = 0; i < zs_size_classes; i++) {
614 class = pool->size_class[i];
615
616 if (class->index != i)
617 continue;
618
619 spin_lock(&class->lock);
620 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
621 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
622 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
623 obj_used = zs_stat_get(class, OBJ_USED);
1120ed54 624 freeable = zs_can_compact(class);
248ca1b0
MK
625 spin_unlock(&class->lock);
626
b4fd07a0 627 objs_per_zspage = class->objs_per_zspage;
248ca1b0
MK
628 pages_used = obj_allocated / objs_per_zspage *
629 class->pages_per_zspage;
630
1120ed54
SS
631 seq_printf(s, " %5u %5u %11lu %12lu %13lu"
632 " %10lu %10lu %16d %8lu\n",
248ca1b0
MK
633 i, class->size, class_almost_full, class_almost_empty,
634 obj_allocated, obj_used, pages_used,
1120ed54 635 class->pages_per_zspage, freeable);
248ca1b0
MK
636
637 total_class_almost_full += class_almost_full;
638 total_class_almost_empty += class_almost_empty;
639 total_objs += obj_allocated;
640 total_used_objs += obj_used;
641 total_pages += pages_used;
1120ed54 642 total_freeable += freeable;
248ca1b0
MK
643 }
644
645 seq_puts(s, "\n");
1120ed54 646 seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
248ca1b0
MK
647 "Total", "", total_class_almost_full,
648 total_class_almost_empty, total_objs,
1120ed54 649 total_used_objs, total_pages, "", total_freeable);
248ca1b0
MK
650
651 return 0;
652}
653
654static int zs_stats_size_open(struct inode *inode, struct file *file)
655{
656 return single_open(file, zs_stats_size_show, inode->i_private);
657}
658
659static const struct file_operations zs_stat_size_ops = {
660 .open = zs_stats_size_open,
661 .read = seq_read,
662 .llseek = seq_lseek,
663 .release = single_release,
664};
665
d34f6157 666static void zs_pool_stat_create(struct zs_pool *pool, const char *name)
248ca1b0
MK
667{
668 struct dentry *entry;
669
4abaac9b
DS
670 if (!zs_stat_root) {
671 pr_warn("no root stat dir, not creating <%s> stat dir\n", name);
d34f6157 672 return;
4abaac9b 673 }
248ca1b0
MK
674
675 entry = debugfs_create_dir(name, zs_stat_root);
676 if (!entry) {
677 pr_warn("debugfs dir <%s> creation failed\n", name);
d34f6157 678 return;
248ca1b0
MK
679 }
680 pool->stat_dentry = entry;
681
682 entry = debugfs_create_file("classes", S_IFREG | S_IRUGO,
683 pool->stat_dentry, pool, &zs_stat_size_ops);
684 if (!entry) {
685 pr_warn("%s: debugfs file entry <%s> creation failed\n",
686 name, "classes");
4abaac9b
DS
687 debugfs_remove_recursive(pool->stat_dentry);
688 pool->stat_dentry = NULL;
248ca1b0 689 }
248ca1b0
MK
690}
691
692static void zs_pool_stat_destroy(struct zs_pool *pool)
693{
694 debugfs_remove_recursive(pool->stat_dentry);
695}
696
697#else /* CONFIG_ZSMALLOC_STAT */
4abaac9b 698static void __init zs_stat_init(void)
248ca1b0 699{
248ca1b0
MK
700}
701
702static void __exit zs_stat_exit(void)
703{
704}
705
d34f6157 706static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name)
248ca1b0 707{
248ca1b0
MK
708}
709
710static inline void zs_pool_stat_destroy(struct zs_pool *pool)
711{
712}
248ca1b0
MK
713#endif
714
48b4800a 715
c3e3e88a
NC
716/*
717 * For each size class, zspages are divided into different groups
718 * depending on how "full" they are. This was done so that we could
719 * easily find empty or nearly empty zspages when we try to shrink
720 * the pool (not yet implemented). This function returns fullness
721 * status of the given page.
722 */
1fc6e27d 723static enum fullness_group get_fullness_group(struct size_class *class,
3783689a 724 struct zspage *zspage)
61989a80 725{
1fc6e27d 726 int inuse, objs_per_zspage;
61989a80 727 enum fullness_group fg;
830e4bc5 728
3783689a 729 inuse = get_zspage_inuse(zspage);
1fc6e27d 730 objs_per_zspage = class->objs_per_zspage;
61989a80
NG
731
732 if (inuse == 0)
733 fg = ZS_EMPTY;
1fc6e27d 734 else if (inuse == objs_per_zspage)
61989a80 735 fg = ZS_FULL;
1fc6e27d 736 else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac)
61989a80
NG
737 fg = ZS_ALMOST_EMPTY;
738 else
739 fg = ZS_ALMOST_FULL;
740
741 return fg;
742}
743
c3e3e88a
NC
744/*
745 * Each size class maintains various freelists and zspages are assigned
746 * to one of these freelists based on the number of live objects they
747 * have. This functions inserts the given zspage into the freelist
748 * identified by <class, fullness_group>.
749 */
251cbb95 750static void insert_zspage(struct size_class *class,
3783689a
MK
751 struct zspage *zspage,
752 enum fullness_group fullness)
61989a80 753{
3783689a 754 struct zspage *head;
61989a80 755
48b4800a 756 zs_stat_inc(class, fullness, 1);
3783689a
MK
757 head = list_first_entry_or_null(&class->fullness_list[fullness],
758 struct zspage, list);
58f17117 759 /*
3783689a
MK
760 * We want to see more ZS_FULL pages and less almost empty/full.
761 * Put pages with higher ->inuse first.
58f17117 762 */
3783689a
MK
763 if (head) {
764 if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {
765 list_add(&zspage->list, &head->list);
766 return;
767 }
768 }
769 list_add(&zspage->list, &class->fullness_list[fullness]);
61989a80
NG
770}
771
c3e3e88a
NC
772/*
773 * This function removes the given zspage from the freelist identified
774 * by <class, fullness_group>.
775 */
251cbb95 776static void remove_zspage(struct size_class *class,
3783689a
MK
777 struct zspage *zspage,
778 enum fullness_group fullness)
61989a80 779{
3783689a 780 VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
48b4800a 781 VM_BUG_ON(is_zspage_isolated(zspage));
61989a80 782
3783689a 783 list_del_init(&zspage->list);
48b4800a 784 zs_stat_dec(class, fullness, 1);
61989a80
NG
785}
786
c3e3e88a
NC
787/*
788 * Each size class maintains zspages in different fullness groups depending
789 * on the number of live objects they contain. When allocating or freeing
790 * objects, the fullness status of the page can change, say, from ALMOST_FULL
791 * to ALMOST_EMPTY when freeing an object. This function checks if such
792 * a status change has occurred for the given page and accordingly moves the
793 * page from the freelist of the old fullness group to that of the new
794 * fullness group.
795 */
c7806261 796static enum fullness_group fix_fullness_group(struct size_class *class,
3783689a 797 struct zspage *zspage)
61989a80
NG
798{
799 int class_idx;
61989a80
NG
800 enum fullness_group currfg, newfg;
801
3783689a
MK
802 get_zspage_mapping(zspage, &class_idx, &currfg);
803 newfg = get_fullness_group(class, zspage);
61989a80
NG
804 if (newfg == currfg)
805 goto out;
806
48b4800a
MK
807 if (!is_zspage_isolated(zspage)) {
808 remove_zspage(class, zspage, currfg);
809 insert_zspage(class, zspage, newfg);
810 }
811
3783689a 812 set_zspage_mapping(zspage, class_idx, newfg);
61989a80
NG
813
814out:
815 return newfg;
816}
817
818/*
819 * We have to decide on how many pages to link together
820 * to form a zspage for each size class. This is important
821 * to reduce wastage due to unusable space left at end of
822 * each zspage which is given as:
888fa374
YX
823 * wastage = Zp % class_size
824 * usage = Zp - wastage
61989a80
NG
825 * where Zp = zspage size = k * PAGE_SIZE where k = 1, 2, ...
826 *
827 * For example, for size class of 3/8 * PAGE_SIZE, we should
828 * link together 3 PAGE_SIZE sized pages to form a zspage
829 * since then we can perfectly fit in 8 such objects.
830 */
2e3b6154 831static int get_pages_per_zspage(int class_size)
61989a80
NG
832{
833 int i, max_usedpc = 0;
834 /* zspage order which gives maximum used size per KB */
835 int max_usedpc_order = 1;
836
84d4faab 837 for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) {
61989a80
NG
838 int zspage_size;
839 int waste, usedpc;
840
841 zspage_size = i * PAGE_SIZE;
842 waste = zspage_size % class_size;
843 usedpc = (zspage_size - waste) * 100 / zspage_size;
844
845 if (usedpc > max_usedpc) {
846 max_usedpc = usedpc;
847 max_usedpc_order = i;
848 }
849 }
850
851 return max_usedpc_order;
852}
853
3783689a 854static struct zspage *get_zspage(struct page *page)
61989a80 855{
48b4800a
MK
856 struct zspage *zspage = (struct zspage *)page->private;
857
858 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
859 return zspage;
61989a80
NG
860}
861
862static struct page *get_next_page(struct page *page)
863{
48b4800a
MK
864 if (unlikely(PageHugeObject(page)))
865 return NULL;
866
867 return page->freelist;
61989a80
NG
868}
869
bfd093f5
MK
870/**
871 * obj_to_location - get (<page>, <obj_idx>) from encoded object value
872 * @page: page object resides in zspage
873 * @obj_idx: object index
67296874 874 */
bfd093f5
MK
875static void obj_to_location(unsigned long obj, struct page **page,
876 unsigned int *obj_idx)
61989a80 877{
bfd093f5
MK
878 obj >>= OBJ_TAG_BITS;
879 *page = pfn_to_page(obj >> OBJ_INDEX_BITS);
880 *obj_idx = (obj & OBJ_INDEX_MASK);
881}
61989a80 882
bfd093f5
MK
883/**
884 * location_to_obj - get obj value encoded from (<page>, <obj_idx>)
885 * @page: page object resides in zspage
886 * @obj_idx: object index
887 */
888static unsigned long location_to_obj(struct page *page, unsigned int obj_idx)
889{
890 unsigned long obj;
61989a80 891
312fcae2 892 obj = page_to_pfn(page) << OBJ_INDEX_BITS;
bfd093f5 893 obj |= obj_idx & OBJ_INDEX_MASK;
312fcae2 894 obj <<= OBJ_TAG_BITS;
61989a80 895
bfd093f5 896 return obj;
61989a80
NG
897}
898
2e40e163
MK
899static unsigned long handle_to_obj(unsigned long handle)
900{
901 return *(unsigned long *)handle;
902}
903
48b4800a 904static unsigned long obj_to_head(struct page *page, void *obj)
312fcae2 905{
48b4800a 906 if (unlikely(PageHugeObject(page))) {
830e4bc5 907 VM_BUG_ON_PAGE(!is_first_page(page), page);
3783689a 908 return page->index;
7b60a685
MK
909 } else
910 return *(unsigned long *)obj;
312fcae2
MK
911}
912
48b4800a
MK
913static inline int testpin_tag(unsigned long handle)
914{
915 return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
916}
917
312fcae2
MK
918static inline int trypin_tag(unsigned long handle)
919{
1b8320b6 920 return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
312fcae2
MK
921}
922
923static void pin_tag(unsigned long handle)
924{
1b8320b6 925 bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
312fcae2
MK
926}
927
928static void unpin_tag(unsigned long handle)
929{
1b8320b6 930 bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
312fcae2
MK
931}
932
f4477e90
NG
933static void reset_page(struct page *page)
934{
48b4800a 935 __ClearPageMovable(page);
18fd06bf 936 ClearPagePrivate(page);
f4477e90 937 set_page_private(page, 0);
48b4800a
MK
938 page_mapcount_reset(page);
939 ClearPageHugeObject(page);
940 page->freelist = NULL;
941}
942
943/*
944 * To prevent zspage destroy during migration, zspage freeing should
945 * hold locks of all pages in the zspage.
946 */
947void lock_zspage(struct zspage *zspage)
948{
949 struct page *page = get_first_page(zspage);
950
951 do {
952 lock_page(page);
953 } while ((page = get_next_page(page)) != NULL);
954}
955
956int trylock_zspage(struct zspage *zspage)
957{
958 struct page *cursor, *fail;
959
960 for (cursor = get_first_page(zspage); cursor != NULL; cursor =
961 get_next_page(cursor)) {
962 if (!trylock_page(cursor)) {
963 fail = cursor;
964 goto unlock;
965 }
966 }
967
968 return 1;
969unlock:
970 for (cursor = get_first_page(zspage); cursor != fail; cursor =
971 get_next_page(cursor))
972 unlock_page(cursor);
973
974 return 0;
f4477e90
NG
975}
976
48b4800a
MK
977static void __free_zspage(struct zs_pool *pool, struct size_class *class,
978 struct zspage *zspage)
61989a80 979{
3783689a 980 struct page *page, *next;
48b4800a
MK
981 enum fullness_group fg;
982 unsigned int class_idx;
983
984 get_zspage_mapping(zspage, &class_idx, &fg);
985
986 assert_spin_locked(&class->lock);
61989a80 987
3783689a 988 VM_BUG_ON(get_zspage_inuse(zspage));
48b4800a 989 VM_BUG_ON(fg != ZS_EMPTY);
61989a80 990
48b4800a 991 next = page = get_first_page(zspage);
3783689a 992 do {
48b4800a
MK
993 VM_BUG_ON_PAGE(!PageLocked(page), page);
994 next = get_next_page(page);
3783689a 995 reset_page(page);
48b4800a 996 unlock_page(page);
91537fee 997 dec_zone_page_state(page, NR_ZSPAGES);
3783689a
MK
998 put_page(page);
999 page = next;
1000 } while (page != NULL);
61989a80 1001
3783689a 1002 cache_free_zspage(pool, zspage);
48b4800a 1003
b4fd07a0 1004 zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
48b4800a
MK
1005 atomic_long_sub(class->pages_per_zspage,
1006 &pool->pages_allocated);
1007}
1008
1009static void free_zspage(struct zs_pool *pool, struct size_class *class,
1010 struct zspage *zspage)
1011{
1012 VM_BUG_ON(get_zspage_inuse(zspage));
1013 VM_BUG_ON(list_empty(&zspage->list));
1014
1015 if (!trylock_zspage(zspage)) {
1016 kick_deferred_free(pool);
1017 return;
1018 }
1019
1020 remove_zspage(class, zspage, ZS_EMPTY);
1021 __free_zspage(pool, class, zspage);
61989a80
NG
1022}
1023
1024/* Initialize a newly allocated zspage */
3783689a 1025static void init_zspage(struct size_class *class, struct zspage *zspage)
61989a80 1026{
bfd093f5 1027 unsigned int freeobj = 1;
61989a80 1028 unsigned long off = 0;
48b4800a 1029 struct page *page = get_first_page(zspage);
830e4bc5 1030
61989a80
NG
1031 while (page) {
1032 struct page *next_page;
1033 struct link_free *link;
af4ee5e9 1034 void *vaddr;
61989a80 1035
3783689a 1036 set_first_obj_offset(page, off);
61989a80 1037
af4ee5e9
MK
1038 vaddr = kmap_atomic(page);
1039 link = (struct link_free *)vaddr + off / sizeof(*link);
5538c562
DS
1040
1041 while ((off += class->size) < PAGE_SIZE) {
3b1d9ca6 1042 link->next = freeobj++ << OBJ_TAG_BITS;
5538c562 1043 link += class->size / sizeof(*link);
61989a80
NG
1044 }
1045
1046 /*
1047 * We now come to the last (full or partial) object on this
1048 * page, which must point to the first object on the next
1049 * page (if present)
1050 */
1051 next_page = get_next_page(page);
bfd093f5 1052 if (next_page) {
3b1d9ca6 1053 link->next = freeobj++ << OBJ_TAG_BITS;
bfd093f5
MK
1054 } else {
1055 /*
3b1d9ca6 1056 * Reset OBJ_TAG_BITS bit to last link to tell
bfd093f5
MK
1057 * whether it's allocated object or not.
1058 */
3b1d9ca6 1059 link->next = -1 << OBJ_TAG_BITS;
bfd093f5 1060 }
af4ee5e9 1061 kunmap_atomic(vaddr);
61989a80 1062 page = next_page;
5538c562 1063 off %= PAGE_SIZE;
61989a80 1064 }
bdb0af7c 1065
bfd093f5 1066 set_freeobj(zspage, 0);
61989a80
NG
1067}
1068
48b4800a
MK
1069static void create_page_chain(struct size_class *class, struct zspage *zspage,
1070 struct page *pages[])
61989a80 1071{
bdb0af7c
MK
1072 int i;
1073 struct page *page;
1074 struct page *prev_page = NULL;
48b4800a 1075 int nr_pages = class->pages_per_zspage;
61989a80
NG
1076
1077 /*
1078 * Allocate individual pages and link them together as:
48b4800a 1079 * 1. all pages are linked together using page->freelist
3783689a 1080 * 2. each sub-page point to zspage using page->private
61989a80 1081 *
3783689a 1082 * we set PG_private to identify the first page (i.e. no other sub-page
22c5cef1 1083 * has this flag set).
61989a80 1084 */
bdb0af7c
MK
1085 for (i = 0; i < nr_pages; i++) {
1086 page = pages[i];
3783689a 1087 set_page_private(page, (unsigned long)zspage);
48b4800a 1088 page->freelist = NULL;
bdb0af7c 1089 if (i == 0) {
3783689a 1090 zspage->first_page = page;
a27545bf 1091 SetPagePrivate(page);
48b4800a
MK
1092 if (unlikely(class->objs_per_zspage == 1 &&
1093 class->pages_per_zspage == 1))
1094 SetPageHugeObject(page);
3783689a 1095 } else {
48b4800a 1096 prev_page->freelist = page;
61989a80 1097 }
61989a80
NG
1098 prev_page = page;
1099 }
bdb0af7c 1100}
61989a80 1101
bdb0af7c
MK
1102/*
1103 * Allocate a zspage for the given size class
1104 */
3783689a
MK
1105static struct zspage *alloc_zspage(struct zs_pool *pool,
1106 struct size_class *class,
1107 gfp_t gfp)
bdb0af7c
MK
1108{
1109 int i;
bdb0af7c 1110 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE];
3783689a
MK
1111 struct zspage *zspage = cache_alloc_zspage(pool, gfp);
1112
1113 if (!zspage)
1114 return NULL;
1115
1116 memset(zspage, 0, sizeof(struct zspage));
48b4800a
MK
1117 zspage->magic = ZSPAGE_MAGIC;
1118 migrate_lock_init(zspage);
61989a80 1119
bdb0af7c
MK
1120 for (i = 0; i < class->pages_per_zspage; i++) {
1121 struct page *page;
61989a80 1122
3783689a 1123 page = alloc_page(gfp);
bdb0af7c 1124 if (!page) {
91537fee
MK
1125 while (--i >= 0) {
1126 dec_zone_page_state(pages[i], NR_ZSPAGES);
bdb0af7c 1127 __free_page(pages[i]);
91537fee 1128 }
3783689a 1129 cache_free_zspage(pool, zspage);
bdb0af7c
MK
1130 return NULL;
1131 }
91537fee
MK
1132
1133 inc_zone_page_state(page, NR_ZSPAGES);
bdb0af7c 1134 pages[i] = page;
61989a80
NG
1135 }
1136
48b4800a 1137 create_page_chain(class, zspage, pages);
3783689a 1138 init_zspage(class, zspage);
bdb0af7c 1139
3783689a 1140 return zspage;
61989a80
NG
1141}
1142
3783689a 1143static struct zspage *find_get_zspage(struct size_class *class)
61989a80
NG
1144{
1145 int i;
3783689a 1146 struct zspage *zspage;
61989a80 1147
48b4800a 1148 for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) {
3783689a
MK
1149 zspage = list_first_entry_or_null(&class->fullness_list[i],
1150 struct zspage, list);
1151 if (zspage)
61989a80
NG
1152 break;
1153 }
1154
3783689a 1155 return zspage;
61989a80
NG
1156}
1157
1b945aee 1158#ifdef CONFIG_PGTABLE_MAPPING
f553646a
SJ
1159static inline int __zs_cpu_up(struct mapping_area *area)
1160{
1161 /*
1162 * Make sure we don't leak memory if a cpu UP notification
1163 * and zs_init() race and both call zs_cpu_up() on the same cpu
1164 */
1165 if (area->vm)
1166 return 0;
1167 area->vm = alloc_vm_area(PAGE_SIZE * 2, NULL);
1168 if (!area->vm)
1169 return -ENOMEM;
1170 return 0;
1171}
1172
1173static inline void __zs_cpu_down(struct mapping_area *area)
1174{
1175 if (area->vm)
1176 free_vm_area(area->vm);
1177 area->vm = NULL;
1178}
1179
1180static inline void *__zs_map_object(struct mapping_area *area,
1181 struct page *pages[2], int off, int size)
1182{
f6f8ed47 1183 BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, pages));
f553646a
SJ
1184 area->vm_addr = area->vm->addr;
1185 return area->vm_addr + off;
1186}
1187
1188static inline void __zs_unmap_object(struct mapping_area *area,
1189 struct page *pages[2], int off, int size)
1190{
1191 unsigned long addr = (unsigned long)area->vm_addr;
f553646a 1192
d95abbbb 1193 unmap_kernel_range(addr, PAGE_SIZE * 2);
f553646a
SJ
1194}
1195
1b945aee 1196#else /* CONFIG_PGTABLE_MAPPING */
f553646a
SJ
1197
1198static inline int __zs_cpu_up(struct mapping_area *area)
1199{
1200 /*
1201 * Make sure we don't leak memory if a cpu UP notification
1202 * and zs_init() race and both call zs_cpu_up() on the same cpu
1203 */
1204 if (area->vm_buf)
1205 return 0;
40f9fb8c 1206 area->vm_buf = kmalloc(ZS_MAX_ALLOC_SIZE, GFP_KERNEL);
f553646a
SJ
1207 if (!area->vm_buf)
1208 return -ENOMEM;
1209 return 0;
1210}
1211
1212static inline void __zs_cpu_down(struct mapping_area *area)
1213{
40f9fb8c 1214 kfree(area->vm_buf);
f553646a
SJ
1215 area->vm_buf = NULL;
1216}
1217
1218static void *__zs_map_object(struct mapping_area *area,
1219 struct page *pages[2], int off, int size)
5f601902 1220{
5f601902
SJ
1221 int sizes[2];
1222 void *addr;
f553646a 1223 char *buf = area->vm_buf;
5f601902 1224
f553646a
SJ
1225 /* disable page faults to match kmap_atomic() return conditions */
1226 pagefault_disable();
1227
1228 /* no read fastpath */
1229 if (area->vm_mm == ZS_MM_WO)
1230 goto out;
5f601902
SJ
1231
1232 sizes[0] = PAGE_SIZE - off;
1233 sizes[1] = size - sizes[0];
1234
5f601902
SJ
1235 /* copy object to per-cpu buffer */
1236 addr = kmap_atomic(pages[0]);
1237 memcpy(buf, addr + off, sizes[0]);
1238 kunmap_atomic(addr);
1239 addr = kmap_atomic(pages[1]);
1240 memcpy(buf + sizes[0], addr, sizes[1]);
1241 kunmap_atomic(addr);
f553646a
SJ
1242out:
1243 return area->vm_buf;
5f601902
SJ
1244}
1245
f553646a
SJ
1246static void __zs_unmap_object(struct mapping_area *area,
1247 struct page *pages[2], int off, int size)
5f601902 1248{
5f601902
SJ
1249 int sizes[2];
1250 void *addr;
2e40e163 1251 char *buf;
5f601902 1252
f553646a
SJ
1253 /* no write fastpath */
1254 if (area->vm_mm == ZS_MM_RO)
1255 goto out;
5f601902 1256
7b60a685 1257 buf = area->vm_buf;
a82cbf07
YX
1258 buf = buf + ZS_HANDLE_SIZE;
1259 size -= ZS_HANDLE_SIZE;
1260 off += ZS_HANDLE_SIZE;
2e40e163 1261
5f601902
SJ
1262 sizes[0] = PAGE_SIZE - off;
1263 sizes[1] = size - sizes[0];
1264
1265 /* copy per-cpu buffer to object */
1266 addr = kmap_atomic(pages[0]);
1267 memcpy(addr + off, buf, sizes[0]);
1268 kunmap_atomic(addr);
1269 addr = kmap_atomic(pages[1]);
1270 memcpy(addr, buf + sizes[0], sizes[1]);
1271 kunmap_atomic(addr);
f553646a
SJ
1272
1273out:
1274 /* enable page faults to match kunmap_atomic() return conditions */
1275 pagefault_enable();
5f601902 1276}
61989a80 1277
1b945aee 1278#endif /* CONFIG_PGTABLE_MAPPING */
f553646a 1279
215c89d0 1280static int zs_cpu_prepare(unsigned int cpu)
61989a80 1281{
61989a80
NG
1282 struct mapping_area *area;
1283
215c89d0
SAS
1284 area = &per_cpu(zs_map_area, cpu);
1285 return __zs_cpu_up(area);
61989a80
NG
1286}
1287
215c89d0 1288static int zs_cpu_dead(unsigned int cpu)
61989a80 1289{
215c89d0 1290 struct mapping_area *area;
40f9fb8c 1291
215c89d0
SAS
1292 area = &per_cpu(zs_map_area, cpu);
1293 __zs_cpu_down(area);
1294 return 0;
b1b00a5b
SS
1295}
1296
35b3445e 1297static void __init init_zs_size_classes(void)
b1b00a5b 1298{
66cdef66 1299 int nr;
c795779d 1300
66cdef66
GM
1301 nr = (ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) / ZS_SIZE_CLASS_DELTA + 1;
1302 if ((ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) % ZS_SIZE_CLASS_DELTA)
1303 nr += 1;
40f9fb8c 1304
66cdef66 1305 zs_size_classes = nr;
61989a80
NG
1306}
1307
64d90465
GM
1308static bool can_merge(struct size_class *prev, int pages_per_zspage,
1309 int objs_per_zspage)
9eec4cd5 1310{
64d90465
GM
1311 if (prev->pages_per_zspage == pages_per_zspage &&
1312 prev->objs_per_zspage == objs_per_zspage)
1313 return true;
9eec4cd5 1314
64d90465 1315 return false;
9eec4cd5
JK
1316}
1317
3783689a 1318static bool zspage_full(struct size_class *class, struct zspage *zspage)
312fcae2 1319{
3783689a 1320 return get_zspage_inuse(zspage) == class->objs_per_zspage;
312fcae2
MK
1321}
1322
66cdef66
GM
1323unsigned long zs_get_total_pages(struct zs_pool *pool)
1324{
1325 return atomic_long_read(&pool->pages_allocated);
1326}
1327EXPORT_SYMBOL_GPL(zs_get_total_pages);
1328
4bbc0bc0 1329/**
66cdef66
GM
1330 * zs_map_object - get address of allocated object from handle.
1331 * @pool: pool from which the object was allocated
1332 * @handle: handle returned from zs_malloc
4bbc0bc0 1333 *
66cdef66
GM
1334 * Before using an object allocated from zs_malloc, it must be mapped using
1335 * this function. When done with the object, it must be unmapped using
1336 * zs_unmap_object.
4bbc0bc0 1337 *
66cdef66
GM
1338 * Only one object can be mapped per cpu at a time. There is no protection
1339 * against nested mappings.
1340 *
1341 * This function returns with preemption and page faults disabled.
4bbc0bc0 1342 */
66cdef66
GM
1343void *zs_map_object(struct zs_pool *pool, unsigned long handle,
1344 enum zs_mapmode mm)
61989a80 1345{
3783689a 1346 struct zspage *zspage;
66cdef66 1347 struct page *page;
bfd093f5
MK
1348 unsigned long obj, off;
1349 unsigned int obj_idx;
61989a80 1350
66cdef66
GM
1351 unsigned int class_idx;
1352 enum fullness_group fg;
1353 struct size_class *class;
1354 struct mapping_area *area;
1355 struct page *pages[2];
2e40e163 1356 void *ret;
61989a80 1357
9eec4cd5 1358 /*
66cdef66
GM
1359 * Because we use per-cpu mapping areas shared among the
1360 * pools/users, we can't allow mapping in interrupt context
1361 * because it can corrupt another users mappings.
9eec4cd5 1362 */
830e4bc5 1363 WARN_ON_ONCE(in_interrupt());
61989a80 1364
312fcae2
MK
1365 /* From now on, migration cannot move the object */
1366 pin_tag(handle);
1367
2e40e163
MK
1368 obj = handle_to_obj(handle);
1369 obj_to_location(obj, &page, &obj_idx);
3783689a 1370 zspage = get_zspage(page);
48b4800a
MK
1371
1372 /* migration cannot move any subpage in this zspage */
1373 migrate_read_lock(zspage);
1374
3783689a 1375 get_zspage_mapping(zspage, &class_idx, &fg);
66cdef66 1376 class = pool->size_class[class_idx];
bfd093f5 1377 off = (class->size * obj_idx) & ~PAGE_MASK;
df8b5bb9 1378
66cdef66
GM
1379 area = &get_cpu_var(zs_map_area);
1380 area->vm_mm = mm;
1381 if (off + class->size <= PAGE_SIZE) {
1382 /* this object is contained entirely within a page */
1383 area->vm_addr = kmap_atomic(page);
2e40e163
MK
1384 ret = area->vm_addr + off;
1385 goto out;
61989a80
NG
1386 }
1387
66cdef66
GM
1388 /* this object spans two pages */
1389 pages[0] = page;
1390 pages[1] = get_next_page(page);
1391 BUG_ON(!pages[1]);
9eec4cd5 1392
2e40e163
MK
1393 ret = __zs_map_object(area, pages, off, class->size);
1394out:
48b4800a 1395 if (likely(!PageHugeObject(page)))
7b60a685
MK
1396 ret += ZS_HANDLE_SIZE;
1397
1398 return ret;
61989a80 1399}
66cdef66 1400EXPORT_SYMBOL_GPL(zs_map_object);
61989a80 1401
66cdef66 1402void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
61989a80 1403{
3783689a 1404 struct zspage *zspage;
66cdef66 1405 struct page *page;
bfd093f5
MK
1406 unsigned long obj, off;
1407 unsigned int obj_idx;
61989a80 1408
66cdef66
GM
1409 unsigned int class_idx;
1410 enum fullness_group fg;
1411 struct size_class *class;
1412 struct mapping_area *area;
9eec4cd5 1413
2e40e163
MK
1414 obj = handle_to_obj(handle);
1415 obj_to_location(obj, &page, &obj_idx);
3783689a
MK
1416 zspage = get_zspage(page);
1417 get_zspage_mapping(zspage, &class_idx, &fg);
66cdef66 1418 class = pool->size_class[class_idx];
bfd093f5 1419 off = (class->size * obj_idx) & ~PAGE_MASK;
61989a80 1420
66cdef66
GM
1421 area = this_cpu_ptr(&zs_map_area);
1422 if (off + class->size <= PAGE_SIZE)
1423 kunmap_atomic(area->vm_addr);
1424 else {
1425 struct page *pages[2];
40f9fb8c 1426
66cdef66
GM
1427 pages[0] = page;
1428 pages[1] = get_next_page(page);
1429 BUG_ON(!pages[1]);
1430
1431 __zs_unmap_object(area, pages, off, class->size);
1432 }
1433 put_cpu_var(zs_map_area);
48b4800a
MK
1434
1435 migrate_read_unlock(zspage);
312fcae2 1436 unpin_tag(handle);
61989a80 1437}
66cdef66 1438EXPORT_SYMBOL_GPL(zs_unmap_object);
61989a80 1439
251cbb95 1440static unsigned long obj_malloc(struct size_class *class,
3783689a 1441 struct zspage *zspage, unsigned long handle)
c7806261 1442{
bfd093f5 1443 int i, nr_page, offset;
c7806261
MK
1444 unsigned long obj;
1445 struct link_free *link;
1446
1447 struct page *m_page;
bfd093f5 1448 unsigned long m_offset;
c7806261
MK
1449 void *vaddr;
1450
312fcae2 1451 handle |= OBJ_ALLOCATED_TAG;
3783689a 1452 obj = get_freeobj(zspage);
bfd093f5
MK
1453
1454 offset = obj * class->size;
1455 nr_page = offset >> PAGE_SHIFT;
1456 m_offset = offset & ~PAGE_MASK;
1457 m_page = get_first_page(zspage);
1458
1459 for (i = 0; i < nr_page; i++)
1460 m_page = get_next_page(m_page);
c7806261
MK
1461
1462 vaddr = kmap_atomic(m_page);
1463 link = (struct link_free *)vaddr + m_offset / sizeof(*link);
3b1d9ca6 1464 set_freeobj(zspage, link->next >> OBJ_TAG_BITS);
48b4800a 1465 if (likely(!PageHugeObject(m_page)))
7b60a685
MK
1466 /* record handle in the header of allocated chunk */
1467 link->handle = handle;
1468 else
3783689a
MK
1469 /* record handle to page->index */
1470 zspage->first_page->index = handle;
1471
c7806261 1472 kunmap_atomic(vaddr);
3783689a 1473 mod_zspage_inuse(zspage, 1);
c7806261
MK
1474 zs_stat_inc(class, OBJ_USED, 1);
1475
bfd093f5
MK
1476 obj = location_to_obj(m_page, obj);
1477
c7806261
MK
1478 return obj;
1479}
1480
1481
61989a80
NG
1482/**
1483 * zs_malloc - Allocate block of given size from pool.
1484 * @pool: pool to allocate from
1485 * @size: size of block to allocate
fd854463 1486 * @gfp: gfp flags when allocating object
61989a80 1487 *
00a61d86 1488 * On success, handle to the allocated object is returned,
c2344348 1489 * otherwise 0.
61989a80
NG
1490 * Allocation requests with size > ZS_MAX_ALLOC_SIZE will fail.
1491 */
d0d8da2d 1492unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
61989a80 1493{
2e40e163 1494 unsigned long handle, obj;
61989a80 1495 struct size_class *class;
48b4800a 1496 enum fullness_group newfg;
3783689a 1497 struct zspage *zspage;
61989a80 1498
7b60a685 1499 if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE))
2e40e163
MK
1500 return 0;
1501
3783689a 1502 handle = cache_alloc_handle(pool, gfp);
2e40e163 1503 if (!handle)
c2344348 1504 return 0;
61989a80 1505
2e40e163
MK
1506 /* extra space in chunk to keep the handle */
1507 size += ZS_HANDLE_SIZE;
9eec4cd5 1508 class = pool->size_class[get_size_class_index(size)];
61989a80
NG
1509
1510 spin_lock(&class->lock);
3783689a 1511 zspage = find_get_zspage(class);
48b4800a
MK
1512 if (likely(zspage)) {
1513 obj = obj_malloc(class, zspage, handle);
1514 /* Now move the zspage to another fullness group, if required */
1515 fix_fullness_group(class, zspage);
1516 record_obj(handle, obj);
61989a80 1517 spin_unlock(&class->lock);
61989a80 1518
48b4800a
MK
1519 return handle;
1520 }
0f050d99 1521
48b4800a
MK
1522 spin_unlock(&class->lock);
1523
1524 zspage = alloc_zspage(pool, class, gfp);
1525 if (!zspage) {
1526 cache_free_handle(pool, handle);
1527 return 0;
61989a80
NG
1528 }
1529
48b4800a 1530 spin_lock(&class->lock);
3783689a 1531 obj = obj_malloc(class, zspage, handle);
48b4800a
MK
1532 newfg = get_fullness_group(class, zspage);
1533 insert_zspage(class, zspage, newfg);
1534 set_zspage_mapping(zspage, class->index, newfg);
2e40e163 1535 record_obj(handle, obj);
48b4800a
MK
1536 atomic_long_add(class->pages_per_zspage,
1537 &pool->pages_allocated);
b4fd07a0 1538 zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage);
48b4800a
MK
1539
1540 /* We completely set up zspage so mark them as movable */
1541 SetZsPageMovable(pool, zspage);
61989a80
NG
1542 spin_unlock(&class->lock);
1543
2e40e163 1544 return handle;
61989a80
NG
1545}
1546EXPORT_SYMBOL_GPL(zs_malloc);
1547
1ee47165 1548static void obj_free(struct size_class *class, unsigned long obj)
61989a80
NG
1549{
1550 struct link_free *link;
3783689a
MK
1551 struct zspage *zspage;
1552 struct page *f_page;
bfd093f5
MK
1553 unsigned long f_offset;
1554 unsigned int f_objidx;
af4ee5e9 1555 void *vaddr;
61989a80 1556
312fcae2 1557 obj &= ~OBJ_ALLOCATED_TAG;
2e40e163 1558 obj_to_location(obj, &f_page, &f_objidx);
bfd093f5 1559 f_offset = (class->size * f_objidx) & ~PAGE_MASK;
3783689a 1560 zspage = get_zspage(f_page);
61989a80 1561
c7806261 1562 vaddr = kmap_atomic(f_page);
61989a80
NG
1563
1564 /* Insert this object in containing zspage's freelist */
af4ee5e9 1565 link = (struct link_free *)(vaddr + f_offset);
3b1d9ca6 1566 link->next = get_freeobj(zspage) << OBJ_TAG_BITS;
af4ee5e9 1567 kunmap_atomic(vaddr);
bfd093f5 1568 set_freeobj(zspage, f_objidx);
3783689a 1569 mod_zspage_inuse(zspage, -1);
0f050d99 1570 zs_stat_dec(class, OBJ_USED, 1);
c7806261
MK
1571}
1572
1573void zs_free(struct zs_pool *pool, unsigned long handle)
1574{
3783689a
MK
1575 struct zspage *zspage;
1576 struct page *f_page;
bfd093f5
MK
1577 unsigned long obj;
1578 unsigned int f_objidx;
c7806261
MK
1579 int class_idx;
1580 struct size_class *class;
1581 enum fullness_group fullness;
48b4800a 1582 bool isolated;
c7806261
MK
1583
1584 if (unlikely(!handle))
1585 return;
1586
312fcae2 1587 pin_tag(handle);
c7806261 1588 obj = handle_to_obj(handle);
c7806261 1589 obj_to_location(obj, &f_page, &f_objidx);
3783689a 1590 zspage = get_zspage(f_page);
c7806261 1591
48b4800a
MK
1592 migrate_read_lock(zspage);
1593
3783689a 1594 get_zspage_mapping(zspage, &class_idx, &fullness);
c7806261
MK
1595 class = pool->size_class[class_idx];
1596
1597 spin_lock(&class->lock);
1ee47165 1598 obj_free(class, obj);
3783689a 1599 fullness = fix_fullness_group(class, zspage);
48b4800a
MK
1600 if (fullness != ZS_EMPTY) {
1601 migrate_read_unlock(zspage);
1602 goto out;
312fcae2 1603 }
48b4800a
MK
1604
1605 isolated = is_zspage_isolated(zspage);
1606 migrate_read_unlock(zspage);
1607 /* If zspage is isolated, zs_page_putback will free the zspage */
1608 if (likely(!isolated))
1609 free_zspage(pool, class, zspage);
1610out:
1611
61989a80 1612 spin_unlock(&class->lock);
312fcae2 1613 unpin_tag(handle);
3783689a 1614 cache_free_handle(pool, handle);
312fcae2
MK
1615}
1616EXPORT_SYMBOL_GPL(zs_free);
1617
251cbb95
MK
1618static void zs_object_copy(struct size_class *class, unsigned long dst,
1619 unsigned long src)
312fcae2
MK
1620{
1621 struct page *s_page, *d_page;
bfd093f5 1622 unsigned int s_objidx, d_objidx;
312fcae2
MK
1623 unsigned long s_off, d_off;
1624 void *s_addr, *d_addr;
1625 int s_size, d_size, size;
1626 int written = 0;
1627
1628 s_size = d_size = class->size;
1629
1630 obj_to_location(src, &s_page, &s_objidx);
1631 obj_to_location(dst, &d_page, &d_objidx);
1632
bfd093f5
MK
1633 s_off = (class->size * s_objidx) & ~PAGE_MASK;
1634 d_off = (class->size * d_objidx) & ~PAGE_MASK;
312fcae2
MK
1635
1636 if (s_off + class->size > PAGE_SIZE)
1637 s_size = PAGE_SIZE - s_off;
1638
1639 if (d_off + class->size > PAGE_SIZE)
1640 d_size = PAGE_SIZE - d_off;
1641
1642 s_addr = kmap_atomic(s_page);
1643 d_addr = kmap_atomic(d_page);
1644
1645 while (1) {
1646 size = min(s_size, d_size);
1647 memcpy(d_addr + d_off, s_addr + s_off, size);
1648 written += size;
1649
1650 if (written == class->size)
1651 break;
1652
495819ea
SS
1653 s_off += size;
1654 s_size -= size;
1655 d_off += size;
1656 d_size -= size;
1657
1658 if (s_off >= PAGE_SIZE) {
312fcae2
MK
1659 kunmap_atomic(d_addr);
1660 kunmap_atomic(s_addr);
1661 s_page = get_next_page(s_page);
312fcae2
MK
1662 s_addr = kmap_atomic(s_page);
1663 d_addr = kmap_atomic(d_page);
1664 s_size = class->size - written;
1665 s_off = 0;
312fcae2
MK
1666 }
1667
495819ea 1668 if (d_off >= PAGE_SIZE) {
312fcae2
MK
1669 kunmap_atomic(d_addr);
1670 d_page = get_next_page(d_page);
312fcae2
MK
1671 d_addr = kmap_atomic(d_page);
1672 d_size = class->size - written;
1673 d_off = 0;
312fcae2
MK
1674 }
1675 }
1676
1677 kunmap_atomic(d_addr);
1678 kunmap_atomic(s_addr);
1679}
1680
1681/*
1682 * Find alloced object in zspage from index object and
1683 * return handle.
1684 */
251cbb95 1685static unsigned long find_alloced_obj(struct size_class *class,
cf675acb 1686 struct page *page, int *obj_idx)
312fcae2
MK
1687{
1688 unsigned long head;
1689 int offset = 0;
cf675acb 1690 int index = *obj_idx;
312fcae2
MK
1691 unsigned long handle = 0;
1692 void *addr = kmap_atomic(page);
1693
3783689a 1694 offset = get_first_obj_offset(page);
312fcae2
MK
1695 offset += class->size * index;
1696
1697 while (offset < PAGE_SIZE) {
48b4800a 1698 head = obj_to_head(page, addr + offset);
312fcae2
MK
1699 if (head & OBJ_ALLOCATED_TAG) {
1700 handle = head & ~OBJ_ALLOCATED_TAG;
1701 if (trypin_tag(handle))
1702 break;
1703 handle = 0;
1704 }
1705
1706 offset += class->size;
1707 index++;
1708 }
1709
1710 kunmap_atomic(addr);
cf675acb
GM
1711
1712 *obj_idx = index;
1713
312fcae2
MK
1714 return handle;
1715}
1716
1717struct zs_compact_control {
3783689a 1718 /* Source spage for migration which could be a subpage of zspage */
312fcae2
MK
1719 struct page *s_page;
1720 /* Destination page for migration which should be a first page
1721 * of zspage. */
1722 struct page *d_page;
1723 /* Starting object index within @s_page which used for live object
1724 * in the subpage. */
41b88e14 1725 int obj_idx;
312fcae2
MK
1726};
1727
1728static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
1729 struct zs_compact_control *cc)
1730{
1731 unsigned long used_obj, free_obj;
1732 unsigned long handle;
1733 struct page *s_page = cc->s_page;
1734 struct page *d_page = cc->d_page;
41b88e14 1735 int obj_idx = cc->obj_idx;
312fcae2
MK
1736 int ret = 0;
1737
1738 while (1) {
cf675acb 1739 handle = find_alloced_obj(class, s_page, &obj_idx);
312fcae2
MK
1740 if (!handle) {
1741 s_page = get_next_page(s_page);
1742 if (!s_page)
1743 break;
41b88e14 1744 obj_idx = 0;
312fcae2
MK
1745 continue;
1746 }
1747
1748 /* Stop if there is no more space */
3783689a 1749 if (zspage_full(class, get_zspage(d_page))) {
312fcae2
MK
1750 unpin_tag(handle);
1751 ret = -ENOMEM;
1752 break;
1753 }
1754
1755 used_obj = handle_to_obj(handle);
3783689a 1756 free_obj = obj_malloc(class, get_zspage(d_page), handle);
251cbb95 1757 zs_object_copy(class, free_obj, used_obj);
41b88e14 1758 obj_idx++;
c102f07c
JL
1759 /*
1760 * record_obj updates handle's value to free_obj and it will
1761 * invalidate lock bit(ie, HANDLE_PIN_BIT) of handle, which
1762 * breaks synchronization using pin_tag(e,g, zs_free) so
1763 * let's keep the lock bit.
1764 */
1765 free_obj |= BIT(HANDLE_PIN_BIT);
312fcae2
MK
1766 record_obj(handle, free_obj);
1767 unpin_tag(handle);
1ee47165 1768 obj_free(class, used_obj);
312fcae2
MK
1769 }
1770
1771 /* Remember last position in this iteration */
1772 cc->s_page = s_page;
41b88e14 1773 cc->obj_idx = obj_idx;
312fcae2
MK
1774
1775 return ret;
1776}
1777
3783689a 1778static struct zspage *isolate_zspage(struct size_class *class, bool source)
312fcae2
MK
1779{
1780 int i;
3783689a
MK
1781 struct zspage *zspage;
1782 enum fullness_group fg[2] = {ZS_ALMOST_EMPTY, ZS_ALMOST_FULL};
312fcae2 1783
3783689a
MK
1784 if (!source) {
1785 fg[0] = ZS_ALMOST_FULL;
1786 fg[1] = ZS_ALMOST_EMPTY;
1787 }
1788
1789 for (i = 0; i < 2; i++) {
1790 zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],
1791 struct zspage, list);
1792 if (zspage) {
48b4800a 1793 VM_BUG_ON(is_zspage_isolated(zspage));
3783689a
MK
1794 remove_zspage(class, zspage, fg[i]);
1795 return zspage;
312fcae2
MK
1796 }
1797 }
1798
3783689a 1799 return zspage;
312fcae2
MK
1800}
1801
860c707d 1802/*
3783689a 1803 * putback_zspage - add @zspage into right class's fullness list
860c707d 1804 * @class: destination class
3783689a 1805 * @zspage: target page
860c707d 1806 *
3783689a 1807 * Return @zspage's fullness_group
860c707d 1808 */
4aa409ca 1809static enum fullness_group putback_zspage(struct size_class *class,
3783689a 1810 struct zspage *zspage)
312fcae2 1811{
312fcae2
MK
1812 enum fullness_group fullness;
1813
48b4800a
MK
1814 VM_BUG_ON(is_zspage_isolated(zspage));
1815
3783689a
MK
1816 fullness = get_fullness_group(class, zspage);
1817 insert_zspage(class, zspage, fullness);
1818 set_zspage_mapping(zspage, class->index, fullness);
839373e6 1819
860c707d 1820 return fullness;
61989a80 1821}
312fcae2 1822
48b4800a
MK
1823#ifdef CONFIG_COMPACTION
1824static struct dentry *zs_mount(struct file_system_type *fs_type,
1825 int flags, const char *dev_name, void *data)
1826{
1827 static const struct dentry_operations ops = {
1828 .d_dname = simple_dname,
1829 };
1830
1831 return mount_pseudo(fs_type, "zsmalloc:", NULL, &ops, ZSMALLOC_MAGIC);
1832}
1833
1834static struct file_system_type zsmalloc_fs = {
1835 .name = "zsmalloc",
1836 .mount = zs_mount,
1837 .kill_sb = kill_anon_super,
1838};
1839
1840static int zsmalloc_mount(void)
1841{
1842 int ret = 0;
1843
1844 zsmalloc_mnt = kern_mount(&zsmalloc_fs);
1845 if (IS_ERR(zsmalloc_mnt))
1846 ret = PTR_ERR(zsmalloc_mnt);
1847
1848 return ret;
1849}
1850
1851static void zsmalloc_unmount(void)
1852{
1853 kern_unmount(zsmalloc_mnt);
1854}
1855
1856static void migrate_lock_init(struct zspage *zspage)
1857{
1858 rwlock_init(&zspage->lock);
1859}
1860
1861static void migrate_read_lock(struct zspage *zspage)
1862{
1863 read_lock(&zspage->lock);
1864}
1865
1866static void migrate_read_unlock(struct zspage *zspage)
1867{
1868 read_unlock(&zspage->lock);
1869}
1870
1871static void migrate_write_lock(struct zspage *zspage)
1872{
1873 write_lock(&zspage->lock);
1874}
1875
1876static void migrate_write_unlock(struct zspage *zspage)
1877{
1878 write_unlock(&zspage->lock);
1879}
1880
1881/* Number of isolated subpage for *page migration* in this zspage */
1882static void inc_zspage_isolation(struct zspage *zspage)
1883{
1884 zspage->isolated++;
1885}
1886
1887static void dec_zspage_isolation(struct zspage *zspage)
1888{
1889 zspage->isolated--;
1890}
1891
1892static void replace_sub_page(struct size_class *class, struct zspage *zspage,
1893 struct page *newpage, struct page *oldpage)
1894{
1895 struct page *page;
1896 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, };
1897 int idx = 0;
1898
1899 page = get_first_page(zspage);
1900 do {
1901 if (page == oldpage)
1902 pages[idx] = newpage;
1903 else
1904 pages[idx] = page;
1905 idx++;
1906 } while ((page = get_next_page(page)) != NULL);
1907
1908 create_page_chain(class, zspage, pages);
1909 set_first_obj_offset(newpage, get_first_obj_offset(oldpage));
1910 if (unlikely(PageHugeObject(oldpage)))
1911 newpage->index = oldpage->index;
1912 __SetPageMovable(newpage, page_mapping(oldpage));
1913}
1914
1915bool zs_page_isolate(struct page *page, isolate_mode_t mode)
1916{
1917 struct zs_pool *pool;
1918 struct size_class *class;
1919 int class_idx;
1920 enum fullness_group fullness;
1921 struct zspage *zspage;
1922 struct address_space *mapping;
1923
1924 /*
1925 * Page is locked so zspage couldn't be destroyed. For detail, look at
1926 * lock_zspage in free_zspage.
1927 */
1928 VM_BUG_ON_PAGE(!PageMovable(page), page);
1929 VM_BUG_ON_PAGE(PageIsolated(page), page);
1930
1931 zspage = get_zspage(page);
1932
1933 /*
1934 * Without class lock, fullness could be stale while class_idx is okay
1935 * because class_idx is constant unless page is freed so we should get
1936 * fullness again under class lock.
1937 */
1938 get_zspage_mapping(zspage, &class_idx, &fullness);
1939 mapping = page_mapping(page);
1940 pool = mapping->private_data;
1941 class = pool->size_class[class_idx];
1942
1943 spin_lock(&class->lock);
1944 if (get_zspage_inuse(zspage) == 0) {
1945 spin_unlock(&class->lock);
1946 return false;
1947 }
1948
1949 /* zspage is isolated for object migration */
1950 if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1951 spin_unlock(&class->lock);
1952 return false;
1953 }
1954
1955 /*
1956 * If this is first time isolation for the zspage, isolate zspage from
1957 * size_class to prevent further object allocation from the zspage.
1958 */
1959 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1960 get_zspage_mapping(zspage, &class_idx, &fullness);
1961 remove_zspage(class, zspage, fullness);
1962 }
1963
1964 inc_zspage_isolation(zspage);
1965 spin_unlock(&class->lock);
1966
1967 return true;
1968}
1969
1970int zs_page_migrate(struct address_space *mapping, struct page *newpage,
1971 struct page *page, enum migrate_mode mode)
1972{
1973 struct zs_pool *pool;
1974 struct size_class *class;
1975 int class_idx;
1976 enum fullness_group fullness;
1977 struct zspage *zspage;
1978 struct page *dummy;
1979 void *s_addr, *d_addr, *addr;
1980 int offset, pos;
1981 unsigned long handle, head;
1982 unsigned long old_obj, new_obj;
1983 unsigned int obj_idx;
1984 int ret = -EAGAIN;
1985
1986 VM_BUG_ON_PAGE(!PageMovable(page), page);
1987 VM_BUG_ON_PAGE(!PageIsolated(page), page);
1988
1989 zspage = get_zspage(page);
1990
1991 /* Concurrent compactor cannot migrate any subpage in zspage */
1992 migrate_write_lock(zspage);
1993 get_zspage_mapping(zspage, &class_idx, &fullness);
1994 pool = mapping->private_data;
1995 class = pool->size_class[class_idx];
1996 offset = get_first_obj_offset(page);
1997
1998 spin_lock(&class->lock);
1999 if (!get_zspage_inuse(zspage)) {
2000 ret = -EBUSY;
2001 goto unlock_class;
2002 }
2003
2004 pos = offset;
2005 s_addr = kmap_atomic(page);
2006 while (pos < PAGE_SIZE) {
2007 head = obj_to_head(page, s_addr + pos);
2008 if (head & OBJ_ALLOCATED_TAG) {
2009 handle = head & ~OBJ_ALLOCATED_TAG;
2010 if (!trypin_tag(handle))
2011 goto unpin_objects;
2012 }
2013 pos += class->size;
2014 }
2015
2016 /*
2017 * Here, any user cannot access all objects in the zspage so let's move.
2018 */
2019 d_addr = kmap_atomic(newpage);
2020 memcpy(d_addr, s_addr, PAGE_SIZE);
2021 kunmap_atomic(d_addr);
2022
2023 for (addr = s_addr + offset; addr < s_addr + pos;
2024 addr += class->size) {
2025 head = obj_to_head(page, addr);
2026 if (head & OBJ_ALLOCATED_TAG) {
2027 handle = head & ~OBJ_ALLOCATED_TAG;
2028 if (!testpin_tag(handle))
2029 BUG();
2030
2031 old_obj = handle_to_obj(handle);
2032 obj_to_location(old_obj, &dummy, &obj_idx);
2033 new_obj = (unsigned long)location_to_obj(newpage,
2034 obj_idx);
2035 new_obj |= BIT(HANDLE_PIN_BIT);
2036 record_obj(handle, new_obj);
2037 }
2038 }
2039
2040 replace_sub_page(class, zspage, newpage, page);
2041 get_page(newpage);
2042
2043 dec_zspage_isolation(zspage);
2044
2045 /*
2046 * Page migration is done so let's putback isolated zspage to
2047 * the list if @page is final isolated subpage in the zspage.
2048 */
2049 if (!is_zspage_isolated(zspage))
2050 putback_zspage(class, zspage);
2051
2052 reset_page(page);
2053 put_page(page);
2054 page = newpage;
2055
dd4123f3 2056 ret = MIGRATEPAGE_SUCCESS;
48b4800a
MK
2057unpin_objects:
2058 for (addr = s_addr + offset; addr < s_addr + pos;
2059 addr += class->size) {
2060 head = obj_to_head(page, addr);
2061 if (head & OBJ_ALLOCATED_TAG) {
2062 handle = head & ~OBJ_ALLOCATED_TAG;
2063 if (!testpin_tag(handle))
2064 BUG();
2065 unpin_tag(handle);
2066 }
2067 }
2068 kunmap_atomic(s_addr);
2069unlock_class:
2070 spin_unlock(&class->lock);
2071 migrate_write_unlock(zspage);
2072
2073 return ret;
2074}
2075
2076void zs_page_putback(struct page *page)
2077{
2078 struct zs_pool *pool;
2079 struct size_class *class;
2080 int class_idx;
2081 enum fullness_group fg;
2082 struct address_space *mapping;
2083 struct zspage *zspage;
2084
2085 VM_BUG_ON_PAGE(!PageMovable(page), page);
2086 VM_BUG_ON_PAGE(!PageIsolated(page), page);
2087
2088 zspage = get_zspage(page);
2089 get_zspage_mapping(zspage, &class_idx, &fg);
2090 mapping = page_mapping(page);
2091 pool = mapping->private_data;
2092 class = pool->size_class[class_idx];
2093
2094 spin_lock(&class->lock);
2095 dec_zspage_isolation(zspage);
2096 if (!is_zspage_isolated(zspage)) {
2097 fg = putback_zspage(class, zspage);
2098 /*
2099 * Due to page_lock, we cannot free zspage immediately
2100 * so let's defer.
2101 */
2102 if (fg == ZS_EMPTY)
2103 schedule_work(&pool->free_work);
2104 }
2105 spin_unlock(&class->lock);
2106}
2107
2108const struct address_space_operations zsmalloc_aops = {
2109 .isolate_page = zs_page_isolate,
2110 .migratepage = zs_page_migrate,
2111 .putback_page = zs_page_putback,
2112};
2113
2114static int zs_register_migration(struct zs_pool *pool)
2115{
2116 pool->inode = alloc_anon_inode(zsmalloc_mnt->mnt_sb);
2117 if (IS_ERR(pool->inode)) {
2118 pool->inode = NULL;
2119 return 1;
2120 }
2121
2122 pool->inode->i_mapping->private_data = pool;
2123 pool->inode->i_mapping->a_ops = &zsmalloc_aops;
2124 return 0;
2125}
2126
2127static void zs_unregister_migration(struct zs_pool *pool)
2128{
2129 flush_work(&pool->free_work);
c3491eca 2130 iput(pool->inode);
48b4800a
MK
2131}
2132
2133/*
2134 * Caller should hold page_lock of all pages in the zspage
2135 * In here, we cannot use zspage meta data.
2136 */
2137static void async_free_zspage(struct work_struct *work)
2138{
2139 int i;
2140 struct size_class *class;
2141 unsigned int class_idx;
2142 enum fullness_group fullness;
2143 struct zspage *zspage, *tmp;
2144 LIST_HEAD(free_pages);
2145 struct zs_pool *pool = container_of(work, struct zs_pool,
2146 free_work);
2147
2148 for (i = 0; i < zs_size_classes; i++) {
2149 class = pool->size_class[i];
2150 if (class->index != i)
2151 continue;
2152
2153 spin_lock(&class->lock);
2154 list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages);
2155 spin_unlock(&class->lock);
2156 }
2157
2158
2159 list_for_each_entry_safe(zspage, tmp, &free_pages, list) {
2160 list_del(&zspage->list);
2161 lock_zspage(zspage);
2162
2163 get_zspage_mapping(zspage, &class_idx, &fullness);
2164 VM_BUG_ON(fullness != ZS_EMPTY);
2165 class = pool->size_class[class_idx];
2166 spin_lock(&class->lock);
2167 __free_zspage(pool, pool->size_class[class_idx], zspage);
2168 spin_unlock(&class->lock);
2169 }
2170};
2171
2172static void kick_deferred_free(struct zs_pool *pool)
2173{
2174 schedule_work(&pool->free_work);
2175}
2176
2177static void init_deferred_free(struct zs_pool *pool)
2178{
2179 INIT_WORK(&pool->free_work, async_free_zspage);
2180}
2181
2182static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)
2183{
2184 struct page *page = get_first_page(zspage);
2185
2186 do {
2187 WARN_ON(!trylock_page(page));
2188 __SetPageMovable(page, pool->inode->i_mapping);
2189 unlock_page(page);
2190 } while ((page = get_next_page(page)) != NULL);
2191}
2192#endif
2193
04f05909
SS
2194/*
2195 *
2196 * Based on the number of unused allocated objects calculate
2197 * and return the number of pages that we can free.
04f05909
SS
2198 */
2199static unsigned long zs_can_compact(struct size_class *class)
2200{
2201 unsigned long obj_wasted;
44f43e99
SS
2202 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
2203 unsigned long obj_used = zs_stat_get(class, OBJ_USED);
04f05909 2204
44f43e99
SS
2205 if (obj_allocated <= obj_used)
2206 return 0;
04f05909 2207
44f43e99 2208 obj_wasted = obj_allocated - obj_used;
b4fd07a0 2209 obj_wasted /= class->objs_per_zspage;
04f05909 2210
6cbf16b3 2211 return obj_wasted * class->pages_per_zspage;
04f05909
SS
2212}
2213
7d3f3938 2214static void __zs_compact(struct zs_pool *pool, struct size_class *class)
312fcae2 2215{
312fcae2 2216 struct zs_compact_control cc;
3783689a
MK
2217 struct zspage *src_zspage;
2218 struct zspage *dst_zspage = NULL;
312fcae2 2219
312fcae2 2220 spin_lock(&class->lock);
3783689a 2221 while ((src_zspage = isolate_zspage(class, true))) {
312fcae2 2222
04f05909
SS
2223 if (!zs_can_compact(class))
2224 break;
2225
41b88e14 2226 cc.obj_idx = 0;
48b4800a 2227 cc.s_page = get_first_page(src_zspage);
312fcae2 2228
3783689a 2229 while ((dst_zspage = isolate_zspage(class, false))) {
48b4800a 2230 cc.d_page = get_first_page(dst_zspage);
312fcae2 2231 /*
0dc63d48
SS
2232 * If there is no more space in dst_page, resched
2233 * and see if anyone had allocated another zspage.
312fcae2
MK
2234 */
2235 if (!migrate_zspage(pool, class, &cc))
2236 break;
2237
4aa409ca 2238 putback_zspage(class, dst_zspage);
312fcae2
MK
2239 }
2240
2241 /* Stop if we couldn't find slot */
3783689a 2242 if (dst_zspage == NULL)
312fcae2
MK
2243 break;
2244
4aa409ca
MK
2245 putback_zspage(class, dst_zspage);
2246 if (putback_zspage(class, src_zspage) == ZS_EMPTY) {
48b4800a 2247 free_zspage(pool, class, src_zspage);
6cbf16b3 2248 pool->stats.pages_compacted += class->pages_per_zspage;
4aa409ca 2249 }
312fcae2 2250 spin_unlock(&class->lock);
312fcae2
MK
2251 cond_resched();
2252 spin_lock(&class->lock);
2253 }
2254
3783689a 2255 if (src_zspage)
4aa409ca 2256 putback_zspage(class, src_zspage);
312fcae2 2257
7d3f3938 2258 spin_unlock(&class->lock);
312fcae2
MK
2259}
2260
2261unsigned long zs_compact(struct zs_pool *pool)
2262{
2263 int i;
312fcae2
MK
2264 struct size_class *class;
2265
2266 for (i = zs_size_classes - 1; i >= 0; i--) {
2267 class = pool->size_class[i];
2268 if (!class)
2269 continue;
2270 if (class->index != i)
2271 continue;
7d3f3938 2272 __zs_compact(pool, class);
312fcae2
MK
2273 }
2274
860c707d 2275 return pool->stats.pages_compacted;
312fcae2
MK
2276}
2277EXPORT_SYMBOL_GPL(zs_compact);
61989a80 2278
7d3f3938
SS
2279void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats)
2280{
2281 memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats));
2282}
2283EXPORT_SYMBOL_GPL(zs_pool_stats);
2284
ab9d306d
SS
2285static unsigned long zs_shrinker_scan(struct shrinker *shrinker,
2286 struct shrink_control *sc)
2287{
2288 unsigned long pages_freed;
2289 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2290 shrinker);
2291
2292 pages_freed = pool->stats.pages_compacted;
2293 /*
2294 * Compact classes and calculate compaction delta.
2295 * Can run concurrently with a manually triggered
2296 * (by user) compaction.
2297 */
2298 pages_freed = zs_compact(pool) - pages_freed;
2299
2300 return pages_freed ? pages_freed : SHRINK_STOP;
2301}
2302
2303static unsigned long zs_shrinker_count(struct shrinker *shrinker,
2304 struct shrink_control *sc)
2305{
2306 int i;
2307 struct size_class *class;
2308 unsigned long pages_to_free = 0;
2309 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2310 shrinker);
2311
ab9d306d
SS
2312 for (i = zs_size_classes - 1; i >= 0; i--) {
2313 class = pool->size_class[i];
2314 if (!class)
2315 continue;
2316 if (class->index != i)
2317 continue;
2318
ab9d306d 2319 pages_to_free += zs_can_compact(class);
ab9d306d
SS
2320 }
2321
2322 return pages_to_free;
2323}
2324
2325static void zs_unregister_shrinker(struct zs_pool *pool)
2326{
2327 if (pool->shrinker_enabled) {
2328 unregister_shrinker(&pool->shrinker);
2329 pool->shrinker_enabled = false;
2330 }
2331}
2332
2333static int zs_register_shrinker(struct zs_pool *pool)
2334{
2335 pool->shrinker.scan_objects = zs_shrinker_scan;
2336 pool->shrinker.count_objects = zs_shrinker_count;
2337 pool->shrinker.batch = 0;
2338 pool->shrinker.seeks = DEFAULT_SEEKS;
2339
2340 return register_shrinker(&pool->shrinker);
2341}
2342
00a61d86 2343/**
66cdef66 2344 * zs_create_pool - Creates an allocation pool to work from.
fd854463 2345 * @name: pool name to be created
166cfda7 2346 *
66cdef66
GM
2347 * This function must be called before anything when using
2348 * the zsmalloc allocator.
166cfda7 2349 *
66cdef66
GM
2350 * On success, a pointer to the newly created pool is returned,
2351 * otherwise NULL.
396b7fd6 2352 */
d0d8da2d 2353struct zs_pool *zs_create_pool(const char *name)
61989a80 2354{
66cdef66
GM
2355 int i;
2356 struct zs_pool *pool;
2357 struct size_class *prev_class = NULL;
61989a80 2358
66cdef66
GM
2359 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2360 if (!pool)
2361 return NULL;
61989a80 2362
48b4800a 2363 init_deferred_free(pool);
66cdef66
GM
2364 pool->size_class = kcalloc(zs_size_classes, sizeof(struct size_class *),
2365 GFP_KERNEL);
2366 if (!pool->size_class) {
2367 kfree(pool);
2368 return NULL;
2369 }
61989a80 2370
2e40e163
MK
2371 pool->name = kstrdup(name, GFP_KERNEL);
2372 if (!pool->name)
2373 goto err;
2374
3783689a 2375 if (create_cache(pool))
2e40e163
MK
2376 goto err;
2377
c60369f0 2378 /*
399d8eeb 2379 * Iterate reversely, because, size of size_class that we want to use
66cdef66 2380 * for merging should be larger or equal to current size.
c60369f0 2381 */
66cdef66
GM
2382 for (i = zs_size_classes - 1; i >= 0; i--) {
2383 int size;
2384 int pages_per_zspage;
64d90465 2385 int objs_per_zspage;
66cdef66 2386 struct size_class *class;
3783689a 2387 int fullness = 0;
c60369f0 2388
66cdef66
GM
2389 size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA;
2390 if (size > ZS_MAX_ALLOC_SIZE)
2391 size = ZS_MAX_ALLOC_SIZE;
2392 pages_per_zspage = get_pages_per_zspage(size);
64d90465 2393 objs_per_zspage = pages_per_zspage * PAGE_SIZE / size;
61989a80 2394
66cdef66
GM
2395 /*
2396 * size_class is used for normal zsmalloc operation such
2397 * as alloc/free for that size. Although it is natural that we
2398 * have one size_class for each size, there is a chance that we
2399 * can get more memory utilization if we use one size_class for
2400 * many different sizes whose size_class have same
2401 * characteristics. So, we makes size_class point to
2402 * previous size_class if possible.
2403 */
2404 if (prev_class) {
64d90465 2405 if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) {
66cdef66
GM
2406 pool->size_class[i] = prev_class;
2407 continue;
2408 }
2409 }
2410
2411 class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
2412 if (!class)
2413 goto err;
2414
2415 class->size = size;
2416 class->index = i;
2417 class->pages_per_zspage = pages_per_zspage;
64d90465 2418 class->objs_per_zspage = objs_per_zspage;
66cdef66
GM
2419 spin_lock_init(&class->lock);
2420 pool->size_class[i] = class;
48b4800a
MK
2421 for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS;
2422 fullness++)
3783689a 2423 INIT_LIST_HEAD(&class->fullness_list[fullness]);
66cdef66
GM
2424
2425 prev_class = class;
61989a80
NG
2426 }
2427
d34f6157
DS
2428 /* debug only, don't abort if it fails */
2429 zs_pool_stat_create(pool, name);
0f050d99 2430
48b4800a
MK
2431 if (zs_register_migration(pool))
2432 goto err;
2433
ab9d306d
SS
2434 /*
2435 * Not critical, we still can use the pool
2436 * and user can trigger compaction manually.
2437 */
2438 if (zs_register_shrinker(pool) == 0)
2439 pool->shrinker_enabled = true;
66cdef66
GM
2440 return pool;
2441
2442err:
2443 zs_destroy_pool(pool);
2444 return NULL;
61989a80 2445}
66cdef66 2446EXPORT_SYMBOL_GPL(zs_create_pool);
61989a80 2447
66cdef66 2448void zs_destroy_pool(struct zs_pool *pool)
61989a80 2449{
66cdef66 2450 int i;
61989a80 2451
ab9d306d 2452 zs_unregister_shrinker(pool);
48b4800a 2453 zs_unregister_migration(pool);
0f050d99
GM
2454 zs_pool_stat_destroy(pool);
2455
66cdef66
GM
2456 for (i = 0; i < zs_size_classes; i++) {
2457 int fg;
2458 struct size_class *class = pool->size_class[i];
61989a80 2459
66cdef66
GM
2460 if (!class)
2461 continue;
61989a80 2462
66cdef66
GM
2463 if (class->index != i)
2464 continue;
61989a80 2465
48b4800a 2466 for (fg = ZS_EMPTY; fg < NR_ZS_FULLNESS; fg++) {
3783689a 2467 if (!list_empty(&class->fullness_list[fg])) {
66cdef66
GM
2468 pr_info("Freeing non-empty class with size %db, fullness group %d\n",
2469 class->size, fg);
2470 }
2471 }
2472 kfree(class);
2473 }
f553646a 2474
3783689a 2475 destroy_cache(pool);
66cdef66 2476 kfree(pool->size_class);
0f050d99 2477 kfree(pool->name);
66cdef66
GM
2478 kfree(pool);
2479}
2480EXPORT_SYMBOL_GPL(zs_destroy_pool);
b7418510 2481
66cdef66
GM
2482static int __init zs_init(void)
2483{
48b4800a
MK
2484 int ret;
2485
2486 ret = zsmalloc_mount();
2487 if (ret)
2488 goto out;
2489
215c89d0
SAS
2490 ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare",
2491 zs_cpu_prepare, zs_cpu_dead);
0f050d99 2492 if (ret)
215c89d0 2493 goto hp_setup_fail;
66cdef66
GM
2494
2495 init_zs_size_classes();
2496
2497#ifdef CONFIG_ZPOOL
2498 zpool_register_driver(&zs_zpool_driver);
2499#endif
0f050d99 2500
4abaac9b
DS
2501 zs_stat_init();
2502
66cdef66 2503 return 0;
0f050d99 2504
215c89d0 2505hp_setup_fail:
48b4800a
MK
2506 zsmalloc_unmount();
2507out:
0f050d99 2508 return ret;
61989a80 2509}
61989a80 2510
66cdef66 2511static void __exit zs_exit(void)
61989a80 2512{
66cdef66
GM
2513#ifdef CONFIG_ZPOOL
2514 zpool_unregister_driver(&zs_zpool_driver);
2515#endif
48b4800a 2516 zsmalloc_unmount();
215c89d0 2517 cpuhp_remove_state(CPUHP_MM_ZS_PREPARE);
0f050d99
GM
2518
2519 zs_stat_exit();
61989a80 2520}
069f101f
BH
2521
2522module_init(zs_init);
2523module_exit(zs_exit);
2524
2525MODULE_LICENSE("Dual BSD/GPL");
2526MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");