]>
Commit | Line | Data |
---|---|---|
715f6251 | 1 | /* |
2 | * This file is part of the SPL: Solaris Porting Layer. | |
3 | * | |
4 | * Copyright (c) 2008 Lawrence Livermore National Security, LLC. | |
5 | * Produced at Lawrence Livermore National Laboratory | |
6 | * Written by: | |
7 | * Brian Behlendorf <behlendorf1@llnl.gov>, | |
8 | * Herb Wartens <wartens2@llnl.gov>, | |
9 | * Jim Garlick <garlick@llnl.gov> | |
10 | * UCRL-CODE-235197 | |
11 | * | |
12 | * This is free software; you can redistribute it and/or modify it | |
13 | * under the terms of the GNU General Public License as published by | |
14 | * the Free Software Foundation; either version 2 of the License, or | |
15 | * (at your option) any later version. | |
16 | * | |
17 | * This is distributed in the hope that it will be useful, but WITHOUT | |
18 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
19 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
20 | * for more details. | |
21 | * | |
22 | * You should have received a copy of the GNU General Public License along | |
23 | * with this program; if not, write to the Free Software Foundation, Inc., | |
24 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
25 | */ | |
26 | ||
09b414e8 | 27 | #ifndef _SPL_KMEM_H |
28 | #define _SPL_KMEM_H | |
f1ca4da6 | 29 | |
30 | #ifdef __cplusplus | |
31 | extern "C" { | |
32 | #endif | |
33 | ||
f1ca4da6 | 34 | #undef DEBUG_KMEM_UNIMPLEMENTED |
35 | ||
f1b59d26 | 36 | #include <linux/module.h> |
f1ca4da6 | 37 | #include <linux/slab.h> |
79b31f36 | 38 | #include <linux/vmalloc.h> |
baf2979e | 39 | #include <linux/mm_compat.h> |
f1ca4da6 | 40 | #include <linux/spinlock.h> |
d6a26c6a | 41 | #include <linux/rwsem.h> |
42 | #include <linux/hash.h> | |
43 | #include <linux/ctype.h> | |
550f1705 | 44 | #include <asm/atomic_compat.h> |
57d86234 | 45 | #include <sys/types.h> |
937879f1 | 46 | #include <sys/debug.h> |
36b313da | 47 | #include <sys/vmsystm.h> |
550f1705 | 48 | |
f1ca4da6 | 49 | /* |
50 | * Memory allocation interfaces | |
51 | */ | |
c89fdee4 | 52 | #define KM_SLEEP GFP_KERNEL |
f1ca4da6 | 53 | #define KM_NOSLEEP GFP_ATOMIC |
54 | #undef KM_PANIC /* No linux analog */ | |
a0f6da3d | 55 | #define KM_PUSHPAGE (KM_SLEEP | __GFP_HIGH) |
f1ca4da6 | 56 | #define KM_VMFLAGS GFP_LEVEL_MASK |
57 | #define KM_FLAGS __GFP_BITS_MASK | |
58 | ||
3d061e9d | 59 | /* |
60 | * Used internally, the kernel does not need to support this flag | |
61 | */ | |
62 | #ifndef __GFP_ZERO | |
a0f6da3d | 63 | # define __GFP_ZERO 0x8000 |
3d061e9d | 64 | #endif |
65 | ||
c89fdee4 BB |
66 | /* |
67 | * __GFP_NOFAIL looks like it will be removed from the kernel perhaps as | |
68 | * early as 2.6.32. To avoid this issue when it occurs in upstream kernels | |
69 | * we retry the allocation here as long as it is not __GFP_WAIT (GFP_ATOMIC). | |
70 | * I would prefer the caller handle the failure case cleanly but we are | |
71 | * trying to emulate Solaris and those are not the Solaris semantics. | |
72 | */ | |
73 | static inline void * | |
74 | kmalloc_nofail(size_t size, gfp_t flags) | |
75 | { | |
76 | void *ptr; | |
77 | ||
78 | do { | |
79 | ptr = kmalloc(size, flags); | |
80 | } while (ptr == NULL && (flags & __GFP_WAIT)); | |
81 | ||
82 | return ptr; | |
83 | } | |
84 | ||
85 | static inline void * | |
86 | kzalloc_nofail(size_t size, gfp_t flags) | |
87 | { | |
88 | void *ptr; | |
89 | ||
90 | do { | |
91 | ptr = kzalloc(size, flags); | |
92 | } while (ptr == NULL && (flags & __GFP_WAIT)); | |
93 | ||
94 | return ptr; | |
95 | } | |
96 | ||
97 | #ifdef HAVE_KMALLOC_NODE | |
98 | static inline void * | |
99 | kmalloc_node_nofail(size_t size, gfp_t flags, int node) | |
100 | { | |
101 | void *ptr; | |
102 | ||
103 | do { | |
104 | ptr = kmalloc_node(size, flags, node); | |
105 | } while (ptr == NULL && (flags & __GFP_WAIT)); | |
106 | ||
107 | return ptr; | |
108 | } | |
109 | #endif /* HAVE_KMALLOC_NODE */ | |
110 | ||
f1ca4da6 | 111 | #ifdef DEBUG_KMEM |
a0f6da3d | 112 | |
c19c06f3 | 113 | extern atomic64_t kmem_alloc_used; |
a0f6da3d | 114 | extern unsigned long long kmem_alloc_max; |
c19c06f3 | 115 | extern atomic64_t vmem_alloc_used; |
a0f6da3d | 116 | extern unsigned long long vmem_alloc_max; |
117 | ||
118 | # define kmem_alloc(size, flags) __kmem_alloc((size), (flags), 0, 0) | |
119 | # define kmem_zalloc(size, flags) __kmem_alloc((size), ((flags) | \ | |
120 | __GFP_ZERO), 0, 0) | |
121 | ||
122 | /* The node alloc functions are only used by the SPL code itself */ | |
123 | # ifdef HAVE_KMALLOC_NODE | |
124 | # define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 1, \ | |
125 | node) | |
126 | # else | |
127 | # define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 0, 0) | |
128 | # endif | |
129 | ||
130 | # define vmem_zalloc(size, flags) vmem_alloc((size), ((flags) | \ | |
131 | __GFP_ZERO)) | |
132 | ||
133 | # ifdef DEBUG_KMEM_TRACKING | |
134 | ||
135 | extern void *kmem_alloc_track(size_t size, int flags, const char *func, | |
136 | int line, int node_alloc, int node); | |
137 | extern void kmem_free_track(void *ptr, size_t size); | |
138 | extern void *vmem_alloc_track(size_t size, int flags, const char *func, | |
139 | int line); | |
140 | extern void vmem_free_track(void *ptr, size_t size); | |
141 | ||
142 | # define __kmem_alloc(size, flags, na, node) kmem_alloc_track((size), \ | |
143 | (flags), __FUNCTION__, \ | |
144 | __LINE__, (na), (node)) | |
145 | # define kmem_free(ptr, size) kmem_free_track((ptr), (size)) | |
146 | # define vmem_alloc(size, flags) vmem_alloc_track((size), \ | |
147 | (flags),__FUNCTION__, \ | |
148 | __LINE__) | |
149 | # define vmem_free(ptr, size) vmem_free_track((ptr), (size)) | |
150 | ||
151 | # else /* DEBUG_KMEM_TRACKING */ | |
152 | ||
153 | extern void *kmem_alloc_debug(size_t size, int flags, const char *func, | |
154 | int line, int node_alloc, int node); | |
155 | extern void kmem_free_debug(void *ptr, size_t size); | |
156 | extern void *vmem_alloc_debug(size_t size, int flags, const char *func, | |
157 | int line); | |
158 | extern void vmem_free_debug(void *ptr, size_t size); | |
159 | ||
160 | # define __kmem_alloc(size, flags, na, node) kmem_alloc_debug((size), \ | |
161 | (flags), __FUNCTION__, \ | |
162 | __LINE__, (na), (node)) | |
163 | # define kmem_free(ptr, size) kmem_free_debug((ptr), (size)) | |
164 | # define vmem_alloc(size, flags) vmem_alloc_debug((size), \ | |
165 | (flags), __FUNCTION__, \ | |
166 | __LINE__) | |
167 | # define vmem_free(ptr, size) vmem_free_debug((ptr), (size)) | |
168 | ||
169 | # endif /* DEBUG_KMEM_TRACKING */ | |
ff449ac4 | 170 | |
c6dc93d6 | 171 | #else /* DEBUG_KMEM */ |
f1ca4da6 | 172 | |
c89fdee4 BB |
173 | # define kmem_alloc(size, flags) kmalloc_nofail((size), (flags)) |
174 | # define kmem_zalloc(size, flags) kzalloc_nofail((size), (flags)) | |
a1cf80b4 | 175 | # define kmem_free(ptr, size) ((void)(size), kfree(ptr)) |
a0f6da3d | 176 | |
177 | # ifdef HAVE_KMALLOC_NODE | |
178 | # define kmem_alloc_node(size, flags, node) \ | |
c89fdee4 | 179 | kmalloc_node_nofail((size), (flags), (node)) |
a0f6da3d | 180 | # else |
181 | # define kmem_alloc_node(size, flags, node) \ | |
c89fdee4 | 182 | kmalloc_nofail((size), (flags)) |
a0f6da3d | 183 | # endif |
184 | ||
a1cf80b4 BB |
185 | # define vmem_alloc(size, flags) __vmalloc((size), ((flags) | \ |
186 | __GFP_HIGHMEM), PAGE_KERNEL) | |
a0f6da3d | 187 | # define vmem_zalloc(size, flags) \ |
79b31f36 | 188 | ({ \ |
c6dc93d6 | 189 | void *_ptr_ = __vmalloc((size),((flags)|__GFP_HIGHMEM),PAGE_KERNEL); \ |
190 | if (_ptr_) \ | |
191 | memset(_ptr_, 0, (size)); \ | |
192 | _ptr_; \ | |
79b31f36 | 193 | }) |
bb01879e | 194 | # define vmem_free(ptr, size) ((void)(size), vfree(ptr)) |
79b31f36 | 195 | |
f1ca4da6 | 196 | #endif /* DEBUG_KMEM */ |
197 | ||
f1ca4da6 | 198 | #ifdef DEBUG_KMEM_UNIMPLEMENTED |
199 | static __inline__ void * | |
200 | kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags) | |
201 | { | |
202 | #error "kmem_alloc_tryhard() not implemented" | |
203 | } | |
204 | #endif /* DEBUG_KMEM_UNIMPLEMENTED */ | |
205 | ||
206 | /* | |
207 | * Slab allocation interfaces | |
208 | */ | |
ea3e6ca9 BB |
209 | enum { |
210 | KMC_BIT_NOTOUCH = 0, /* Don't update ages */ | |
211 | KMC_BIT_NODEBUG = 1, /* Default behavior */ | |
212 | KMC_BIT_NOMAGAZINE = 2, /* XXX: Unsupported */ | |
213 | KMC_BIT_NOHASH = 3, /* XXX: Unsupported */ | |
214 | KMC_BIT_QCACHE = 4, /* XXX: Unsupported */ | |
215 | KMC_BIT_KMEM = 5, /* Use kmem cache */ | |
216 | KMC_BIT_VMEM = 6, /* Use vmem cache */ | |
217 | KMC_BIT_OFFSLAB = 7, /* Objects not on slab */ | |
218 | KMC_BIT_REAPING = 16, /* Reaping in progress */ | |
219 | KMC_BIT_DESTROY = 17, /* Destroy in progress */ | |
220 | }; | |
221 | ||
222 | #define KMC_NOTOUCH (1 << KMC_BIT_NOTOUCH) | |
223 | #define KMC_NODEBUG (1 << KMC_BIT_NODEBUG) | |
224 | #define KMC_NOMAGAZINE (1 << KMC_BIT_NOMAGAZINE) | |
225 | #define KMC_NOHASH (1 << KMC_BIT_NOHASH) | |
226 | #define KMC_QCACHE (1 << KMC_BIT_QCACHE) | |
227 | #define KMC_KMEM (1 << KMC_BIT_KMEM) | |
228 | #define KMC_VMEM (1 << KMC_BIT_VMEM) | |
229 | #define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB) | |
230 | #define KMC_REAPING (1 << KMC_BIT_REAPING) | |
231 | #define KMC_DESTROY (1 << KMC_BIT_DESTROY) | |
232 | ||
233 | #define KMC_REAP_CHUNK INT_MAX | |
234 | #define KMC_DEFAULT_SEEKS 1 | |
f1ca4da6 | 235 | |
f1ca4da6 | 236 | #ifdef DEBUG_KMEM_UNIMPLEMENTED |
237 | static __inline__ void kmem_init(void) { | |
238 | #error "kmem_init() not implemented" | |
239 | } | |
240 | ||
241 | static __inline__ void kmem_thread_init(void) { | |
242 | #error "kmem_thread_init() not implemented" | |
243 | } | |
244 | ||
245 | static __inline__ void kmem_mp_init(void) { | |
246 | #error "kmem_mp_init() not implemented" | |
247 | } | |
248 | ||
249 | static __inline__ void kmem_reap_idspace(void) { | |
250 | #error "kmem_reap_idspace() not implemented" | |
251 | } | |
252 | ||
253 | static __inline__ size_t kmem_avail(void) { | |
254 | #error "kmem_avail() not implemented" | |
255 | } | |
256 | ||
257 | static __inline__ size_t kmem_maxavail(void) { | |
258 | #error "kmem_maxavail() not implemented" | |
259 | } | |
260 | ||
2fb9b26a | 261 | static __inline__ uint64_t kmem_cache_stat(spl_kmem_cache_t *cache) { |
f1ca4da6 | 262 | #error "kmem_cache_stat() not implemented" |
263 | } | |
264 | #endif /* DEBUG_KMEM_UNIMPLEMENTED */ | |
265 | ||
266 | /* XXX - Used by arc.c to adjust its memory footprint. We may want | |
267 | * to use this hook in the future to adjust behavior based on | |
268 | * debug levels. For now it's safe to always return 0. | |
269 | */ | |
270 | static __inline__ int | |
271 | kmem_debugging(void) | |
272 | { | |
273 | return 0; | |
274 | } | |
275 | ||
c19c06f3 | 276 | extern int kmem_set_warning(int flag); |
277 | ||
ff449ac4 | 278 | extern struct list_head spl_kmem_cache_list; |
279 | extern struct rw_semaphore spl_kmem_cache_sem; | |
2fb9b26a | 280 | |
4afaaefa | 281 | #define SKM_MAGIC 0x2e2e2e2e |
2fb9b26a | 282 | #define SKO_MAGIC 0x20202020 |
283 | #define SKS_MAGIC 0x22222222 | |
284 | #define SKC_MAGIC 0x2c2c2c2c | |
285 | ||
37db7d8c BB |
286 | #define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */ |
287 | #define SPL_KMEM_CACHE_REAP 0 /* Default reap everything */ | |
ea3e6ca9 BB |
288 | #define SPL_KMEM_CACHE_OBJ_PER_SLAB 32 /* Target objects per slab */ |
289 | #define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 8 /* Minimum objects per slab */ | |
290 | #define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */ | |
2fb9b26a | 291 | |
292 | typedef int (*spl_kmem_ctor_t)(void *, void *, int); | |
293 | typedef void (*spl_kmem_dtor_t)(void *, void *); | |
294 | typedef void (*spl_kmem_reclaim_t)(void *); | |
295 | ||
4afaaefa | 296 | typedef struct spl_kmem_magazine { |
9b1b8e4c | 297 | uint32_t skm_magic; /* Sanity magic */ |
4afaaefa | 298 | uint32_t skm_avail; /* Available objects */ |
299 | uint32_t skm_size; /* Magazine size */ | |
300 | uint32_t skm_refill; /* Batch refill size */ | |
9b1b8e4c BB |
301 | struct spl_kmem_cache *skm_cache; /* Owned by cache */ |
302 | struct delayed_work skm_work; /* Magazine reclaim work */ | |
4afaaefa | 303 | unsigned long skm_age; /* Last cache access */ |
304 | void *skm_objs[0]; /* Object pointers */ | |
305 | } spl_kmem_magazine_t; | |
306 | ||
2fb9b26a | 307 | typedef struct spl_kmem_obj { |
308 | uint32_t sko_magic; /* Sanity magic */ | |
2fb9b26a | 309 | void *sko_addr; /* Buffer address */ |
310 | struct spl_kmem_slab *sko_slab; /* Owned by slab */ | |
311 | struct list_head sko_list; /* Free object list linkage */ | |
2fb9b26a | 312 | } spl_kmem_obj_t; |
313 | ||
314 | typedef struct spl_kmem_slab { | |
315 | uint32_t sks_magic; /* Sanity magic */ | |
316 | uint32_t sks_objs; /* Objects per slab */ | |
317 | struct spl_kmem_cache *sks_cache; /* Owned by cache */ | |
318 | struct list_head sks_list; /* Slab list linkage */ | |
319 | struct list_head sks_free_list; /* Free object list */ | |
320 | unsigned long sks_age; /* Last modify jiffie */ | |
4afaaefa | 321 | uint32_t sks_ref; /* Ref count used objects */ |
2fb9b26a | 322 | } spl_kmem_slab_t; |
323 | ||
324 | typedef struct spl_kmem_cache { | |
ea3e6ca9 BB |
325 | uint32_t skc_magic; /* Sanity magic */ |
326 | uint32_t skc_name_size; /* Name length */ | |
327 | char *skc_name; /* Name string */ | |
4afaaefa | 328 | spl_kmem_magazine_t *skc_mag[NR_CPUS]; /* Per-CPU warm cache */ |
329 | uint32_t skc_mag_size; /* Magazine size */ | |
330 | uint32_t skc_mag_refill; /* Magazine refill count */ | |
ea3e6ca9 BB |
331 | spl_kmem_ctor_t skc_ctor; /* Constructor */ |
332 | spl_kmem_dtor_t skc_dtor; /* Destructor */ | |
333 | spl_kmem_reclaim_t skc_reclaim; /* Reclaimator */ | |
334 | void *skc_private; /* Private data */ | |
335 | void *skc_vmp; /* Unused */ | |
31a033ec | 336 | unsigned long skc_flags; /* Flags */ |
2fb9b26a | 337 | uint32_t skc_obj_size; /* Object size */ |
48e0606a | 338 | uint32_t skc_obj_align; /* Object alignment */ |
a1502d76 | 339 | uint32_t skc_slab_objs; /* Objects per slab */ |
ea3e6ca9 BB |
340 | uint32_t skc_slab_size; /* Slab size */ |
341 | uint32_t skc_delay; /* Slab reclaim interval */ | |
37db7d8c | 342 | uint32_t skc_reap; /* Slab reclaim count */ |
ea3e6ca9 BB |
343 | atomic_t skc_ref; /* Ref count callers */ |
344 | struct delayed_work skc_work; /* Slab reclaim work */ | |
ea3e6ca9 | 345 | struct list_head skc_list; /* List of caches linkage */ |
2fb9b26a | 346 | struct list_head skc_complete_list;/* Completely alloc'ed */ |
347 | struct list_head skc_partial_list; /* Partially alloc'ed */ | |
d46630e0 | 348 | spinlock_t skc_lock; /* Cache lock */ |
2fb9b26a | 349 | uint64_t skc_slab_fail; /* Slab alloc failures */ |
350 | uint64_t skc_slab_create;/* Slab creates */ | |
351 | uint64_t skc_slab_destroy;/* Slab destroys */ | |
d46630e0 | 352 | uint64_t skc_slab_total; /* Slab total current */ |
ea3e6ca9 | 353 | uint64_t skc_slab_alloc; /* Slab alloc current */ |
d46630e0 | 354 | uint64_t skc_slab_max; /* Slab max historic */ |
355 | uint64_t skc_obj_total; /* Obj total current */ | |
356 | uint64_t skc_obj_alloc; /* Obj alloc current */ | |
357 | uint64_t skc_obj_max; /* Obj max historic */ | |
2fb9b26a | 358 | } spl_kmem_cache_t; |
7afde631 | 359 | #define kmem_cache_t spl_kmem_cache_t |
2fb9b26a | 360 | |
361 | extern spl_kmem_cache_t * | |
362 | spl_kmem_cache_create(char *name, size_t size, size_t align, | |
363 | spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim, | |
f1ca4da6 | 364 | void *priv, void *vmp, int flags); |
365 | ||
2fb9b26a | 366 | extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc); |
367 | extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags); | |
368 | extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj); | |
369 | extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc); | |
370 | extern void spl_kmem_reap(void); | |
f1ca4da6 | 371 | |
d1ff2312 | 372 | int spl_kmem_init_kallsyms_lookup(void); |
2fb9b26a | 373 | int spl_kmem_init(void); |
374 | void spl_kmem_fini(void); | |
5d86345d | 375 | |
f1ca4da6 | 376 | #define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \ |
2fb9b26a | 377 | spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) |
378 | #define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc) | |
379 | #define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags) | |
380 | #define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj) | |
381 | #define kmem_cache_reap_now(skc) spl_kmem_cache_reap_now(skc) | |
382 | #define kmem_reap() spl_kmem_reap() | |
a1502d76 | 383 | #define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \ |
384 | ((ptr) < (void *)VMALLOC_END)) | |
f1ca4da6 | 385 | |
386 | #ifdef __cplusplus | |
387 | } | |
388 | #endif | |
389 | ||
09b414e8 | 390 | #endif /* _SPL_KMEM_H */ |