]>
Commit | Line | Data |
---|---|---|
10946b02 | 1 | /* |
716154c5 BB |
2 | * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. |
3 | * Copyright (C) 2007 The Regents of the University of California. | |
4 | * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). | |
5 | * Written by Brian Behlendorf <behlendorf1@llnl.gov>. | |
715f6251 | 6 | * UCRL-CODE-235197 |
7 | * | |
716154c5 | 8 | * This file is part of the SPL, Solaris Porting Layer. |
3d6af2dd | 9 | * For details, see <http://zfsonlinux.org/>. |
716154c5 BB |
10 | * |
11 | * The SPL is free software; you can redistribute it and/or modify it | |
12 | * under the terms of the GNU General Public License as published by the | |
13 | * Free Software Foundation; either version 2 of the License, or (at your | |
14 | * option) any later version. | |
715f6251 | 15 | * |
716154c5 | 16 | * The SPL is distributed in the hope that it will be useful, but WITHOUT |
715f6251 | 17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
19 | * for more details. | |
20 | * | |
21 | * You should have received a copy of the GNU General Public License along | |
716154c5 | 22 | * with the SPL. If not, see <http://www.gnu.org/licenses/>. |
10946b02 | 23 | */ |
715f6251 | 24 | |
09b414e8 | 25 | #ifndef _SPL_KMEM_H |
26 | #define _SPL_KMEM_H | |
f1ca4da6 | 27 | |
10946b02 | 28 | #include <sys/debug.h> |
f1ca4da6 | 29 | #include <linux/slab.h> |
10946b02 AX |
30 | #include <linux/sched.h> |
31 | ||
32 | extern int kmem_debugging(void); | |
33 | extern char *kmem_vasprintf(const char *fmt, va_list ap); | |
34 | extern char *kmem_asprintf(const char *fmt, ...); | |
35 | extern char *strdup(const char *str); | |
36 | extern void strfree(char *str); | |
550f1705 | 37 | |
f1ca4da6 | 38 | /* |
39 | * Memory allocation interfaces | |
40 | */ | |
10946b02 AX |
41 | #define KM_SLEEP 0x0000 /* can block for memory; success guaranteed */ |
42 | #define KM_NOSLEEP 0x0001 /* cannot block for memory; may fail */ | |
43 | #define KM_PUSHPAGE 0x0004 /* can block for memory; may use reserve */ | |
44 | #define KM_ZERO 0x1000 /* zero the allocation */ | |
45 | #define KM_VMEM 0x2000 /* caller is vmem_* wrapper */ | |
f1ca4da6 | 46 | |
10946b02 | 47 | #define KM_PUBLIC_MASK (KM_SLEEP | KM_NOSLEEP | KM_PUSHPAGE) |
3d061e9d | 48 | |
2ea56c1d AX |
49 | static int spl_fstrans_check(void); |
50 | ||
eb0f407a | 51 | /* |
10946b02 AX |
52 | * Convert a KM_* flags mask to its Linux GFP_* counterpart. The conversion |
53 | * function is context aware which means that KM_SLEEP allocations can be | |
54 | * safely used in syncing contexts which have set PF_FSTRANS. | |
eb0f407a | 55 | */ |
10946b02 AX |
56 | static inline gfp_t |
57 | kmem_flags_convert(int flags) | |
eb0f407a | 58 | { |
10946b02 AX |
59 | gfp_t lflags = __GFP_NOWARN | __GFP_COMP; |
60 | ||
61 | if (flags & KM_NOSLEEP) { | |
62 | lflags |= GFP_ATOMIC | __GFP_NORETRY; | |
63 | } else { | |
64 | lflags |= GFP_KERNEL; | |
2ea56c1d | 65 | if (spl_fstrans_check()) |
10946b02 | 66 | lflags &= ~(__GFP_IO|__GFP_FS); |
eb0f407a | 67 | } |
eb0f407a | 68 | |
10946b02 AX |
69 | if (flags & KM_PUSHPAGE) |
70 | lflags |= __GFP_HIGH; | |
eb0f407a | 71 | |
10946b02 AX |
72 | if (flags & KM_ZERO) |
73 | lflags |= __GFP_ZERO; | |
c89fdee4 | 74 | |
10946b02 | 75 | return (lflags); |
c89fdee4 BB |
76 | } |
77 | ||
10946b02 AX |
78 | typedef struct { |
79 | struct task_struct *fstrans_thread; | |
80 | unsigned int saved_flags; | |
81 | } fstrans_cookie_t; | |
c89fdee4 | 82 | |
2ea56c1d AX |
83 | /* |
84 | * Introduced in Linux 3.9, however this cannot be solely relied on before | |
85 | * Linux 3.18 as it doesn't turn off __GFP_FS as it should. | |
86 | */ | |
81dab2ed | 87 | #ifdef PF_MEMALLOC_NOIO |
2ea56c1d AX |
88 | #define __SPL_PF_MEMALLOC_NOIO (PF_MEMALLOC_NOIO) |
89 | #else | |
90 | #define __SPL_PF_MEMALLOC_NOIO (0) | |
91 | #endif | |
92 | ||
93 | /* | |
94 | * PF_FSTRANS is removed from Linux 4.12 | |
95 | */ | |
96 | #ifdef PF_FSTRANS | |
97 | #define __SPL_PF_FSTRANS (PF_FSTRANS) | |
81dab2ed | 98 | #else |
2ea56c1d | 99 | #define __SPL_PF_FSTRANS (0) |
81dab2ed AX |
100 | #endif |
101 | ||
2ea56c1d AX |
102 | #define SPL_FSTRANS (__SPL_PF_FSTRANS|__SPL_PF_MEMALLOC_NOIO) |
103 | ||
10946b02 AX |
104 | static inline fstrans_cookie_t |
105 | spl_fstrans_mark(void) | |
c89fdee4 | 106 | { |
10946b02 | 107 | fstrans_cookie_t cookie; |
eb0f407a | 108 | |
2ea56c1d AX |
109 | BUILD_BUG_ON(SPL_FSTRANS == 0); |
110 | ||
10946b02 | 111 | cookie.fstrans_thread = current; |
81dab2ed AX |
112 | cookie.saved_flags = current->flags & SPL_FSTRANS; |
113 | current->flags |= SPL_FSTRANS; | |
c89fdee4 | 114 | |
10946b02 | 115 | return (cookie); |
10129680 BB |
116 | } |
117 | ||
10946b02 AX |
118 | static inline void |
119 | spl_fstrans_unmark(fstrans_cookie_t cookie) | |
10129680 | 120 | { |
10946b02 | 121 | ASSERT3P(cookie.fstrans_thread, ==, current); |
81dab2ed | 122 | ASSERT((current->flags & SPL_FSTRANS) == SPL_FSTRANS); |
eb0f407a | 123 | |
81dab2ed | 124 | current->flags &= ~SPL_FSTRANS; |
10946b02 | 125 | current->flags |= cookie.saved_flags; |
10129680 BB |
126 | } |
127 | ||
10946b02 AX |
128 | static inline int |
129 | spl_fstrans_check(void) | |
10129680 | 130 | { |
2ea56c1d AX |
131 | return (current->flags & SPL_FSTRANS); |
132 | } | |
133 | ||
134 | /* | |
135 | * specifically used to check PF_FSTRANS flag, cannot be relied on for | |
136 | * checking spl_fstrans_mark(). | |
137 | */ | |
138 | static inline int | |
139 | __spl_pf_fstrans_check(void) | |
140 | { | |
141 | return (current->flags & __SPL_PF_FSTRANS); | |
10129680 | 142 | } |
c89fdee4 | 143 | |
10946b02 AX |
144 | #ifdef HAVE_ATOMIC64_T |
145 | #define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used) | |
146 | #define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used) | |
147 | #define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used) | |
148 | #define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size) | |
10129680 | 149 | extern atomic64_t kmem_alloc_used; |
d04c8a56 | 150 | extern unsigned long long kmem_alloc_max; |
10946b02 AX |
151 | #else /* HAVE_ATOMIC64_T */ |
152 | #define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used) | |
153 | #define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used) | |
154 | #define kmem_alloc_used_read() atomic_read(&kmem_alloc_used) | |
155 | #define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size) | |
10129680 BB |
156 | extern atomic_t kmem_alloc_used; |
157 | extern unsigned long long kmem_alloc_max; | |
10946b02 | 158 | #endif /* HAVE_ATOMIC64_T */ |
10129680 | 159 | |
10946b02 AX |
160 | extern unsigned int spl_kmem_alloc_warn; |
161 | extern unsigned int spl_kmem_alloc_max; | |
10129680 | 162 | |
10946b02 AX |
163 | #define kmem_alloc(sz, fl) spl_kmem_alloc((sz), (fl), __func__, __LINE__) |
164 | #define kmem_zalloc(sz, fl) spl_kmem_zalloc((sz), (fl), __func__, __LINE__) | |
165 | #define kmem_free(ptr, sz) spl_kmem_free((ptr), (sz)) | |
a0f6da3d | 166 | |
10946b02 AX |
167 | extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line); |
168 | extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line); | |
169 | extern void spl_kmem_free(const void *ptr, size_t sz); | |
f1ca4da6 | 170 | |
9e4fb5c2 | 171 | /* |
10946b02 | 172 | * The following functions are only available for internal use. |
9e4fb5c2 | 173 | */ |
10946b02 AX |
174 | extern void *spl_kmem_alloc_impl(size_t size, int flags, int node); |
175 | extern void *spl_kmem_alloc_debug(size_t size, int flags, int node); | |
176 | extern void *spl_kmem_alloc_track(size_t size, int flags, | |
177 | const char *func, int line, int node); | |
178 | extern void spl_kmem_free_impl(const void *buf, size_t size); | |
179 | extern void spl_kmem_free_debug(const void *buf, size_t size); | |
180 | extern void spl_kmem_free_track(const void *buf, size_t size); | |
181 | ||
182 | extern int spl_kmem_init(void); | |
183 | extern void spl_kmem_fini(void); | |
9e4fb5c2 | 184 | |
09b414e8 | 185 | #endif /* _SPL_KMEM_H */ |