]>
Commit | Line | Data |
---|---|---|
5d752600 MK |
1 | /* |
2 | * memfd_create system call and file sealing support | |
3 | * | |
4 | * Code was originally included in shmem.c, and broken out to facilitate | |
5 | * use by hugetlbfs as well as tmpfs. | |
6 | * | |
7 | * This file is released under the GPL. | |
8 | */ | |
9 | ||
10 | #include <linux/fs.h> | |
11 | #include <linux/vfs.h> | |
12 | #include <linux/pagemap.h> | |
13 | #include <linux/file.h> | |
14 | #include <linux/mm.h> | |
15 | #include <linux/sched/signal.h> | |
16 | #include <linux/khugepaged.h> | |
17 | #include <linux/syscalls.h> | |
18 | #include <linux/hugetlb.h> | |
19 | #include <linux/shmem_fs.h> | |
20 | #include <linux/memfd.h> | |
21 | #include <uapi/linux/memfd.h> | |
22 | ||
23 | /* | |
2313216f | 24 | * We need a tag: a new tag would expand every xa_node by 8 bytes, |
5d752600 MK |
25 | * so reuse a tag which we firmly believe is never set or cleared on tmpfs |
26 | * or hugetlbfs because they are memory only filesystems. | |
27 | */ | |
28 | #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE | |
29 | #define LAST_SCAN 4 /* about 150ms max */ | |
30 | ||
ef3038a5 | 31 | static void memfd_tag_pins(struct xa_state *xas) |
5d752600 | 32 | { |
5d752600 | 33 | struct page *page; |
f2b277c4 HD |
34 | int latency = 0; |
35 | int cache_count; | |
5d752600 MK |
36 | |
37 | lru_add_drain(); | |
5d752600 | 38 | |
ef3038a5 MW |
39 | xas_lock_irq(xas); |
40 | xas_for_each(xas, page, ULONG_MAX) { | |
f2b277c4 HD |
41 | cache_count = 1; |
42 | if (!xa_is_value(page) && | |
43 | PageTransHuge(page) && !PageHuge(page)) | |
44 | cache_count = HPAGE_PMD_NR; | |
45 | ||
46 | if (!xa_is_value(page) && | |
47 | page_count(page) - total_mapcount(page) != cache_count) | |
ef3038a5 | 48 | xas_set_mark(xas, MEMFD_TAG_PINNED); |
f2b277c4 HD |
49 | if (cache_count != 1) |
50 | xas_set(xas, page->index + cache_count); | |
5d752600 | 51 | |
f2b277c4 HD |
52 | latency += cache_count; |
53 | if (latency < XA_CHECK_SCHED) | |
ef3038a5 | 54 | continue; |
f2b277c4 | 55 | latency = 0; |
ef3038a5 MW |
56 | |
57 | xas_pause(xas); | |
58 | xas_unlock_irq(xas); | |
59 | cond_resched(); | |
60 | xas_lock_irq(xas); | |
5d752600 | 61 | } |
ef3038a5 | 62 | xas_unlock_irq(xas); |
5d752600 MK |
63 | } |
64 | ||
65 | /* | |
66 | * Setting SEAL_WRITE requires us to verify there's no pending writer. However, | |
67 | * via get_user_pages(), drivers might have some pending I/O without any active | |
68 | * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages | |
69 | * and see whether it has an elevated ref-count. If so, we tag them and wait for | |
70 | * them to be dropped. | |
71 | * The caller must guarantee that no new user will acquire writable references | |
72 | * to those pages to avoid races. | |
73 | */ | |
74 | static int memfd_wait_for_pins(struct address_space *mapping) | |
75 | { | |
2313216f | 76 | XA_STATE(xas, &mapping->i_pages, 0); |
5d752600 MK |
77 | struct page *page; |
78 | int error, scan; | |
79 | ||
ef3038a5 | 80 | memfd_tag_pins(&xas); |
5d752600 MK |
81 | |
82 | error = 0; | |
83 | for (scan = 0; scan <= LAST_SCAN; scan++) { | |
f2b277c4 HD |
84 | int latency = 0; |
85 | int cache_count; | |
2313216f MW |
86 | |
87 | if (!xas_marked(&xas, MEMFD_TAG_PINNED)) | |
5d752600 MK |
88 | break; |
89 | ||
90 | if (!scan) | |
91 | lru_add_drain_all(); | |
92 | else if (schedule_timeout_killable((HZ << scan) / 200)) | |
93 | scan = LAST_SCAN; | |
94 | ||
2313216f MW |
95 | xas_set(&xas, 0); |
96 | xas_lock_irq(&xas); | |
97 | xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) { | |
98 | bool clear = true; | |
f2b277c4 HD |
99 | |
100 | cache_count = 1; | |
101 | if (!xa_is_value(page) && | |
102 | PageTransHuge(page) && !PageHuge(page)) | |
103 | cache_count = HPAGE_PMD_NR; | |
104 | ||
105 | if (!xa_is_value(page) && cache_count != | |
106 | page_count(page) - total_mapcount(page)) { | |
5d752600 MK |
107 | /* |
108 | * On the last scan, we clean up all those tags | |
109 | * we inserted; but make a note that we still | |
110 | * found pages pinned. | |
111 | */ | |
2313216f MW |
112 | if (scan == LAST_SCAN) |
113 | error = -EBUSY; | |
114 | else | |
115 | clear = false; | |
5d752600 | 116 | } |
2313216f MW |
117 | if (clear) |
118 | xas_clear_mark(&xas, MEMFD_TAG_PINNED); | |
f2b277c4 HD |
119 | |
120 | latency += cache_count; | |
121 | if (latency < XA_CHECK_SCHED) | |
2313216f | 122 | continue; |
f2b277c4 | 123 | latency = 0; |
5d752600 | 124 | |
2313216f MW |
125 | xas_pause(&xas); |
126 | xas_unlock_irq(&xas); | |
127 | cond_resched(); | |
128 | xas_lock_irq(&xas); | |
5d752600 | 129 | } |
2313216f | 130 | xas_unlock_irq(&xas); |
5d752600 MK |
131 | } |
132 | ||
133 | return error; | |
134 | } | |
135 | ||
136 | static unsigned int *memfd_file_seals_ptr(struct file *file) | |
137 | { | |
138 | if (shmem_file(file)) | |
139 | return &SHMEM_I(file_inode(file))->seals; | |
140 | ||
141 | #ifdef CONFIG_HUGETLBFS | |
142 | if (is_file_hugepages(file)) | |
143 | return &HUGETLBFS_I(file_inode(file))->seals; | |
144 | #endif | |
145 | ||
146 | return NULL; | |
147 | } | |
148 | ||
149 | #define F_ALL_SEALS (F_SEAL_SEAL | \ | |
150 | F_SEAL_SHRINK | \ | |
151 | F_SEAL_GROW | \ | |
ab3948f5 JFG |
152 | F_SEAL_WRITE | \ |
153 | F_SEAL_FUTURE_WRITE) | |
5d752600 MK |
154 | |
155 | static int memfd_add_seals(struct file *file, unsigned int seals) | |
156 | { | |
157 | struct inode *inode = file_inode(file); | |
158 | unsigned int *file_seals; | |
159 | int error; | |
160 | ||
161 | /* | |
162 | * SEALING | |
163 | * Sealing allows multiple parties to share a tmpfs or hugetlbfs file | |
164 | * but restrict access to a specific subset of file operations. Seals | |
165 | * can only be added, but never removed. This way, mutually untrusted | |
166 | * parties can share common memory regions with a well-defined policy. | |
167 | * A malicious peer can thus never perform unwanted operations on a | |
168 | * shared object. | |
169 | * | |
170 | * Seals are only supported on special tmpfs or hugetlbfs files and | |
171 | * always affect the whole underlying inode. Once a seal is set, it | |
172 | * may prevent some kinds of access to the file. Currently, the | |
173 | * following seals are defined: | |
174 | * SEAL_SEAL: Prevent further seals from being set on this file | |
175 | * SEAL_SHRINK: Prevent the file from shrinking | |
176 | * SEAL_GROW: Prevent the file from growing | |
177 | * SEAL_WRITE: Prevent write access to the file | |
178 | * | |
179 | * As we don't require any trust relationship between two parties, we | |
180 | * must prevent seals from being removed. Therefore, sealing a file | |
181 | * only adds a given set of seals to the file, it never touches | |
182 | * existing seals. Furthermore, the "setting seals"-operation can be | |
183 | * sealed itself, which basically prevents any further seal from being | |
184 | * added. | |
185 | * | |
186 | * Semantics of sealing are only defined on volatile files. Only | |
187 | * anonymous tmpfs and hugetlbfs files support sealing. More | |
188 | * importantly, seals are never written to disk. Therefore, there's | |
189 | * no plan to support it on other file types. | |
190 | */ | |
191 | ||
192 | if (!(file->f_mode & FMODE_WRITE)) | |
193 | return -EPERM; | |
194 | if (seals & ~(unsigned int)F_ALL_SEALS) | |
195 | return -EINVAL; | |
196 | ||
197 | inode_lock(inode); | |
198 | ||
199 | file_seals = memfd_file_seals_ptr(file); | |
200 | if (!file_seals) { | |
201 | error = -EINVAL; | |
202 | goto unlock; | |
203 | } | |
204 | ||
205 | if (*file_seals & F_SEAL_SEAL) { | |
206 | error = -EPERM; | |
207 | goto unlock; | |
208 | } | |
209 | ||
210 | if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) { | |
211 | error = mapping_deny_writable(file->f_mapping); | |
212 | if (error) | |
213 | goto unlock; | |
214 | ||
215 | error = memfd_wait_for_pins(file->f_mapping); | |
216 | if (error) { | |
217 | mapping_allow_writable(file->f_mapping); | |
218 | goto unlock; | |
219 | } | |
220 | } | |
221 | ||
222 | *file_seals |= seals; | |
223 | error = 0; | |
224 | ||
225 | unlock: | |
226 | inode_unlock(inode); | |
227 | return error; | |
228 | } | |
229 | ||
230 | static int memfd_get_seals(struct file *file) | |
231 | { | |
232 | unsigned int *seals = memfd_file_seals_ptr(file); | |
233 | ||
234 | return seals ? *seals : -EINVAL; | |
235 | } | |
236 | ||
237 | long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg) | |
238 | { | |
239 | long error; | |
240 | ||
241 | switch (cmd) { | |
242 | case F_ADD_SEALS: | |
243 | /* disallow upper 32bit */ | |
244 | if (arg > UINT_MAX) | |
245 | return -EINVAL; | |
246 | ||
247 | error = memfd_add_seals(file, arg); | |
248 | break; | |
249 | case F_GET_SEALS: | |
250 | error = memfd_get_seals(file); | |
251 | break; | |
252 | default: | |
253 | error = -EINVAL; | |
254 | break; | |
255 | } | |
256 | ||
257 | return error; | |
258 | } | |
259 | ||
260 | #define MFD_NAME_PREFIX "memfd:" | |
261 | #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) | |
262 | #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) | |
263 | ||
264 | #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB) | |
265 | ||
266 | SYSCALL_DEFINE2(memfd_create, | |
267 | const char __user *, uname, | |
268 | unsigned int, flags) | |
269 | { | |
270 | unsigned int *file_seals; | |
271 | struct file *file; | |
272 | int fd, error; | |
273 | char *name; | |
274 | long len; | |
275 | ||
276 | if (!(flags & MFD_HUGETLB)) { | |
277 | if (flags & ~(unsigned int)MFD_ALL_FLAGS) | |
278 | return -EINVAL; | |
279 | } else { | |
280 | /* Allow huge page size encoding in flags. */ | |
281 | if (flags & ~(unsigned int)(MFD_ALL_FLAGS | | |
282 | (MFD_HUGE_MASK << MFD_HUGE_SHIFT))) | |
283 | return -EINVAL; | |
284 | } | |
285 | ||
286 | /* length includes terminating zero */ | |
287 | len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); | |
288 | if (len <= 0) | |
289 | return -EFAULT; | |
290 | if (len > MFD_NAME_MAX_LEN + 1) | |
291 | return -EINVAL; | |
292 | ||
293 | name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL); | |
294 | if (!name) | |
295 | return -ENOMEM; | |
296 | ||
297 | strcpy(name, MFD_NAME_PREFIX); | |
298 | if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { | |
299 | error = -EFAULT; | |
300 | goto err_name; | |
301 | } | |
302 | ||
303 | /* terminating-zero may have changed after strnlen_user() returned */ | |
304 | if (name[len + MFD_NAME_PREFIX_LEN - 1]) { | |
305 | error = -EFAULT; | |
306 | goto err_name; | |
307 | } | |
308 | ||
309 | fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); | |
310 | if (fd < 0) { | |
311 | error = fd; | |
312 | goto err_name; | |
313 | } | |
314 | ||
315 | if (flags & MFD_HUGETLB) { | |
83c1fd76 | 316 | file = hugetlb_file_setup(name, 0, VM_NORESERVE, |
5d752600 MK |
317 | HUGETLB_ANONHUGE_INODE, |
318 | (flags >> MFD_HUGE_SHIFT) & | |
319 | MFD_HUGE_MASK); | |
320 | } else | |
321 | file = shmem_file_setup(name, 0, VM_NORESERVE); | |
322 | if (IS_ERR(file)) { | |
323 | error = PTR_ERR(file); | |
324 | goto err_fd; | |
325 | } | |
326 | file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; | |
c9c554f2 | 327 | file->f_flags |= O_LARGEFILE; |
5d752600 MK |
328 | |
329 | if (flags & MFD_ALLOW_SEALING) { | |
330 | file_seals = memfd_file_seals_ptr(file); | |
331 | *file_seals &= ~F_SEAL_SEAL; | |
332 | } | |
333 | ||
334 | fd_install(fd, file); | |
335 | kfree(name); | |
336 | return fd; | |
337 | ||
338 | err_fd: | |
339 | put_unused_fd(fd); | |
340 | err_name: | |
341 | kfree(name); | |
342 | return error; | |
343 | } |