]>
Commit | Line | Data |
---|---|---|
5d752600 MK |
1 | /* |
2 | * memfd_create system call and file sealing support | |
3 | * | |
4 | * Code was originally included in shmem.c, and broken out to facilitate | |
5 | * use by hugetlbfs as well as tmpfs. | |
6 | * | |
7 | * This file is released under the GPL. | |
8 | */ | |
9 | ||
10 | #include <linux/fs.h> | |
11 | #include <linux/vfs.h> | |
12 | #include <linux/pagemap.h> | |
13 | #include <linux/file.h> | |
14 | #include <linux/mm.h> | |
15 | #include <linux/sched/signal.h> | |
16 | #include <linux/khugepaged.h> | |
17 | #include <linux/syscalls.h> | |
18 | #include <linux/hugetlb.h> | |
19 | #include <linux/shmem_fs.h> | |
20 | #include <linux/memfd.h> | |
21 | #include <uapi/linux/memfd.h> | |
22 | ||
23 | /* | |
2313216f | 24 | * We need a tag: a new tag would expand every xa_node by 8 bytes, |
5d752600 MK |
25 | * so reuse a tag which we firmly believe is never set or cleared on tmpfs |
26 | * or hugetlbfs because they are memory only filesystems. | |
27 | */ | |
28 | #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE | |
29 | #define LAST_SCAN 4 /* about 150ms max */ | |
30 | ||
31 | static void memfd_tag_pins(struct address_space *mapping) | |
32 | { | |
33 | struct radix_tree_iter iter; | |
34 | void __rcu **slot; | |
35 | pgoff_t start; | |
36 | struct page *page; | |
37 | ||
38 | lru_add_drain(); | |
39 | start = 0; | |
40 | rcu_read_lock(); | |
41 | ||
42 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { | |
43 | page = radix_tree_deref_slot(slot); | |
44 | if (!page || radix_tree_exception(page)) { | |
45 | if (radix_tree_deref_retry(page)) { | |
46 | slot = radix_tree_iter_retry(&iter); | |
47 | continue; | |
48 | } | |
49 | } else if (page_count(page) - page_mapcount(page) > 1) { | |
50 | xa_lock_irq(&mapping->i_pages); | |
51 | radix_tree_tag_set(&mapping->i_pages, iter.index, | |
52 | MEMFD_TAG_PINNED); | |
53 | xa_unlock_irq(&mapping->i_pages); | |
54 | } | |
55 | ||
56 | if (need_resched()) { | |
57 | slot = radix_tree_iter_resume(slot, &iter); | |
58 | cond_resched_rcu(); | |
59 | } | |
60 | } | |
61 | rcu_read_unlock(); | |
62 | } | |
63 | ||
64 | /* | |
65 | * Setting SEAL_WRITE requires us to verify there's no pending writer. However, | |
66 | * via get_user_pages(), drivers might have some pending I/O without any active | |
67 | * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages | |
68 | * and see whether it has an elevated ref-count. If so, we tag them and wait for | |
69 | * them to be dropped. | |
70 | * The caller must guarantee that no new user will acquire writable references | |
71 | * to those pages to avoid races. | |
72 | */ | |
73 | static int memfd_wait_for_pins(struct address_space *mapping) | |
74 | { | |
2313216f | 75 | XA_STATE(xas, &mapping->i_pages, 0); |
5d752600 MK |
76 | struct page *page; |
77 | int error, scan; | |
78 | ||
79 | memfd_tag_pins(mapping); | |
80 | ||
81 | error = 0; | |
82 | for (scan = 0; scan <= LAST_SCAN; scan++) { | |
2313216f MW |
83 | unsigned int tagged = 0; |
84 | ||
85 | if (!xas_marked(&xas, MEMFD_TAG_PINNED)) | |
5d752600 MK |
86 | break; |
87 | ||
88 | if (!scan) | |
89 | lru_add_drain_all(); | |
90 | else if (schedule_timeout_killable((HZ << scan) / 200)) | |
91 | scan = LAST_SCAN; | |
92 | ||
2313216f MW |
93 | xas_set(&xas, 0); |
94 | xas_lock_irq(&xas); | |
95 | xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) { | |
96 | bool clear = true; | |
97 | if (xa_is_value(page)) | |
98 | continue; | |
99 | if (page_count(page) - page_mapcount(page) != 1) { | |
5d752600 MK |
100 | /* |
101 | * On the last scan, we clean up all those tags | |
102 | * we inserted; but make a note that we still | |
103 | * found pages pinned. | |
104 | */ | |
2313216f MW |
105 | if (scan == LAST_SCAN) |
106 | error = -EBUSY; | |
107 | else | |
108 | clear = false; | |
5d752600 | 109 | } |
2313216f MW |
110 | if (clear) |
111 | xas_clear_mark(&xas, MEMFD_TAG_PINNED); | |
112 | if (++tagged % XA_CHECK_SCHED) | |
113 | continue; | |
5d752600 | 114 | |
2313216f MW |
115 | xas_pause(&xas); |
116 | xas_unlock_irq(&xas); | |
117 | cond_resched(); | |
118 | xas_lock_irq(&xas); | |
5d752600 | 119 | } |
2313216f | 120 | xas_unlock_irq(&xas); |
5d752600 MK |
121 | } |
122 | ||
123 | return error; | |
124 | } | |
125 | ||
126 | static unsigned int *memfd_file_seals_ptr(struct file *file) | |
127 | { | |
128 | if (shmem_file(file)) | |
129 | return &SHMEM_I(file_inode(file))->seals; | |
130 | ||
131 | #ifdef CONFIG_HUGETLBFS | |
132 | if (is_file_hugepages(file)) | |
133 | return &HUGETLBFS_I(file_inode(file))->seals; | |
134 | #endif | |
135 | ||
136 | return NULL; | |
137 | } | |
138 | ||
139 | #define F_ALL_SEALS (F_SEAL_SEAL | \ | |
140 | F_SEAL_SHRINK | \ | |
141 | F_SEAL_GROW | \ | |
142 | F_SEAL_WRITE) | |
143 | ||
144 | static int memfd_add_seals(struct file *file, unsigned int seals) | |
145 | { | |
146 | struct inode *inode = file_inode(file); | |
147 | unsigned int *file_seals; | |
148 | int error; | |
149 | ||
150 | /* | |
151 | * SEALING | |
152 | * Sealing allows multiple parties to share a tmpfs or hugetlbfs file | |
153 | * but restrict access to a specific subset of file operations. Seals | |
154 | * can only be added, but never removed. This way, mutually untrusted | |
155 | * parties can share common memory regions with a well-defined policy. | |
156 | * A malicious peer can thus never perform unwanted operations on a | |
157 | * shared object. | |
158 | * | |
159 | * Seals are only supported on special tmpfs or hugetlbfs files and | |
160 | * always affect the whole underlying inode. Once a seal is set, it | |
161 | * may prevent some kinds of access to the file. Currently, the | |
162 | * following seals are defined: | |
163 | * SEAL_SEAL: Prevent further seals from being set on this file | |
164 | * SEAL_SHRINK: Prevent the file from shrinking | |
165 | * SEAL_GROW: Prevent the file from growing | |
166 | * SEAL_WRITE: Prevent write access to the file | |
167 | * | |
168 | * As we don't require any trust relationship between two parties, we | |
169 | * must prevent seals from being removed. Therefore, sealing a file | |
170 | * only adds a given set of seals to the file, it never touches | |
171 | * existing seals. Furthermore, the "setting seals"-operation can be | |
172 | * sealed itself, which basically prevents any further seal from being | |
173 | * added. | |
174 | * | |
175 | * Semantics of sealing are only defined on volatile files. Only | |
176 | * anonymous tmpfs and hugetlbfs files support sealing. More | |
177 | * importantly, seals are never written to disk. Therefore, there's | |
178 | * no plan to support it on other file types. | |
179 | */ | |
180 | ||
181 | if (!(file->f_mode & FMODE_WRITE)) | |
182 | return -EPERM; | |
183 | if (seals & ~(unsigned int)F_ALL_SEALS) | |
184 | return -EINVAL; | |
185 | ||
186 | inode_lock(inode); | |
187 | ||
188 | file_seals = memfd_file_seals_ptr(file); | |
189 | if (!file_seals) { | |
190 | error = -EINVAL; | |
191 | goto unlock; | |
192 | } | |
193 | ||
194 | if (*file_seals & F_SEAL_SEAL) { | |
195 | error = -EPERM; | |
196 | goto unlock; | |
197 | } | |
198 | ||
199 | if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) { | |
200 | error = mapping_deny_writable(file->f_mapping); | |
201 | if (error) | |
202 | goto unlock; | |
203 | ||
204 | error = memfd_wait_for_pins(file->f_mapping); | |
205 | if (error) { | |
206 | mapping_allow_writable(file->f_mapping); | |
207 | goto unlock; | |
208 | } | |
209 | } | |
210 | ||
211 | *file_seals |= seals; | |
212 | error = 0; | |
213 | ||
214 | unlock: | |
215 | inode_unlock(inode); | |
216 | return error; | |
217 | } | |
218 | ||
219 | static int memfd_get_seals(struct file *file) | |
220 | { | |
221 | unsigned int *seals = memfd_file_seals_ptr(file); | |
222 | ||
223 | return seals ? *seals : -EINVAL; | |
224 | } | |
225 | ||
226 | long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg) | |
227 | { | |
228 | long error; | |
229 | ||
230 | switch (cmd) { | |
231 | case F_ADD_SEALS: | |
232 | /* disallow upper 32bit */ | |
233 | if (arg > UINT_MAX) | |
234 | return -EINVAL; | |
235 | ||
236 | error = memfd_add_seals(file, arg); | |
237 | break; | |
238 | case F_GET_SEALS: | |
239 | error = memfd_get_seals(file); | |
240 | break; | |
241 | default: | |
242 | error = -EINVAL; | |
243 | break; | |
244 | } | |
245 | ||
246 | return error; | |
247 | } | |
248 | ||
249 | #define MFD_NAME_PREFIX "memfd:" | |
250 | #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) | |
251 | #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) | |
252 | ||
253 | #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB) | |
254 | ||
255 | SYSCALL_DEFINE2(memfd_create, | |
256 | const char __user *, uname, | |
257 | unsigned int, flags) | |
258 | { | |
259 | unsigned int *file_seals; | |
260 | struct file *file; | |
261 | int fd, error; | |
262 | char *name; | |
263 | long len; | |
264 | ||
265 | if (!(flags & MFD_HUGETLB)) { | |
266 | if (flags & ~(unsigned int)MFD_ALL_FLAGS) | |
267 | return -EINVAL; | |
268 | } else { | |
269 | /* Allow huge page size encoding in flags. */ | |
270 | if (flags & ~(unsigned int)(MFD_ALL_FLAGS | | |
271 | (MFD_HUGE_MASK << MFD_HUGE_SHIFT))) | |
272 | return -EINVAL; | |
273 | } | |
274 | ||
275 | /* length includes terminating zero */ | |
276 | len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); | |
277 | if (len <= 0) | |
278 | return -EFAULT; | |
279 | if (len > MFD_NAME_MAX_LEN + 1) | |
280 | return -EINVAL; | |
281 | ||
282 | name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL); | |
283 | if (!name) | |
284 | return -ENOMEM; | |
285 | ||
286 | strcpy(name, MFD_NAME_PREFIX); | |
287 | if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { | |
288 | error = -EFAULT; | |
289 | goto err_name; | |
290 | } | |
291 | ||
292 | /* terminating-zero may have changed after strnlen_user() returned */ | |
293 | if (name[len + MFD_NAME_PREFIX_LEN - 1]) { | |
294 | error = -EFAULT; | |
295 | goto err_name; | |
296 | } | |
297 | ||
298 | fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); | |
299 | if (fd < 0) { | |
300 | error = fd; | |
301 | goto err_name; | |
302 | } | |
303 | ||
304 | if (flags & MFD_HUGETLB) { | |
305 | struct user_struct *user = NULL; | |
306 | ||
307 | file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user, | |
308 | HUGETLB_ANONHUGE_INODE, | |
309 | (flags >> MFD_HUGE_SHIFT) & | |
310 | MFD_HUGE_MASK); | |
311 | } else | |
312 | file = shmem_file_setup(name, 0, VM_NORESERVE); | |
313 | if (IS_ERR(file)) { | |
314 | error = PTR_ERR(file); | |
315 | goto err_fd; | |
316 | } | |
317 | file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; | |
c9c554f2 | 318 | file->f_flags |= O_LARGEFILE; |
5d752600 MK |
319 | |
320 | if (flags & MFD_ALLOW_SEALING) { | |
321 | file_seals = memfd_file_seals_ptr(file); | |
322 | *file_seals &= ~F_SEAL_SEAL; | |
323 | } | |
324 | ||
325 | fd_install(fd, file); | |
326 | kfree(name); | |
327 | return fd; | |
328 | ||
329 | err_fd: | |
330 | put_unused_fd(fd); | |
331 | err_name: | |
332 | kfree(name); | |
333 | return error; | |
334 | } |