1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright IBM Corporation, 2021
5 * Author: Mike Rapoport <rppt@linux.ibm.com>
10 #include <linux/swap.h>
11 #include <linux/mount.h>
12 #include <linux/memfd.h>
13 #include <linux/bitops.h>
14 #include <linux/printk.h>
15 #include <linux/pagemap.h>
16 #include <linux/syscalls.h>
17 #include <linux/pseudo_fs.h>
18 #include <linux/secretmem.h>
19 #include <linux/set_memory.h>
20 #include <linux/sched/signal.h>
22 #include <uapi/linux/magic.h>
24 #include <asm/tlbflush.h>
29 #define pr_fmt(fmt) "secretmem: " fmt
32 * Define mode and flag masks to allow validation of the system call
35 #define SECRETMEM_MODE_MASK (0x0)
36 #define SECRETMEM_FLAGS_MASK SECRETMEM_MODE_MASK
38 static bool secretmem_enable __ro_after_init
;
39 module_param_named(enable
, secretmem_enable
, bool, 0400);
40 MODULE_PARM_DESC(secretmem_enable
,
41 "Enable secretmem and memfd_secret(2) system call");
43 static atomic_t secretmem_users
;
45 bool secretmem_active(void)
47 return !!atomic_read(&secretmem_users
);
50 static vm_fault_t
secretmem_fault(struct vm_fault
*vmf
)
52 struct address_space
*mapping
= vmf
->vma
->vm_file
->f_mapping
;
53 struct inode
*inode
= file_inode(vmf
->vma
->vm_file
);
54 pgoff_t offset
= vmf
->pgoff
;
55 gfp_t gfp
= vmf
->gfp_mask
;
60 if (((loff_t
)vmf
->pgoff
<< PAGE_SHIFT
) >= i_size_read(inode
))
61 return vmf_error(-EINVAL
);
64 page
= find_lock_page(mapping
, offset
);
66 page
= alloc_page(gfp
| __GFP_ZERO
);
70 err
= set_direct_map_invalid_noflush(page
);
73 return vmf_error(err
);
76 __SetPageUptodate(page
);
77 err
= add_to_page_cache_lru(page
, mapping
, offset
, gfp
);
81 * If a split of large page was required, it
82 * already happened when we marked the page invalid
83 * which guarantees that this call won't fail
85 set_direct_map_default_noflush(page
);
89 return vmf_error(err
);
92 addr
= (unsigned long)page_address(page
);
93 flush_tlb_kernel_range(addr
, addr
+ PAGE_SIZE
);
97 return VM_FAULT_LOCKED
;
100 static const struct vm_operations_struct secretmem_vm_ops
= {
101 .fault
= secretmem_fault
,
104 static int secretmem_release(struct inode
*inode
, struct file
*file
)
106 atomic_dec(&secretmem_users
);
110 static int secretmem_mmap(struct file
*file
, struct vm_area_struct
*vma
)
112 unsigned long len
= vma
->vm_end
- vma
->vm_start
;
114 if ((vma
->vm_flags
& (VM_SHARED
| VM_MAYSHARE
)) == 0)
117 if (mlock_future_check(vma
->vm_mm
, vma
->vm_flags
| VM_LOCKED
, len
))
120 vma
->vm_flags
|= VM_LOCKED
| VM_DONTDUMP
;
121 vma
->vm_ops
= &secretmem_vm_ops
;
126 bool vma_is_secretmem(struct vm_area_struct
*vma
)
128 return vma
->vm_ops
== &secretmem_vm_ops
;
131 static const struct file_operations secretmem_fops
= {
132 .release
= secretmem_release
,
133 .mmap
= secretmem_mmap
,
136 static bool secretmem_isolate_page(struct page
*page
, isolate_mode_t mode
)
141 static int secretmem_migratepage(struct address_space
*mapping
,
142 struct page
*newpage
, struct page
*page
,
143 enum migrate_mode mode
)
148 static void secretmem_freepage(struct page
*page
)
150 set_direct_map_default_noflush(page
);
151 clear_highpage(page
);
154 const struct address_space_operations secretmem_aops
= {
155 .set_page_dirty
= __set_page_dirty_no_writeback
,
156 .freepage
= secretmem_freepage
,
157 .migratepage
= secretmem_migratepage
,
158 .isolate_page
= secretmem_isolate_page
,
161 static struct vfsmount
*secretmem_mnt
;
163 static struct file
*secretmem_file_create(unsigned long flags
)
165 struct file
*file
= ERR_PTR(-ENOMEM
);
168 inode
= alloc_anon_inode(secretmem_mnt
->mnt_sb
);
170 return ERR_CAST(inode
);
172 file
= alloc_file_pseudo(inode
, secretmem_mnt
, "secretmem",
173 O_RDWR
, &secretmem_fops
);
177 mapping_set_gfp_mask(inode
->i_mapping
, GFP_HIGHUSER
);
178 mapping_set_unevictable(inode
->i_mapping
);
180 inode
->i_mapping
->a_ops
= &secretmem_aops
;
182 /* pretend we are a normal file with zero size */
183 inode
->i_mode
|= S_IFREG
;
193 SYSCALL_DEFINE1(memfd_secret
, unsigned int, flags
)
198 /* make sure local flags do not confict with global fcntl.h */
199 BUILD_BUG_ON(SECRETMEM_FLAGS_MASK
& O_CLOEXEC
);
201 if (!secretmem_enable
)
204 if (flags
& ~(SECRETMEM_FLAGS_MASK
| O_CLOEXEC
))
206 if (atomic_read(&secretmem_users
) < 0)
209 fd
= get_unused_fd_flags(flags
& O_CLOEXEC
);
213 file
= secretmem_file_create(flags
);
219 file
->f_flags
|= O_LARGEFILE
;
221 atomic_inc(&secretmem_users
);
222 fd_install(fd
, file
);
230 static int secretmem_init_fs_context(struct fs_context
*fc
)
232 return init_pseudo(fc
, SECRETMEM_MAGIC
) ? 0 : -ENOMEM
;
235 static struct file_system_type secretmem_fs
= {
237 .init_fs_context
= secretmem_init_fs_context
,
238 .kill_sb
= kill_anon_super
,
241 static int secretmem_init(void)
245 if (!secretmem_enable
)
248 secretmem_mnt
= kern_mount(&secretmem_fs
);
249 if (IS_ERR(secretmem_mnt
))
250 ret
= PTR_ERR(secretmem_mnt
);
252 /* prevent secretmem mappings from ever getting PROT_EXEC */
253 secretmem_mnt
->mnt_flags
|= MNT_NOEXEC
;
257 fs_initcall(secretmem_init
);