[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / sys_x86_64.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/syscalls.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/smp.h>
#include <linux/sem.h>
#include <linux/msg.h>
#include <linux/shm.h>
#include <linux/stat.h>
#include <linux/mman.h>
#include <linux/file.h>
#include <linux/utsname.h>
#include <linux/personality.h>
#include <linux/random.h>
#include <linux/uaccess.h>
#include <linux/elf.h>

#include <asm/elf.h>
#include <asm/compat.h>
#include <asm/ia32.h>
#include <asm/syscalls.h>
#include <asm/mpx.h>

/*
 * Align a virtual address to avoid aliasing in the I$ on AMD F15h.
 */
static unsigned long get_align_mask(void)
{
	/* handle 32- and 64-bit case with a single conditional */
	if (va_align.flags < 0 || !(va_align.flags & (2 - mmap_is_ia32())))
		return 0;

	if (!(current->flags & PF_RANDOMIZE))
		return 0;

	return va_align.mask;
}

/*
 * To avoid aliasing in the I$ on AMD F15h, the bits defined by the
 * va_align.bits, [12:upper_bit), are set to a random value instead of
 * zeroing them. This random value is computed once per boot. This form
 * of ASLR is known as "per-boot ASLR".
 *
 * To achieve this, the random value is added to the info.align_offset
 * value before calling vm_unmapped_area() or ORed directly to the
 * address.
 */
static unsigned long get_align_bits(void)
{
	return va_align.bits & get_align_mask();
}

unsigned long align_vdso_addr(unsigned long addr)
{
	unsigned long align_mask = get_align_mask();
	addr = (addr + align_mask) & ~align_mask;
	return addr | get_align_bits();
}

static int __init control_va_addr_alignment(char *str)
{
	/* guard against enabling this on other CPU families */
	if (va_align.flags < 0)
		return 1;

	if (*str == 0)
		return 1;

	if (*str == '=')
		str++;

	if (!strcmp(str, "32"))
		va_align.flags = ALIGN_VA_32;
	else if (!strcmp(str, "64"))
		va_align.flags = ALIGN_VA_64;
	else if (!strcmp(str, "off"))
		va_align.flags = 0;
	else if (!strcmp(str, "on"))
		va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
	else
		return 0;

	return 1;
}
__setup("align_va_addr", control_va_addr_alignment);

SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
		unsigned long, prot, unsigned long, flags,
		unsigned long, fd, unsigned long, off)
{
	long error;
	error = -EINVAL;
	if (off & ~PAGE_MASK)
		goto out;

	error = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
out:
	return error;
}

static void find_start_end(unsigned long addr, unsigned long flags,
		unsigned long *begin, unsigned long *end)
{
	if (!in_compat_syscall() && (flags & MAP_32BIT)) {
		/* This is usually used needed to map code in small
		   model, so it needs to be in the first 31bit. Limit
		   it to that.  This means we need to move the
		   unmapped base down for this case. This can give
		   conflicts with the heap, but we assume that glibc
		   malloc knows how to fall back to mmap. Give it 1GB
		   of playground for now. -AK */
		*begin = 0x40000000;
		*end = 0x80000000;
		if (current->flags & PF_RANDOMIZE) {
			*begin = randomize_page(*begin, 0x02000000);
		}
		return;
	}

	*begin	= get_mmap_base(1);
	if (in_compat_syscall())
		*end = task_size_32bit();
	else
		*end = task_size_64bit(addr > DEFAULT_MAP_WINDOW);
}

unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	struct vm_unmapped_area_info info;
	unsigned long begin, end;

	addr = mpx_unmapped_area_check(addr, len, flags);
	if (IS_ERR_VALUE(addr))
		return addr;

	if (flags & MAP_FIXED)
		return addr;

	find_start_end(addr, flags, &begin, &end);

	if (len > end)
		return -ENOMEM;

	if (addr) {
		addr = PAGE_ALIGN(addr);
		vma = find_vma(mm, addr);
		if (end - len >= addr &&
		    (!vma || addr + len <= vm_start_gap(vma)))
			return addr;
	}

	info.flags = 0;
	info.length = len;
	info.low_limit = begin;
	info.high_limit = end;
	info.align_mask = 0;
	info.align_offset = pgoff << PAGE_SHIFT;
	if (filp) {
		info.align_mask = get_align_mask();
		info.align_offset += get_align_bits();
	}
	return vm_unmapped_area(&info);
}

unsigned long
arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
			  const unsigned long len, const unsigned long pgoff,
			  const unsigned long flags)
{
	struct vm_area_struct *vma;
	struct mm_struct *mm = current->mm;
	unsigned long addr = addr0;
	struct vm_unmapped_area_info info;

	addr = mpx_unmapped_area_check(addr, len, flags);
	if (IS_ERR_VALUE(addr))
		return addr;

	/* requested length too big for entire address space */
	if (len > TASK_SIZE)
		return -ENOMEM;

	/* No address checking. See comment at mmap_address_hint_valid() */
	if (flags & MAP_FIXED)
		return addr;

	/* for MAP_32BIT mappings we force the legacy mmap base */
	if (!in_compat_syscall() && (flags & MAP_32BIT))
		goto bottomup;

	/* requesting a specific address */
	if (addr) {
		addr &= PAGE_MASK;
		if (!mmap_address_hint_valid(addr, len))
			goto get_unmapped_area;

		vma = find_vma(mm, addr);
		if (!vma || addr + len <= vm_start_gap(vma))
			return addr;
	}
get_unmapped_area:

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = get_mmap_base(0);

	/*
	 * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
	 * in the full address space.
	 *
	 * !in_compat_syscall() check to avoid high addresses for x32.
	 */
	if (addr > DEFAULT_MAP_WINDOW && !in_compat_syscall())
		info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW;

	info.align_mask = 0;
	info.align_offset = pgoff << PAGE_SHIFT;
	if (filp) {
		info.align_mask = get_align_mask();
		info.align_offset += get_align_bits();
	}
	addr = vm_unmapped_area(&info);
	if (!(addr & ~PAGE_MASK))
		return addr;
	VM_BUG_ON(addr != -ENOMEM);

bottomup:
	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
	return arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	#include <linux/errno.h>
1da177e4 LT	3	#include <linux/sched.h>
01042607	4	#include <linux/sched/mm.h>
1da177e4 LT	5	#include <linux/syscalls.h>
1da177e4 LT	6	#include <linux/mm.h>
4e950f6f	7	#include <linux/fs.h>
1da177e4	8	#include <linux/smp.h>
1da177e4 LT	9	#include <linux/sem.h>
	10	#include <linux/msg.h>
	11	#include <linux/shm.h>
	12	#include <linux/stat.h>
	13	#include <linux/mman.h>
	14	#include <linux/file.h>
	15	#include <linux/utsname.h>
	16	#include <linux/personality.h>
cc503c1b	17	#include <linux/random.h>
e9c8abb6	18	#include <linux/uaccess.h>
910b2c51	19	#include <linux/elf.h>
1da177e4	20
1b028f78 DS	21	#include <asm/elf.h>
1b028f78 DS	22	#include <asm/compat.h>
1da177e4	23	#include <asm/ia32.h>
bbc1f698	24	#include <asm/syscalls.h>
44b04912	25	#include <asm/mpx.h>
1da177e4	26
dfb09f9b BP	27	/*
dfb09f9b BP	28	* Align a virtual address to avoid aliasing in the I$ on AMD F15h.
dfb09f9b	29	*/
f9902472	30	static unsigned long get_align_mask(void)
dfb09f9b	31	{
dfb09f9b BP	32	/* handle 32- and 64-bit case with a single conditional */
dfb09f9b BP	33	if (va_align.flags < 0 \|\| !(va_align.flags & (2 - mmap_is_ia32())))
f9902472	34	return 0;
dfb09f9b BP	35
dfb09f9b BP	36	if (!(current->flags & PF_RANDOMIZE))
f9902472	37	return 0;
dfb09f9b	38
f9902472 ML	39	return va_align.mask;
f9902472 ML	40	}
dfb09f9b	41
4e26d11f HMG	42	/*
	43	* To avoid aliasing in the I$ on AMD F15h, the bits defined by the
	44	* va_align.bits, [12:upper_bit), are set to a random value instead of
	45	* zeroing them. This random value is computed once per boot. This form
	46	* of ASLR is known as "per-boot ASLR".
	47	*
	48	* To achieve this, the random value is added to the info.align_offset
	49	* value before calling vm_unmapped_area() or ORed directly to the
	50	* address.
	51	*/
	52	static unsigned long get_align_bits(void)
	53	{
	54	return va_align.bits & get_align_mask();
	55	}
	56
f9902472 ML	57	unsigned long align_vdso_addr(unsigned long addr)
	58	{
	59	unsigned long align_mask = get_align_mask();
4e26d11f HMG	60	addr = (addr + align_mask) & ~align_mask;
4e26d11f HMG	61	return addr \| get_align_bits();
dfb09f9b BP	62	}
	63
	64	static int __init control_va_addr_alignment(char *str)
	65	{
	66	/* guard against enabling this on other CPU families */
	67	if (va_align.flags < 0)
	68	return 1;
	69
	70	if (*str == 0)
	71	return 1;
	72
	73	if (*str == '=')
	74	str++;
	75
	76	if (!strcmp(str, "32"))
	77	va_align.flags = ALIGN_VA_32;
	78	else if (!strcmp(str, "64"))
	79	va_align.flags = ALIGN_VA_64;
	80	else if (!strcmp(str, "off"))
	81	va_align.flags = 0;
	82	else if (!strcmp(str, "on"))
	83	va_align.flags = ALIGN_VA_32 \| ALIGN_VA_64;
	84	else
	85	return 0;
	86
	87	return 1;
	88	}
	89	__setup("align_va_addr", control_va_addr_alignment);
	90
0ac676fb JB	91	SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
	92	unsigned long, prot, unsigned long, flags,
	93	unsigned long, fd, unsigned long, off)
1da177e4 LT	94	{
1da177e4 LT	95	long error;
1da177e4 LT	96	error = -EINVAL;
	97	if (off & ~PAGE_MASK)
	98	goto out;
	99
f8b72560	100	error = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT);
1da177e4 LT	101	out:
	102	return error;
	103	}
	104
b569bab7 KS	105	static void find_start_end(unsigned long addr, unsigned long flags,
b569bab7 KS	106	unsigned long begin, unsigned long end)
1da177e4	107	{
3e6ef9c8	108	if (!in_compat_syscall() && (flags & MAP_32BIT)) {
1da177e4 LT	109	/* This is usually used needed to map code in small
	110	model, so it needs to be in the first 31bit. Limit
	111	it to that. This means we need to move the
	112	unmapped base down for this case. This can give
	113	conflicts with the heap, but we assume that glibc
	114	malloc knows how to fall back to mmap. Give it 1GB
e9c8abb6 GP	115	of playground for now. -AK */
	116	*begin = 0x40000000;
	117	*end = 0x80000000;
cc503c1b	118	if (current->flags & PF_RANDOMIZE) {
9c6f0902	119	begin = randomize_page(begin, 0x02000000);
cc503c1b	120	}
1b028f78	121	return;
84929801	122	}
1b028f78 DS	123
1b028f78 DS	124	*begin = get_mmap_base(1);
b569bab7 KS	125	if (in_compat_syscall())
	126	*end = task_size_32bit();
	127	else
	128	*end = task_size_64bit(addr > DEFAULT_MAP_WINDOW);
e9c8abb6	129	}
1da177e4 LT	130
	131	unsigned long
	132	arch_get_unmapped_area(struct file *filp, unsigned long addr,
	133	unsigned long len, unsigned long pgoff, unsigned long flags)
	134	{
	135	struct mm_struct *mm = current->mm;
	136	struct vm_area_struct *vma;
f9902472	137	struct vm_unmapped_area_info info;
1da177e4	138	unsigned long begin, end;
e9c8abb6	139
44b04912 KS	140	addr = mpx_unmapped_area_check(addr, len, flags);
	141	if (IS_ERR_VALUE(addr))
	142	return addr;
	143
11300a64 BH	144	if (flags & MAP_FIXED)
	145	return addr;
	146
b569bab7	147	find_start_end(addr, flags, &begin, &end);
1da177e4 LT	148
	149	if (len > end)
	150	return -ENOMEM;
	151
	152	if (addr) {
	153	addr = PAGE_ALIGN(addr);
	154	vma = find_vma(mm, addr);
	155	if (end - len >= addr &&
1be7107f	156	(!vma \|\| addr + len <= vm_start_gap(vma)))
1da177e4 LT	157	return addr;
1da177e4 LT	158	}
1363c3cd	159
f9902472 ML	160	info.flags = 0;
	161	info.length = len;
	162	info.low_limit = begin;
	163	info.high_limit = end;
4e26d11f	164	info.align_mask = 0;
7d025059	165	info.align_offset = pgoff << PAGE_SHIFT;
4e26d11f HMG	166	if (filp) {
	167	info.align_mask = get_align_mask();
	168	info.align_offset += get_align_bits();
	169	}
f9902472	170	return vm_unmapped_area(&info);
1da177e4 LT	171	}
1da177e4 LT	172
cc503c1b JK	173	unsigned long
	174	arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
	175	const unsigned long len, const unsigned long pgoff,
	176	const unsigned long flags)
	177	{
	178	struct vm_area_struct *vma;
	179	struct mm_struct *mm = current->mm;
f9902472 ML	180	unsigned long addr = addr0;
f9902472 ML	181	struct vm_unmapped_area_info info;
cc503c1b	182
44b04912 KS	183	addr = mpx_unmapped_area_check(addr, len, flags);
	184	if (IS_ERR_VALUE(addr))
	185	return addr;
	186
cc503c1b JK	187	/* requested length too big for entire address space */
	188	if (len > TASK_SIZE)
	189	return -ENOMEM;
	190
1e0f25db	191	/* No address checking. See comment at mmap_address_hint_valid() */
cc503c1b JK	192	if (flags & MAP_FIXED)
	193	return addr;
	194
e3e81aca	195	/* for MAP_32BIT mappings we force the legacy mmap base */
3e6ef9c8	196	if (!in_compat_syscall() && (flags & MAP_32BIT))
cc503c1b JK	197	goto bottomup;
	198
	199	/* requesting a specific address */
	200	if (addr) {
1e0f25db KS	201	addr &= PAGE_MASK;
	202	if (!mmap_address_hint_valid(addr, len))
	203	goto get_unmapped_area;
	204
cc503c1b	205	vma = find_vma(mm, addr);
1e0f25db	206	if (!vma \|\| addr + len <= vm_start_gap(vma))
cc503c1b JK	207	return addr;
cc503c1b JK	208	}
1e0f25db	209	get_unmapped_area:
cc503c1b	210
f9902472 ML	211	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	212	info.length = len;
	213	info.low_limit = PAGE_SIZE;
1b028f78	214	info.high_limit = get_mmap_base(0);
b569bab7 KS	215
	216	/*
	217	* If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
	218	* in the full address space.
	219	*
	220	* !in_compat_syscall() check to avoid high addresses for x32.
	221	*/
	222	if (addr > DEFAULT_MAP_WINDOW && !in_compat_syscall())
	223	info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW;
	224
4e26d11f	225	info.align_mask = 0;
7d025059	226	info.align_offset = pgoff << PAGE_SHIFT;
4e26d11f HMG	227	if (filp) {
	228	info.align_mask = get_align_mask();
	229	info.align_offset += get_align_bits();
	230	}
f9902472 ML	231	addr = vm_unmapped_area(&info);
	232	if (!(addr & ~PAGE_MASK))
	233	return addr;
	234	VM_BUG_ON(addr != -ENOMEM);
b716ad95	235
cc503c1b JK	236	bottomup:
	237	/*
	238	* A failed mmap() very likely causes application failure,
	239	* so fall back to the bottom-up function here. This scenario
	240	* can happen with large stack limits and large mmap()
	241	* allocations.
	242	*/
f9902472	243	return arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
cc503c1b	244	}