[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kernel / module.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*  Kernel module help for x86.
    Copyright (C) 2001 Rusty Russell.

*/

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/moduleloader.h>
#include <linux/elf.h>
#include <linux/vmalloc.h>
#include <linux/fs.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/kasan.h>
#include <linux/bug.h>
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/jump_label.h>
#include <linux/random.h>
#include <linux/memory.h>

#include <asm/text-patching.h>
#include <asm/page.h>
#include <asm/setup.h>
#include <asm/unwind.h>

#if 0
#define DEBUGP(fmt, ...)				\
	printk(KERN_DEBUG fmt, ##__VA_ARGS__)
#else
#define DEBUGP(fmt, ...)				\
do {							\
	if (0)						\
		printk(KERN_DEBUG fmt, ##__VA_ARGS__);	\
} while (0)
#endif

#ifdef CONFIG_RANDOMIZE_BASE
static unsigned long module_load_offset;

/* Mutex protects the module_load_offset. */
static DEFINE_MUTEX(module_kaslr_mutex);

static unsigned long int get_module_load_offset(void)
{
	if (kaslr_enabled()) {
		mutex_lock(&module_kaslr_mutex);
		/*
		 * Calculate the module_load_offset the first time this
		 * code is called. Once calculated it stays the same until
		 * reboot.
		 */
		if (module_load_offset == 0)
			module_load_offset =
				(get_random_int() % 1024 + 1) * PAGE_SIZE;
		mutex_unlock(&module_kaslr_mutex);
	}
	return module_load_offset;
}
#else
static unsigned long int get_module_load_offset(void)
{
	return 0;
}
#endif

void *module_alloc(unsigned long size)
{
	gfp_t gfp_mask = GFP_KERNEL;
	void *p;

	if (PAGE_ALIGN(size) > MODULES_LEN)
		return NULL;

	p = __vmalloc_node_range(size, MODULE_ALIGN,
				    MODULES_VADDR + get_module_load_offset(),
				    MODULES_END, gfp_mask,
				    PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE,
				    __builtin_return_address(0));
	if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) {
		vfree(p);
		return NULL;
	}

	return p;
}

#ifdef CONFIG_X86_32
int apply_relocate(Elf32_Shdr *sechdrs,
		   const char *strtab,
		   unsigned int symindex,
		   unsigned int relsec,
		   struct module *me)
{
	unsigned int i;
	Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr;
	Elf32_Sym *sym;
	uint32_t *location;

	DEBUGP("Applying relocate section %u to %u\n",
	       relsec, sechdrs[relsec].sh_info);
	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
		/* This is where to make the change */
		location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
			+ rel[i].r_offset;
		/* This is the symbol it is referring to.  Note that all
		   undefined symbols have been resolved.  */
		sym = (Elf32_Sym *)sechdrs[symindex].sh_addr
			+ ELF32_R_SYM(rel[i].r_info);

		switch (ELF32_R_TYPE(rel[i].r_info)) {
		case R_386_32:
			/* We add the value into the location given */
			*location += sym->st_value;
			break;
		case R_386_PC32:
		case R_386_PLT32:
			/* Add the value, subtract its position */
			*location += sym->st_value - (uint32_t)location;
			break;
		default:
			pr_err("%s: Unknown relocation: %u\n",
			       me->name, ELF32_R_TYPE(rel[i].r_info));
			return -ENOEXEC;
		}
	}
	return 0;
}
#else /*X86_64*/
static int __apply_relocate_add(Elf64_Shdr *sechdrs,
		   const char *strtab,
		   unsigned int symindex,
		   unsigned int relsec,
		   struct module *me,
		   void *(*write)(void *dest, const void *src, size_t len))
{
	unsigned int i;
	Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;
	Elf64_Sym *sym;
	void *loc;
	u64 val;

	DEBUGP("Applying relocate section %u to %u\n",
	       relsec, sechdrs[relsec].sh_info);
	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
		/* This is where to make the change */
		loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
			+ rel[i].r_offset;

		/* This is the symbol it is referring to.  Note that all
		   undefined symbols have been resolved.  */
		sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
			+ ELF64_R_SYM(rel[i].r_info);

		DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
		       (int)ELF64_R_TYPE(rel[i].r_info),
		       sym->st_value, rel[i].r_addend, (u64)loc);

		val = sym->st_value + rel[i].r_addend;

		switch (ELF64_R_TYPE(rel[i].r_info)) {
		case R_X86_64_NONE:
			break;
		case R_X86_64_64:
			if (*(u64 *)loc != 0)
				goto invalid_relocation;
			write(loc, &val, 8);
			break;
		case R_X86_64_32:
			if (*(u32 *)loc != 0)
				goto invalid_relocation;
			write(loc, &val, 4);
			if (val != *(u32 *)loc)
				goto overflow;
			break;
		case R_X86_64_32S:
			if (*(s32 *)loc != 0)
				goto invalid_relocation;
			write(loc, &val, 4);
			if ((s64)val != *(s32 *)loc)
				goto overflow;
			break;
		case R_X86_64_PC32:
		case R_X86_64_PLT32:
			if (*(u32 *)loc != 0)
				goto invalid_relocation;
			val -= (u64)loc;
			write(loc, &val, 4);
#if 0
			if ((s64)val != *(s32 *)loc)
				goto overflow;
#endif
			break;
		case R_X86_64_PC64:
			if (*(u64 *)loc != 0)
				goto invalid_relocation;
			val -= (u64)loc;
			write(loc, &val, 8);
			break;
		default:
			pr_err("%s: Unknown rela relocation: %llu\n",
			       me->name, ELF64_R_TYPE(rel[i].r_info));
			return -ENOEXEC;
		}
	}
	return 0;

invalid_relocation:
	pr_err("x86/modules: Skipping invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n",
	       (int)ELF64_R_TYPE(rel[i].r_info), loc, val);
	return -ENOEXEC;

overflow:
	pr_err("overflow in relocation type %d val %Lx\n",
	       (int)ELF64_R_TYPE(rel[i].r_info), val);
	pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
	       me->name);
	return -ENOEXEC;
}

int apply_relocate_add(Elf64_Shdr *sechdrs,
		   const char *strtab,
		   unsigned int symindex,
		   unsigned int relsec,
		   struct module *me)
{
	int ret;
	bool early = me->state == MODULE_STATE_UNFORMED;
	void *(*write)(void *, const void *, size_t) = memcpy;

	if (!early) {
		write = text_poke;
		mutex_lock(&text_mutex);
	}

	ret = __apply_relocate_add(sechdrs, strtab, symindex, relsec, me,
				   write);

	if (!early) {
		text_poke_sync();
		mutex_unlock(&text_mutex);
	}

	return ret;
}

#endif

int module_finalize(const Elf_Ehdr *hdr,
		    const Elf_Shdr *sechdrs,
		    struct module *me)
{
	const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
		*para = NULL, *orc = NULL, *orc_ip = NULL,
		*retpolines = NULL, *returns = NULL;
	char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;

	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
		if (!strcmp(".text", secstrings + s->sh_name))
			text = s;
		if (!strcmp(".altinstructions", secstrings + s->sh_name))
			alt = s;
		if (!strcmp(".smp_locks", secstrings + s->sh_name))
			locks = s;
		if (!strcmp(".parainstructions", secstrings + s->sh_name))
			para = s;
		if (!strcmp(".orc_unwind", secstrings + s->sh_name))
			orc = s;
		if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name))
			orc_ip = s;
		if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
			retpolines = s;
		if (!strcmp(".return_sites", secstrings + s->sh_name))
			returns = s;
	}

	/*
	 * See alternative_instructions() for the ordering rules between the
	 * various patching types.
	 */
	if (para) {
		void *pseg = (void *)para->sh_addr;
		apply_paravirt(pseg, pseg + para->sh_size);
	}
	if (retpolines) {
		void *rseg = (void *)retpolines->sh_addr;
		apply_retpolines(rseg, rseg + retpolines->sh_size);
	}
	if (returns) {
		void *rseg = (void *)returns->sh_addr;
		apply_returns(rseg, rseg + returns->sh_size);
	}
	if (alt) {
		/* patch .altinstructions */
		void *aseg = (void *)alt->sh_addr;
		apply_alternatives(aseg, aseg + alt->sh_size);
	}
	if (locks && text) {
		void *lseg = (void *)locks->sh_addr;
		void *tseg = (void *)text->sh_addr;
		alternatives_smp_module_add(me, me->name,
					    lseg, lseg + locks->sh_size,
					    tseg, tseg + text->sh_size);
	}

	/* make jump label nops */
	jump_label_apply_nops(me);

	if (orc && orc_ip)
		unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size,
				   (void *)orc->sh_addr, orc->sh_size);

	return 0;
}

void module_arch_cleanup(struct module *mod)
{
	alternatives_smp_module_del(mod);
}
Commit	Line	Data
1a59d1b8	1	// SPDX-License-Identifier: GPL-2.0-or-later
2d5bf28f AW	2	/* Kernel module help for x86.
	3	Copyright (C) 2001 Rusty Russell.
	4
2d5bf28f	5	*/
c767a54b JP	6
	7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	8
2d5bf28f AW	9	#include <linux/moduleloader.h>
	10	#include <linux/elf.h>
	11	#include <linux/vmalloc.h>
	12	#include <linux/fs.h>
	13	#include <linux/string.h>
	14	#include <linux/kernel.h>
bebf56a1	15	#include <linux/kasan.h>
2d5bf28f AW	16	#include <linux/bug.h>
2d5bf28f AW	17	#include <linux/mm.h>
5a0e3ad6	18	#include <linux/gfp.h>
d430d3d7	19	#include <linux/jump_label.h>
e2b32e67	20	#include <linux/random.h>
5b384f93	21	#include <linux/memory.h>
2d5bf28f	22
35de5b06	23	#include <asm/text-patching.h>
2d5bf28f	24	#include <asm/page.h>
78cac48c	25	#include <asm/setup.h>
ee9f8fce	26	#include <asm/unwind.h>
2d5bf28f AW	27
2d5bf28f AW	28	#if 0
c767a54b JP	29	#define DEBUGP(fmt, ...) \
c767a54b JP	30	printk(KERN_DEBUG fmt, ##__VA_ARGS__)
2d5bf28f	31	#else
c767a54b JP	32	#define DEBUGP(fmt, ...) \
	33	do { \
	34	if (0) \
	35	printk(KERN_DEBUG fmt, ##__VA_ARGS__); \
	36	} while (0)
2d5bf28f AW	37	#endif
2d5bf28f AW	38
e2b32e67 KC	39	#ifdef CONFIG_RANDOMIZE_BASE
e2b32e67 KC	40	static unsigned long module_load_offset;
e2b32e67	41
9dd721c6 KC	42	/* Mutex protects the module_load_offset. */
	43	static DEFINE_MUTEX(module_kaslr_mutex);
	44
e2b32e67 KC	45	static unsigned long int get_module_load_offset(void)
e2b32e67 KC	46	{
78cac48c	47	if (kaslr_enabled()) {
9dd721c6	48	mutex_lock(&module_kaslr_mutex);
e2b32e67 KC	49	/*
	50	* Calculate the module_load_offset the first time this
	51	* code is called. Once calculated it stays the same until
	52	* reboot.
	53	*/
	54	if (module_load_offset == 0)
	55	module_load_offset =
	56	(get_random_int() % 1024 + 1) * PAGE_SIZE;
9dd721c6	57	mutex_unlock(&module_kaslr_mutex);
e2b32e67 KC	58	}
	59	return module_load_offset;
	60	}
	61	#else
	62	static unsigned long int get_module_load_offset(void)
	63	{
	64	return 0;
	65	}
	66	#endif
	67
0fdc83b9 AW	68	void *module_alloc(unsigned long size)
0fdc83b9 AW	69	{
2ee84b78	70	gfp_t gfp_mask = GFP_KERNEL;
bebf56a1 AR	71	void *p;
bebf56a1 AR	72
d0a21265	73	if (PAGE_ALIGN(size) > MODULES_LEN)
0fdc83b9	74	return NULL;
bebf56a1 AR	75
bebf56a1 AR	76	p = __vmalloc_node_range(size, MODULE_ALIGN,
e2b32e67	77	MODULES_VADDR + get_module_load_offset(),
2ee84b78 KW	78	MODULES_END, gfp_mask,
2ee84b78 KW	79	PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE,
e2b32e67	80	__builtin_return_address(0));
2ee84b78	81	if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) {
bebf56a1 AR	82	vfree(p);
	83	return NULL;
	84	}
	85
	86	return p;
0fdc83b9	87	}
0fdc83b9	88
0fdc83b9 AW	89	#ifdef CONFIG_X86_32
	90	int apply_relocate(Elf32_Shdr *sechdrs,
	91	const char *strtab,
	92	unsigned int symindex,
	93	unsigned int relsec,
	94	struct module *me)
	95	{
	96	unsigned int i;
	97	Elf32_Rel rel = (void )sechdrs[relsec].sh_addr;
	98	Elf32_Sym *sym;
	99	uint32_t *location;
	100
c767a54b JP	101	DEBUGP("Applying relocate section %u to %u\n",
c767a54b JP	102	relsec, sechdrs[relsec].sh_info);
0fdc83b9 AW	103	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
	104	/* This is where to make the change */
	105	location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
	106	+ rel[i].r_offset;
	107	/* This is the symbol it is referring to. Note that all
	108	undefined symbols have been resolved. */
	109	sym = (Elf32_Sym *)sechdrs[symindex].sh_addr
	110	+ ELF32_R_SYM(rel[i].r_info);
	111
	112	switch (ELF32_R_TYPE(rel[i].r_info)) {
	113	case R_386_32:
	114	/* We add the value into the location given */
	115	*location += sym->st_value;
	116	break;
	117	case R_386_PC32:
bb73d071	118	case R_386_PLT32:
2e76c283	119	/* Add the value, subtract its position */
0fdc83b9 AW	120	*location += sym->st_value - (uint32_t)location;
	121	break;
	122	default:
c767a54b	123	pr_err("%s: Unknown relocation: %u\n",
0fdc83b9 AW	124	me->name, ELF32_R_TYPE(rel[i].r_info));
	125	return -ENOEXEC;
	126	}
	127	}
	128	return 0;
	129	}
0fdc83b9	130	#else /X86_64/
88fc078a	131	static int __apply_relocate_add(Elf64_Shdr *sechdrs,
0fdc83b9 AW	132	const char *strtab,
	133	unsigned int symindex,
	134	unsigned int relsec,
88fc078a PZ	135	struct module *me,
88fc078a PZ	136	void (write)(void dest, const void src, size_t len))
0fdc83b9 AW	137	{
	138	unsigned int i;
	139	Elf64_Rela rel = (void )sechdrs[relsec].sh_addr;
	140	Elf64_Sym *sym;
	141	void *loc;
	142	u64 val;
	143
c767a54b JP	144	DEBUGP("Applying relocate section %u to %u\n",
c767a54b JP	145	relsec, sechdrs[relsec].sh_info);
0fdc83b9 AW	146	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
	147	/* This is where to make the change */
	148	loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
	149	+ rel[i].r_offset;
	150
	151	/* This is the symbol it is referring to. Note that all
	152	undefined symbols have been resolved. */
	153	sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
	154	+ ELF64_R_SYM(rel[i].r_info);
	155
	156	DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
c767a54b JP	157	(int)ELF64_R_TYPE(rel[i].r_info),
c767a54b JP	158	sym->st_value, rel[i].r_addend, (u64)loc);
0fdc83b9 AW	159
	160	val = sym->st_value + rel[i].r_addend;
	161
	162	switch (ELF64_R_TYPE(rel[i].r_info)) {
	163	case R_X86_64_NONE:
	164	break;
	165	case R_X86_64_64:
eda9cec4 JP	166	if ((u64 )loc != 0)
eda9cec4 JP	167	goto invalid_relocation;
88fc078a	168	write(loc, &val, 8);
0fdc83b9 AW	169	break;
0fdc83b9 AW	170	case R_X86_64_32:
eda9cec4 JP	171	if ((u32 )loc != 0)
eda9cec4 JP	172	goto invalid_relocation;
88fc078a	173	write(loc, &val, 4);
0fdc83b9 AW	174	if (val != (u32 )loc)
	175	goto overflow;
	176	break;
	177	case R_X86_64_32S:
eda9cec4 JP	178	if ((s32 )loc != 0)
eda9cec4 JP	179	goto invalid_relocation;
88fc078a	180	write(loc, &val, 4);
0fdc83b9 AW	181	if ((s64)val != (s32 )loc)
	182	goto overflow;
	183	break;
	184	case R_X86_64_PC32:
b21ebf2f	185	case R_X86_64_PLT32:
eda9cec4 JP	186	if ((u32 )loc != 0)
eda9cec4 JP	187	goto invalid_relocation;
0fdc83b9	188	val -= (u64)loc;
88fc078a	189	write(loc, &val, 4);
0fdc83b9 AW	190	#if 0
	191	if ((s64)val != (s32 )loc)
	192	goto overflow;
	193	#endif
	194	break;
b40a142b AB	195	case R_X86_64_PC64:
	196	if ((u64 )loc != 0)
	197	goto invalid_relocation;
	198	val -= (u64)loc;
88fc078a	199	write(loc, &val, 8);
b40a142b	200	break;
0fdc83b9	201	default:
c767a54b	202	pr_err("%s: Unknown rela relocation: %llu\n",
0fdc83b9 AW	203	me->name, ELF64_R_TYPE(rel[i].r_info));
	204	return -ENOEXEC;
	205	}
	206	}
	207	return 0;
	208
eda9cec4 JP	209	invalid_relocation:
	210	pr_err("x86/modules: Skipping invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n",
	211	(int)ELF64_R_TYPE(rel[i].r_info), loc, val);
	212	return -ENOEXEC;
	213
0fdc83b9	214	overflow:
c767a54b	215	pr_err("overflow in relocation type %d val %Lx\n",
0fdc83b9	216	(int)ELF64_R_TYPE(rel[i].r_info), val);
c767a54b	217	pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
0fdc83b9 AW	218	me->name);
	219	return -ENOEXEC;
	220	}
88fc078a PZ	221
	222	int apply_relocate_add(Elf64_Shdr *sechdrs,
	223	const char *strtab,
	224	unsigned int symindex,
	225	unsigned int relsec,
	226	struct module *me)
	227	{
	228	int ret;
	229	bool early = me->state == MODULE_STATE_UNFORMED;
	230	void (write)(void , const void , size_t) = memcpy;
	231
5b384f93	232	if (!early) {
88fc078a	233	write = text_poke;
5b384f93 JP	234	mutex_lock(&text_mutex);
5b384f93 JP	235	}
88fc078a PZ	236
	237	ret = __apply_relocate_add(sechdrs, strtab, symindex, relsec, me,
	238	write);
	239
5b384f93	240	if (!early) {
88fc078a	241	text_poke_sync();
5b384f93 JP	242	mutex_unlock(&text_mutex);
5b384f93 JP	243	}
88fc078a PZ	244
	245	return ret;
	246	}
	247
0fdc83b9 AW	248	#endif
0fdc83b9 AW	249
2d5bf28f AW	250	int module_finalize(const Elf_Ehdr *hdr,
	251	const Elf_Shdr *sechdrs,
	252	struct module *me)
	253	{
	254	const Elf_Shdr s, text = NULL, alt = NULL, locks = NULL,
81211f23	255	para = NULL, orc = NULL, *orc_ip = NULL,
196de01a	256	retpolines = NULL, returns = NULL;
2d5bf28f AW	257	char secstrings = (void )hdr + sechdrs[hdr->e_shstrndx].sh_offset;
	258
	259	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
	260	if (!strcmp(".text", secstrings + s->sh_name))
	261	text = s;
	262	if (!strcmp(".altinstructions", secstrings + s->sh_name))
	263	alt = s;
	264	if (!strcmp(".smp_locks", secstrings + s->sh_name))
	265	locks = s;
	266	if (!strcmp(".parainstructions", secstrings + s->sh_name))
	267	para = s;
ee9f8fce JP	268	if (!strcmp(".orc_unwind", secstrings + s->sh_name))
	269	orc = s;
	270	if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name))
	271	orc_ip = s;
81211f23 PZ	272	if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
81211f23 PZ	273	retpolines = s;
196de01a PZ	274	if (!strcmp(".return_sites", secstrings + s->sh_name))
196de01a PZ	275	returns = s;
2d5bf28f AW	276	}
2d5bf28f AW	277
292cfe1d PZ	278	/*
	279	* See alternative_instructions() for the ordering rules between the
	280	* various patching types.
	281	*/
	282	if (para) {
	283	void pseg = (void )para->sh_addr;
	284	apply_paravirt(pseg, pseg + para->sh_size);
	285	}
81211f23 PZ	286	if (retpolines) {
	287	void rseg = (void )retpolines->sh_addr;
	288	apply_retpolines(rseg, rseg + retpolines->sh_size);
	289	}
196de01a PZ	290	if (returns) {
	291	void rseg = (void )returns->sh_addr;
	292	apply_returns(rseg, rseg + returns->sh_size);
	293	}
2d5bf28f AW	294	if (alt) {
	295	/* patch .altinstructions */
	296	void aseg = (void )alt->sh_addr;
	297	apply_alternatives(aseg, aseg + alt->sh_size);
	298	}
	299	if (locks && text) {
	300	void lseg = (void )locks->sh_addr;
	301	void tseg = (void )text->sh_addr;
	302	alternatives_smp_module_add(me, me->name,
	303	lseg, lseg + locks->sh_size,
	304	tseg, tseg + text->sh_size);
	305	}
	306
d9f5ab7b JB	307	/* make jump label nops */
	308	jump_label_apply_nops(me);
	309
ee9f8fce JP	310	if (orc && orc_ip)
	311	unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size,
	312	(void *)orc->sh_addr, orc->sh_size);
	313
5336377d	314	return 0;
2d5bf28f AW	315	}
	316
	317	void module_arch_cleanup(struct module *mod)
	318	{
	319	alternatives_smp_module_del(mod);
2d5bf28f	320	}