2 * AArch64 loadable module support.
4 * Copyright (C) 2012 ARM Limited
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 * Author: Will Deacon <will.deacon@arm.com>
21 #include <linux/bitops.h>
22 #include <linux/elf.h>
23 #include <linux/gfp.h>
24 #include <linux/kasan.h>
25 #include <linux/kernel.h>
27 #include <linux/moduleloader.h>
28 #include <linux/vmalloc.h>
29 #include <asm/alternative.h>
31 #include <asm/sections.h>
33 void *module_alloc(unsigned long size
)
35 u64 module_alloc_end
= module_alloc_base
+ MODULES_VSIZE
;
36 gfp_t gfp_mask
= GFP_KERNEL
;
39 /* Silence the initial allocation */
40 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS
))
41 gfp_mask
|= __GFP_NOWARN
;
43 if (IS_ENABLED(CONFIG_KASAN
))
44 /* don't exceed the static module region - see below */
45 module_alloc_end
= MODULES_END
;
47 p
= __vmalloc_node_range(size
, MODULE_ALIGN
, module_alloc_base
,
48 module_alloc_end
, gfp_mask
, PAGE_KERNEL_EXEC
, 0,
49 NUMA_NO_NODE
, __builtin_return_address(0));
51 if (!p
&& IS_ENABLED(CONFIG_ARM64_MODULE_PLTS
) &&
52 !IS_ENABLED(CONFIG_KASAN
))
54 * KASAN can only deal with module allocations being served
55 * from the reserved module region, since the remainder of
56 * the vmalloc region is already backed by zero shadow pages,
57 * and punching holes into it is non-trivial. Since the module
58 * region is not randomized when KASAN is enabled, it is even
59 * less likely that the module region gets exhausted, so we
60 * can simply omit this fallback in that case.
62 p
= __vmalloc_node_range(size
, MODULE_ALIGN
, module_alloc_base
,
63 module_alloc_base
+ SZ_4G
, GFP_KERNEL
,
64 PAGE_KERNEL_EXEC
, 0, NUMA_NO_NODE
,
65 __builtin_return_address(0));
67 if (p
&& (kasan_module_alloc(p
, size
) < 0)) {
75 enum aarch64_reloc_op
{
82 static u64
do_reloc(enum aarch64_reloc_op reloc_op
, __le32
*place
, u64 val
)
88 return val
- (u64
)place
;
90 return (val
& ~0xfff) - ((u64
)place
& ~0xfff);
95 pr_err("do_reloc: unknown relocation operation %d\n", reloc_op
);
99 static int reloc_data(enum aarch64_reloc_op op
, void *place
, u64 val
, int len
)
101 s64 sval
= do_reloc(op
, place
, val
);
105 *(s16
*)place
= sval
;
106 if (sval
< S16_MIN
|| sval
> U16_MAX
)
110 *(s32
*)place
= sval
;
111 if (sval
< S32_MIN
|| sval
> U32_MAX
)
115 *(s64
*)place
= sval
;
118 pr_err("Invalid length (%d) for data relocation\n", len
);
124 enum aarch64_insn_movw_imm_type
{
125 AARCH64_INSN_IMM_MOVNZ
,
126 AARCH64_INSN_IMM_MOVKZ
,
129 static int reloc_insn_movw(enum aarch64_reloc_op op
, __le32
*place
, u64 val
,
130 int lsb
, enum aarch64_insn_movw_imm_type imm_type
)
134 u32 insn
= le32_to_cpu(*place
);
136 sval
= do_reloc(op
, place
, val
);
139 if (imm_type
== AARCH64_INSN_IMM_MOVNZ
) {
141 * For signed MOVW relocations, we have to manipulate the
142 * instruction encoding depending on whether or not the
143 * immediate is less than zero.
147 /* >=0: Set the instruction to MOVZ (opcode 10b). */
151 * <0: Set the instruction to MOVN (opcode 00b).
152 * Since we've masked the opcode already, we
153 * don't need to do anything other than
154 * inverting the new immediate field.
160 /* Update the instruction with the new encoding. */
161 insn
= aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16
, insn
, imm
);
162 *place
= cpu_to_le32(insn
);
170 static int reloc_insn_imm(enum aarch64_reloc_op op
, __le32
*place
, u64 val
,
171 int lsb
, int len
, enum aarch64_insn_imm_type imm_type
)
175 u32 insn
= le32_to_cpu(*place
);
177 /* Calculate the relocation value. */
178 sval
= do_reloc(op
, place
, val
);
181 /* Extract the value bits and shift them to bit 0. */
182 imm_mask
= (BIT(lsb
+ len
) - 1) >> lsb
;
183 imm
= sval
& imm_mask
;
185 /* Update the instruction's immediate field. */
186 insn
= aarch64_insn_encode_immediate(imm_type
, insn
, imm
);
187 *place
= cpu_to_le32(insn
);
190 * Extract the upper value bits (including the sign bit) and
191 * shift them to bit 0.
193 sval
= (s64
)(sval
& ~(imm_mask
>> 1)) >> (len
- 1);
196 * Overflow has occurred if the upper bits are not all equal to
197 * the sign bit of the value.
199 if ((u64
)(sval
+ 1) >= 2)
205 int apply_relocate_add(Elf64_Shdr
*sechdrs
,
207 unsigned int symindex
,
217 Elf64_Rela
*rel
= (void *)sechdrs
[relsec
].sh_addr
;
219 for (i
= 0; i
< sechdrs
[relsec
].sh_size
/ sizeof(*rel
); i
++) {
220 /* loc corresponds to P in the AArch64 ELF document. */
221 loc
= (void *)sechdrs
[sechdrs
[relsec
].sh_info
].sh_addr
224 /* sym is the ELF symbol we're referring to. */
225 sym
= (Elf64_Sym
*)sechdrs
[symindex
].sh_addr
226 + ELF64_R_SYM(rel
[i
].r_info
);
228 /* val corresponds to (S + A) in the AArch64 ELF document. */
229 val
= sym
->st_value
+ rel
[i
].r_addend
;
231 /* Check for overflow by default. */
232 overflow_check
= true;
234 /* Perform the static relocation. */
235 switch (ELF64_R_TYPE(rel
[i
].r_info
)) {
236 /* Null relocations. */
242 /* Data relocations. */
243 case R_AARCH64_ABS64
:
244 overflow_check
= false;
245 ovf
= reloc_data(RELOC_OP_ABS
, loc
, val
, 64);
247 case R_AARCH64_ABS32
:
248 ovf
= reloc_data(RELOC_OP_ABS
, loc
, val
, 32);
250 case R_AARCH64_ABS16
:
251 ovf
= reloc_data(RELOC_OP_ABS
, loc
, val
, 16);
253 case R_AARCH64_PREL64
:
254 overflow_check
= false;
255 ovf
= reloc_data(RELOC_OP_PREL
, loc
, val
, 64);
257 case R_AARCH64_PREL32
:
258 ovf
= reloc_data(RELOC_OP_PREL
, loc
, val
, 32);
260 case R_AARCH64_PREL16
:
261 ovf
= reloc_data(RELOC_OP_PREL
, loc
, val
, 16);
264 /* MOVW instruction relocations. */
265 case R_AARCH64_MOVW_UABS_G0_NC
:
266 overflow_check
= false;
267 case R_AARCH64_MOVW_UABS_G0
:
268 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 0,
269 AARCH64_INSN_IMM_MOVKZ
);
271 case R_AARCH64_MOVW_UABS_G1_NC
:
272 overflow_check
= false;
273 case R_AARCH64_MOVW_UABS_G1
:
274 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 16,
275 AARCH64_INSN_IMM_MOVKZ
);
277 case R_AARCH64_MOVW_UABS_G2_NC
:
278 overflow_check
= false;
279 case R_AARCH64_MOVW_UABS_G2
:
280 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 32,
281 AARCH64_INSN_IMM_MOVKZ
);
283 case R_AARCH64_MOVW_UABS_G3
:
284 /* We're using the top bits so we can't overflow. */
285 overflow_check
= false;
286 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 48,
287 AARCH64_INSN_IMM_MOVKZ
);
289 case R_AARCH64_MOVW_SABS_G0
:
290 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 0,
291 AARCH64_INSN_IMM_MOVNZ
);
293 case R_AARCH64_MOVW_SABS_G1
:
294 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 16,
295 AARCH64_INSN_IMM_MOVNZ
);
297 case R_AARCH64_MOVW_SABS_G2
:
298 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 32,
299 AARCH64_INSN_IMM_MOVNZ
);
301 case R_AARCH64_MOVW_PREL_G0_NC
:
302 overflow_check
= false;
303 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 0,
304 AARCH64_INSN_IMM_MOVKZ
);
306 case R_AARCH64_MOVW_PREL_G0
:
307 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 0,
308 AARCH64_INSN_IMM_MOVNZ
);
310 case R_AARCH64_MOVW_PREL_G1_NC
:
311 overflow_check
= false;
312 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 16,
313 AARCH64_INSN_IMM_MOVKZ
);
315 case R_AARCH64_MOVW_PREL_G1
:
316 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 16,
317 AARCH64_INSN_IMM_MOVNZ
);
319 case R_AARCH64_MOVW_PREL_G2_NC
:
320 overflow_check
= false;
321 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 32,
322 AARCH64_INSN_IMM_MOVKZ
);
324 case R_AARCH64_MOVW_PREL_G2
:
325 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 32,
326 AARCH64_INSN_IMM_MOVNZ
);
328 case R_AARCH64_MOVW_PREL_G3
:
329 /* We're using the top bits so we can't overflow. */
330 overflow_check
= false;
331 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 48,
332 AARCH64_INSN_IMM_MOVNZ
);
335 /* Immediate instruction relocations. */
336 case R_AARCH64_LD_PREL_LO19
:
337 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 19,
338 AARCH64_INSN_IMM_19
);
340 case R_AARCH64_ADR_PREL_LO21
:
341 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 0, 21,
342 AARCH64_INSN_IMM_ADR
);
344 #ifndef CONFIG_ARM64_ERRATUM_843419
345 case R_AARCH64_ADR_PREL_PG_HI21_NC
:
346 overflow_check
= false;
347 case R_AARCH64_ADR_PREL_PG_HI21
:
348 ovf
= reloc_insn_imm(RELOC_OP_PAGE
, loc
, val
, 12, 21,
349 AARCH64_INSN_IMM_ADR
);
352 case R_AARCH64_ADD_ABS_LO12_NC
:
353 case R_AARCH64_LDST8_ABS_LO12_NC
:
354 overflow_check
= false;
355 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 0, 12,
356 AARCH64_INSN_IMM_12
);
358 case R_AARCH64_LDST16_ABS_LO12_NC
:
359 overflow_check
= false;
360 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 1, 11,
361 AARCH64_INSN_IMM_12
);
363 case R_AARCH64_LDST32_ABS_LO12_NC
:
364 overflow_check
= false;
365 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 2, 10,
366 AARCH64_INSN_IMM_12
);
368 case R_AARCH64_LDST64_ABS_LO12_NC
:
369 overflow_check
= false;
370 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 3, 9,
371 AARCH64_INSN_IMM_12
);
373 case R_AARCH64_LDST128_ABS_LO12_NC
:
374 overflow_check
= false;
375 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 4, 8,
376 AARCH64_INSN_IMM_12
);
378 case R_AARCH64_TSTBR14
:
379 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 14,
380 AARCH64_INSN_IMM_14
);
382 case R_AARCH64_CONDBR19
:
383 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 19,
384 AARCH64_INSN_IMM_19
);
386 case R_AARCH64_JUMP26
:
387 case R_AARCH64_CALL26
:
388 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 26,
389 AARCH64_INSN_IMM_26
);
391 if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS
) &&
393 val
= module_emit_plt_entry(me
, loc
, &rel
[i
], sym
);
394 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2,
395 26, AARCH64_INSN_IMM_26
);
400 pr_err("module %s: unsupported RELA relocation: %llu\n",
401 me
->name
, ELF64_R_TYPE(rel
[i
].r_info
));
405 if (overflow_check
&& ovf
== -ERANGE
)
413 pr_err("module %s: overflow in relocation type %d val %Lx\n",
414 me
->name
, (int)ELF64_R_TYPE(rel
[i
].r_info
), val
);
418 int module_finalize(const Elf_Ehdr
*hdr
,
419 const Elf_Shdr
*sechdrs
,
422 const Elf_Shdr
*s
, *se
;
423 const char *secstrs
= (void *)hdr
+ sechdrs
[hdr
->e_shstrndx
].sh_offset
;
425 for (s
= sechdrs
, se
= sechdrs
+ hdr
->e_shnum
; s
< se
; s
++) {
426 if (strcmp(".altinstructions", secstrs
+ s
->sh_name
) == 0) {
427 apply_alternatives((void *)s
->sh_addr
, s
->sh_size
);
429 #ifdef CONFIG_ARM64_MODULE_PLTS
430 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE
) &&
431 !strcmp(".text.ftrace_trampoline", secstrs
+ s
->sh_name
))
432 me
->arch
.ftrace_trampoline
= (void *)s
->sh_addr
;