2 * AArch64 loadable module support.
4 * Copyright (C) 2012 ARM Limited
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 * Author: Will Deacon <will.deacon@arm.com>
21 #include <linux/bitops.h>
22 #include <linux/elf.h>
23 #include <linux/gfp.h>
24 #include <linux/kasan.h>
25 #include <linux/kernel.h>
27 #include <linux/moduleloader.h>
28 #include <linux/vmalloc.h>
29 #include <asm/alternative.h>
31 #include <asm/sections.h>
33 void *module_alloc(unsigned long size
)
37 p
= __vmalloc_node_range(size
, MODULE_ALIGN
, MODULES_VADDR
, MODULES_END
,
38 GFP_KERNEL
, PAGE_KERNEL_EXEC
, 0,
39 NUMA_NO_NODE
, __builtin_return_address(0));
41 if (p
&& (kasan_module_alloc(p
, size
) < 0)) {
49 enum aarch64_reloc_op
{
56 static u64
do_reloc(enum aarch64_reloc_op reloc_op
, void *place
, u64 val
)
62 return val
- (u64
)place
;
64 return (val
& ~0xfff) - ((u64
)place
& ~0xfff);
69 pr_err("do_reloc: unknown relocation operation %d\n", reloc_op
);
73 static int reloc_data(enum aarch64_reloc_op op
, void *place
, u64 val
, int len
)
75 s64 sval
= do_reloc(op
, place
, val
);
80 if (sval
< S16_MIN
|| sval
> U16_MAX
)
85 if (sval
< S32_MIN
|| sval
> U32_MAX
)
92 pr_err("Invalid length (%d) for data relocation\n", len
);
98 enum aarch64_insn_movw_imm_type
{
99 AARCH64_INSN_IMM_MOVNZ
,
100 AARCH64_INSN_IMM_MOVKZ
,
103 static int reloc_insn_movw(enum aarch64_reloc_op op
, void *place
, u64 val
,
104 int lsb
, enum aarch64_insn_movw_imm_type imm_type
)
108 u32 insn
= le32_to_cpu(*(u32
*)place
);
110 sval
= do_reloc(op
, place
, val
);
113 if (imm_type
== AARCH64_INSN_IMM_MOVNZ
) {
115 * For signed MOVW relocations, we have to manipulate the
116 * instruction encoding depending on whether or not the
117 * immediate is less than zero.
121 /* >=0: Set the instruction to MOVZ (opcode 10b). */
125 * <0: Set the instruction to MOVN (opcode 00b).
126 * Since we've masked the opcode already, we
127 * don't need to do anything other than
128 * inverting the new immediate field.
134 /* Update the instruction with the new encoding. */
135 insn
= aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16
, insn
, imm
);
136 *(u32
*)place
= cpu_to_le32(insn
);
144 static int reloc_insn_imm(enum aarch64_reloc_op op
, void *place
, u64 val
,
145 int lsb
, int len
, enum aarch64_insn_imm_type imm_type
)
149 u32 insn
= le32_to_cpu(*(u32
*)place
);
151 /* Calculate the relocation value. */
152 sval
= do_reloc(op
, place
, val
);
155 /* Extract the value bits and shift them to bit 0. */
156 imm_mask
= (BIT(lsb
+ len
) - 1) >> lsb
;
157 imm
= sval
& imm_mask
;
159 /* Update the instruction's immediate field. */
160 insn
= aarch64_insn_encode_immediate(imm_type
, insn
, imm
);
161 *(u32
*)place
= cpu_to_le32(insn
);
164 * Extract the upper value bits (including the sign bit) and
165 * shift them to bit 0.
167 sval
= (s64
)(sval
& ~(imm_mask
>> 1)) >> (len
- 1);
170 * Overflow has occurred if the upper bits are not all equal to
171 * the sign bit of the value.
173 if ((u64
)(sval
+ 1) >= 2)
179 int apply_relocate_add(Elf64_Shdr
*sechdrs
,
181 unsigned int symindex
,
191 Elf64_Rela
*rel
= (void *)sechdrs
[relsec
].sh_addr
;
193 for (i
= 0; i
< sechdrs
[relsec
].sh_size
/ sizeof(*rel
); i
++) {
194 /* loc corresponds to P in the AArch64 ELF document. */
195 loc
= (void *)sechdrs
[sechdrs
[relsec
].sh_info
].sh_addr
198 /* sym is the ELF symbol we're referring to. */
199 sym
= (Elf64_Sym
*)sechdrs
[symindex
].sh_addr
200 + ELF64_R_SYM(rel
[i
].r_info
);
202 /* val corresponds to (S + A) in the AArch64 ELF document. */
203 val
= sym
->st_value
+ rel
[i
].r_addend
;
205 /* Check for overflow by default. */
206 overflow_check
= true;
208 /* Perform the static relocation. */
209 switch (ELF64_R_TYPE(rel
[i
].r_info
)) {
210 /* Null relocations. */
216 /* Data relocations. */
217 case R_AARCH64_ABS64
:
218 overflow_check
= false;
219 ovf
= reloc_data(RELOC_OP_ABS
, loc
, val
, 64);
221 case R_AARCH64_ABS32
:
222 ovf
= reloc_data(RELOC_OP_ABS
, loc
, val
, 32);
224 case R_AARCH64_ABS16
:
225 ovf
= reloc_data(RELOC_OP_ABS
, loc
, val
, 16);
227 case R_AARCH64_PREL64
:
228 overflow_check
= false;
229 ovf
= reloc_data(RELOC_OP_PREL
, loc
, val
, 64);
231 case R_AARCH64_PREL32
:
232 ovf
= reloc_data(RELOC_OP_PREL
, loc
, val
, 32);
234 case R_AARCH64_PREL16
:
235 ovf
= reloc_data(RELOC_OP_PREL
, loc
, val
, 16);
238 /* MOVW instruction relocations. */
239 case R_AARCH64_MOVW_UABS_G0_NC
:
240 overflow_check
= false;
241 case R_AARCH64_MOVW_UABS_G0
:
242 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 0,
243 AARCH64_INSN_IMM_MOVKZ
);
245 case R_AARCH64_MOVW_UABS_G1_NC
:
246 overflow_check
= false;
247 case R_AARCH64_MOVW_UABS_G1
:
248 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 16,
249 AARCH64_INSN_IMM_MOVKZ
);
251 case R_AARCH64_MOVW_UABS_G2_NC
:
252 overflow_check
= false;
253 case R_AARCH64_MOVW_UABS_G2
:
254 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 32,
255 AARCH64_INSN_IMM_MOVKZ
);
257 case R_AARCH64_MOVW_UABS_G3
:
258 /* We're using the top bits so we can't overflow. */
259 overflow_check
= false;
260 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 48,
261 AARCH64_INSN_IMM_MOVKZ
);
263 case R_AARCH64_MOVW_SABS_G0
:
264 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 0,
265 AARCH64_INSN_IMM_MOVNZ
);
267 case R_AARCH64_MOVW_SABS_G1
:
268 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 16,
269 AARCH64_INSN_IMM_MOVNZ
);
271 case R_AARCH64_MOVW_SABS_G2
:
272 ovf
= reloc_insn_movw(RELOC_OP_ABS
, loc
, val
, 32,
273 AARCH64_INSN_IMM_MOVNZ
);
275 case R_AARCH64_MOVW_PREL_G0_NC
:
276 overflow_check
= false;
277 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 0,
278 AARCH64_INSN_IMM_MOVKZ
);
280 case R_AARCH64_MOVW_PREL_G0
:
281 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 0,
282 AARCH64_INSN_IMM_MOVNZ
);
284 case R_AARCH64_MOVW_PREL_G1_NC
:
285 overflow_check
= false;
286 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 16,
287 AARCH64_INSN_IMM_MOVKZ
);
289 case R_AARCH64_MOVW_PREL_G1
:
290 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 16,
291 AARCH64_INSN_IMM_MOVNZ
);
293 case R_AARCH64_MOVW_PREL_G2_NC
:
294 overflow_check
= false;
295 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 32,
296 AARCH64_INSN_IMM_MOVKZ
);
298 case R_AARCH64_MOVW_PREL_G2
:
299 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 32,
300 AARCH64_INSN_IMM_MOVNZ
);
302 case R_AARCH64_MOVW_PREL_G3
:
303 /* We're using the top bits so we can't overflow. */
304 overflow_check
= false;
305 ovf
= reloc_insn_movw(RELOC_OP_PREL
, loc
, val
, 48,
306 AARCH64_INSN_IMM_MOVNZ
);
309 /* Immediate instruction relocations. */
310 case R_AARCH64_LD_PREL_LO19
:
311 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 19,
312 AARCH64_INSN_IMM_19
);
314 case R_AARCH64_ADR_PREL_LO21
:
315 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 0, 21,
316 AARCH64_INSN_IMM_ADR
);
318 #ifndef CONFIG_ARM64_ERRATUM_843419
319 case R_AARCH64_ADR_PREL_PG_HI21_NC
:
320 overflow_check
= false;
321 case R_AARCH64_ADR_PREL_PG_HI21
:
322 ovf
= reloc_insn_imm(RELOC_OP_PAGE
, loc
, val
, 12, 21,
323 AARCH64_INSN_IMM_ADR
);
326 case R_AARCH64_ADD_ABS_LO12_NC
:
327 case R_AARCH64_LDST8_ABS_LO12_NC
:
328 overflow_check
= false;
329 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 0, 12,
330 AARCH64_INSN_IMM_12
);
332 case R_AARCH64_LDST16_ABS_LO12_NC
:
333 overflow_check
= false;
334 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 1, 11,
335 AARCH64_INSN_IMM_12
);
337 case R_AARCH64_LDST32_ABS_LO12_NC
:
338 overflow_check
= false;
339 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 2, 10,
340 AARCH64_INSN_IMM_12
);
342 case R_AARCH64_LDST64_ABS_LO12_NC
:
343 overflow_check
= false;
344 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 3, 9,
345 AARCH64_INSN_IMM_12
);
347 case R_AARCH64_LDST128_ABS_LO12_NC
:
348 overflow_check
= false;
349 ovf
= reloc_insn_imm(RELOC_OP_ABS
, loc
, val
, 4, 8,
350 AARCH64_INSN_IMM_12
);
352 case R_AARCH64_TSTBR14
:
353 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 14,
354 AARCH64_INSN_IMM_14
);
356 case R_AARCH64_CONDBR19
:
357 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 19,
358 AARCH64_INSN_IMM_19
);
360 case R_AARCH64_JUMP26
:
361 case R_AARCH64_CALL26
:
362 ovf
= reloc_insn_imm(RELOC_OP_PREL
, loc
, val
, 2, 26,
363 AARCH64_INSN_IMM_26
);
367 pr_err("module %s: unsupported RELA relocation: %llu\n",
368 me
->name
, ELF64_R_TYPE(rel
[i
].r_info
));
372 if (overflow_check
&& ovf
== -ERANGE
)
380 pr_err("module %s: overflow in relocation type %d val %Lx\n",
381 me
->name
, (int)ELF64_R_TYPE(rel
[i
].r_info
), val
);
385 int module_finalize(const Elf_Ehdr
*hdr
,
386 const Elf_Shdr
*sechdrs
,
389 const Elf_Shdr
*s
, *se
;
390 const char *secstrs
= (void *)hdr
+ sechdrs
[hdr
->e_shstrndx
].sh_offset
;
392 for (s
= sechdrs
, se
= sechdrs
+ hdr
->e_shnum
; s
< se
; s
++) {
393 if (strcmp(".altinstructions", secstrs
+ s
->sh_name
) == 0) {
394 apply_alternatives((void *)s
->sh_addr
, s
->sh_size
);