]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - tools/objtool/check.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
objtool: Introduce CFI hash
[mirror_ubuntu-jammy-kernel.git] / tools / objtool / check.c
CommitLineData
1ccea77e 1// SPDX-License-Identifier: GPL-2.0-or-later
dcc914f4
JP		 2/*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
dcc914f4
JP		 4 */
5
6#include <string.h>
7#include <stdlib.h>
26d618bb 8#include <inttypes.h>
e70e567e 9#include <sys/mman.h>
dcc914f4 10
7786032e
VG		 11#include <arch/elf.h>
12#include <objtool/builtin.h>
13#include <objtool/cfi.h>
14#include <objtool/arch.h>
15#include <objtool/check.h>
16#include <objtool/special.h>
17#include <objtool/warn.h>
18#include <objtool/endianness.h>
dcc914f4 19
ee819aed 20#include <linux/objtool.h>
dcc914f4
JP		 21#include <linux/hashtable.h>
22#include <linux/kernel.h>
1e7e4788 23#include <linux/static_call_types.h>
dcc914f4 24
dcc914f4
JP		 25struct alternative {
26 struct list_head list;
27 struct instruction *insn;
764eef4b 28 bool skip_orig;
dcc914f4
JP		 29};
30
e70e567e
PZ		 31static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33static struct cfi_init_state initial_func_cfi;
34static struct cfi_state init_cfi;
35static struct cfi_state func_cfi;
dcc914f4 36
627fce14
JP		 37struct instruction *find_insn(struct objtool_file *file,
38 struct section *sec, unsigned long offset)
dcc914f4
JP		 39{
40 struct instruction *insn;
41
87ecb582 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
dcc914f4
JP		 43 if (insn->sec == sec && insn->offset == offset)
44 return insn;
87ecb582 45 }
dcc914f4
JP		 46
47 return NULL;
48}
49
50static struct instruction *next_insn_same_sec(struct objtool_file *file,
51 struct instruction *insn)
52{
53 struct instruction *next = list_next_entry(insn, list);
54
baa41469 55 if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
dcc914f4
JP		 56 return NULL;
57
58 return next;
59}
60
13810435
JP		 61static struct instruction *next_insn_same_func(struct objtool_file *file,
62 struct instruction *insn)
63{
64 struct instruction *next = list_next_entry(insn, list);
65 struct symbol *func = insn->func;
66
67 if (!func)
68 return NULL;
69
70 if (&next->list != &file->insn_list && next->func == func)
71 return next;
72
73 /* Check if we're already in the subfunction: */
74 if (func == func->cfunc)
75 return NULL;
76
77 /* Move to the subfunction: */
78 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
79}
80
1119d265
JP		 81static struct instruction *prev_insn_same_sym(struct objtool_file *file,
82 struct instruction *insn)
83{
84 struct instruction *prev = list_prev_entry(insn, list);
85
86 if (&prev->list != &file->insn_list && prev->func == insn->func)
87 return prev;
88
89 return NULL;
90}
91
f0f70adb 92#define func_for_each_insn(file, func, insn) \
13810435
JP		 93 for (insn = find_insn(file, func->sec, func->offset); \
94 insn; \
95 insn = next_insn_same_func(file, insn))
96
dbf4aeb0
PZ		 97#define sym_for_each_insn(file, sym, insn) \
98 for (insn = find_insn(file, sym->sec, sym->offset); \
dcc914f4 99 insn && &insn->list != &file->insn_list && \
dbf4aeb0
PZ		 100 insn->sec == sym->sec && \
101 insn->offset < sym->offset + sym->len; \
dcc914f4
JP		 102 insn = list_next_entry(insn, list))
103
dbf4aeb0 104#define sym_for_each_insn_continue_reverse(file, sym, insn) \
dcc914f4
JP		 105 for (insn = list_prev_entry(insn, list); \
106 &insn->list != &file->insn_list && \
dbf4aeb0 107 insn->sec == sym->sec && insn->offset >= sym->offset; \
dcc914f4
JP		 108 insn = list_prev_entry(insn, list))
109
110#define sec_for_each_insn_from(file, insn) \
111 for (; insn; insn = next_insn_same_sec(file, insn))
112
baa41469
JP		 113#define sec_for_each_insn_continue(file, insn) \
114 for (insn = next_insn_same_sec(file, insn); insn; \
115 insn = next_insn_same_sec(file, insn))
dcc914f4 116
99033461
JP		 117static bool is_jump_table_jump(struct instruction *insn)
118{
119 struct alt_group *alt_group = insn->alt_group;
120
121 if (insn->jump_table)
122 return true;
123
124 /* Retpoline alternative for a jump table? */
125 return alt_group && alt_group->orig_group &&
126 alt_group->orig_group->first_insn->jump_table;
127}
128
0c1ddd33
JP		 129static bool is_sibling_call(struct instruction *insn)
130{
ecf11ba4
JP		 131 /*
132 * Assume only ELF functions can make sibling calls. This ensures
133 * sibling call detection consistency between vmlinux.o and individual
134 * objects.
135 */
136 if (!insn->func)
137 return false;
138
0c1ddd33
JP		 139 /* An indirect jump is either a sibling call or a jump to a table. */
140 if (insn->type == INSN_JUMP_DYNAMIC)
99033461 141 return !is_jump_table_jump(insn);
0c1ddd33 142
0c1ddd33 143 /* add_jump_destinations() sets insn->call_dest for sibling calls. */
ecf11ba4 144 return (is_static_jump(insn) && insn->call_dest);
0c1ddd33
JP		 145}
146
dcc914f4
JP		 147/*
148 * This checks to see if the given function is a "noreturn" function.
149 *
150 * For global functions which are outside the scope of this object file, we
151 * have to keep a manual list of them.
152 *
153 * For local functions, we have to detect them manually by simply looking for
154 * the lack of a return instruction.
dcc914f4 155 */
8e25c9f8
JP		 156static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
157 int recursion)
dcc914f4
JP		 158{
159 int i;
160 struct instruction *insn;
161 bool empty = true;
162
163 /*
164 * Unfortunately these have to be hard coded because the noreturn
165 * attribute isn't provided in ELF data.
166 */
167 static const char * const global_noreturns[] = {
168 "__stack_chk_fail",
169 "panic",
170 "do_exit",
171 "do_task_dead",
172 "__module_put_and_exit",
173 "complete_and_exit",
dcc914f4
JP		 174 "__reiserfs_panic",
175 "lbug_with_loc",
176 "fortify_panic",
b394d468 177 "usercopy_abort",
684fb246 178 "machine_real_restart",
4fa5ecda 179 "rewind_stack_do_exit",
33adf80f 180 "kunit_try_catch_throw",
c26acfbb 181 "xen_start_kernel",
f12babfb 182 "cpu_bringup_and_idle",
dcc914f4
JP		 183 };
184
c9bab22b
JP		 185 if (!func)
186 return false;
187
dcc914f4 188 if (func->bind == STB_WEAK)
8e25c9f8 189 return false;
dcc914f4
JP		 190
191 if (func->bind == STB_GLOBAL)
192 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
193 if (!strcmp(func->name, global_noreturns[i]))
8e25c9f8 194 return true;
dcc914f4 195
13810435 196 if (!func->len)
8e25c9f8 197 return false;
dcc914f4 198
13810435
JP		 199 insn = find_insn(file, func->sec, func->offset);
200 if (!insn->func)
8e25c9f8 201 return false;
13810435 202
f0f70adb 203 func_for_each_insn(file, func, insn) {
dcc914f4
JP		 204 empty = false;
205
206 if (insn->type == INSN_RETURN)
8e25c9f8 207 return false;
dcc914f4
JP		 208 }
209
210 if (empty)
8e25c9f8 211 return false;
dcc914f4
JP		 212
213 /*
214 * A function can have a sibling call instead of a return. In that
215 * case, the function's dead-end status depends on whether the target
216 * of the sibling call returns.
217 */
f0f70adb 218 func_for_each_insn(file, func, insn) {
0c1ddd33 219 if (is_sibling_call(insn)) {
dcc914f4 220 struct instruction *dest = insn->jump_dest;
dcc914f4
JP		 221
222 if (!dest)
223 /* sibling call to another file */
8e25c9f8 224 return false;
dcc914f4 225
0c1ddd33
JP		 226 /* local sibling call */
227 if (recursion == 5) {
228 /*
229 * Infinite recursion: two functions have
230 * sibling calls to each other. This is a very
231 * rare case. It means they aren't dead ends.
232 */
233 return false;
dcc914f4 234 }
dcc914f4 235
0c1ddd33
JP		 236 return __dead_end_function(file, dest->func, recursion+1);
237 }
dcc914f4
JP		 238 }
239
8e25c9f8 240 return true;
dcc914f4
JP		 241}
242
8e25c9f8 243static bool dead_end_function(struct objtool_file *file, struct symbol *func)
dcc914f4
JP		 244{
245 return __dead_end_function(file, func, 0);
246}
247
e7c0219b 248static void init_cfi_state(struct cfi_state *cfi)
baa41469
JP		 249{
250 int i;
251
dd88a0a0 252 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 253 cfi->regs[i].base = CFI_UNDEFINED;
254 cfi->vals[i].base = CFI_UNDEFINED;
dd88a0a0 255 }
e7c0219b
PZ		 256 cfi->cfa.base = CFI_UNDEFINED;
257 cfi->drap_reg = CFI_UNDEFINED;
258 cfi->drap_offset = -1;
259}
260
932f8e98 261static void init_insn_state(struct insn_state *state, struct section *sec)
e7c0219b
PZ		 262{
263 memset(state, 0, sizeof(*state));
264 init_cfi_state(&state->cfi);
932f8e98
PZ		 265
266 /*
267 * We need the full vmlinux for noinstr validation, otherwise we can
268 * not correctly determine insn->call_dest->sec (external symbols do
269 * not have a section).
270 */
41425ebe 271 if (vmlinux && noinstr && sec)
932f8e98 272 state->noinstr = sec->noinstr;
baa41469
JP		 273}
274
e70e567e
PZ		 275static struct cfi_state *cfi_alloc(void)
276{
277 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
278 if (!cfi) {
279 WARN("calloc failed");
280 exit(1);
281 }
282 nr_cfi++;
283 return cfi;
284}
285
286static int cfi_bits;
287static struct hlist_head *cfi_hash;
288
289static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
290{
291 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
292 (void *)cfi2 + sizeof(cfi2->hash),
293 sizeof(struct cfi_state) - sizeof(struct hlist_node));
294}
295
296static inline u32 cfi_key(struct cfi_state *cfi)
297{
298 return jhash((void *)cfi + sizeof(cfi->hash),
299 sizeof(*cfi) - sizeof(cfi->hash), 0);
300}
301
302static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
303{
304 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
305 struct cfi_state *obj;
306
307 hlist_for_each_entry(obj, head, hash) {
308 if (!cficmp(cfi, obj)) {
309 nr_cfi_cache++;
310 return obj;
311 }
312 }
313
314 obj = cfi_alloc();
315 *obj = *cfi;
316 hlist_add_head(&obj->hash, head);
317
318 return obj;
319}
320
321static void cfi_hash_add(struct cfi_state *cfi)
322{
323 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
324
325 hlist_add_head(&cfi->hash, head);
326}
327
328static void *cfi_hash_alloc(unsigned long size)
329{
330 cfi_bits = max(10, ilog2(size));
331 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
332 PROT_READ|PROT_WRITE,
333 MAP_PRIVATE|MAP_ANON, -1, 0);
334 if (cfi_hash == (void *)-1L) {
335 WARN("mmap fail cfi_hash");
336 cfi_hash = NULL;
337 } else if (stats) {
338 printf("cfi_bits: %d\n", cfi_bits);
339 }
340
341 return cfi_hash;
342}
343
344static unsigned long nr_insns;
345static unsigned long nr_insns_visited;
346
dcc914f4
JP		 347/*
348 * Call the arch-specific instruction decoder for all the instructions and add
349 * them to the global instruction list.
350 */
351static int decode_instructions(struct objtool_file *file)
352{
353 struct section *sec;
354 struct symbol *func;
355 unsigned long offset;
356 struct instruction *insn;
357 int ret;
358
baa41469 359 for_each_sec(file, sec) {
dcc914f4
JP		 360
361 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
362 continue;
363
627fce14
JP		 364 if (strcmp(sec->name, ".altinstr_replacement") &&
365 strcmp(sec->name, ".altinstr_aux") &&
366 strncmp(sec->name, ".discard.", 9))
367 sec->text = true;
368
0cc9ac8d
TG		 369 if (!strcmp(sec->name, ".noinstr.text") ||
370 !strcmp(sec->name, ".entry.text"))
c4a33939
PZ		 371 sec->noinstr = true;
372
fe255fe6 373 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
dcc914f4 374 insn = malloc(sizeof(*insn));
baa41469
JP		 375 if (!insn) {
376 WARN("malloc failed");
377 return -1;
378 }
dcc914f4 379 memset(insn, 0, sizeof(*insn));
dcc914f4 380 INIT_LIST_HEAD(&insn->alts);
65ea47dc 381 INIT_LIST_HEAD(&insn->stack_ops);
baa41469 382
dcc914f4
JP		 383 insn->sec = sec;
384 insn->offset = offset;
385
386 ret = arch_decode_instruction(file->elf, sec, offset,
fe255fe6 387 sec->sh.sh_size - offset,
dcc914f4 388 &insn->len, &insn->type,
baa41469 389 &insn->immediate,
65ea47dc 390 &insn->stack_ops);
dcc914f4 391 if (ret)
b7037983 392 goto err;
dcc914f4 393
87ecb582 394 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
dcc914f4 395 list_add_tail(&insn->list, &file->insn_list);
1e11f3fd 396 nr_insns++;
dcc914f4
JP		 397 }
398
399 list_for_each_entry(func, &sec->symbol_list, list) {
e10cd8fe 400 if (func->type != STT_FUNC || func->alias != func)
dcc914f4
JP		 401 continue;
402
403 if (!find_insn(file, sec, func->offset)) {
404 WARN("%s(): can't find starting instruction",
405 func->name);
406 return -1;
407 }
408
dbf4aeb0 409 sym_for_each_insn(file, func, insn)
e10cd8fe 410 insn->func = func;
dcc914f4
JP		 411 }
412 }
413
1e11f3fd
PZ		 414 if (stats)
415 printf("nr_insns: %lu\n", nr_insns);
416
dcc914f4 417 return 0;
b7037983
KB		 418
419err:
420 free(insn);
421 return ret;
dcc914f4
JP		 422}
423
6b5dd716
ST		 424static struct instruction *find_last_insn(struct objtool_file *file,
425 struct section *sec)
426{
427 struct instruction *insn = NULL;
428 unsigned int offset;
fe255fe6 429 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
6b5dd716 430
fe255fe6 431 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
6b5dd716
ST		 432 insn = find_insn(file, sec, offset);
433
434 return insn;
435}
436
dcc914f4 437/*
649ea4d5 438 * Mark "ud2" instructions and manually annotated dead ends.
dcc914f4
JP		 439 */
440static int add_dead_ends(struct objtool_file *file)
441{
442 struct section *sec;
f1974222 443 struct reloc *reloc;
dcc914f4 444 struct instruction *insn;
dcc914f4 445
649ea4d5
JP		 446 /*
447 * By default, "ud2" is a dead end unless otherwise annotated, because
448 * GCC 7 inserts it for certain divide-by-zero cases.
449 */
450 for_each_insn(file, insn)
451 if (insn->type == INSN_BUG)
452 insn->dead_end = true;
453
454 /*
455 * Check for manually annotated dead ends.
456 */
dcc914f4
JP		 457 sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
458 if (!sec)
649ea4d5 459 goto reachable;
dcc914f4 460
f1974222
MH		 461 list_for_each_entry(reloc, &sec->reloc_list, list) {
462 if (reloc->sym->type != STT_SECTION) {
dcc914f4
JP		 463 WARN("unexpected relocation symbol type in %s", sec->name);
464 return -1;
465 }
f1974222 466 insn = find_insn(file, reloc->sym->sec, reloc->addend);
dcc914f4
JP		 467 if (insn)
468 insn = list_prev_entry(insn, list);
fe255fe6 469 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 470 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 471 if (!insn) {
26d618bb 472 WARN("can't find unreachable insn at %s+0x%" PRIx64,
f1974222 473 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 474 return -1;
475 }
476 } else {
26d618bb 477 WARN("can't find unreachable insn at %s+0x%" PRIx64,
f1974222 478 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 479 return -1;
480 }
481
482 insn->dead_end = true;
483 }
484
649ea4d5
JP		 485reachable:
486 /*
487 * These manually annotated reachable checks are needed for GCC 4.4,
488 * where the Linux unreachable() macro isn't supported. In that case
489 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
490 * not a dead end.
491 */
492 sec = find_section_by_name(file->elf, ".rela.discard.reachable");
493 if (!sec)
494 return 0;
495
f1974222
MH		 496 list_for_each_entry(reloc, &sec->reloc_list, list) {
497 if (reloc->sym->type != STT_SECTION) {
649ea4d5
JP		 498 WARN("unexpected relocation symbol type in %s", sec->name);
499 return -1;
500 }
f1974222 501 insn = find_insn(file, reloc->sym->sec, reloc->addend);
649ea4d5
JP		 502 if (insn)
503 insn = list_prev_entry(insn, list);
fe255fe6 504 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 505 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 506 if (!insn) {
26d618bb 507 WARN("can't find reachable insn at %s+0x%" PRIx64,
f1974222 508 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 509 return -1;
510 }
511 } else {
26d618bb 512 WARN("can't find reachable insn at %s+0x%" PRIx64,
f1974222 513 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 514 return -1;
515 }
516
517 insn->dead_end = false;
518 }
519
dcc914f4
JP		 520 return 0;
521}
522
1e7e4788
JP		 523static int create_static_call_sections(struct objtool_file *file)
524{
ef47cc01 525 struct section *sec;
1e7e4788
JP		 526 struct static_call_site *site;
527 struct instruction *insn;
528 struct symbol *key_sym;
529 char *key_name, *tmp;
530 int idx;
531
532 sec = find_section_by_name(file->elf, ".static_call_sites");
533 if (sec) {
534 INIT_LIST_HEAD(&file->static_call_list);
535 WARN("file already has .static_call_sites section, skipping");
536 return 0;
537 }
538
539 if (list_empty(&file->static_call_list))
540 return 0;
541
542 idx = 0;
43d5430a 543 list_for_each_entry(insn, &file->static_call_list, call_node)
1e7e4788
JP		 544 idx++;
545
546 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
547 sizeof(struct static_call_site), idx);
548 if (!sec)
549 return -1;
550
1e7e4788 551 idx = 0;
43d5430a 552 list_for_each_entry(insn, &file->static_call_list, call_node) {
1e7e4788
JP		 553
554 site = (struct static_call_site *)sec->data->d_buf + idx;
555 memset(site, 0, sizeof(struct static_call_site));
556
557 /* populate reloc for 'addr' */
ef47cc01
PZ		 558 if (elf_add_reloc_to_insn(file->elf, sec,
559 idx * sizeof(struct static_call_site),
560 R_X86_64_PC32,
561 insn->sec, insn->offset))
44f6a7c0 562 return -1;
1e7e4788
JP		 563
564 /* find key symbol */
565 key_name = strdup(insn->call_dest->name);
566 if (!key_name) {
567 perror("strdup");
568 return -1;
569 }
570 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
571 STATIC_CALL_TRAMP_PREFIX_LEN)) {
572 WARN("static_call: trampoline name malformed: %s", key_name);
573 return -1;
574 }
575 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
576 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
577
578 key_sym = find_symbol_by_name(file->elf, tmp);
579 if (!key_sym) {
73f44fe1
JP		 580 if (!module) {
581 WARN("static_call: can't find static_call_key symbol: %s", tmp);
582 return -1;
583 }
584
585 /*
586 * For modules(), the key might not be exported, which
587 * means the module can make static calls but isn't
588 * allowed to change them.
589 *
590 * In that case we temporarily set the key to be the
591 * trampoline address. This is fixed up in
592 * static_call_add_module().
593 */
594 key_sym = insn->call_dest;
1e7e4788
JP		 595 }
596 free(key_name);
597
598 /* populate reloc for 'key' */
ef47cc01
PZ		 599 if (elf_add_reloc(file->elf, sec,
600 idx * sizeof(struct static_call_site) + 4,
601 R_X86_64_PC32, key_sym,
602 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
1e7e4788 603 return -1;
1e7e4788
JP		 604
605 idx++;
606 }
607
1e7e4788
JP		 608 return 0;
609}
610
a77a9f03
PZ		 611static int create_retpoline_sites_sections(struct objtool_file *file)
612{
613 struct instruction *insn;
614 struct section *sec;
615 int idx;
616
617 sec = find_section_by_name(file->elf, ".retpoline_sites");
618 if (sec) {
619 WARN("file already has .retpoline_sites, skipping");
620 return 0;
621 }
622
623 idx = 0;
624 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
625 idx++;
626
627 if (!idx)
628 return 0;
629
630 sec = elf_create_section(file->elf, ".retpoline_sites", 0,
631 sizeof(int), idx);
632 if (!sec) {
633 WARN("elf_create_section: .retpoline_sites");
634 return -1;
635 }
636
637 idx = 0;
638 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
639
640 int *site = (int *)sec->data->d_buf + idx;
641 *site = 0;
642
643 if (elf_add_reloc_to_insn(file->elf, sec,
644 idx * sizeof(int),
645 R_X86_64_PC32,
646 insn->sec, insn->offset)) {
647 WARN("elf_add_reloc_to_insn: .retpoline_sites");
648 return -1;
649 }
650
651 idx++;
652 }
653
654 return 0;
655}
656
99d00215
PZ		 657static int create_mcount_loc_sections(struct objtool_file *file)
658{
ef47cc01 659 struct section *sec;
99d00215
PZ		 660 unsigned long *loc;
661 struct instruction *insn;
662 int idx;
663
664 sec = find_section_by_name(file->elf, "__mcount_loc");
665 if (sec) {
666 INIT_LIST_HEAD(&file->mcount_loc_list);
667 WARN("file already has __mcount_loc section, skipping");
668 return 0;
669 }
670
671 if (list_empty(&file->mcount_loc_list))
672 return 0;
673
674 idx = 0;
80c90288 675 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
99d00215
PZ		 676 idx++;
677
678 sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
679 if (!sec)
680 return -1;
681
99d00215 682 idx = 0;
80c90288 683 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
99d00215
PZ		 684
685 loc = (unsigned long *)sec->data->d_buf + idx;
686 memset(loc, 0, sizeof(unsigned long));
687
ef47cc01
PZ		 688 if (elf_add_reloc_to_insn(file->elf, sec,
689 idx * sizeof(unsigned long),
690 R_X86_64_64,
691 insn->sec, insn->offset))
99d00215 692 return -1;
99d00215
PZ		 693
694 idx++;
695 }
696
99d00215
PZ		 697 return 0;
698}
699
dcc914f4
JP		 700/*
701 * Warnings shouldn't be reported for ignored functions.
702 */
703static void add_ignores(struct objtool_file *file)
704{
705 struct instruction *insn;
706 struct section *sec;
707 struct symbol *func;
f1974222 708 struct reloc *reloc;
dcc914f4 709
aaf5c623
PZ		 710 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
711 if (!sec)
712 return;
dcc914f4 713
f1974222
MH		 714 list_for_each_entry(reloc, &sec->reloc_list, list) {
715 switch (reloc->sym->type) {
aaf5c623 716 case STT_FUNC:
f1974222 717 func = reloc->sym;
aaf5c623
PZ		 718 break;
719
720 case STT_SECTION:
f1974222 721 func = find_func_by_offset(reloc->sym->sec, reloc->addend);
7acfe531 722 if (!func)
dcc914f4 723 continue;
aaf5c623 724 break;
dcc914f4 725
aaf5c623 726 default:
f1974222 727 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
aaf5c623 728 continue;
dcc914f4 729 }
aaf5c623 730
f0f70adb 731 func_for_each_insn(file, func, insn)
aaf5c623 732 insn->ignore = true;
dcc914f4
JP		 733 }
734}
735
ea24213d
PZ		 736/*
737 * This is a whitelist of functions that is allowed to be called with AC set.
738 * The list is meant to be minimal and only contains compiler instrumentation
739 * ABI and a few functions used to implement *_{to,from}_user() functions.
740 *
741 * These functions must not directly change AC, but may PUSHF/POPF.
742 */
743static const char *uaccess_safe_builtin[] = {
744 /* KASAN */
745 "kasan_report",
f00748bf 746 "kasan_check_range",
ea24213d
PZ		 747 /* KASAN out-of-line */
748 "__asan_loadN_noabort",
749 "__asan_load1_noabort",
750 "__asan_load2_noabort",
751 "__asan_load4_noabort",
752 "__asan_load8_noabort",
753 "__asan_load16_noabort",
754 "__asan_storeN_noabort",
755 "__asan_store1_noabort",
756 "__asan_store2_noabort",
757 "__asan_store4_noabort",
758 "__asan_store8_noabort",
759 "__asan_store16_noabort",
b0b8e56b
JH		 760 "__kasan_check_read",
761 "__kasan_check_write",
ea24213d
PZ		 762 /* KASAN in-line */
763 "__asan_report_load_n_noabort",
764 "__asan_report_load1_noabort",
765 "__asan_report_load2_noabort",
766 "__asan_report_load4_noabort",
767 "__asan_report_load8_noabort",
768 "__asan_report_load16_noabort",
769 "__asan_report_store_n_noabort",
770 "__asan_report_store1_noabort",
771 "__asan_report_store2_noabort",
772 "__asan_report_store4_noabort",
773 "__asan_report_store8_noabort",
774 "__asan_report_store16_noabort",
5f5c9712 775 /* KCSAN */
9967683c 776 "__kcsan_check_access",
5f5c9712
ME		 777 "kcsan_found_watchpoint",
778 "kcsan_setup_watchpoint",
9967683c 779 "kcsan_check_scoped_accesses",
50a19ad4
ME		 780 "kcsan_disable_current",
781 "kcsan_enable_current_nowarn",
5f5c9712
ME		 782 /* KCSAN/TSAN */
783 "__tsan_func_entry",
784 "__tsan_func_exit",
785 "__tsan_read_range",
786 "__tsan_write_range",
787 "__tsan_read1",
788 "__tsan_read2",
789 "__tsan_read4",
790 "__tsan_read8",
791 "__tsan_read16",
792 "__tsan_write1",
793 "__tsan_write2",
794 "__tsan_write4",
795 "__tsan_write8",
796 "__tsan_write16",
a81b3759
ME		 797 "__tsan_read_write1",
798 "__tsan_read_write2",
799 "__tsan_read_write4",
800 "__tsan_read_write8",
801 "__tsan_read_write16",
883957b1
ME		 802 "__tsan_atomic8_load",
803 "__tsan_atomic16_load",
804 "__tsan_atomic32_load",
805 "__tsan_atomic64_load",
806 "__tsan_atomic8_store",
807 "__tsan_atomic16_store",
808 "__tsan_atomic32_store",
809 "__tsan_atomic64_store",
810 "__tsan_atomic8_exchange",
811 "__tsan_atomic16_exchange",
812 "__tsan_atomic32_exchange",
813 "__tsan_atomic64_exchange",
814 "__tsan_atomic8_fetch_add",
815 "__tsan_atomic16_fetch_add",
816 "__tsan_atomic32_fetch_add",
817 "__tsan_atomic64_fetch_add",
818 "__tsan_atomic8_fetch_sub",
819 "__tsan_atomic16_fetch_sub",
820 "__tsan_atomic32_fetch_sub",
821 "__tsan_atomic64_fetch_sub",
822 "__tsan_atomic8_fetch_and",
823 "__tsan_atomic16_fetch_and",
824 "__tsan_atomic32_fetch_and",
825 "__tsan_atomic64_fetch_and",
826 "__tsan_atomic8_fetch_or",
827 "__tsan_atomic16_fetch_or",
828 "__tsan_atomic32_fetch_or",
829 "__tsan_atomic64_fetch_or",
830 "__tsan_atomic8_fetch_xor",
831 "__tsan_atomic16_fetch_xor",
832 "__tsan_atomic32_fetch_xor",
833 "__tsan_atomic64_fetch_xor",
834 "__tsan_atomic8_fetch_nand",
835 "__tsan_atomic16_fetch_nand",
836 "__tsan_atomic32_fetch_nand",
837 "__tsan_atomic64_fetch_nand",
838 "__tsan_atomic8_compare_exchange_strong",
839 "__tsan_atomic16_compare_exchange_strong",
840 "__tsan_atomic32_compare_exchange_strong",
841 "__tsan_atomic64_compare_exchange_strong",
842 "__tsan_atomic8_compare_exchange_weak",
843 "__tsan_atomic16_compare_exchange_weak",
844 "__tsan_atomic32_compare_exchange_weak",
845 "__tsan_atomic64_compare_exchange_weak",
846 "__tsan_atomic8_compare_exchange_val",
847 "__tsan_atomic16_compare_exchange_val",
848 "__tsan_atomic32_compare_exchange_val",
849 "__tsan_atomic64_compare_exchange_val",
850 "__tsan_atomic_thread_fence",
851 "__tsan_atomic_signal_fence",
ea24213d
PZ		 852 /* KCOV */
853 "write_comp_data",
ae033f08 854 "check_kcov_mode",
ea24213d
PZ		 855 "__sanitizer_cov_trace_pc",
856 "__sanitizer_cov_trace_const_cmp1",
857 "__sanitizer_cov_trace_const_cmp2",
858 "__sanitizer_cov_trace_const_cmp4",
859 "__sanitizer_cov_trace_const_cmp8",
860 "__sanitizer_cov_trace_cmp1",
861 "__sanitizer_cov_trace_cmp2",
862 "__sanitizer_cov_trace_cmp4",
863 "__sanitizer_cov_trace_cmp8",
36b1c700 864 "__sanitizer_cov_trace_switch",
ea24213d
PZ		 865 /* UBSAN */
866 "ubsan_type_mismatch_common",
867 "__ubsan_handle_type_mismatch",
868 "__ubsan_handle_type_mismatch_v1",
9a50dcaf 869 "__ubsan_handle_shift_out_of_bounds",
ea24213d
PZ		 870 /* misc */
871 "csum_partial_copy_generic",
ec6347bb
DW		 872 "copy_mc_fragile",
873 "copy_mc_fragile_handle_tail",
5da8e4a6 874 "copy_mc_enhanced_fast_string",
ea24213d
PZ		 875 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
876 NULL
877};
878
879static void add_uaccess_safe(struct objtool_file *file)
880{
881 struct symbol *func;
882 const char **name;
883
884 if (!uaccess)
885 return;
886
887 for (name = uaccess_safe_builtin; *name; name++) {
888 func = find_symbol_by_name(file->elf, *name);
889 if (!func)
890 continue;
891
e10cd8fe 892 func->uaccess_safe = true;
dcc914f4
JP		 893 }
894}
895
258c7605
JP		 896/*
897 * FIXME: For now, just ignore any alternatives which add retpolines. This is
898 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
899 * But it at least allows objtool to understand the control flow *around* the
900 * retpoline.
901 */
ff05ab23 902static int add_ignore_alternatives(struct objtool_file *file)
258c7605
JP		 903{
904 struct section *sec;
f1974222 905 struct reloc *reloc;
258c7605
JP		 906 struct instruction *insn;
907
ff05ab23 908 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
258c7605
JP		 909 if (!sec)
910 return 0;
911
f1974222
MH		 912 list_for_each_entry(reloc, &sec->reloc_list, list) {
913 if (reloc->sym->type != STT_SECTION) {
258c7605
JP		 914 WARN("unexpected relocation symbol type in %s", sec->name);
915 return -1;
916 }
917
f1974222 918 insn = find_insn(file, reloc->sym->sec, reloc->addend);
258c7605 919 if (!insn) {
ff05ab23 920 WARN("bad .discard.ignore_alts entry");
258c7605
JP		 921 return -1;
922 }
923
924 insn->ignore_alts = true;
925 }
926
927 return 0;
928}
929
530b4ddd
PZ		 930__weak bool arch_is_retpoline(struct symbol *sym)
931{
932 return false;
933}
934
7bd2a600
PZ		 935#define NEGATIVE_RELOC ((void *)-1L)
936
937static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
938{
939 if (insn->reloc == NEGATIVE_RELOC)
940 return NULL;
941
942 if (!insn->reloc) {
943 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
944 insn->offset, insn->len);
945 if (!insn->reloc) {
946 insn->reloc = NEGATIVE_RELOC;
947 return NULL;
948 }
949 }
950
951 return insn->reloc;
952}
953
3f86188b
PZ		 954static void remove_insn_ops(struct instruction *insn)
955{
956 struct stack_op *op, *tmp;
957
958 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
959 list_del(&op->list);
960 free(op);
961 }
962}
963
8e102fbe
PZ		 964static void annotate_call_site(struct objtool_file *file,
965 struct instruction *insn, bool sibling)
3f86188b
PZ		 966{
967 struct reloc *reloc = insn_reloc(file, insn);
8e102fbe 968 struct symbol *sym = insn->call_dest;
3f86188b 969
8e102fbe
PZ		 970 if (!sym)
971 sym = reloc->sym;
972
973 /*
974 * Alternative replacement code is just template code which is
975 * sometimes copied to the original instruction. For now, don't
976 * annotate it. (In the future we might consider annotating the
977 * original instruction if/when it ever makes sense to do so.)
978 */
979 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
3f86188b
PZ		 980 return;
981
8e102fbe
PZ		 982 if (sym->static_call_tramp) {
983 list_add_tail(&insn->call_node, &file->static_call_list);
984 return;
3f86188b
PZ		 985 }
986
a77a9f03
PZ		 987 if (sym->retpoline_thunk) {
988 list_add_tail(&insn->call_node, &file->retpoline_call_list);
989 return;
990 }
991
3f86188b
PZ		 992 /*
993 * Many compilers cannot disable KCOV with a function attribute
994 * so they need a little help, NOP out any KCOV calls from noinstr
995 * text.
996 */
8e102fbe 997 if (insn->sec->noinstr && sym->kcov) {
3f86188b
PZ		 998 if (reloc) {
999 reloc->type = R_NONE;
1000 elf_write_reloc(file->elf, reloc);
1001 }
1002
1003 elf_write_insn(file->elf, insn->sec,
1004 insn->offset, insn->len,
1005 sibling ? arch_ret_insn(insn->len)
1006 : arch_nop_insn(insn->len));
1007
1008 insn->type = sibling ? INSN_RETURN : INSN_NOP;
9509c1a9
PZ		 1009
1010 if (sibling) {
1011 /*
1012 * We've replaced the tail-call JMP insn by two new
1013 * insn: RET; INT3, except we only have a single struct
1014 * insn here. Mark it retpoline_safe to avoid the SLS
1015 * warning, instead of adding another insn.
1016 */
1017 insn->retpoline_safe = true;
1018 }
1019
8e102fbe 1020 return;
3f86188b
PZ		 1021 }
1022
8e102fbe 1023 if (mcount && sym->fentry) {
3f86188b
PZ		 1024 if (sibling)
1025 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1026
1027 if (reloc) {
1028 reloc->type = R_NONE;
1029 elf_write_reloc(file->elf, reloc);
1030 }
1031
1032 elf_write_insn(file->elf, insn->sec,
1033 insn->offset, insn->len,
1034 arch_nop_insn(insn->len));
1035
1036 insn->type = INSN_NOP;
1037
80c90288 1038 list_add_tail(&insn->call_node, &file->mcount_loc_list);
8e102fbe 1039 return;
3f86188b 1040 }
8e102fbe
PZ		 1041}
1042
1043static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1044 struct symbol *dest, bool sibling)
1045{
1046 insn->call_dest = dest;
1047 if (!dest)
1048 return;
3f86188b
PZ		 1049
1050 /*
1051 * Whatever stack impact regular CALLs have, should be undone
1052 * by the RETURN of the called function.
1053 *
1054 * Annotated intra-function calls retain the stack_ops but
1055 * are converted to JUMP, see read_intra_function_calls().
1056 */
1057 remove_insn_ops(insn);
8e102fbe
PZ		 1058
1059 annotate_call_site(file, insn, sibling);
3f86188b
PZ		 1060}
1061
a77a9f03
PZ		 1062static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1063{
1064 /*
1065 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1066 * so convert them accordingly.
1067 */
1068 switch (insn->type) {
1069 case INSN_CALL:
1070 insn->type = INSN_CALL_DYNAMIC;
1071 break;
1072 case INSN_JUMP_UNCONDITIONAL:
1073 insn->type = INSN_JUMP_DYNAMIC;
1074 break;
1075 case INSN_JUMP_CONDITIONAL:
1076 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1077 break;
1078 default:
1079 return;
1080 }
1081
1082 insn->retpoline_safe = true;
1083
1084 /*
1085 * Whatever stack impact regular CALLs have, should be undone
1086 * by the RETURN of the called function.
1087 *
1088 * Annotated intra-function calls retain the stack_ops but
1089 * are converted to JUMP, see read_intra_function_calls().
1090 */
1091 remove_insn_ops(insn);
1092
1093 annotate_call_site(file, insn, false);
1094}
dcc914f4
JP		 1095/*
1096 * Find the destination instructions for all jumps.
1097 */
1098static int add_jump_destinations(struct objtool_file *file)
1099{
1100 struct instruction *insn;
f1974222 1101 struct reloc *reloc;
dcc914f4
JP		 1102 struct section *dest_sec;
1103 unsigned long dest_off;
1104
1105 for_each_insn(file, insn) {
a2296140 1106 if (!is_static_jump(insn))
dcc914f4
JP		 1107 continue;
1108
7bd2a600 1109 reloc = insn_reloc(file, insn);
f1974222 1110 if (!reloc) {
dcc914f4 1111 dest_sec = insn->sec;
bfb08f22 1112 dest_off = arch_jump_destination(insn);
f1974222
MH		 1113 } else if (reloc->sym->type == STT_SECTION) {
1114 dest_sec = reloc->sym->sec;
1115 dest_off = arch_dest_reloc_offset(reloc->addend);
2e532bd1 1116 } else if (reloc->sym->retpoline_thunk) {
a77a9f03 1117 add_retpoline_call(file, insn);
39b73533 1118 continue;
ecf11ba4
JP		 1119 } else if (insn->func) {
1120 /* internal or external sibling call (with reloc) */
3f86188b 1121 add_call_dest(file, insn, reloc->sym, true);
dcc914f4 1122 continue;
ecf11ba4
JP		 1123 } else if (reloc->sym->sec->idx) {
1124 dest_sec = reloc->sym->sec;
1125 dest_off = reloc->sym->sym.st_value +
1126 arch_dest_reloc_offset(reloc->addend);
1127 } else {
1128 /* non-func asm code jumping to another file */
1129 continue;
dcc914f4
JP		 1130 }
1131
1132 insn->jump_dest = find_insn(file, dest_sec, dest_off);
1133 if (!insn->jump_dest) {
1134
1135 /*
1136 * This is a special case where an alt instruction
1137 * jumps past the end of the section. These are
1138 * handled later in handle_group_alt().
1139 */
1140 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1141 continue;
1142
1143 WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1144 insn->sec, insn->offset, dest_sec->name,
1145 dest_off);
1146 return -1;
1147 }
cd77849a
JP		 1148
1149 /*
54262aa2 1150 * Cross-function jump.
cd77849a
JP		 1151 */
1152 if (insn->func && insn->jump_dest->func &&
54262aa2
PZ		 1153 insn->func != insn->jump_dest->func) {
1154
1155 /*
1156 * For GCC 8+, create parent/child links for any cold
1157 * subfunctions. This is _mostly_ redundant with a
1158 * similar initialization in read_symbols().
1159 *
1160 * If a function has aliases, we want the *first* such
1161 * function in the symbol table to be the subfunction's
1162 * parent. In that case we overwrite the
1163 * initialization done in read_symbols().
1164 *
1165 * However this code can't completely replace the
1166 * read_symbols() code because this doesn't detect the
1167 * case where the parent function's only reference to a
e7c2bc37 1168 * subfunction is through a jump table.
54262aa2 1169 */
34ca59e1
JP		 1170 if (!strstr(insn->func->name, ".cold") &&
1171 strstr(insn->jump_dest->func->name, ".cold")) {
54262aa2
PZ		 1172 insn->func->cfunc = insn->jump_dest->func;
1173 insn->jump_dest->func->pfunc = insn->func;
1174
1175 } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
1176 insn->jump_dest->offset == insn->jump_dest->func->offset) {
ecf11ba4 1177 /* internal sibling call (without reloc) */
3f86188b 1178 add_call_dest(file, insn, insn->jump_dest->func, true);
54262aa2 1179 }
cd77849a 1180 }
dcc914f4
JP		 1181 }
1182
1183 return 0;
1184}
1185
2b232a22
JT		 1186static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1187{
1188 struct symbol *call_dest;
1189
1190 call_dest = find_func_by_offset(sec, offset);
1191 if (!call_dest)
1192 call_dest = find_symbol_by_offset(sec, offset);
1193
1194 return call_dest;
1195}
1196
dcc914f4
JP		 1197/*
1198 * Find the destination instructions for all calls.
1199 */
1200static int add_call_destinations(struct objtool_file *file)
1201{
1202 struct instruction *insn;
1203 unsigned long dest_off;
3f86188b 1204 struct symbol *dest;
f1974222 1205 struct reloc *reloc;
dcc914f4
JP		 1206
1207 for_each_insn(file, insn) {
1208 if (insn->type != INSN_CALL)
1209 continue;
1210
7bd2a600 1211 reloc = insn_reloc(file, insn);
f1974222 1212 if (!reloc) {
bfb08f22 1213 dest_off = arch_jump_destination(insn);
3f86188b
PZ		 1214 dest = find_call_destination(insn->sec, dest_off);
1215
1216 add_call_dest(file, insn, dest, false);
a845c7cf 1217
7acfe531
JP		 1218 if (insn->ignore)
1219 continue;
1220
1221 if (!insn->call_dest) {
8aa8eb2a 1222 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
dcc914f4
JP		 1223 return -1;
1224 }
a845c7cf 1225
7acfe531
JP		 1226 if (insn->func && insn->call_dest->type != STT_FUNC) {
1227 WARN_FUNC("unsupported call to non-function",
1228 insn->sec, insn->offset);
1229 return -1;
1230 }
1231
f1974222
MH		 1232 } else if (reloc->sym->type == STT_SECTION) {
1233 dest_off = arch_dest_reloc_offset(reloc->addend);
3f86188b
PZ		 1234 dest = find_call_destination(reloc->sym->sec, dest_off);
1235 if (!dest) {
bfb08f22 1236 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
dcc914f4 1237 insn->sec, insn->offset,
f1974222 1238 reloc->sym->sec->name,
bfb08f22 1239 dest_off);
dcc914f4
JP		 1240 return -1;
1241 }
bcb1b6ff 1242
3f86188b
PZ		 1243 add_call_dest(file, insn, dest, false);
1244
2e532bd1 1245 } else if (reloc->sym->retpoline_thunk) {
a77a9f03 1246 add_retpoline_call(file, insn);
bcb1b6ff 1247
dcc914f4 1248 } else
3f86188b 1249 add_call_dest(file, insn, reloc->sym, false);
dcc914f4
JP		 1250 }
1251
1252 return 0;
1253}
1254
1255/*
c9c324dc
JP		 1256 * The .alternatives section requires some extra special care over and above
1257 * other special sections because alternatives are patched in place.
dcc914f4
JP		 1258 */
1259static int handle_group_alt(struct objtool_file *file,
1260 struct special_alt *special_alt,
1261 struct instruction *orig_insn,
1262 struct instruction **new_insn)
1263{
c9c324dc 1264 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
b23cc71c 1265 struct alt_group *orig_alt_group, *new_alt_group;
dcc914f4
JP		 1266 unsigned long dest_off;
1267
b23cc71c
JP		 1268
1269 orig_alt_group = malloc(sizeof(*orig_alt_group));
1270 if (!orig_alt_group) {
1271 WARN("malloc failed");
1272 return -1;
1273 }
c9c324dc
JP		 1274 orig_alt_group->cfi = calloc(special_alt->orig_len,
1275 sizeof(struct cfi_state *));
1276 if (!orig_alt_group->cfi) {
1277 WARN("calloc failed");
1278 return -1;
1279 }
1280
dcc914f4
JP		 1281 last_orig_insn = NULL;
1282 insn = orig_insn;
1283 sec_for_each_insn_from(file, insn) {
1284 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1285 break;
1286
b23cc71c 1287 insn->alt_group = orig_alt_group;
dcc914f4
JP		 1288 last_orig_insn = insn;
1289 }
b23cc71c
JP		 1290 orig_alt_group->orig_group = NULL;
1291 orig_alt_group->first_insn = orig_insn;
1292 orig_alt_group->last_insn = last_orig_insn;
dcc914f4 1293
17bc3391 1294
c9c324dc
JP		 1295 new_alt_group = malloc(sizeof(*new_alt_group));
1296 if (!new_alt_group) {
1297 WARN("malloc failed");
1298 return -1;
dcc914f4 1299 }
dcc914f4 1300
c9c324dc
JP		 1301 if (special_alt->new_len < special_alt->orig_len) {
1302 /*
1303 * Insert a fake nop at the end to make the replacement
1304 * alt_group the same size as the original. This is needed to
1305 * allow propagate_alt_cfi() to do its magic. When the last
1306 * instruction affects the stack, the instruction after it (the
1307 * nop) will propagate the new state to the shared CFI array.
1308 */
1309 nop = malloc(sizeof(*nop));
1310 if (!nop) {
1311 WARN("malloc failed");
17bc3391
JP		 1312 return -1;
1313 }
c9c324dc
JP		 1314 memset(nop, 0, sizeof(*nop));
1315 INIT_LIST_HEAD(&nop->alts);
1316 INIT_LIST_HEAD(&nop->stack_ops);
c9c324dc
JP		 1317
1318 nop->sec = special_alt->new_sec;
1319 nop->offset = special_alt->new_off + special_alt->new_len;
1320 nop->len = special_alt->orig_len - special_alt->new_len;
1321 nop->type = INSN_NOP;
1322 nop->func = orig_insn->func;
1323 nop->alt_group = new_alt_group;
1324 nop->ignore = orig_insn->ignore_alts;
dcc914f4 1325 }
17bc3391 1326
c9c324dc
JP		 1327 if (!special_alt->new_len) {
1328 *new_insn = nop;
1329 goto end;
dcc914f4
JP		 1330 }
1331
dcc914f4
JP		 1332 insn = *new_insn;
1333 sec_for_each_insn_from(file, insn) {
45245f51
JT		 1334 struct reloc *alt_reloc;
1335
dcc914f4
JP		 1336 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1337 break;
1338
1339 last_new_insn = insn;
1340
a845c7cf 1341 insn->ignore = orig_insn->ignore_alts;
a4d09dde 1342 insn->func = orig_insn->func;
b23cc71c 1343 insn->alt_group = new_alt_group;
a845c7cf 1344
dc419723
JP		 1345 /*
1346 * Since alternative replacement code is copy/pasted by the
1347 * kernel after applying relocations, generally such code can't
1348 * have relative-address relocation references to outside the
1349 * .altinstr_replacement section, unless the arch's
1350 * alternatives code can adjust the relative offsets
1351 * accordingly.
dc419723 1352 */
7bd2a600 1353 alt_reloc = insn_reloc(file, insn);
45245f51
JT		 1354 if (alt_reloc &&
1355 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
dc419723
JP		 1356
1357 WARN_FUNC("unsupported relocation in alternatives section",
1358 insn->sec, insn->offset);
1359 return -1;
1360 }
1361
a2296140 1362 if (!is_static_jump(insn))
dcc914f4
JP		 1363 continue;
1364
1365 if (!insn->immediate)
1366 continue;
1367
bfb08f22 1368 dest_off = arch_jump_destination(insn);
c9c324dc
JP		 1369 if (dest_off == special_alt->new_off + special_alt->new_len)
1370 insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
dcc914f4
JP		 1371
1372 if (!insn->jump_dest) {
1373 WARN_FUNC("can't find alternative jump destination",
1374 insn->sec, insn->offset);
1375 return -1;
1376 }
1377 }
1378
1379 if (!last_new_insn) {
1380 WARN_FUNC("can't find last new alternative instruction",
1381 special_alt->new_sec, special_alt->new_off);
1382 return -1;
1383 }
1384
c9c324dc
JP		 1385 if (nop)
1386 list_add(&nop->list, &last_new_insn->list);
1387end:
b23cc71c
JP		 1388 new_alt_group->orig_group = orig_alt_group;
1389 new_alt_group->first_insn = *new_insn;
c9c324dc
JP		 1390 new_alt_group->last_insn = nop ? : last_new_insn;
1391 new_alt_group->cfi = orig_alt_group->cfi;
dcc914f4
JP		 1392 return 0;
1393}
1394
1395/*
1396 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1397 * If the original instruction is a jump, make the alt entry an effective nop
1398 * by just skipping the original instruction.
1399 */
1400static int handle_jump_alt(struct objtool_file *file,
1401 struct special_alt *special_alt,
1402 struct instruction *orig_insn,
1403 struct instruction **new_insn)
1404{
48001d26
PZ		 1405 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1406 orig_insn->type != INSN_NOP) {
e2d9494b 1407
dcc914f4
JP		 1408 WARN_FUNC("unsupported instruction at jump label",
1409 orig_insn->sec, orig_insn->offset);
1410 return -1;
1411 }
1412
6d37b83c
PZ		 1413 if (special_alt->key_addend & 2) {
1414 struct reloc *reloc = insn_reloc(file, orig_insn);
1415
1416 if (reloc) {
1417 reloc->type = R_NONE;
1418 elf_write_reloc(file->elf, reloc);
1419 }
1420 elf_write_insn(file->elf, orig_insn->sec,
1421 orig_insn->offset, orig_insn->len,
1422 arch_nop_insn(orig_insn->len));
1423 orig_insn->type = INSN_NOP;
48001d26
PZ		 1424 }
1425
1426 if (orig_insn->type == INSN_NOP) {
1427 if (orig_insn->len == 2)
1428 file->jl_nop_short++;
1429 else
1430 file->jl_nop_long++;
1431
1432 return 0;
6d37b83c
PZ		 1433 }
1434
e2d9494b
PZ		 1435 if (orig_insn->len == 2)
1436 file->jl_short++;
1437 else
1438 file->jl_long++;
1439
dcc914f4
JP		 1440 *new_insn = list_next_entry(orig_insn, list);
1441 return 0;
1442}
1443
1444/*
1445 * Read all the special sections which have alternate instructions which can be
1446 * patched in or redirected to at runtime. Each instruction having alternate
1447 * instruction(s) has them added to its insn->alts list, which will be
1448 * traversed in validate_branch().
1449 */
1450static int add_special_section_alts(struct objtool_file *file)
1451{
1452 struct list_head special_alts;
1453 struct instruction *orig_insn, *new_insn;
1454 struct special_alt *special_alt, *tmp;
1455 struct alternative *alt;
1456 int ret;
1457
1458 ret = special_get_alts(file->elf, &special_alts);
1459 if (ret)
1460 return ret;
1461
1462 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
dcc914f4
JP		 1463
1464 orig_insn = find_insn(file, special_alt->orig_sec,
1465 special_alt->orig_off);
1466 if (!orig_insn) {
1467 WARN_FUNC("special: can't find orig instruction",
1468 special_alt->orig_sec, special_alt->orig_off);
1469 ret = -1;
1470 goto out;
1471 }
1472
1473 new_insn = NULL;
1474 if (!special_alt->group || special_alt->new_len) {
1475 new_insn = find_insn(file, special_alt->new_sec,
1476 special_alt->new_off);
1477 if (!new_insn) {
1478 WARN_FUNC("special: can't find new instruction",
1479 special_alt->new_sec,
1480 special_alt->new_off);
1481 ret = -1;
1482 goto out;
1483 }
1484 }
1485
1486 if (special_alt->group) {
7170cf47
JT		 1487 if (!special_alt->orig_len) {
1488 WARN_FUNC("empty alternative entry",
1489 orig_insn->sec, orig_insn->offset);
1490 continue;
1491 }
1492
dcc914f4
JP		 1493 ret = handle_group_alt(file, special_alt, orig_insn,
1494 &new_insn);
1495 if (ret)
1496 goto out;
1497 } else if (special_alt->jump_or_nop) {
1498 ret = handle_jump_alt(file, special_alt, orig_insn,
1499 &new_insn);
1500 if (ret)
1501 goto out;
1502 }
1503
258c7605
JP		 1504 alt = malloc(sizeof(*alt));
1505 if (!alt) {
1506 WARN("malloc failed");
1507 ret = -1;
1508 goto out;
1509 }
1510
dcc914f4 1511 alt->insn = new_insn;
764eef4b 1512 alt->skip_orig = special_alt->skip_orig;
ea24213d 1513 orig_insn->ignore_alts |= special_alt->skip_alt;
dcc914f4
JP		 1514 list_add_tail(&alt->list, &orig_insn->alts);
1515
1516 list_del(&special_alt->list);
1517 free(special_alt);
1518 }
1519
e2d9494b
PZ		 1520 if (stats) {
1521 printf("jl\\\tNOP\tJMP\n");
1522 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1523 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1524 }
1525
dcc914f4
JP		 1526out:
1527 return ret;
1528}
1529
e7c2bc37 1530static int add_jump_table(struct objtool_file *file, struct instruction *insn,
f1974222 1531 struct reloc *table)
dcc914f4 1532{
f1974222 1533 struct reloc *reloc = table;
e7c2bc37 1534 struct instruction *dest_insn;
dcc914f4 1535 struct alternative *alt;
fd35c88b
JP		 1536 struct symbol *pfunc = insn->func->pfunc;
1537 unsigned int prev_offset = 0;
dcc914f4 1538
e7c2bc37 1539 /*
f1974222 1540 * Each @reloc is a switch table relocation which points to the target
e7c2bc37
JP		 1541 * instruction.
1542 */
f1974222 1543 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
bd98c813
JH		 1544
1545 /* Check for the end of the table: */
f1974222 1546 if (reloc != table && reloc->jump_table_start)
dcc914f4
JP		 1547 break;
1548
e7c2bc37 1549 /* Make sure the table entries are consecutive: */
f1974222 1550 if (prev_offset && reloc->offset != prev_offset + 8)
fd35c88b
JP		 1551 break;
1552
1553 /* Detect function pointers from contiguous objects: */
f1974222
MH		 1554 if (reloc->sym->sec == pfunc->sec &&
1555 reloc->addend == pfunc->offset)
fd35c88b
JP		 1556 break;
1557
f1974222 1558 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
e7c2bc37 1559 if (!dest_insn)
dcc914f4
JP		 1560 break;
1561
e7c2bc37 1562 /* Make sure the destination is in the same function: */
e65050b9 1563 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
13810435 1564 break;
dcc914f4
JP		 1565
1566 alt = malloc(sizeof(*alt));
1567 if (!alt) {
1568 WARN("malloc failed");
1569 return -1;
1570 }
1571
e7c2bc37 1572 alt->insn = dest_insn;
dcc914f4 1573 list_add_tail(&alt->list, &insn->alts);
f1974222 1574 prev_offset = reloc->offset;
fd35c88b
JP		 1575 }
1576
1577 if (!prev_offset) {
1578 WARN_FUNC("can't find switch jump table",
1579 insn->sec, insn->offset);
1580 return -1;
dcc914f4
JP		 1581 }
1582
1583 return 0;
1584}
1585
1586/*
d871f7b5
RG		 1587 * find_jump_table() - Given a dynamic jump, find the switch jump table
1588 * associated with it.
dcc914f4 1589 */
f1974222 1590static struct reloc *find_jump_table(struct objtool_file *file,
dcc914f4
JP		 1591 struct symbol *func,
1592 struct instruction *insn)
1593{
d871f7b5 1594 struct reloc *table_reloc;
113d4bc9 1595 struct instruction *dest_insn, *orig_insn = insn;
dcc914f4 1596
99ce7962
PZ		 1597 /*
1598 * Backward search using the @first_jump_src links, these help avoid
1599 * much of the 'in between' code. Which avoids us getting confused by
1600 * it.
1601 */
7dec80cc 1602 for (;
1119d265
JP		 1603 insn && insn->func && insn->func->pfunc == func;
1604 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
99ce7962 1605
7dec80cc 1606 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
dcc914f4
JP		 1607 break;
1608
1609 /* allow small jumps within the range */
1610 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1611 insn->jump_dest &&
1612 (insn->jump_dest->offset <= insn->offset ||
1613 insn->jump_dest->offset > orig_insn->offset))
1614 break;
1615
d871f7b5 1616 table_reloc = arch_find_switch_table(file, insn);
f1974222 1617 if (!table_reloc)
e7c2bc37 1618 continue;
f1974222 1619 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
113d4bc9
JP		 1620 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1621 continue;
7dec80cc 1622
f1974222 1623 return table_reloc;
dcc914f4
JP		 1624 }
1625
1626 return NULL;
1627}
1628
bd98c813
JH		 1629/*
1630 * First pass: Mark the head of each jump table so that in the next pass,
1631 * we know when a given jump table ends and the next one starts.
1632 */
1633static void mark_func_jump_tables(struct objtool_file *file,
1634 struct symbol *func)
dcc914f4 1635{
bd98c813 1636 struct instruction *insn, *last = NULL;
f1974222 1637 struct reloc *reloc;
dcc914f4 1638
f0f70adb 1639 func_for_each_insn(file, func, insn) {
99ce7962
PZ		 1640 if (!last)
1641 last = insn;
1642
1643 /*
1644 * Store back-pointers for unconditional forward jumps such
e7c2bc37 1645 * that find_jump_table() can back-track using those and
99ce7962
PZ		 1646 * avoid some potentially confusing code.
1647 */
1648 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1649 insn->offset > last->offset &&
1650 insn->jump_dest->offset > insn->offset &&
1651 !insn->jump_dest->first_jump_src) {
1652
1653 insn->jump_dest->first_jump_src = insn;
1654 last = insn->jump_dest;
1655 }
1656
dcc914f4
JP		 1657 if (insn->type != INSN_JUMP_DYNAMIC)
1658 continue;
1659
f1974222
MH		 1660 reloc = find_jump_table(file, func, insn);
1661 if (reloc) {
1662 reloc->jump_table_start = true;
1663 insn->jump_table = reloc;
dcc914f4 1664 }
dcc914f4 1665 }
bd98c813
JH		 1666}
1667
1668static int add_func_jump_tables(struct objtool_file *file,
1669 struct symbol *func)
1670{
1671 struct instruction *insn;
1672 int ret;
1673
f0f70adb 1674 func_for_each_insn(file, func, insn) {
bd98c813
JH		 1675 if (!insn->jump_table)
1676 continue;
dcc914f4 1677
bd98c813 1678 ret = add_jump_table(file, insn, insn->jump_table);
dcc914f4
JP		 1679 if (ret)
1680 return ret;
1681 }
1682
1683 return 0;
1684}
1685
1686/*
1687 * For some switch statements, gcc generates a jump table in the .rodata
1688 * section which contains a list of addresses within the function to jump to.
1689 * This finds these jump tables and adds them to the insn->alts lists.
1690 */
e7c2bc37 1691static int add_jump_table_alts(struct objtool_file *file)
dcc914f4
JP		 1692{
1693 struct section *sec;
1694 struct symbol *func;
1695 int ret;
1696
4a60aa05 1697 if (!file->rodata)
dcc914f4
JP		 1698 return 0;
1699
baa41469 1700 for_each_sec(file, sec) {
dcc914f4
JP		 1701 list_for_each_entry(func, &sec->symbol_list, list) {
1702 if (func->type != STT_FUNC)
1703 continue;
1704
bd98c813 1705 mark_func_jump_tables(file, func);
e7c2bc37 1706 ret = add_func_jump_tables(file, func);
dcc914f4
JP		 1707 if (ret)
1708 return ret;
1709 }
1710 }
1711
1712 return 0;
1713}
1714
b735bd3e
JP		 1715static void set_func_state(struct cfi_state *state)
1716{
1717 state->cfa = initial_func_cfi.cfa;
1718 memcpy(&state->regs, &initial_func_cfi.regs,
1719 CFI_NUM_REGS * sizeof(struct cfi_reg));
1720 state->stack_size = initial_func_cfi.cfa.offset;
1721}
1722
39358a03
JP		 1723static int read_unwind_hints(struct objtool_file *file)
1724{
e70e567e 1725 struct cfi_state cfi = init_cfi;
f1974222 1726 struct section *sec, *relocsec;
39358a03
JP		 1727 struct unwind_hint *hint;
1728 struct instruction *insn;
e70e567e 1729 struct reloc *reloc;
39358a03
JP		 1730 int i;
1731
1732 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1733 if (!sec)
1734 return 0;
1735
f1974222
MH		 1736 relocsec = sec->reloc;
1737 if (!relocsec) {
39358a03
JP		 1738 WARN("missing .rela.discard.unwind_hints section");
1739 return -1;
1740 }
1741
fe255fe6 1742 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
39358a03
JP		 1743 WARN("struct unwind_hint size mismatch");
1744 return -1;
1745 }
1746
1747 file->hints = true;
1748
fe255fe6 1749 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
39358a03
JP		 1750 hint = (struct unwind_hint *)sec->data->d_buf + i;
1751
f1974222
MH		 1752 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1753 if (!reloc) {
1754 WARN("can't find reloc for unwind_hints[%d]", i);
39358a03
JP		 1755 return -1;
1756 }
1757
f1974222 1758 insn = find_insn(file, reloc->sym->sec, reloc->addend);
39358a03
JP		 1759 if (!insn) {
1760 WARN("can't find insn for unwind_hints[%d]", i);
1761 return -1;
1762 }
1763
b735bd3e 1764 insn->hint = true;
39358a03 1765
b735bd3e 1766 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
e70e567e 1767 insn->cfi = &func_cfi;
39358a03
JP		 1768 continue;
1769 }
1770
e70e567e
PZ		 1771 if (insn->cfi)
1772 cfi = *(insn->cfi);
1773
1774 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
39358a03
JP		 1775 WARN_FUNC("unsupported unwind_hint sp base reg %d",
1776 insn->sec, insn->offset, hint->sp_reg);
1777 return -1;
1778 }
1779
e70e567e
PZ		 1780 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1781 cfi.type = hint->type;
1782 cfi.end = hint->end;
1783
1784 insn->cfi = cfi_hash_find_or_add(&cfi);
39358a03
JP		 1785 }
1786
1787 return 0;
1788}
1789
b5bc2231
PZ		 1790static int read_retpoline_hints(struct objtool_file *file)
1791{
63474dc4 1792 struct section *sec;
b5bc2231 1793 struct instruction *insn;
f1974222 1794 struct reloc *reloc;
b5bc2231 1795
63474dc4 1796 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
b5bc2231
PZ		 1797 if (!sec)
1798 return 0;
1799
f1974222
MH		 1800 list_for_each_entry(reloc, &sec->reloc_list, list) {
1801 if (reloc->sym->type != STT_SECTION) {
63474dc4 1802 WARN("unexpected relocation symbol type in %s", sec->name);
b5bc2231
PZ		 1803 return -1;
1804 }
1805
f1974222 1806 insn = find_insn(file, reloc->sym->sec, reloc->addend);
b5bc2231 1807 if (!insn) {
63474dc4 1808 WARN("bad .discard.retpoline_safe entry");
b5bc2231
PZ		 1809 return -1;
1810 }
1811
1812 if (insn->type != INSN_JUMP_DYNAMIC &&
1813 insn->type != INSN_CALL_DYNAMIC) {
63474dc4 1814 WARN_FUNC("retpoline_safe hint not an indirect jump/call",
b5bc2231
PZ		 1815 insn->sec, insn->offset);
1816 return -1;
1817 }
1818
1819 insn->retpoline_safe = true;
1820 }
1821
1822 return 0;
1823}
1824
c4a33939
PZ		 1825static int read_instr_hints(struct objtool_file *file)
1826{
1827 struct section *sec;
1828 struct instruction *insn;
f1974222 1829 struct reloc *reloc;
c4a33939
PZ		 1830
1831 sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
1832 if (!sec)
1833 return 0;
1834
f1974222
MH		 1835 list_for_each_entry(reloc, &sec->reloc_list, list) {
1836 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 1837 WARN("unexpected relocation symbol type in %s", sec->name);
1838 return -1;
1839 }
1840
f1974222 1841 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 1842 if (!insn) {
1843 WARN("bad .discard.instr_end entry");
1844 return -1;
1845 }
1846
1847 insn->instr--;
1848 }
1849
1850 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
1851 if (!sec)
1852 return 0;
1853
f1974222
MH		 1854 list_for_each_entry(reloc, &sec->reloc_list, list) {
1855 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 1856 WARN("unexpected relocation symbol type in %s", sec->name);
1857 return -1;
1858 }
1859
f1974222 1860 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 1861 if (!insn) {
1862 WARN("bad .discard.instr_begin entry");
1863 return -1;
1864 }
1865
1866 insn->instr++;
1867 }
1868
1869 return 0;
1870}
1871
8aa8eb2a
AC		 1872static int read_intra_function_calls(struct objtool_file *file)
1873{
1874 struct instruction *insn;
1875 struct section *sec;
f1974222 1876 struct reloc *reloc;
8aa8eb2a
AC		 1877
1878 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
1879 if (!sec)
1880 return 0;
1881
f1974222 1882 list_for_each_entry(reloc, &sec->reloc_list, list) {
8aa8eb2a
AC		 1883 unsigned long dest_off;
1884
f1974222 1885 if (reloc->sym->type != STT_SECTION) {
8aa8eb2a
AC		 1886 WARN("unexpected relocation symbol type in %s",
1887 sec->name);
1888 return -1;
1889 }
1890
f1974222 1891 insn = find_insn(file, reloc->sym->sec, reloc->addend);
8aa8eb2a
AC		 1892 if (!insn) {
1893 WARN("bad .discard.intra_function_call entry");
1894 return -1;
1895 }
1896
1897 if (insn->type != INSN_CALL) {
1898 WARN_FUNC("intra_function_call not a direct call",
1899 insn->sec, insn->offset);
1900 return -1;
1901 }
1902
1903 /*
1904 * Treat intra-function CALLs as JMPs, but with a stack_op.
1905 * See add_call_destinations(), which strips stack_ops from
1906 * normal CALLs.
1907 */
1908 insn->type = INSN_JUMP_UNCONDITIONAL;
1909
1910 dest_off = insn->offset + insn->len + insn->immediate;
1911 insn->jump_dest = find_insn(file, insn->sec, dest_off);
1912 if (!insn->jump_dest) {
1913 WARN_FUNC("can't find call dest at %s+0x%lx",
1914 insn->sec, insn->offset,
1915 insn->sec->name, dest_off);
1916 return -1;
1917 }
1918 }
1919
1920 return 0;
1921}
1922
2e532bd1 1923static int classify_symbols(struct objtool_file *file)
1e7e4788
JP		 1924{
1925 struct section *sec;
1926 struct symbol *func;
1927
1928 for_each_sec(file, sec) {
1929 list_for_each_entry(func, &sec->symbol_list, list) {
2e532bd1
PZ		 1930 if (func->bind != STB_GLOBAL)
1931 continue;
1932
1933 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
1e7e4788
JP		 1934 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
1935 func->static_call_tramp = true;
2e532bd1
PZ		 1936
1937 if (arch_is_retpoline(func))
1938 func->retpoline_thunk = true;
1939
1940 if (!strcmp(func->name, "__fentry__"))
1941 func->fentry = true;
1942
1943 if (!strncmp(func->name, "__sanitizer_cov_", 16))
1944 func->kcov = true;
1e7e4788
JP		 1945 }
1946 }
1947
1948 return 0;
1949}
1950
4a60aa05
AX		 1951static void mark_rodata(struct objtool_file *file)
1952{
1953 struct section *sec;
1954 bool found = false;
1955
1956 /*
87b512de
JP		 1957 * Search for the following rodata sections, each of which can
1958 * potentially contain jump tables:
1959 *
1960 * - .rodata: can contain GCC switch tables
1961 * - .rodata.<func>: same, if -fdata-sections is being used
1962 * - .rodata..c_jump_table: contains C annotated jump tables
1963 *
1964 * .rodata.str1.* sections are ignored; they don't contain jump tables.
4a60aa05
AX		 1965 */
1966 for_each_sec(file, sec) {
1ee44470
MS		 1967 if (!strncmp(sec->name, ".rodata", 7) &&
1968 !strstr(sec->name, ".str1.")) {
4a60aa05
AX		 1969 sec->rodata = true;
1970 found = true;
1971 }
1972 }
1973
1974 file->rodata = found;
1975}
1976
dcc914f4
JP		 1977static int decode_sections(struct objtool_file *file)
1978{
1979 int ret;
1980
4a60aa05
AX		 1981 mark_rodata(file);
1982
dcc914f4
JP		 1983 ret = decode_instructions(file);
1984 if (ret)
1985 return ret;
1986
1987 ret = add_dead_ends(file);
1988 if (ret)
1989 return ret;
1990
1991 add_ignores(file);
ea24213d 1992 add_uaccess_safe(file);
dcc914f4 1993
ff05ab23 1994 ret = add_ignore_alternatives(file);
258c7605
JP		 1995 if (ret)
1996 return ret;
1997
a958c4fe
PZ		 1998 /*
1999 * Must be before add_{jump_call}_destination.
2000 */
2e532bd1 2001 ret = classify_symbols(file);
5b06fd3b
PZ		 2002 if (ret)
2003 return ret;
2004
43d5430a
PZ		 2005 /*
2006 * Must be before add_special_section_alts() as that depends on
2007 * jump_dest being set.
2008 */
dcc914f4
JP		 2009 ret = add_jump_destinations(file);
2010 if (ret)
2011 return ret;
2012
a845c7cf 2013 ret = add_special_section_alts(file);
dcc914f4
JP		 2014 if (ret)
2015 return ret;
2016
a958c4fe
PZ		 2017 /*
2018 * Must be before add_call_destination(); it changes INSN_CALL to
2019 * INSN_JUMP.
2020 */
8aa8eb2a
AC		 2021 ret = read_intra_function_calls(file);
2022 if (ret)
2023 return ret;
2024
a845c7cf 2025 ret = add_call_destinations(file);
dcc914f4
JP		 2026 if (ret)
2027 return ret;
2028
e7c2bc37 2029 ret = add_jump_table_alts(file);
dcc914f4
JP		 2030 if (ret)
2031 return ret;
2032
39358a03
JP		 2033 ret = read_unwind_hints(file);
2034 if (ret)
2035 return ret;
2036
b5bc2231
PZ		 2037 ret = read_retpoline_hints(file);
2038 if (ret)
2039 return ret;
2040
c4a33939
PZ		 2041 ret = read_instr_hints(file);
2042 if (ret)
2043 return ret;
2044
dcc914f4
JP		 2045 return 0;
2046}
2047
2048static bool is_fentry_call(struct instruction *insn)
2049{
2e532bd1
PZ		 2050 if (insn->type == INSN_CALL &&
2051 insn->call_dest &&
2052 insn->call_dest->fentry)
dcc914f4
JP		 2053 return true;
2054
2055 return false;
2056}
2057
e25eea89 2058static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
dcc914f4 2059{
e7c0219b 2060 struct cfi_state *cfi = &state->cfi;
baa41469
JP		 2061 int i;
2062
e7c0219b 2063 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
e25eea89
PZ		 2064 return true;
2065
b735bd3e 2066 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
baa41469
JP		 2067 return true;
2068
b735bd3e 2069 if (cfi->stack_size != initial_func_cfi.cfa.offset)
e25eea89
PZ		 2070 return true;
2071
2072 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 2073 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2074 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
baa41469 2075 return true;
e25eea89 2076 }
baa41469
JP		 2077
2078 return false;
2079}
2080
fb084fde
JT		 2081static bool check_reg_frame_pos(const struct cfi_reg *reg,
2082 int expected_offset)
2083{
2084 return reg->base == CFI_CFA &&
2085 reg->offset == expected_offset;
2086}
2087
baa41469
JP		 2088static bool has_valid_stack_frame(struct insn_state *state)
2089{
e7c0219b
PZ		 2090 struct cfi_state *cfi = &state->cfi;
2091
fb084fde
JT		 2092 if (cfi->cfa.base == CFI_BP &&
2093 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2094 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
baa41469
JP		 2095 return true;
2096
e7c0219b 2097 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
baa41469
JP		 2098 return true;
2099
2100 return false;
dcc914f4
JP		 2101}
2102
e7c0219b
PZ		 2103static int update_cfi_state_regs(struct instruction *insn,
2104 struct cfi_state *cfi,
65ea47dc 2105 struct stack_op *op)
627fce14 2106{
e7c0219b 2107 struct cfi_reg *cfa = &cfi->cfa;
627fce14 2108
d8dd25a4 2109 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
627fce14
JP		 2110 return 0;
2111
2112 /* push */
ea24213d 2113 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
627fce14
JP		 2114 cfa->offset += 8;
2115
2116 /* pop */
ea24213d 2117 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
627fce14
JP		 2118 cfa->offset -= 8;
2119
2120 /* add immediate to sp */
2121 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2122 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2123 cfa->offset -= op->src.offset;
2124
2125 return 0;
2126}
2127
e7c0219b 2128static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
dcc914f4 2129{
bf4d1a83 2130 if (arch_callee_saved_reg(reg) &&
e7c0219b
PZ		 2131 cfi->regs[reg].base == CFI_UNDEFINED) {
2132 cfi->regs[reg].base = base;
2133 cfi->regs[reg].offset = offset;
baa41469 2134 }
dcc914f4
JP		 2135}
2136
e7c0219b 2137static void restore_reg(struct cfi_state *cfi, unsigned char reg)
dcc914f4 2138{
e7c0219b
PZ		 2139 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2140 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
baa41469
JP		 2141}
2142
2143/*
2144 * A note about DRAP stack alignment:
2145 *
2146 * GCC has the concept of a DRAP register, which is used to help keep track of
2147 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2148 * register. The typical DRAP pattern is:
2149 *
2150 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2151 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2152 * 41 ff 72 f8 pushq -0x8(%r10)
2153 * 55 push %rbp
2154 * 48 89 e5 mov %rsp,%rbp
2155 * (more pushes)
2156 * 41 52 push %r10
2157 * ...
2158 * 41 5a pop %r10
2159 * (more pops)
2160 * 5d pop %rbp
2161 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2162 * c3 retq
2163 *
2164 * There are some variations in the epilogues, like:
2165 *
2166 * 5b pop %rbx
2167 * 41 5a pop %r10
2168 * 41 5c pop %r12
2169 * 41 5d pop %r13
2170 * 41 5e pop %r14
2171 * c9 leaveq
2172 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2173 * c3 retq
2174 *
2175 * and:
2176 *
2177 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2178 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2179 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2180 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2181 * c9 leaveq
2182 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2183 * c3 retq
2184 *
2185 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2186 * restored beforehand:
2187 *
2188 * 41 55 push %r13
2189 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2190 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2191 * ...
2192 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2193 * 41 5d pop %r13
2194 * c3 retq
2195 */
d54dba41
PZ		 2196static int update_cfi_state(struct instruction *insn,
2197 struct instruction *next_insn,
2198 struct cfi_state *cfi, struct stack_op *op)
baa41469 2199{
e7c0219b
PZ		 2200 struct cfi_reg *cfa = &cfi->cfa;
2201 struct cfi_reg *regs = cfi->regs;
baa41469
JP		 2202
2203 /* stack operations don't make sense with an undefined CFA */
2204 if (cfa->base == CFI_UNDEFINED) {
2205 if (insn->func) {
2206 WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2207 return -1;
2208 }
2209 return 0;
2210 }
2211
ee819aed
JT		 2212 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2213 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
e7c0219b 2214 return update_cfi_state_regs(insn, cfi, op);
627fce14 2215
baa41469
JP		 2216 switch (op->dest.type) {
2217
2218 case OP_DEST_REG:
2219 switch (op->src.type) {
2220
2221 case OP_SRC_REG:
0d0970ee
JP		 2222 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2223 cfa->base == CFI_SP &&
fb084fde 2224 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
0d0970ee
JP		 2225
2226 /* mov %rsp, %rbp */
2227 cfa->base = op->dest.reg;
e7c0219b 2228 cfi->bp_scratch = false;
0d0970ee 2229 }
dd88a0a0 2230
0d0970ee 2231 else if (op->src.reg == CFI_SP &&
e7c0219b 2232 op->dest.reg == CFI_BP && cfi->drap) {
dd88a0a0 2233
0d0970ee
JP		 2234 /* drap: mov %rsp, %rbp */
2235 regs[CFI_BP].base = CFI_BP;
e7c0219b
PZ		 2236 regs[CFI_BP].offset = -cfi->stack_size;
2237 cfi->bp_scratch = false;
0d0970ee 2238 }
dd88a0a0 2239
0d0970ee
JP		 2240 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2241
2242 /*
2243 * mov %rsp, %reg
2244 *
2245 * This is needed for the rare case where GCC
2246 * does:
2247 *
2248 * mov %rsp, %rax
2249 * ...
2250 * mov %rax, %rsp
2251 */
e7c0219b
PZ		 2252 cfi->vals[op->dest.reg].base = CFI_CFA;
2253 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
dd88a0a0
JP		 2254 }
2255
3c1f0583 2256 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
ffc7e74f 2257 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
3c1f0583
JP		 2258
2259 /*
2260 * mov %rbp, %rsp
2261 *
2262 * Restore the original stack pointer (Clang).
2263 */
e7c0219b 2264 cfi->stack_size = -cfi->regs[CFI_BP].offset;
3c1f0583
JP		 2265 }
2266
dd88a0a0
JP		 2267 else if (op->dest.reg == cfa->base) {
2268
2269 /* mov %reg, %rsp */
2270 if (cfa->base == CFI_SP &&
e7c0219b 2271 cfi->vals[op->src.reg].base == CFI_CFA) {
dd88a0a0
JP		 2272
2273 /*
2274 * This is needed for the rare case
2275 * where GCC does something dumb like:
2276 *
2277 * lea 0x8(%rsp), %rcx
2278 * ...
2279 * mov %rcx, %rsp
2280 */
e7c0219b
PZ		 2281 cfa->offset = -cfi->vals[op->src.reg].offset;
2282 cfi->stack_size = cfa->offset;
dd88a0a0 2283
aafeb14e
PZ		 2284 } else if (cfa->base == CFI_SP &&
2285 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2286 cfi->vals[op->src.reg].offset == cfa->offset) {
2287
2288 /*
2289 * Stack swizzle:
2290 *
2291 * 1: mov %rsp, (%[tos])
2292 * 2: mov %[tos], %rsp
2293 * ...
2294 * 3: pop %rsp
2295 *
2296 * Where:
2297 *
2298 * 1 - places a pointer to the previous
2299 * stack at the Top-of-Stack of the
2300 * new stack.
2301 *
2302 * 2 - switches to the new stack.
2303 *
2304 * 3 - pops the Top-of-Stack to restore
2305 * the original stack.
2306 *
2307 * Note: we set base to SP_INDIRECT
2308 * here and preserve offset. Therefore
2309 * when the unwinder reaches ToS it
2310 * will dereference SP and then add the
2311 * offset to find the next frame, IOW:
2312 * (%rsp) + offset.
2313 */
2314 cfa->base = CFI_SP_INDIRECT;
2315
dd88a0a0
JP		 2316 } else {
2317 cfa->base = CFI_UNDEFINED;
2318 cfa->offset = 0;
2319 }
baa41469
JP		 2320 }
2321
724c8a23
PZ		 2322 else if (op->dest.reg == CFI_SP &&
2323 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2324 cfi->vals[op->src.reg].offset == cfa->offset) {
2325
2326 /*
2327 * The same stack swizzle case 2) as above. But
2328 * because we can't change cfa->base, case 3)
2329 * will become a regular POP. Pretend we're a
2330 * PUSH so things don't go unbalanced.
2331 */
2332 cfi->stack_size += 8;
2333 }
2334
2335
baa41469
JP		 2336 break;
2337
2338 case OP_SRC_ADD:
2339 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2340
2341 /* add imm, %rsp */
e7c0219b 2342 cfi->stack_size -= op->src.offset;
baa41469
JP		 2343 if (cfa->base == CFI_SP)
2344 cfa->offset -= op->src.offset;
2345 break;
2346 }
2347
2348 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2349
2350 /* lea disp(%rbp), %rsp */
e7c0219b 2351 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
baa41469
JP		 2352 break;
2353 }
2354
468af56a
JT		 2355 if (!cfi->drap && op->src.reg == CFI_SP &&
2356 op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2357 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2358
2359 /* lea disp(%rsp), %rbp */
2360 cfa->base = CFI_BP;
2361 cfa->offset -= op->src.offset;
2362 cfi->bp_scratch = false;
2363 break;
2364 }
2365
dd88a0a0 2366 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
baa41469
JP		 2367
2368 /* drap: lea disp(%rsp), %drap */
e7c0219b 2369 cfi->drap_reg = op->dest.reg;
dd88a0a0
JP		 2370
2371 /*
2372 * lea disp(%rsp), %reg
2373 *
2374 * This is needed for the rare case where GCC
2375 * does something dumb like:
2376 *
2377 * lea 0x8(%rsp), %rcx
2378 * ...
2379 * mov %rcx, %rsp
2380 */
e7c0219b
PZ		 2381 cfi->vals[op->dest.reg].base = CFI_CFA;
2382 cfi->vals[op->dest.reg].offset = \
2383 -cfi->stack_size + op->src.offset;
dd88a0a0 2384
baa41469
JP		 2385 break;
2386 }
2387
e7c0219b
PZ		 2388 if (cfi->drap && op->dest.reg == CFI_SP &&
2389 op->src.reg == cfi->drap_reg) {
baa41469
JP		 2390
2391 /* drap: lea disp(%drap), %rsp */
2392 cfa->base = CFI_SP;
e7c0219b
PZ		 2393 cfa->offset = cfi->stack_size = -op->src.offset;
2394 cfi->drap_reg = CFI_UNDEFINED;
2395 cfi->drap = false;
baa41469
JP		 2396 break;
2397 }
2398
d54dba41 2399 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
baa41469
JP		 2400 WARN_FUNC("unsupported stack register modification",
2401 insn->sec, insn->offset);
2402 return -1;
2403 }
2404
2405 break;
2406
2407 case OP_SRC_AND:
2408 if (op->dest.reg != CFI_SP ||
e7c0219b
PZ		 2409 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2410 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
baa41469
JP		 2411 WARN_FUNC("unsupported stack pointer realignment",
2412 insn->sec, insn->offset);
2413 return -1;
2414 }
2415
e7c0219b 2416 if (cfi->drap_reg != CFI_UNDEFINED) {
baa41469 2417 /* drap: and imm, %rsp */
e7c0219b
PZ		 2418 cfa->base = cfi->drap_reg;
2419 cfa->offset = cfi->stack_size = 0;
2420 cfi->drap = true;
baa41469
JP		 2421 }
2422
2423 /*
2424 * Older versions of GCC (4.8ish) realign the stack
2425 * without DRAP, with a frame pointer.
2426 */
2427
2428 break;
2429
2430 case OP_SRC_POP:
ea24213d 2431 case OP_SRC_POPF:
aafeb14e
PZ		 2432 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2433
2434 /* pop %rsp; # restore from a stack swizzle */
2435 cfa->base = CFI_SP;
2436 break;
2437 }
2438
e7c0219b 2439 if (!cfi->drap && op->dest.reg == cfa->base) {
baa41469
JP		 2440
2441 /* pop %rbp */
2442 cfa->base = CFI_SP;
2443 }
2444
e7c0219b
PZ		 2445 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2446 op->dest.reg == cfi->drap_reg &&
2447 cfi->drap_offset == -cfi->stack_size) {
baa41469 2448
bf4d1a83 2449 /* drap: pop %drap */
e7c0219b 2450 cfa->base = cfi->drap_reg;
bf4d1a83 2451 cfa->offset = 0;
e7c0219b 2452 cfi->drap_offset = -1;
baa41469 2453
ffc7e74f 2454 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
baa41469 2455
bf4d1a83 2456 /* pop %reg */
e7c0219b 2457 restore_reg(cfi, op->dest.reg);
baa41469
JP		 2458 }
2459
e7c0219b 2460 cfi->stack_size -= 8;
baa41469
JP		 2461 if (cfa->base == CFI_SP)
2462 cfa->offset -= 8;
2463
2464 break;
2465
2466 case OP_SRC_REG_INDIRECT:
201ef5a9
JT		 2467 if (!cfi->drap && op->dest.reg == cfa->base &&
2468 op->dest.reg == CFI_BP) {
2469
2470 /* mov disp(%rsp), %rbp */
2471 cfa->base = CFI_SP;
2472 cfa->offset = cfi->stack_size;
2473 }
2474
e7c0219b
PZ		 2475 if (cfi->drap && op->src.reg == CFI_BP &&
2476 op->src.offset == cfi->drap_offset) {
bf4d1a83
JP		 2477
2478 /* drap: mov disp(%rbp), %drap */
e7c0219b 2479 cfa->base = cfi->drap_reg;
bf4d1a83 2480 cfa->offset = 0;
e7c0219b 2481 cfi->drap_offset = -1;
bf4d1a83
JP		 2482 }
2483
e7c0219b 2484 if (cfi->drap && op->src.reg == CFI_BP &&
baa41469
JP		 2485 op->src.offset == regs[op->dest.reg].offset) {
2486
2487 /* drap: mov disp(%rbp), %reg */
e7c0219b 2488 restore_reg(cfi, op->dest.reg);
baa41469
JP		 2489
2490 } else if (op->src.reg == cfa->base &&
2491 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2492
2493 /* mov disp(%rbp), %reg */
2494 /* mov disp(%rsp), %reg */
e7c0219b 2495 restore_reg(cfi, op->dest.reg);
201ef5a9
JT		 2496
2497 } else if (op->src.reg == CFI_SP &&
2498 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2499
2500 /* mov disp(%rsp), %reg */
2501 restore_reg(cfi, op->dest.reg);
baa41469
JP		 2502 }
2503
2504 break;
2505
2506 default:
2507 WARN_FUNC("unknown stack-related instruction",
2508 insn->sec, insn->offset);
2509 return -1;
2510 }
2511
2512 break;
2513
2514 case OP_DEST_PUSH:
ea24213d 2515 case OP_DEST_PUSHF:
e7c0219b 2516 cfi->stack_size += 8;
baa41469
JP		 2517 if (cfa->base == CFI_SP)
2518 cfa->offset += 8;
2519
2520 if (op->src.type != OP_SRC_REG)
2521 break;
2522
e7c0219b
PZ		 2523 if (cfi->drap) {
2524 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 2525
2526 /* drap: push %drap */
2527 cfa->base = CFI_BP_INDIRECT;
e7c0219b 2528 cfa->offset = -cfi->stack_size;
baa41469 2529
bf4d1a83 2530 /* save drap so we know when to restore it */
e7c0219b 2531 cfi->drap_offset = -cfi->stack_size;
baa41469 2532
e7c0219b 2533 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
baa41469
JP		 2534
2535 /* drap: push %rbp */
e7c0219b 2536 cfi->stack_size = 0;
baa41469 2537
f4f80398 2538 } else {
baa41469
JP		 2539
2540 /* drap: push %reg */
e7c0219b 2541 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
baa41469
JP		 2542 }
2543
2544 } else {
2545
2546 /* push %reg */
e7c0219b 2547 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
baa41469
JP		 2548 }
2549
2550 /* detect when asm code uses rbp as a scratch register */
867ac9d7 2551 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
baa41469 2552 cfa->base != CFI_BP)
e7c0219b 2553 cfi->bp_scratch = true;
baa41469
JP		 2554 break;
2555
2556 case OP_DEST_REG_INDIRECT:
2557
e7c0219b
PZ		 2558 if (cfi->drap) {
2559 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 2560
2561 /* drap: mov %drap, disp(%rbp) */
2562 cfa->base = CFI_BP_INDIRECT;
2563 cfa->offset = op->dest.offset;
2564
bf4d1a83 2565 /* save drap offset so we know when to restore it */
e7c0219b 2566 cfi->drap_offset = op->dest.offset;
f4f80398 2567 } else {
baa41469
JP		 2568
2569 /* drap: mov reg, disp(%rbp) */
e7c0219b 2570 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
baa41469
JP		 2571 }
2572
2573 } else if (op->dest.reg == cfa->base) {
2574
2575 /* mov reg, disp(%rbp) */
2576 /* mov reg, disp(%rsp) */
e7c0219b
PZ		 2577 save_reg(cfi, op->src.reg, CFI_CFA,
2578 op->dest.offset - cfi->cfa.offset);
201ef5a9
JT		 2579
2580 } else if (op->dest.reg == CFI_SP) {
2581
2582 /* mov reg, disp(%rsp) */
2583 save_reg(cfi, op->src.reg, CFI_CFA,
2584 op->dest.offset - cfi->stack_size);
aafeb14e
PZ		 2585
2586 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2587
2588 /* mov %rsp, (%reg); # setup a stack swizzle. */
2589 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2590 cfi->vals[op->dest.reg].offset = cfa->offset;
baa41469
JP		 2591 }
2592
2593 break;
2594
baa41469 2595 case OP_DEST_MEM:
ea24213d 2596 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
baa41469
JP		 2597 WARN_FUNC("unknown stack-related memory operation",
2598 insn->sec, insn->offset);
2599 return -1;
2600 }
2601
2602 /* pop mem */
e7c0219b 2603 cfi->stack_size -= 8;
baa41469
JP		 2604 if (cfa->base == CFI_SP)
2605 cfa->offset -= 8;
2606
2607 break;
2608
2609 default:
2610 WARN_FUNC("unknown stack-related instruction",
2611 insn->sec, insn->offset);
2612 return -1;
2613 }
2614
2615 return 0;
2616}
2617
c9c324dc
JP		 2618/*
2619 * The stack layouts of alternatives instructions can sometimes diverge when
2620 * they have stack modifications. That's fine as long as the potential stack
2621 * layouts don't conflict at any given potential instruction boundary.
2622 *
2623 * Flatten the CFIs of the different alternative code streams (both original
2624 * and replacement) into a single shared CFI array which can be used to detect
2625 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2626 */
2627static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
65ea47dc 2628{
c9c324dc
JP		 2629 struct cfi_state **alt_cfi;
2630 int group_off;
65ea47dc 2631
c9c324dc
JP		 2632 if (!insn->alt_group)
2633 return 0;
65ea47dc 2634
e70e567e
PZ		 2635 if (!insn->cfi) {
2636 WARN("CFI missing");
2637 return -1;
2638 }
2639
c9c324dc
JP		 2640 alt_cfi = insn->alt_group->cfi;
2641 group_off = insn->offset - insn->alt_group->first_insn->offset;
65ea47dc 2642
c9c324dc 2643 if (!alt_cfi[group_off]) {
e70e567e 2644 alt_cfi[group_off] = insn->cfi;
c9c324dc 2645 } else {
e70e567e 2646 if (cficmp(alt_cfi[group_off], insn->cfi)) {
c9c324dc
JP		 2647 WARN_FUNC("stack layout conflict in alternatives",
2648 insn->sec, insn->offset);
ab3852ab
PZ		 2649 return -1;
2650 }
c9c324dc
JP		 2651 }
2652
2653 return 0;
2654}
2655
d54dba41
PZ		 2656static int handle_insn_ops(struct instruction *insn,
2657 struct instruction *next_insn,
2658 struct insn_state *state)
c9c324dc
JP		 2659{
2660 struct stack_op *op;
2661
2662 list_for_each_entry(op, &insn->stack_ops, list) {
2663
d54dba41 2664 if (update_cfi_state(insn, next_insn, &state->cfi, op))
c9c324dc 2665 return 1;
ab3852ab 2666
ba08abca
PZ		 2667 if (!insn->alt_group)
2668 continue;
2669
65ea47dc
JT		 2670 if (op->dest.type == OP_DEST_PUSHF) {
2671 if (!state->uaccess_stack) {
2672 state->uaccess_stack = 1;
2673 } else if (state->uaccess_stack >> 31) {
2674 WARN_FUNC("PUSHF stack exhausted",
2675 insn->sec, insn->offset);
2676 return 1;
2677 }
2678 state->uaccess_stack <<= 1;
2679 state->uaccess_stack |= state->uaccess;
2680 }
2681
2682 if (op->src.type == OP_SRC_POPF) {
2683 if (state->uaccess_stack) {
2684 state->uaccess = state->uaccess_stack & 1;
2685 state->uaccess_stack >>= 1;
2686 if (state->uaccess_stack == 1)
2687 state->uaccess_stack = 0;
2688 }
2689 }
2690 }
2691
2692 return 0;
2693}
2694
e7c0219b 2695static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
baa41469 2696{
e70e567e 2697 struct cfi_state *cfi1 = insn->cfi;
baa41469
JP		 2698 int i;
2699
e70e567e
PZ		 2700 if (!cfi1) {
2701 WARN("CFI missing");
2702 return false;
2703 }
2704
e7c0219b
PZ		 2705 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2706
baa41469
JP		 2707 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2708 insn->sec, insn->offset,
e7c0219b
PZ		 2709 cfi1->cfa.base, cfi1->cfa.offset,
2710 cfi2->cfa.base, cfi2->cfa.offset);
baa41469 2711
e7c0219b 2712 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
baa41469 2713 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b 2714 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
baa41469
JP		 2715 sizeof(struct cfi_reg)))
2716 continue;
2717
2718 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2719 insn->sec, insn->offset,
e7c0219b
PZ		 2720 i, cfi1->regs[i].base, cfi1->regs[i].offset,
2721 i, cfi2->regs[i].base, cfi2->regs[i].offset);
baa41469
JP		 2722 break;
2723 }
2724
e7c0219b
PZ		 2725 } else if (cfi1->type != cfi2->type) {
2726
627fce14 2727 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
e7c0219b
PZ		 2728 insn->sec, insn->offset, cfi1->type, cfi2->type);
2729
2730 } else if (cfi1->drap != cfi2->drap ||
2731 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2732 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
627fce14 2733
bf4d1a83 2734 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
baa41469 2735 insn->sec, insn->offset,
e7c0219b
PZ		 2736 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2737 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
baa41469
JP		 2738
2739 } else
2740 return true;
2741
2742 return false;
dcc914f4
JP		 2743}
2744
ea24213d
PZ		 2745static inline bool func_uaccess_safe(struct symbol *func)
2746{
2747 if (func)
e10cd8fe 2748 return func->uaccess_safe;
ea24213d
PZ		 2749
2750 return false;
2751}
2752
0c1ddd33 2753static inline const char *call_dest_name(struct instruction *insn)
ea24213d
PZ		 2754{
2755 if (insn->call_dest)
2756 return insn->call_dest->name;
2757
2758 return "{dynamic}";
2759}
2760
6b643a07
PZ		 2761static inline bool noinstr_call_dest(struct symbol *func)
2762{
2763 /*
2764 * We can't deal with indirect function calls at present;
2765 * assume they're instrumented.
2766 */
2767 if (!func)
2768 return false;
2769
2770 /*
2771 * If the symbol is from a noinstr section; we good.
2772 */
2773 if (func->sec->noinstr)
2774 return true;
2775
2776 /*
2777 * The __ubsan_handle_*() calls are like WARN(), they only happen when
2778 * something 'BAD' happened. At the risk of taking the machine down,
2779 * let them proceed to get the message out.
2780 */
2781 if (!strncmp(func->name, "__ubsan_handle_", 15))
2782 return true;
2783
2784 return false;
2785}
2786
ea24213d
PZ		 2787static int validate_call(struct instruction *insn, struct insn_state *state)
2788{
c4a33939 2789 if (state->noinstr && state->instr <= 0 &&
6b643a07 2790 !noinstr_call_dest(insn->call_dest)) {
c4a33939
PZ		 2791 WARN_FUNC("call to %s() leaves .noinstr.text section",
2792 insn->sec, insn->offset, call_dest_name(insn));
2793 return 1;
2794 }
2795
ea24213d
PZ		 2796 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
2797 WARN_FUNC("call to %s() with UACCESS enabled",
0c1ddd33 2798 insn->sec, insn->offset, call_dest_name(insn));
ea24213d
PZ		 2799 return 1;
2800 }
2801
2f0f9e9a
PZ		 2802 if (state->df) {
2803 WARN_FUNC("call to %s() with DF set",
0c1ddd33 2804 insn->sec, insn->offset, call_dest_name(insn));
2f0f9e9a
PZ		 2805 return 1;
2806 }
2807
ea24213d
PZ		 2808 return 0;
2809}
2810
54262aa2
PZ		 2811static int validate_sibling_call(struct instruction *insn, struct insn_state *state)
2812{
e25eea89 2813 if (has_modified_stack_frame(insn, state)) {
54262aa2
PZ		 2814 WARN_FUNC("sibling call from callable instruction with modified stack frame",
2815 insn->sec, insn->offset);
2816 return 1;
2817 }
2818
ea24213d 2819 return validate_call(insn, state);
54262aa2
PZ		 2820}
2821
a92e92d1
PZ		 2822static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
2823{
c4a33939
PZ		 2824 if (state->noinstr && state->instr > 0) {
2825 WARN_FUNC("return with instrumentation enabled",
2826 insn->sec, insn->offset);
2827 return 1;
2828 }
2829
a92e92d1
PZ		 2830 if (state->uaccess && !func_uaccess_safe(func)) {
2831 WARN_FUNC("return with UACCESS enabled",
2832 insn->sec, insn->offset);
2833 return 1;
2834 }
2835
2836 if (!state->uaccess && func_uaccess_safe(func)) {
2837 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
2838 insn->sec, insn->offset);
2839 return 1;
2840 }
2841
2842 if (state->df) {
2843 WARN_FUNC("return with DF set",
2844 insn->sec, insn->offset);
2845 return 1;
2846 }
2847
e25eea89 2848 if (func && has_modified_stack_frame(insn, state)) {
a92e92d1
PZ		 2849 WARN_FUNC("return with modified stack frame",
2850 insn->sec, insn->offset);
2851 return 1;
2852 }
2853
e7c0219b 2854 if (state->cfi.bp_scratch) {
b2966952
JP		 2855 WARN_FUNC("BP used as a scratch register",
2856 insn->sec, insn->offset);
a92e92d1
PZ		 2857 return 1;
2858 }
2859
2860 return 0;
2861}
2862
c9c324dc
JP		 2863static struct instruction *next_insn_to_validate(struct objtool_file *file,
2864 struct instruction *insn)
7117f16b 2865{
b23cc71c 2866 struct alt_group *alt_group = insn->alt_group;
7117f16b 2867
c9c324dc
JP		 2868 /*
2869 * Simulate the fact that alternatives are patched in-place. When the
2870 * end of a replacement alt_group is reached, redirect objtool flow to
2871 * the end of the original alt_group.
2872 */
2873 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
2874 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
2875
2876 return next_insn_same_sec(file, insn);
7117f16b
PZ		 2877}
2878
dcc914f4
JP		 2879/*
2880 * Follow the branch starting at the given instruction, and recursively follow
2881 * any other branches (jumps). Meanwhile, track the frame pointer state at
2882 * each instruction and validate all the rules described in
2883 * tools/objtool/Documentation/stack-validation.txt.
2884 */
c705cecc 2885static int validate_branch(struct objtool_file *file, struct symbol *func,
b7460462 2886 struct instruction *insn, struct insn_state state)
dcc914f4
JP		 2887{
2888 struct alternative *alt;
e70e567e 2889 struct instruction *next_insn, *prev_insn = NULL;
dcc914f4 2890 struct section *sec;
882a0db9 2891 u8 visited;
dcc914f4
JP		 2892 int ret;
2893
dcc914f4 2894 sec = insn->sec;
dcc914f4 2895
dcc914f4 2896 while (1) {
c9c324dc 2897 next_insn = next_insn_to_validate(file, insn);
39358a03 2898
13810435 2899 if (file->c_file && func && insn->func && func != insn->func->pfunc) {
ee97638b
JP		 2900 WARN("%s() falls through to next function %s()",
2901 func->name, insn->func->name);
2902 return 1;
dcc914f4
JP		 2903 }
2904
4855022a
JP		 2905 if (func && insn->ignore) {
2906 WARN_FUNC("BUG: why am I validating an ignored function?",
2907 sec, insn->offset);
12b25729 2908 return 1;
4855022a
JP		 2909 }
2910
882a0db9 2911 visited = 1 << state.uaccess;
dcc914f4 2912 if (insn->visited) {
e7c0219b 2913 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
dcc914f4 2914 return 1;
dcc914f4 2915
882a0db9 2916 if (insn->visited & visited)
ea24213d 2917 return 0;
e70e567e
PZ		 2918 } else {
2919 nr_insns_visited++;
dcc914f4
JP		 2920 }
2921
c4a33939
PZ		 2922 if (state.noinstr)
2923 state.instr += insn->instr;
2924
e70e567e
PZ		 2925 if (insn->hint) {
2926 state.cfi = *insn->cfi;
2927 } else {
2928 /* XXX track if we actually changed state.cfi */
2929
2930 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
2931 insn->cfi = prev_insn->cfi;
2932 nr_cfi_reused++;
2933 } else {
2934 insn->cfi = cfi_hash_find_or_add(&state.cfi);
2935 }
2936 }
dcc914f4 2937
882a0db9 2938 insn->visited |= visited;
baa41469 2939
c9c324dc
JP		 2940 if (propagate_alt_cfi(file, insn))
2941 return 1;
2942
7117f16b 2943 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
764eef4b
PZ		 2944 bool skip_orig = false;
2945
a845c7cf 2946 list_for_each_entry(alt, &insn->alts, list) {
764eef4b
PZ		 2947 if (alt->skip_orig)
2948 skip_orig = true;
2949
c705cecc 2950 ret = validate_branch(file, func, alt->insn, state);
7697eee3
PZ		 2951 if (ret) {
2952 if (backtrace)
2953 BT_FUNC("(alt)", insn);
2954 return ret;
2955 }
a845c7cf 2956 }
764eef4b
PZ		 2957
2958 if (skip_orig)
2959 return 0;
dcc914f4
JP		 2960 }
2961
d54dba41 2962 if (handle_insn_ops(insn, next_insn, &state))
60041bcd
PZ		 2963 return 1;
2964
dcc914f4
JP		 2965 switch (insn->type) {
2966
dcc914f4 2967 case INSN_RETURN:
98256722
PZ		 2968 if (next_insn && next_insn->type == INSN_TRAP) {
2969 next_insn->ignore = true;
2970 } else if (sls && !insn->retpoline_safe) {
2971 WARN_FUNC("missing int3 after ret",
2972 insn->sec, insn->offset);
2973 }
a92e92d1 2974 return validate_return(func, insn, &state);
dcc914f4
JP		 2975
2976 case INSN_CALL:
ea24213d
PZ		 2977 case INSN_CALL_DYNAMIC:
2978 ret = validate_call(insn, &state);
2979 if (ret)
2980 return ret;
dcc914f4 2981
c9bab22b
JP		 2982 if (!no_fp && func && !is_fentry_call(insn) &&
2983 !has_valid_stack_frame(&state)) {
dcc914f4
JP		 2984 WARN_FUNC("call without frame pointer save/setup",
2985 sec, insn->offset);
2986 return 1;
2987 }
c9bab22b
JP		 2988
2989 if (dead_end_function(file, insn->call_dest))
2990 return 0;
2991
dcc914f4
JP		 2992 break;
2993
2994 case INSN_JUMP_CONDITIONAL:
2995 case INSN_JUMP_UNCONDITIONAL:
ecf11ba4 2996 if (is_sibling_call(insn)) {
54262aa2 2997 ret = validate_sibling_call(insn, &state);
dcc914f4 2998 if (ret)
54262aa2 2999 return ret;
4855022a 3000
0c1ddd33 3001 } else if (insn->jump_dest) {
c705cecc
JP		 3002 ret = validate_branch(file, func,
3003 insn->jump_dest, state);
7697eee3
PZ		 3004 if (ret) {
3005 if (backtrace)
3006 BT_FUNC("(branch)", insn);
3007 return ret;
3008 }
4855022a 3009 }
dcc914f4
JP		 3010
3011 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3012 return 0;
3013
3014 break;
3015
3016 case INSN_JUMP_DYNAMIC:
98256722
PZ		 3017 if (next_insn && next_insn->type == INSN_TRAP) {
3018 next_insn->ignore = true;
3019 } else if (sls && !insn->retpoline_safe) {
3020 WARN_FUNC("missing int3 after indirect jump",
3021 insn->sec, insn->offset);
3022 }
3023
3024 /* fallthrough */
b68b9907 3025 case INSN_JUMP_DYNAMIC_CONDITIONAL:
ecf11ba4 3026 if (is_sibling_call(insn)) {
54262aa2
PZ		 3027 ret = validate_sibling_call(insn, &state);
3028 if (ret)
3029 return ret;
dcc914f4
JP		 3030 }
3031
b68b9907
JP		 3032 if (insn->type == INSN_JUMP_DYNAMIC)
3033 return 0;
3034
3035 break;
dcc914f4 3036
39358a03
JP		 3037 case INSN_CONTEXT_SWITCH:
3038 if (func && (!next_insn || !next_insn->hint)) {
3039 WARN_FUNC("unsupported instruction in callable function",
3040 sec, insn->offset);
3041 return 1;
3042 }
3043 return 0;
3044
ea24213d
PZ		 3045 case INSN_STAC:
3046 if (state.uaccess) {
3047 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3048 return 1;
3049 }
3050
3051 state.uaccess = true;
3052 break;
3053
3054 case INSN_CLAC:
c705cecc 3055 if (!state.uaccess && func) {
ea24213d
PZ		 3056 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3057 return 1;
3058 }
3059
3060 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3061 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3062 return 1;
3063 }
3064
3065 state.uaccess = false;
baa41469
JP		 3066 break;
3067
2f0f9e9a 3068 case INSN_STD:
6f567c93 3069 if (state.df) {
2f0f9e9a 3070 WARN_FUNC("recursive STD", sec, insn->offset);
6f567c93
JP		 3071 return 1;
3072 }
2f0f9e9a
PZ		 3073
3074 state.df = true;
3075 break;
3076
3077 case INSN_CLD:
6f567c93 3078 if (!state.df && func) {
2f0f9e9a 3079 WARN_FUNC("redundant CLD", sec, insn->offset);
6f567c93
JP		 3080 return 1;
3081 }
2f0f9e9a
PZ		 3082
3083 state.df = false;
baa41469
JP		 3084 break;
3085
dcc914f4
JP		 3086 default:
3087 break;
3088 }
3089
3090 if (insn->dead_end)
3091 return 0;
3092
00d96180 3093 if (!next_insn) {
e7c0219b 3094 if (state.cfi.cfa.base == CFI_UNDEFINED)
00d96180 3095 return 0;
dcc914f4
JP		 3096 WARN("%s: unexpected end of section", sec->name);
3097 return 1;
3098 }
00d96180 3099
e70e567e 3100 prev_insn = insn;
00d96180 3101 insn = next_insn;
dcc914f4
JP		 3102 }
3103
3104 return 0;
3105}
3106
932f8e98 3107static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
39358a03
JP		 3108{
3109 struct instruction *insn;
39358a03 3110 struct insn_state state;
932f8e98 3111 int ret, warnings = 0;
39358a03
JP		 3112
3113 if (!file->hints)
3114 return 0;
3115
932f8e98 3116 init_insn_state(&state, sec);
39358a03 3117
932f8e98
PZ		 3118 if (sec) {
3119 insn = find_insn(file, sec, 0);
3120 if (!insn)
3121 return 0;
3122 } else {
3123 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3124 }
3125
3126 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
39358a03 3127 if (insn->hint && !insn->visited) {
c705cecc 3128 ret = validate_branch(file, insn->func, insn, state);
7697eee3
PZ		 3129 if (ret && backtrace)
3130 BT_FUNC("<=== (hint)", insn);
39358a03
JP		 3131 warnings += ret;
3132 }
932f8e98
PZ		 3133
3134 insn = list_next_entry(insn, list);
39358a03
JP		 3135 }
3136
3137 return warnings;
3138}
3139
b5bc2231
PZ		 3140static int validate_retpoline(struct objtool_file *file)
3141{
3142 struct instruction *insn;
3143 int warnings = 0;
3144
3145 for_each_insn(file, insn) {
3146 if (insn->type != INSN_JUMP_DYNAMIC &&
3147 insn->type != INSN_CALL_DYNAMIC)
3148 continue;
3149
3150 if (insn->retpoline_safe)
3151 continue;
3152
ca41b97e
PZ		 3153 /*
3154 * .init.text code is ran before userspace and thus doesn't
3155 * strictly need retpolines, except for modules which are
3156 * loaded late, they very much do need retpoline in their
3157 * .init.text
3158 */
3159 if (!strcmp(insn->sec->name, ".init.text") && !module)
3160 continue;
3161
b5bc2231
PZ		 3162 WARN_FUNC("indirect %s found in RETPOLINE build",
3163 insn->sec, insn->offset,
3164 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3165
3166 warnings++;
3167 }
3168
3169 return warnings;
3170}
3171
dcc914f4
JP		 3172static bool is_kasan_insn(struct instruction *insn)
3173{
3174 return (insn->type == INSN_CALL &&
3175 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3176}
3177
3178static bool is_ubsan_insn(struct instruction *insn)
3179{
3180 return (insn->type == INSN_CALL &&
3181 !strcmp(insn->call_dest->name,
3182 "__ubsan_handle_builtin_unreachable"));
3183}
3184
14db1f0a 3185static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
dcc914f4
JP		 3186{
3187 int i;
14db1f0a 3188 struct instruction *prev_insn;
dcc914f4 3189
baa41469
JP		 3190 if (insn->ignore || insn->type == INSN_NOP)
3191 return true;
3192
3193 /*
3194 * Ignore any unused exceptions. This can happen when a whitelisted
3195 * function has an exception table entry.
0e2bb2bc
JP		 3196 *
3197 * Also ignore alternative replacement instructions. This can happen
3198 * when a whitelisted function uses one of the ALTERNATIVE macros.
baa41469 3199 */
0e2bb2bc
JP		 3200 if (!strcmp(insn->sec->name, ".fixup") ||
3201 !strcmp(insn->sec->name, ".altinstr_replacement") ||
3202 !strcmp(insn->sec->name, ".altinstr_aux"))
dcc914f4
JP		 3203 return true;
3204
bd841d61
JP		 3205 if (!insn->func)
3206 return false;
3207
3208 /*
3209 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3210 * __builtin_unreachable(). The BUG() macro has an unreachable() after
3211 * the UD2, which causes GCC's undefined trap logic to emit another UD2
3212 * (or occasionally a JMP to UD2).
14db1f0a
IH		 3213 *
3214 * It may also insert a UD2 after calling a __noreturn function.
bd841d61 3215 */
14db1f0a
IH		 3216 prev_insn = list_prev_entry(insn, list);
3217 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
bd841d61
JP		 3218 (insn->type == INSN_BUG ||
3219 (insn->type == INSN_JUMP_UNCONDITIONAL &&
3220 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3221 return true;
3222
dcc914f4
JP		 3223 /*
3224 * Check if this (or a subsequent) instruction is related to
3225 * CONFIG_UBSAN or CONFIG_KASAN.
3226 *
3227 * End the search at 5 instructions to avoid going into the weeds.
3228 */
3229 for (i = 0; i < 5; i++) {
3230
3231 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3232 return true;
3233
fe24e271
JP		 3234 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3235 if (insn->jump_dest &&
3236 insn->jump_dest->func == insn->func) {
3237 insn = insn->jump_dest;
3238 continue;
3239 }
3240
3241 break;
dcc914f4
JP		 3242 }
3243
baa41469 3244 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
dcc914f4 3245 break;
fe24e271 3246
dcc914f4
JP		 3247 insn = list_next_entry(insn, list);
3248 }
3249
3250 return false;
3251}
3252
4b5e2e7f
PZ		 3253static int validate_symbol(struct objtool_file *file, struct section *sec,
3254 struct symbol *sym, struct insn_state *state)
dcc914f4 3255{
dcc914f4 3256 struct instruction *insn;
4b5e2e7f
PZ		 3257 int ret;
3258
3259 if (!sym->len) {
3260 WARN("%s() is missing an ELF size annotation", sym->name);
3261 return 1;
3262 }
3263
3264 if (sym->pfunc != sym || sym->alias != sym)
3265 return 0;
3266
3267 insn = find_insn(file, sec, sym->offset);
3268 if (!insn || insn->ignore || insn->visited)
3269 return 0;
3270
3271 state->uaccess = sym->uaccess_safe;
3272
3273 ret = validate_branch(file, insn->func, insn, *state);
3274 if (ret && backtrace)
3275 BT_FUNC("<=== (sym)", insn);
3276 return ret;
3277}
3278
3279static int validate_section(struct objtool_file *file, struct section *sec)
3280{
baa41469 3281 struct insn_state state;
4b5e2e7f
PZ		 3282 struct symbol *func;
3283 int warnings = 0;
dcc914f4 3284
350994bf
PZ		 3285 list_for_each_entry(func, &sec->symbol_list, list) {
3286 if (func->type != STT_FUNC)
3287 continue;
e10cd8fe 3288
932f8e98 3289 init_insn_state(&state, sec);
b735bd3e 3290 set_func_state(&state.cfi);
0699e551 3291
4b5e2e7f 3292 warnings += validate_symbol(file, sec, func, &state);
dcc914f4
JP		 3293 }
3294
dcc914f4
JP		 3295 return warnings;
3296}
3297
c4a33939
PZ		 3298static int validate_vmlinux_functions(struct objtool_file *file)
3299{
3300 struct section *sec;
932f8e98 3301 int warnings = 0;
c4a33939
PZ		 3302
3303 sec = find_section_by_name(file->elf, ".noinstr.text");
0cc9ac8d
TG		 3304 if (sec) {
3305 warnings += validate_section(file, sec);
3306 warnings += validate_unwind_hints(file, sec);
3307 }
c4a33939 3308
0cc9ac8d
TG		 3309 sec = find_section_by_name(file->elf, ".entry.text");
3310 if (sec) {
3311 warnings += validate_section(file, sec);
3312 warnings += validate_unwind_hints(file, sec);
3313 }
932f8e98
PZ		 3314
3315 return warnings;
c4a33939
PZ		 3316}
3317
350994bf
PZ		 3318static int validate_functions(struct objtool_file *file)
3319{
3320 struct section *sec;
3321 int warnings = 0;
3322
da837bd6
PZ		 3323 for_each_sec(file, sec) {
3324 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3325 continue;
3326
350994bf 3327 warnings += validate_section(file, sec);
da837bd6 3328 }
350994bf
PZ		 3329
3330 return warnings;
3331}
3332
baa41469 3333static int validate_reachable_instructions(struct objtool_file *file)
dcc914f4
JP		 3334{
3335 struct instruction *insn;
baa41469
JP		 3336
3337 if (file->ignore_unreachables)
3338 return 0;
dcc914f4
JP		 3339
3340 for_each_insn(file, insn) {
14db1f0a 3341 if (insn->visited || ignore_unreachable_insn(file, insn))
baa41469
JP		 3342 continue;
3343
baa41469
JP		 3344 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3345 return 1;
dcc914f4
JP		 3346 }
3347
baa41469 3348 return 0;
dcc914f4
JP		 3349}
3350
d44becb9 3351int check(struct objtool_file *file)
dcc914f4 3352{
dcc914f4
JP		 3353 int ret, warnings = 0;
3354
baa41469 3355 arch_initial_func_cfi_state(&initial_func_cfi);
e70e567e
PZ		 3356 init_cfi_state(&init_cfi);
3357 init_cfi_state(&func_cfi);
3358 set_func_state(&func_cfi);
3359
3360 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
3361 goto out;
3362
3363 cfi_hash_add(&init_cfi);
3364 cfi_hash_add(&func_cfi);
baa41469 3365
6545eb03 3366 ret = decode_sections(file);
dcc914f4
JP		 3367 if (ret < 0)
3368 goto out;
e70e567e 3369
dcc914f4
JP		 3370 warnings += ret;
3371
6545eb03 3372 if (list_empty(&file->insn_list))
dcc914f4 3373 goto out;
dcc914f4 3374
c4a33939 3375 if (vmlinux && !validate_dup) {
6545eb03 3376 ret = validate_vmlinux_functions(file);
c4a33939
PZ		 3377 if (ret < 0)
3378 goto out;
3379
3380 warnings += ret;
3381 goto out;
3382 }
3383
b5bc2231 3384 if (retpoline) {
6545eb03 3385 ret = validate_retpoline(file);
b5bc2231
PZ		 3386 if (ret < 0)
3387 return ret;
3388 warnings += ret;
3389 }
3390
6545eb03 3391 ret = validate_functions(file);
dcc914f4
JP		 3392 if (ret < 0)
3393 goto out;
3394 warnings += ret;
3395
6545eb03 3396 ret = validate_unwind_hints(file, NULL);
39358a03
JP		 3397 if (ret < 0)
3398 goto out;
3399 warnings += ret;
3400
baa41469 3401 if (!warnings) {
6545eb03 3402 ret = validate_reachable_instructions(file);
baa41469
JP		 3403 if (ret < 0)
3404 goto out;
3405 warnings += ret;
3406 }
3407
6545eb03 3408 ret = create_static_call_sections(file);
1e7e4788
JP		 3409 if (ret < 0)
3410 goto out;
3411 warnings += ret;
3412
a77a9f03
PZ		 3413 if (retpoline) {
3414 ret = create_retpoline_sites_sections(file);
3415 if (ret < 0)
3416 goto out;
3417 warnings += ret;
3418 }
3419
99d00215
PZ		 3420 if (mcount) {
3421 ret = create_mcount_loc_sections(file);
3422 if (ret < 0)
3423 goto out;
3424 warnings += ret;
3425 }
3426
e70e567e
PZ		 3427 if (stats) {
3428 printf("nr_insns_visited: %ld\n", nr_insns_visited);
3429 printf("nr_cfi: %ld\n", nr_cfi);
3430 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
3431 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
3432 }
3433
dcc914f4 3434out:
655cf865
JP		 3435 /*
3436 * For now, don't fail the kernel build on fatal warnings. These
3437 * errors are still fairly common due to the growing matrix of
3438 * supported toolchains and their recent pace of change.
3439 */
dcc914f4
JP		 3440 return 0;
3441}
Ubuntu Jammy Linux kernel mirror