1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2014 Red Hat Inc.
7 * Vivek Goyal <vgoyal@redhat.com>
10 #define pr_fmt(fmt) "kexec-bzImage64: " fmt
12 #include <linux/string.h>
13 #include <linux/printk.h>
14 #include <linux/errno.h>
15 #include <linux/slab.h>
16 #include <linux/kexec.h>
17 #include <linux/kernel.h>
19 #include <linux/efi.h>
20 #include <linux/verification.h>
22 #include <asm/bootparam.h>
23 #include <asm/setup.h>
24 #include <asm/crash.h>
26 #include <asm/e820/api.h>
27 #include <asm/kexec-bzimage64.h>
29 #define MAX_ELFCOREHDR_STR_LEN 30 /* elfcorehdr=0x<64bit-value> */
32 * Defines lowest physical address for various segments. Not sure where
33 * exactly these limits came from. Current bzimage64 loader in kexec-tools
34 * uses these so I am retaining it. It can be changed over time as we gain
37 #define MIN_PURGATORY_ADDR 0x3000
38 #define MIN_BOOTPARAM_ADDR 0x3000
39 #define MIN_KERNEL_LOAD_ADDR 0x100000
40 #define MIN_INITRD_LOAD_ADDR 0x1000000
43 * This is a place holder for all boot loader specific data structure which
44 * gets allocated in one call but gets freed much later during cleanup
45 * time. Right now there is only one field but it can grow as need be.
47 struct bzimage64_data
{
49 * Temporary buffer to hold bootparams buffer. This should be
50 * freed once the bootparam segment has been loaded.
55 static int setup_initrd(struct boot_params
*params
,
56 unsigned long initrd_load_addr
, unsigned long initrd_len
)
58 params
->hdr
.ramdisk_image
= initrd_load_addr
& 0xffffffffUL
;
59 params
->hdr
.ramdisk_size
= initrd_len
& 0xffffffffUL
;
61 params
->ext_ramdisk_image
= initrd_load_addr
>> 32;
62 params
->ext_ramdisk_size
= initrd_len
>> 32;
67 static int setup_cmdline(struct kimage
*image
, struct boot_params
*params
,
68 unsigned long bootparams_load_addr
,
69 unsigned long cmdline_offset
, char *cmdline
,
70 unsigned long cmdline_len
)
72 char *cmdline_ptr
= ((char *)params
) + cmdline_offset
;
73 unsigned long cmdline_ptr_phys
, len
= 0;
74 uint32_t cmdline_low_32
, cmdline_ext_32
;
76 if (image
->type
== KEXEC_TYPE_CRASH
) {
77 len
= sprintf(cmdline_ptr
,
78 "elfcorehdr=0x%lx ", image
->elf_load_addr
);
80 memcpy(cmdline_ptr
+ len
, cmdline
, cmdline_len
);
83 cmdline_ptr
[cmdline_len
- 1] = '\0';
85 pr_debug("Final command line is: %s\n", cmdline_ptr
);
86 cmdline_ptr_phys
= bootparams_load_addr
+ cmdline_offset
;
87 cmdline_low_32
= cmdline_ptr_phys
& 0xffffffffUL
;
88 cmdline_ext_32
= cmdline_ptr_phys
>> 32;
90 params
->hdr
.cmd_line_ptr
= cmdline_low_32
;
92 params
->ext_cmd_line_ptr
= cmdline_ext_32
;
97 static int setup_e820_entries(struct boot_params
*params
)
99 unsigned int nr_e820_entries
;
101 nr_e820_entries
= e820_table_kexec
->nr_entries
;
103 /* TODO: Pass entries more than E820_MAX_ENTRIES_ZEROPAGE in bootparams setup data */
104 if (nr_e820_entries
> E820_MAX_ENTRIES_ZEROPAGE
)
105 nr_e820_entries
= E820_MAX_ENTRIES_ZEROPAGE
;
107 params
->e820_entries
= nr_e820_entries
;
108 memcpy(¶ms
->e820_table
, &e820_table_kexec
->entries
, nr_e820_entries
*sizeof(struct e820_entry
));
114 static int setup_efi_info_memmap(struct boot_params
*params
,
115 unsigned long params_load_addr
,
116 unsigned int efi_map_offset
,
117 unsigned int efi_map_sz
)
119 void *efi_map
= (void *)params
+ efi_map_offset
;
120 unsigned long efi_map_phys_addr
= params_load_addr
+ efi_map_offset
;
121 struct efi_info
*ei
= ¶ms
->efi_info
;
126 efi_runtime_map_copy(efi_map
, efi_map_sz
);
128 ei
->efi_memmap
= efi_map_phys_addr
& 0xffffffff;
129 ei
->efi_memmap_hi
= efi_map_phys_addr
>> 32;
130 ei
->efi_memmap_size
= efi_map_sz
;
136 prepare_add_efi_setup_data(struct boot_params
*params
,
137 unsigned long params_load_addr
,
138 unsigned int efi_setup_data_offset
)
140 unsigned long setup_data_phys
;
141 struct setup_data
*sd
= (void *)params
+ efi_setup_data_offset
;
142 struct efi_setup_data
*esd
= (void *)sd
+ sizeof(struct setup_data
);
144 esd
->fw_vendor
= efi_fw_vendor
;
145 esd
->tables
= efi_config_table
;
146 esd
->smbios
= efi
.smbios
;
148 sd
->type
= SETUP_EFI
;
149 sd
->len
= sizeof(struct efi_setup_data
);
152 setup_data_phys
= params_load_addr
+ efi_setup_data_offset
;
153 sd
->next
= params
->hdr
.setup_data
;
154 params
->hdr
.setup_data
= setup_data_phys
;
160 setup_efi_state(struct boot_params
*params
, unsigned long params_load_addr
,
161 unsigned int efi_map_offset
, unsigned int efi_map_sz
,
162 unsigned int efi_setup_data_offset
)
164 struct efi_info
*current_ei
= &boot_params
.efi_info
;
165 struct efi_info
*ei
= ¶ms
->efi_info
;
167 if (!efi_enabled(EFI_RUNTIME_SERVICES
))
170 if (!current_ei
->efi_memmap_size
)
173 params
->secure_boot
= boot_params
.secure_boot
;
174 ei
->efi_loader_signature
= current_ei
->efi_loader_signature
;
175 ei
->efi_systab
= current_ei
->efi_systab
;
176 ei
->efi_systab_hi
= current_ei
->efi_systab_hi
;
178 ei
->efi_memdesc_version
= current_ei
->efi_memdesc_version
;
179 ei
->efi_memdesc_size
= efi_get_runtime_map_desc_size();
181 setup_efi_info_memmap(params
, params_load_addr
, efi_map_offset
,
183 prepare_add_efi_setup_data(params
, params_load_addr
,
184 efi_setup_data_offset
);
187 #endif /* CONFIG_EFI */
190 setup_boot_parameters(struct kimage
*image
, struct boot_params
*params
,
191 unsigned long params_load_addr
,
192 unsigned int efi_map_offset
, unsigned int efi_map_sz
,
193 unsigned int efi_setup_data_offset
)
195 unsigned int nr_e820_entries
;
196 unsigned long long mem_k
, start
, end
;
199 /* Get subarch from existing bootparams */
200 params
->hdr
.hardware_subarch
= boot_params
.hdr
.hardware_subarch
;
202 /* Copying screen_info will do? */
203 memcpy(¶ms
->screen_info
, &screen_info
, sizeof(struct screen_info
));
205 /* Fill in memsize later */
206 params
->screen_info
.ext_mem_k
= 0;
207 params
->alt_mem_k
= 0;
209 /* Always fill in RSDP: it is either 0 or a valid value */
210 params
->acpi_rsdp_addr
= boot_params
.acpi_rsdp_addr
;
212 /* Default APM info */
213 memset(¶ms
->apm_bios_info
, 0, sizeof(params
->apm_bios_info
));
215 /* Default drive info */
216 memset(¶ms
->hd0_info
, 0, sizeof(params
->hd0_info
));
217 memset(¶ms
->hd1_info
, 0, sizeof(params
->hd1_info
));
219 if (image
->type
== KEXEC_TYPE_CRASH
) {
220 ret
= crash_setup_memmap_entries(image
, params
);
224 setup_e820_entries(params
);
226 nr_e820_entries
= params
->e820_entries
;
228 for (i
= 0; i
< nr_e820_entries
; i
++) {
229 if (params
->e820_table
[i
].type
!= E820_TYPE_RAM
)
231 start
= params
->e820_table
[i
].addr
;
232 end
= params
->e820_table
[i
].addr
+ params
->e820_table
[i
].size
- 1;
234 if ((start
<= 0x100000) && end
> 0x100000) {
235 mem_k
= (end
>> 10) - (0x100000 >> 10);
236 params
->screen_info
.ext_mem_k
= mem_k
;
237 params
->alt_mem_k
= mem_k
;
239 params
->screen_info
.ext_mem_k
= 0xfc00; /* 64M*/
240 if (mem_k
> 0xffffffff)
241 params
->alt_mem_k
= 0xffffffff;
246 /* Setup EFI state */
247 setup_efi_state(params
, params_load_addr
, efi_map_offset
, efi_map_sz
,
248 efi_setup_data_offset
);
251 memcpy(params
->eddbuf
, boot_params
.eddbuf
,
252 EDDMAXNR
* sizeof(struct edd_info
));
253 params
->eddbuf_entries
= boot_params
.eddbuf_entries
;
255 memcpy(params
->edd_mbr_sig_buffer
, boot_params
.edd_mbr_sig_buffer
,
256 EDD_MBR_SIG_MAX
* sizeof(unsigned int));
261 static int bzImage64_probe(const char *buf
, unsigned long len
)
264 struct setup_header
*header
;
266 /* kernel should be at least two sectors long */
268 pr_err("File is too short to be a bzImage\n");
272 header
= (struct setup_header
*)(buf
+ offsetof(struct boot_params
, hdr
));
273 if (memcmp((char *)&header
->header
, "HdrS", 4) != 0) {
274 pr_err("Not a bzImage\n");
278 if (header
->boot_flag
!= 0xAA55) {
279 pr_err("No x86 boot sector present\n");
283 if (header
->version
< 0x020C) {
284 pr_err("Must be at least protocol version 2.12\n");
288 if (!(header
->loadflags
& LOADED_HIGH
)) {
289 pr_err("zImage not a bzImage\n");
293 if (!(header
->xloadflags
& XLF_KERNEL_64
)) {
294 pr_err("Not a bzImage64. XLF_KERNEL_64 is not set.\n");
298 if (!(header
->xloadflags
& XLF_CAN_BE_LOADED_ABOVE_4G
)) {
299 pr_err("XLF_CAN_BE_LOADED_ABOVE_4G is not set.\n");
304 * Can't handle 32bit EFI as it does not allow loading kernel
305 * above 4G. This should be handled by 32bit bzImage loader
307 if (efi_enabled(EFI_RUNTIME_SERVICES
) && !efi_enabled(EFI_64BIT
)) {
308 pr_debug("EFI is 32 bit. Can't load kernel above 4G.\n");
312 if (!(header
->xloadflags
& XLF_5LEVEL
) && pgtable_l5_enabled()) {
313 pr_err("bzImage cannot handle 5-level paging mode.\n");
317 /* I've got a bzImage */
318 pr_debug("It's a relocatable bzImage64\n");
324 static void *bzImage64_load(struct kimage
*image
, char *kernel
,
325 unsigned long kernel_len
, char *initrd
,
326 unsigned long initrd_len
, char *cmdline
,
327 unsigned long cmdline_len
)
330 struct setup_header
*header
;
331 int setup_sects
, kern16_size
, ret
= 0;
332 unsigned long setup_header_size
, params_cmdline_sz
;
333 struct boot_params
*params
;
334 unsigned long bootparam_load_addr
, kernel_load_addr
, initrd_load_addr
;
335 struct bzimage64_data
*ldata
;
336 struct kexec_entry64_regs regs64
;
338 unsigned int setup_hdr_offset
= offsetof(struct boot_params
, hdr
);
339 unsigned int efi_map_offset
, efi_map_sz
, efi_setup_data_offset
;
340 struct kexec_buf kbuf
= { .image
= image
, .buf_max
= ULONG_MAX
,
342 struct kexec_buf pbuf
= { .image
= image
, .buf_min
= MIN_PURGATORY_ADDR
,
343 .buf_max
= ULONG_MAX
, .top_down
= true };
345 header
= (struct setup_header
*)(kernel
+ setup_hdr_offset
);
346 setup_sects
= header
->setup_sects
;
347 if (setup_sects
== 0)
350 kern16_size
= (setup_sects
+ 1) * 512;
351 if (kernel_len
< kern16_size
) {
352 pr_err("bzImage truncated\n");
353 return ERR_PTR(-ENOEXEC
);
356 if (cmdline_len
> header
->cmdline_size
) {
357 pr_err("Kernel command line too long\n");
358 return ERR_PTR(-EINVAL
);
362 * In case of crash dump, we will append elfcorehdr=<addr> to
363 * command line. Make sure it does not overflow
365 if (cmdline_len
+ MAX_ELFCOREHDR_STR_LEN
> header
->cmdline_size
) {
366 pr_debug("Appending elfcorehdr=<addr> to command line exceeds maximum allowed length\n");
367 return ERR_PTR(-EINVAL
);
370 /* Allocate and load backup region */
371 if (image
->type
== KEXEC_TYPE_CRASH
) {
372 ret
= crash_load_segments(image
);
378 * Load purgatory. For 64bit entry point, purgatory code can be
381 ret
= kexec_load_purgatory(image
, &pbuf
);
383 pr_err("Loading purgatory failed\n");
387 pr_debug("Loaded purgatory at 0x%lx\n", pbuf
.mem
);
391 * Load Bootparams and cmdline and space for efi stuff.
393 * Allocate memory together for multiple data structures so
394 * that they all can go in single area/segment and we don't
395 * have to create separate segment for each. Keeps things
398 efi_map_sz
= efi_get_runtime_map_size();
399 params_cmdline_sz
= sizeof(struct boot_params
) + cmdline_len
+
400 MAX_ELFCOREHDR_STR_LEN
;
401 params_cmdline_sz
= ALIGN(params_cmdline_sz
, 16);
402 kbuf
.bufsz
= params_cmdline_sz
+ ALIGN(efi_map_sz
, 16) +
403 sizeof(struct setup_data
) +
404 sizeof(struct efi_setup_data
);
406 params
= kzalloc(kbuf
.bufsz
, GFP_KERNEL
);
408 return ERR_PTR(-ENOMEM
);
409 efi_map_offset
= params_cmdline_sz
;
410 efi_setup_data_offset
= efi_map_offset
+ ALIGN(efi_map_sz
, 16);
412 /* Copy setup header onto bootparams. Documentation/x86/boot.rst */
413 setup_header_size
= 0x0202 + kernel
[0x0201] - setup_hdr_offset
;
415 /* Is there a limit on setup header size? */
416 memcpy(¶ms
->hdr
, (kernel
+ setup_hdr_offset
), setup_header_size
);
418 kbuf
.buffer
= params
;
419 kbuf
.memsz
= kbuf
.bufsz
;
421 kbuf
.buf_min
= MIN_BOOTPARAM_ADDR
;
422 ret
= kexec_add_buffer(&kbuf
);
424 goto out_free_params
;
425 bootparam_load_addr
= kbuf
.mem
;
426 pr_debug("Loaded boot_param, command line and misc at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
427 bootparam_load_addr
, kbuf
.bufsz
, kbuf
.bufsz
);
430 kbuf
.buffer
= kernel
+ kern16_size
;
431 kbuf
.bufsz
= kernel_len
- kern16_size
;
432 kbuf
.memsz
= PAGE_ALIGN(header
->init_size
);
433 kbuf
.buf_align
= header
->kernel_alignment
;
434 kbuf
.buf_min
= MIN_KERNEL_LOAD_ADDR
;
435 kbuf
.mem
= KEXEC_BUF_MEM_UNKNOWN
;
436 ret
= kexec_add_buffer(&kbuf
);
438 goto out_free_params
;
439 kernel_load_addr
= kbuf
.mem
;
441 pr_debug("Loaded 64bit kernel at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
442 kernel_load_addr
, kbuf
.bufsz
, kbuf
.memsz
);
444 /* Load initrd high */
446 kbuf
.buffer
= initrd
;
447 kbuf
.bufsz
= kbuf
.memsz
= initrd_len
;
448 kbuf
.buf_align
= PAGE_SIZE
;
449 kbuf
.buf_min
= MIN_INITRD_LOAD_ADDR
;
450 kbuf
.mem
= KEXEC_BUF_MEM_UNKNOWN
;
451 ret
= kexec_add_buffer(&kbuf
);
453 goto out_free_params
;
454 initrd_load_addr
= kbuf
.mem
;
456 pr_debug("Loaded initrd at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
457 initrd_load_addr
, initrd_len
, initrd_len
);
459 setup_initrd(params
, initrd_load_addr
, initrd_len
);
462 setup_cmdline(image
, params
, bootparam_load_addr
,
463 sizeof(struct boot_params
), cmdline
, cmdline_len
);
465 /* bootloader info. Do we need a separate ID for kexec kernel loader? */
466 params
->hdr
.type_of_loader
= 0x0D << 4;
467 params
->hdr
.loadflags
= 0;
469 /* Setup purgatory regs for entry */
470 ret
= kexec_purgatory_get_set_symbol(image
, "entry64_regs", ®s64
,
473 goto out_free_params
;
475 regs64
.rbx
= 0; /* Bootstrap Processor */
476 regs64
.rsi
= bootparam_load_addr
;
477 regs64
.rip
= kernel_load_addr
+ 0x200;
478 stack
= kexec_purgatory_get_symbol_addr(image
, "stack_end");
480 pr_err("Could not find address of symbol stack_end\n");
482 goto out_free_params
;
485 regs64
.rsp
= (unsigned long)stack
;
486 ret
= kexec_purgatory_get_set_symbol(image
, "entry64_regs", ®s64
,
489 goto out_free_params
;
491 ret
= setup_boot_parameters(image
, params
, bootparam_load_addr
,
492 efi_map_offset
, efi_map_sz
,
493 efi_setup_data_offset
);
495 goto out_free_params
;
497 /* Allocate loader specific data */
498 ldata
= kzalloc(sizeof(struct bzimage64_data
), GFP_KERNEL
);
501 goto out_free_params
;
505 * Store pointer to params so that it could be freed after loading
506 * params segment has been loaded and contents have been copied
509 ldata
->bootparams_buf
= params
;
517 /* This cleanup function is called after various segments have been loaded */
518 static int bzImage64_cleanup(void *loader_data
)
520 struct bzimage64_data
*ldata
= loader_data
;
525 kfree(ldata
->bootparams_buf
);
526 ldata
->bootparams_buf
= NULL
;
531 #ifdef CONFIG_KEXEC_BZIMAGE_VERIFY_SIG
532 static int bzImage64_verify_sig(const char *kernel
, unsigned long kernel_len
)
536 ret
= verify_pefile_signature(kernel
, kernel_len
,
537 VERIFY_USE_SECONDARY_KEYRING
,
538 VERIFYING_KEXEC_PE_SIGNATURE
);
539 if (ret
== -ENOKEY
&& IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING
)) {
540 ret
= verify_pefile_signature(kernel
, kernel_len
,
541 VERIFY_USE_PLATFORM_KEYRING
,
542 VERIFYING_KEXEC_PE_SIGNATURE
);
548 const struct kexec_file_ops kexec_bzImage64_ops
= {
549 .probe
= bzImage64_probe
,
550 .load
= bzImage64_load
,
551 .cleanup
= bzImage64_cleanup
,
552 #ifdef CONFIG_KEXEC_BZIMAGE_VERIFY_SIG
553 .verify_sig
= bzImage64_verify_sig
,