[mirror_ubuntu-jammy-kernel.git] / arch / powerpc / kernel / kexec_elf_64.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Load ELF vmlinux file for the kexec_file_load syscall.
 *
 * Copyright (C) 2004  Adam Litke (agl@us.ibm.com)
 * Copyright (C) 2004  IBM Corp.
 * Copyright (C) 2005  R Sharada (sharada@in.ibm.com)
 * Copyright (C) 2006  Mohan Kumar M (mohan@in.ibm.com)
 * Copyright (C) 2016  IBM Corporation
 *
 * Based on kexec-tools' kexec-elf-exec.c and kexec-elf-ppc64.c.
 * Heavily modified for the kernel by
 * Thiago Jung Bauermann <bauerman@linux.vnet.ibm.com>.
 */

#define pr_fmt(fmt)	"kexec_elf: " fmt

#include <linux/elf.h>
#include <linux/kexec.h>
#include <linux/libfdt.h>
#include <linux/module.h>
#include <linux/of_fdt.h>
#include <linux/slab.h>
#include <linux/types.h>

#define PURGATORY_STACK_SIZE	(16 * 1024)

#define elf_addr_to_cpu	elf64_to_cpu

#ifndef Elf_Rel
#define Elf_Rel		Elf64_Rel
#endif /* Elf_Rel */

struct elf_info {
	/*
	 * Where the ELF binary contents are kept.
	 * Memory managed by the user of the struct.
	 */
	const char *buffer;

	const struct elfhdr *ehdr;
	const struct elf_phdr *proghdrs;
	struct elf_shdr *sechdrs;
};

static inline bool elf_is_elf_file(const struct elfhdr *ehdr)
{
       return memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0;
}

static uint64_t elf64_to_cpu(const struct elfhdr *ehdr, uint64_t value)
{
	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
		value = le64_to_cpu(value);
	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
		value = be64_to_cpu(value);

	return value;
}

static uint16_t elf16_to_cpu(const struct elfhdr *ehdr, uint16_t value)
{
	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
		value = le16_to_cpu(value);
	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
		value = be16_to_cpu(value);

	return value;
}

static uint32_t elf32_to_cpu(const struct elfhdr *ehdr, uint32_t value)
{
	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
		value = le32_to_cpu(value);
	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
		value = be32_to_cpu(value);

	return value;
}

/**
 * elf_is_ehdr_sane - check that it is safe to use the ELF header
 * @buf_len:	size of the buffer in which the ELF file is loaded.
 */
static bool elf_is_ehdr_sane(const struct elfhdr *ehdr, size_t buf_len)
{
	if (ehdr->e_phnum > 0 && ehdr->e_phentsize != sizeof(struct elf_phdr)) {
		pr_debug("Bad program header size.\n");
		return false;
	} else if (ehdr->e_shnum > 0 &&
		   ehdr->e_shentsize != sizeof(struct elf_shdr)) {
		pr_debug("Bad section header size.\n");
		return false;
	} else if (ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
		   ehdr->e_version != EV_CURRENT) {
		pr_debug("Unknown ELF version.\n");
		return false;
	}

	if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
		size_t phdr_size;

		/*
		 * e_phnum is at most 65535 so calculating the size of the
		 * program header cannot overflow.
		 */
		phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;

		/* Sanity check the program header table location. */
		if (ehdr->e_phoff + phdr_size < ehdr->e_phoff) {
			pr_debug("Program headers at invalid location.\n");
			return false;
		} else if (ehdr->e_phoff + phdr_size > buf_len) {
			pr_debug("Program headers truncated.\n");
			return false;
		}
	}

	if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
		size_t shdr_size;

		/*
		 * e_shnum is at most 65536 so calculating
		 * the size of the section header cannot overflow.
		 */
		shdr_size = sizeof(struct elf_shdr) * ehdr->e_shnum;

		/* Sanity check the section header table location. */
		if (ehdr->e_shoff + shdr_size < ehdr->e_shoff) {
			pr_debug("Section headers at invalid location.\n");
			return false;
		} else if (ehdr->e_shoff + shdr_size > buf_len) {
			pr_debug("Section headers truncated.\n");
			return false;
		}
	}

	return true;
}

static int elf_read_ehdr(const char *buf, size_t len, struct elfhdr *ehdr)
{
	struct elfhdr *buf_ehdr;

	if (len < sizeof(*buf_ehdr)) {
		pr_debug("Buffer is too small to hold ELF header.\n");
		return -ENOEXEC;
	}

	memset(ehdr, 0, sizeof(*ehdr));
	memcpy(ehdr->e_ident, buf, sizeof(ehdr->e_ident));
	if (!elf_is_elf_file(ehdr)) {
		pr_debug("No ELF header magic.\n");
		return -ENOEXEC;
	}

	if (ehdr->e_ident[EI_CLASS] != ELF_CLASS) {
		pr_debug("Not a supported ELF class.\n");
		return -ENOEXEC;
	} else  if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
		ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
		pr_debug("Not a supported ELF data format.\n");
		return -ENOEXEC;
	}

	buf_ehdr = (struct elfhdr *) buf;
	if (elf16_to_cpu(ehdr, buf_ehdr->e_ehsize) != sizeof(*buf_ehdr)) {
		pr_debug("Bad ELF header size.\n");
		return -ENOEXEC;
	}

	ehdr->e_type      = elf16_to_cpu(ehdr, buf_ehdr->e_type);
	ehdr->e_machine   = elf16_to_cpu(ehdr, buf_ehdr->e_machine);
	ehdr->e_version   = elf32_to_cpu(ehdr, buf_ehdr->e_version);
	ehdr->e_entry     = elf_addr_to_cpu(ehdr, buf_ehdr->e_entry);
	ehdr->e_phoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_phoff);
	ehdr->e_shoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_shoff);
	ehdr->e_flags     = elf32_to_cpu(ehdr, buf_ehdr->e_flags);
	ehdr->e_phentsize = elf16_to_cpu(ehdr, buf_ehdr->e_phentsize);
	ehdr->e_phnum     = elf16_to_cpu(ehdr, buf_ehdr->e_phnum);
	ehdr->e_shentsize = elf16_to_cpu(ehdr, buf_ehdr->e_shentsize);
	ehdr->e_shnum     = elf16_to_cpu(ehdr, buf_ehdr->e_shnum);
	ehdr->e_shstrndx  = elf16_to_cpu(ehdr, buf_ehdr->e_shstrndx);

	return elf_is_ehdr_sane(ehdr, len) ? 0 : -ENOEXEC;
}

/**
 * elf_is_phdr_sane - check that it is safe to use the program header
 * @buf_len:	size of the buffer in which the ELF file is loaded.
 */
static bool elf_is_phdr_sane(const struct elf_phdr *phdr, size_t buf_len)
{

	if (phdr->p_offset + phdr->p_filesz < phdr->p_offset) {
		pr_debug("ELF segment location wraps around.\n");
		return false;
	} else if (phdr->p_offset + phdr->p_filesz > buf_len) {
		pr_debug("ELF segment not in file.\n");
		return false;
	} else if (phdr->p_paddr + phdr->p_memsz < phdr->p_paddr) {
		pr_debug("ELF segment address wraps around.\n");
		return false;
	}

	return true;
}

static int elf_read_phdr(const char *buf, size_t len, struct elf_info *elf_info,
			 int idx)
{
	/* Override the const in proghdrs, we are the ones doing the loading. */
	struct elf_phdr *phdr = (struct elf_phdr *) &elf_info->proghdrs[idx];
	const char *pbuf;
	struct elf_phdr *buf_phdr;

	pbuf = buf + elf_info->ehdr->e_phoff + (idx * sizeof(*buf_phdr));
	buf_phdr = (struct elf_phdr *) pbuf;

	phdr->p_type   = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_type);
	phdr->p_offset = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_offset);
	phdr->p_paddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_paddr);
	phdr->p_vaddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_vaddr);
	phdr->p_flags  = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_flags);

	/*
	 * The following fields have a type equivalent to Elf_Addr
	 * both in 32 bit and 64 bit ELF.
	 */
	phdr->p_filesz = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_filesz);
	phdr->p_memsz  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_memsz);
	phdr->p_align  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_align);

	return elf_is_phdr_sane(phdr, len) ? 0 : -ENOEXEC;
}

/**
 * elf_read_phdrs - read the program headers from the buffer
 *
 * This function assumes that the program header table was checked for sanity.
 * Use elf_is_ehdr_sane() if it wasn't.
 */
static int elf_read_phdrs(const char *buf, size_t len,
			  struct elf_info *elf_info)
{
	size_t phdr_size, i;
	const struct elfhdr *ehdr = elf_info->ehdr;

	/*
	 * e_phnum is at most 65535 so calculating the size of the
	 * program header cannot overflow.
	 */
	phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;

	elf_info->proghdrs = kzalloc(phdr_size, GFP_KERNEL);
	if (!elf_info->proghdrs)
		return -ENOMEM;

	for (i = 0; i < ehdr->e_phnum; i++) {
		int ret;

		ret = elf_read_phdr(buf, len, elf_info, i);
		if (ret) {
			kfree(elf_info->proghdrs);
			elf_info->proghdrs = NULL;
			return ret;
		}
	}

	return 0;
}

/**
 * elf_is_shdr_sane - check that it is safe to use the section header
 * @buf_len:	size of the buffer in which the ELF file is loaded.
 */
static bool elf_is_shdr_sane(const struct elf_shdr *shdr, size_t buf_len)
{
	bool size_ok;

	/* SHT_NULL headers have undefined values, so we can't check them. */
	if (shdr->sh_type == SHT_NULL)
		return true;

	/* Now verify sh_entsize */
	switch (shdr->sh_type) {
	case SHT_SYMTAB:
		size_ok = shdr->sh_entsize == sizeof(Elf_Sym);
		break;
	case SHT_RELA:
		size_ok = shdr->sh_entsize == sizeof(Elf_Rela);
		break;
	case SHT_DYNAMIC:
		size_ok = shdr->sh_entsize == sizeof(Elf_Dyn);
		break;
	case SHT_REL:
		size_ok = shdr->sh_entsize == sizeof(Elf_Rel);
		break;
	case SHT_NOTE:
	case SHT_PROGBITS:
	case SHT_HASH:
	case SHT_NOBITS:
	default:
		/*
		 * This is a section whose entsize requirements
		 * I don't care about.  If I don't know about
		 * the section I can't care about it's entsize
		 * requirements.
		 */
		size_ok = true;
		break;
	}

	if (!size_ok) {
		pr_debug("ELF section with wrong entry size.\n");
		return false;
	} else if (shdr->sh_addr + shdr->sh_size < shdr->sh_addr) {
		pr_debug("ELF section address wraps around.\n");
		return false;
	}

	if (shdr->sh_type != SHT_NOBITS) {
		if (shdr->sh_offset + shdr->sh_size < shdr->sh_offset) {
			pr_debug("ELF section location wraps around.\n");
			return false;
		} else if (shdr->sh_offset + shdr->sh_size > buf_len) {
			pr_debug("ELF section not in file.\n");
			return false;
		}
	}

	return true;
}

static int elf_read_shdr(const char *buf, size_t len, struct elf_info *elf_info,
			 int idx)
{
	struct elf_shdr *shdr = &elf_info->sechdrs[idx];
	const struct elfhdr *ehdr = elf_info->ehdr;
	const char *sbuf;
	struct elf_shdr *buf_shdr;

	sbuf = buf + ehdr->e_shoff + idx * sizeof(*buf_shdr);
	buf_shdr = (struct elf_shdr *) sbuf;

	shdr->sh_name      = elf32_to_cpu(ehdr, buf_shdr->sh_name);
	shdr->sh_type      = elf32_to_cpu(ehdr, buf_shdr->sh_type);
	shdr->sh_addr      = elf_addr_to_cpu(ehdr, buf_shdr->sh_addr);
	shdr->sh_offset    = elf_addr_to_cpu(ehdr, buf_shdr->sh_offset);
	shdr->sh_link      = elf32_to_cpu(ehdr, buf_shdr->sh_link);
	shdr->sh_info      = elf32_to_cpu(ehdr, buf_shdr->sh_info);

	/*
	 * The following fields have a type equivalent to Elf_Addr
	 * both in 32 bit and 64 bit ELF.
	 */
	shdr->sh_flags     = elf_addr_to_cpu(ehdr, buf_shdr->sh_flags);
	shdr->sh_size      = elf_addr_to_cpu(ehdr, buf_shdr->sh_size);
	shdr->sh_addralign = elf_addr_to_cpu(ehdr, buf_shdr->sh_addralign);
	shdr->sh_entsize   = elf_addr_to_cpu(ehdr, buf_shdr->sh_entsize);

	return elf_is_shdr_sane(shdr, len) ? 0 : -ENOEXEC;
}

/**
 * elf_read_shdrs - read the section headers from the buffer
 *
 * This function assumes that the section header table was checked for sanity.
 * Use elf_is_ehdr_sane() if it wasn't.
 */
static int elf_read_shdrs(const char *buf, size_t len,
			  struct elf_info *elf_info)
{
	size_t shdr_size, i;

	/*
	 * e_shnum is at most 65536 so calculating
	 * the size of the section header cannot overflow.
	 */
	shdr_size = sizeof(struct elf_shdr) * elf_info->ehdr->e_shnum;

	elf_info->sechdrs = kzalloc(shdr_size, GFP_KERNEL);
	if (!elf_info->sechdrs)
		return -ENOMEM;

	for (i = 0; i < elf_info->ehdr->e_shnum; i++) {
		int ret;

		ret = elf_read_shdr(buf, len, elf_info, i);
		if (ret) {
			kfree(elf_info->sechdrs);
			elf_info->sechdrs = NULL;
			return ret;
		}
	}

	return 0;
}

/**
 * elf_read_from_buffer - read ELF file and sets up ELF header and ELF info
 * @buf:	Buffer to read ELF file from.
 * @len:	Size of @buf.
 * @ehdr:	Pointer to existing struct which will be populated.
 * @elf_info:	Pointer to existing struct which will be populated.
 *
 * This function allows reading ELF files with different byte order than
 * the kernel, byte-swapping the fields as needed.
 *
 * Return:
 * On success returns 0, and the caller should call elf_free_info(elf_info) to
 * free the memory allocated for the section and program headers.
 */
int elf_read_from_buffer(const char *buf, size_t len, struct elfhdr *ehdr,
			 struct elf_info *elf_info)
{
	int ret;

	ret = elf_read_ehdr(buf, len, ehdr);
	if (ret)
		return ret;

	elf_info->buffer = buf;
	elf_info->ehdr = ehdr;
	if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
		ret = elf_read_phdrs(buf, len, elf_info);
		if (ret)
			return ret;
	}
	if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
		ret = elf_read_shdrs(buf, len, elf_info);
		if (ret) {
			kfree(elf_info->proghdrs);
			return ret;
		}
	}

	return 0;
}

/**
 * elf_free_info - free memory allocated by elf_read_from_buffer
 */
void elf_free_info(struct elf_info *elf_info)
{
	kfree(elf_info->proghdrs);
	kfree(elf_info->sechdrs);
	memset(elf_info, 0, sizeof(*elf_info));
}
/**
 * build_elf_exec_info - read ELF executable and check that we can use it
 */
static int build_elf_exec_info(const char *buf, size_t len, struct elfhdr *ehdr,
			       struct elf_info *elf_info)
{
	int i;
	int ret;

	ret = elf_read_from_buffer(buf, len, ehdr, elf_info);
	if (ret)
		return ret;

	/* Big endian vmlinux has type ET_DYN. */
	if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) {
		pr_err("Not an ELF executable.\n");
		goto error;
	} else if (!elf_info->proghdrs) {
		pr_err("No ELF program header.\n");
		goto error;
	}

	for (i = 0; i < ehdr->e_phnum; i++) {
		/*
		 * Kexec does not support loading interpreters.
		 * In addition this check keeps us from attempting
		 * to kexec ordinay executables.
		 */
		if (elf_info->proghdrs[i].p_type == PT_INTERP) {
			pr_err("Requires an ELF interpreter.\n");
			goto error;
		}
	}

	return 0;
error:
	elf_free_info(elf_info);
	return -ENOEXEC;
}

static int elf64_probe(const char *buf, unsigned long len)
{
	struct elfhdr ehdr;
	struct elf_info elf_info;
	int ret;

	ret = build_elf_exec_info(buf, len, &ehdr, &elf_info);
	if (ret)
		return ret;

	elf_free_info(&elf_info);

	return elf_check_arch(&ehdr) ? 0 : -ENOEXEC;
}

/**
 * elf_exec_load - load ELF executable image
 * @lowest_load_addr:	On return, will be the address where the first PT_LOAD
 *			section will be loaded in memory.
 *
 * Return:
 * 0 on success, negative value on failure.
 */
static int elf_exec_load(struct kimage *image, struct elfhdr *ehdr,
			 struct elf_info *elf_info,
			 unsigned long *lowest_load_addr)
{
	unsigned long base = 0, lowest_addr = UINT_MAX;
	int ret;
	size_t i;
	struct kexec_buf kbuf = { .image = image, .buf_max = ppc64_rma_size,
				  .top_down = false };

	/* Read in the PT_LOAD segments. */
	for (i = 0; i < ehdr->e_phnum; i++) {
		unsigned long load_addr;
		size_t size;
		const struct elf_phdr *phdr;

		phdr = &elf_info->proghdrs[i];
		if (phdr->p_type != PT_LOAD)
			continue;

		size = phdr->p_filesz;
		if (size > phdr->p_memsz)
			size = phdr->p_memsz;

		kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
		kbuf.bufsz = size;
		kbuf.memsz = phdr->p_memsz;
		kbuf.buf_align = phdr->p_align;
		kbuf.buf_min = phdr->p_paddr + base;
		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
		ret = kexec_add_buffer(&kbuf);
		if (ret)
			goto out;
		load_addr = kbuf.mem;

		if (load_addr < lowest_addr)
			lowest_addr = load_addr;
	}

	/* Update entry point to reflect new load address. */
	ehdr->e_entry += base;

	*lowest_load_addr = lowest_addr;
	ret = 0;
 out:
	return ret;
}

static void *elf64_load(struct kimage *image, char *kernel_buf,
			unsigned long kernel_len, char *initrd,
			unsigned long initrd_len, char *cmdline,
			unsigned long cmdline_len)
{
	int ret;
	unsigned int fdt_size;
	unsigned long kernel_load_addr;
	unsigned long initrd_load_addr = 0, fdt_load_addr;
	void *fdt;
	const void *slave_code;
	struct elfhdr ehdr;
	struct elf_info elf_info;
	struct kexec_buf kbuf = { .image = image, .buf_min = 0,
				  .buf_max = ppc64_rma_size };
	struct kexec_buf pbuf = { .image = image, .buf_min = 0,
				  .buf_max = ppc64_rma_size, .top_down = true,
				  .mem = KEXEC_BUF_MEM_UNKNOWN };

	ret = build_elf_exec_info(kernel_buf, kernel_len, &ehdr, &elf_info);
	if (ret)
		goto out;

	ret = elf_exec_load(image, &ehdr, &elf_info, &kernel_load_addr);
	if (ret)
		goto out;

	pr_debug("Loaded the kernel at 0x%lx\n", kernel_load_addr);

	ret = kexec_load_purgatory(image, &pbuf);
	if (ret) {
		pr_err("Loading purgatory failed.\n");
		goto out;
	}

	pr_debug("Loaded purgatory at 0x%lx\n", pbuf.mem);

	if (initrd != NULL) {
		kbuf.buffer = initrd;
		kbuf.bufsz = kbuf.memsz = initrd_len;
		kbuf.buf_align = PAGE_SIZE;
		kbuf.top_down = false;
		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
		ret = kexec_add_buffer(&kbuf);
		if (ret)
			goto out;
		initrd_load_addr = kbuf.mem;

		pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr);
	}

	fdt_size = fdt_totalsize(initial_boot_params) * 2;
	fdt = kmalloc(fdt_size, GFP_KERNEL);
	if (!fdt) {
		pr_err("Not enough memory for the device tree.\n");
		ret = -ENOMEM;
		goto out;
	}
	ret = fdt_open_into(initial_boot_params, fdt, fdt_size);
	if (ret < 0) {
		pr_err("Error setting up the new device tree.\n");
		ret = -EINVAL;
		goto out;
	}

	ret = setup_new_fdt(image, fdt, initrd_load_addr, initrd_len, cmdline);
	if (ret)
		goto out;

	fdt_pack(fdt);

	kbuf.buffer = fdt;
	kbuf.bufsz = kbuf.memsz = fdt_size;
	kbuf.buf_align = PAGE_SIZE;
	kbuf.top_down = true;
	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
	ret = kexec_add_buffer(&kbuf);
	if (ret)
		goto out;
	fdt_load_addr = kbuf.mem;

	pr_debug("Loaded device tree at 0x%lx\n", fdt_load_addr);

	slave_code = elf_info.buffer + elf_info.proghdrs[0].p_offset;
	ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
			      fdt_load_addr);
	if (ret)
		pr_err("Error setting up the purgatory.\n");

out:
	elf_free_info(&elf_info);

	/* Make kimage_file_post_load_cleanup free the fdt buffer for us. */
	return ret ? ERR_PTR(ret) : fdt;
}

const struct kexec_file_ops kexec_elf64_ops = {
	.probe = elf64_probe,
	.load = elf64_load,
};
Commit	Line	Data
8e8e69d6	1	// SPDX-License-Identifier: GPL-2.0-only
a0458284 TJB	2	/*
	3	* Load ELF vmlinux file for the kexec_file_load syscall.
	4	*
	5	* Copyright (C) 2004 Adam Litke (agl@us.ibm.com)
	6	* Copyright (C) 2004 IBM Corp.
	7	* Copyright (C) 2005 R Sharada (sharada@in.ibm.com)
	8	* Copyright (C) 2006 Mohan Kumar M (mohan@in.ibm.com)
	9	* Copyright (C) 2016 IBM Corporation
	10	*
	11	* Based on kexec-tools' kexec-elf-exec.c and kexec-elf-ppc64.c.
	12	* Heavily modified for the kernel by
	13	* Thiago Jung Bauermann <bauerman@linux.vnet.ibm.com>.
a0458284 TJB	14	*/
	15
	16	#define pr_fmt(fmt) "kexec_elf: " fmt
	17
	18	#include <linux/elf.h>
	19	#include <linux/kexec.h>
	20	#include <linux/libfdt.h>
	21	#include <linux/module.h>
	22	#include <linux/of_fdt.h>
	23	#include <linux/slab.h>
	24	#include <linux/types.h>
	25
	26	#define PURGATORY_STACK_SIZE (16 * 1024)
	27
	28	#define elf_addr_to_cpu elf64_to_cpu
	29
	30	#ifndef Elf_Rel
	31	#define Elf_Rel Elf64_Rel
	32	#endif /* Elf_Rel */
	33
	34	struct elf_info {
	35	/*
	36	* Where the ELF binary contents are kept.
	37	* Memory managed by the user of the struct.
	38	*/
	39	const char *buffer;
	40
	41	const struct elfhdr *ehdr;
	42	const struct elf_phdr *proghdrs;
	43	struct elf_shdr *sechdrs;
	44	};
	45
	46	static inline bool elf_is_elf_file(const struct elfhdr *ehdr)
	47	{
	48	return memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0;
	49	}
	50
	51	static uint64_t elf64_to_cpu(const struct elfhdr *ehdr, uint64_t value)
	52	{
	53	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
	54	value = le64_to_cpu(value);
	55	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
	56	value = be64_to_cpu(value);
	57
	58	return value;
	59	}
	60
	61	static uint16_t elf16_to_cpu(const struct elfhdr *ehdr, uint16_t value)
	62	{
	63	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
	64	value = le16_to_cpu(value);
	65	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
	66	value = be16_to_cpu(value);
	67
	68	return value;
	69	}
	70
	71	static uint32_t elf32_to_cpu(const struct elfhdr *ehdr, uint32_t value)
	72	{
	73	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
	74	value = le32_to_cpu(value);
	75	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
	76	value = be32_to_cpu(value);
	77
78	return value;
79	}
80
81	/**
82	* elf_is_ehdr_sane - check that it is safe to use the ELF header
83	* @buf_len: size of the buffer in which the ELF file is loaded.
84	*/
85	static bool elf_is_ehdr_sane(const struct elfhdr *ehdr, size_t buf_len)
86	{
87	if (ehdr->e_phnum > 0 && ehdr->e_phentsize != sizeof(struct elf_phdr)) {
88	pr_debug("Bad program header size.\n");
89	return false;
90	} else if (ehdr->e_shnum > 0 &&
91	ehdr->e_shentsize != sizeof(struct elf_shdr)) {
92	pr_debug("Bad section header size.\n");
93	return false;
94	} else if (ehdr->e_ident[EI_VERSION] != EV_CURRENT \|\|
95	ehdr->e_version != EV_CURRENT) {
96	pr_debug("Unknown ELF version.\n");
97	return false;
98	}
99
100	if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
101	size_t phdr_size;
102
103	/*
104	* e_phnum is at most 65535 so calculating the size of the
105	* program header cannot overflow.
106	*/
107	phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
108
109	/* Sanity check the program header table location. */
110	if (ehdr->e_phoff + phdr_size < ehdr->e_phoff) {
111	pr_debug("Program headers at invalid location.\n");
112	return false;
113	} else if (ehdr->e_phoff + phdr_size > buf_len) {
114	pr_debug("Program headers truncated.\n");
115	return false;
116	}
117	}
118
119	if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
120	size_t shdr_size;
121
122	/*
123	* e_shnum is at most 65536 so calculating
124	* the size of the section header cannot overflow.
125	*/
126	shdr_size = sizeof(struct elf_shdr) * ehdr->e_shnum;
127
128	/* Sanity check the section header table location. */
129	if (ehdr->e_shoff + shdr_size < ehdr->e_shoff) {
130	pr_debug("Section headers at invalid location.\n");
131	return false;
132	} else if (ehdr->e_shoff + shdr_size > buf_len) {
133	pr_debug("Section headers truncated.\n");
134	return false;
135	}
136	}
137
138	return true;
139	}
140
141	static int elf_read_ehdr(const char buf, size_t len, struct elfhdr ehdr)
142	{
143	struct elfhdr *buf_ehdr;
144
145	if (len < sizeof(*buf_ehdr)) {
146	pr_debug("Buffer is too small to hold ELF header.\n");
147	return -ENOEXEC;
148	}
149
150	memset(ehdr, 0, sizeof(*ehdr));
151	memcpy(ehdr->e_ident, buf, sizeof(ehdr->e_ident));
152	if (!elf_is_elf_file(ehdr)) {
153	pr_debug("No ELF header magic.\n");
154	return -ENOEXEC;
155	}
156
157	if (ehdr->e_ident[EI_CLASS] != ELF_CLASS) {
158	pr_debug("Not a supported ELF class.\n");
159	return -ENOEXEC;
160	} else if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
161	ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
162	pr_debug("Not a supported ELF data format.\n");
163	return -ENOEXEC;
164	}
165
166	buf_ehdr = (struct elfhdr *) buf;
167	if (elf16_to_cpu(ehdr, buf_ehdr->e_ehsize) != sizeof(*buf_ehdr)) {
168	pr_debug("Bad ELF header size.\n");
169	return -ENOEXEC;
170	}
171
172	ehdr->e_type = elf16_to_cpu(ehdr, buf_ehdr->e_type);
173	ehdr->e_machine = elf16_to_cpu(ehdr, buf_ehdr->e_machine);
174	ehdr->e_version = elf32_to_cpu(ehdr, buf_ehdr->e_version);
175	ehdr->e_entry = elf_addr_to_cpu(ehdr, buf_ehdr->e_entry);
176	ehdr->e_phoff = elf_addr_to_cpu(ehdr, buf_ehdr->e_phoff);
177	ehdr->e_shoff = elf_addr_to_cpu(ehdr, buf_ehdr->e_shoff);
178	ehdr->e_flags = elf32_to_cpu(ehdr, buf_ehdr->e_flags);
179	ehdr->e_phentsize = elf16_to_cpu(ehdr, buf_ehdr->e_phentsize);
180	ehdr->e_phnum = elf16_to_cpu(ehdr, buf_ehdr->e_phnum);
181	ehdr->e_shentsize = elf16_to_cpu(ehdr, buf_ehdr->e_shentsize);
182	ehdr->e_shnum = elf16_to_cpu(ehdr, buf_ehdr->e_shnum);
183	ehdr->e_shstrndx = elf16_to_cpu(ehdr, buf_ehdr->e_shstrndx);
184
185	return elf_is_ehdr_sane(ehdr, len) ? 0 : -ENOEXEC;
186	}
187
188	/**
189	* elf_is_phdr_sane - check that it is safe to use the program header
190	* @buf_len: size of the buffer in which the ELF file is loaded.
191	*/
192	static bool elf_is_phdr_sane(const struct elf_phdr *phdr, size_t buf_len)
193	{
194
195	if (phdr->p_offset + phdr->p_filesz < phdr->p_offset) {
196	pr_debug("ELF segment location wraps around.\n");
197	return false;
198	} else if (phdr->p_offset + phdr->p_filesz > buf_len) {
199	pr_debug("ELF segment not in file.\n");
200	return false;
201	} else if (phdr->p_paddr + phdr->p_memsz < phdr->p_paddr) {
202	pr_debug("ELF segment address wraps around.\n");
203	return false;
204	}
205
206	return true;
207	}
208
209	static int elf_read_phdr(const char buf, size_t len, struct elf_info elf_info,
210	int idx)
211	{
212	/* Override the const in proghdrs, we are the ones doing the loading. */
213	struct elf_phdr phdr = (struct elf_phdr ) &elf_info->proghdrs[idx];
214	const char *pbuf;
215	struct elf_phdr *buf_phdr;
216
217	pbuf = buf + elf_info->ehdr->e_phoff + (idx * sizeof(*buf_phdr));
218	buf_phdr = (struct elf_phdr *) pbuf;
219
220	phdr->p_type = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_type);
221	phdr->p_offset = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_offset);
222	phdr->p_paddr = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_paddr);
223	phdr->p_vaddr = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_vaddr);
224	phdr->p_flags = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_flags);
225
226	/*
227	* The following fields have a type equivalent to Elf_Addr
228	* both in 32 bit and 64 bit ELF.
229	*/
230	phdr->p_filesz = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_filesz);
231	phdr->p_memsz = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_memsz);
232	phdr->p_align = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_align);
233
234	return elf_is_phdr_sane(phdr, len) ? 0 : -ENOEXEC;
235	}
236
237	/**
238	* elf_read_phdrs - read the program headers from the buffer
239	*
240	* This function assumes that the program header table was checked for sanity.
241	* Use elf_is_ehdr_sane() if it wasn't.
242	*/
243	static int elf_read_phdrs(const char *buf, size_t len,
244	struct elf_info *elf_info)
245	{
246	size_t phdr_size, i;
247	const struct elfhdr *ehdr = elf_info->ehdr;
248
249	/*
250	* e_phnum is at most 65535 so calculating the size of the
251	* program header cannot overflow.
252	*/
253	phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
254
255	elf_info->proghdrs = kzalloc(phdr_size, GFP_KERNEL);
256	if (!elf_info->proghdrs)
257	return -ENOMEM;
258
259	for (i = 0; i < ehdr->e_phnum; i++) {
260	int ret;
261
262	ret = elf_read_phdr(buf, len, elf_info, i);
263	if (ret) {
264	kfree(elf_info->proghdrs);
265	elf_info->proghdrs = NULL;
266	return ret;
267	}
268	}
269
270	return 0;
271	}
272
273	/**
274	* elf_is_shdr_sane - check that it is safe to use the section header
275	* @buf_len: size of the buffer in which the ELF file is loaded.
276	*/
277	static bool elf_is_shdr_sane(const struct elf_shdr *shdr, size_t buf_len)
278	{
279	bool size_ok;
280
281	/* SHT_NULL headers have undefined values, so we can't check them. */
282	if (shdr->sh_type == SHT_NULL)
283	return true;
284
285	/* Now verify sh_entsize */
286	switch (shdr->sh_type) {
287	case SHT_SYMTAB:
288	size_ok = shdr->sh_entsize == sizeof(Elf_Sym);
289	break;
290	case SHT_RELA:
291	size_ok = shdr->sh_entsize == sizeof(Elf_Rela);
292	break;
293	case SHT_DYNAMIC:
294	size_ok = shdr->sh_entsize == sizeof(Elf_Dyn);
295	break;
296	case SHT_REL:
297	size_ok = shdr->sh_entsize == sizeof(Elf_Rel);
298	break;
299	case SHT_NOTE:
300	case SHT_PROGBITS:
301	case SHT_HASH:
302	case SHT_NOBITS:
303	default:
304	/*
305	* This is a section whose entsize requirements
306	* I don't care about. If I don't know about
307	* the section I can't care about it's entsize
308	* requirements.
309	*/
310	size_ok = true;
311	break;
312	}
313
314	if (!size_ok) {
315	pr_debug("ELF section with wrong entry size.\n");
316	return false;
317	} else if (shdr->sh_addr + shdr->sh_size < shdr->sh_addr) {
318	pr_debug("ELF section address wraps around.\n");
319	return false;
320	}
321
322	if (shdr->sh_type != SHT_NOBITS) {
323	if (shdr->sh_offset + shdr->sh_size < shdr->sh_offset) {
324	pr_debug("ELF section location wraps around.\n");
325	return false;
326	} else if (shdr->sh_offset + shdr->sh_size > buf_len) {
327	pr_debug("ELF section not in file.\n");
328	return false;
329	}
330	}
331
332	return true;
333	}
334
335	static int elf_read_shdr(const char buf, size_t len, struct elf_info elf_info,
336	int idx)
337	{
338	struct elf_shdr *shdr = &elf_info->sechdrs[idx];
339	const struct elfhdr *ehdr = elf_info->ehdr;
340	const char *sbuf;
341	struct elf_shdr *buf_shdr;
342
343	sbuf = buf + ehdr->e_shoff + idx * sizeof(*buf_shdr);
344	buf_shdr = (struct elf_shdr *) sbuf;
345
346	shdr->sh_name = elf32_to_cpu(ehdr, buf_shdr->sh_name);
347	shdr->sh_type = elf32_to_cpu(ehdr, buf_shdr->sh_type);
348	shdr->sh_addr = elf_addr_to_cpu(ehdr, buf_shdr->sh_addr);
349	shdr->sh_offset = elf_addr_to_cpu(ehdr, buf_shdr->sh_offset);
350	shdr->sh_link = elf32_to_cpu(ehdr, buf_shdr->sh_link);
351	shdr->sh_info = elf32_to_cpu(ehdr, buf_shdr->sh_info);
352
353	/*
354	* The following fields have a type equivalent to Elf_Addr
355	* both in 32 bit and 64 bit ELF.
356	*/
357	shdr->sh_flags = elf_addr_to_cpu(ehdr, buf_shdr->sh_flags);
358	shdr->sh_size = elf_addr_to_cpu(ehdr, buf_shdr->sh_size);
359	shdr->sh_addralign = elf_addr_to_cpu(ehdr, buf_shdr->sh_addralign);
360	shdr->sh_entsize = elf_addr_to_cpu(ehdr, buf_shdr->sh_entsize);
361
362	return elf_is_shdr_sane(shdr, len) ? 0 : -ENOEXEC;
363	}
364
365	/**
366	* elf_read_shdrs - read the section headers from the buffer
367	*
368	* This function assumes that the section header table was checked for sanity.
369	* Use elf_is_ehdr_sane() if it wasn't.
370	*/
371	static int elf_read_shdrs(const char *buf, size_t len,
372	struct elf_info *elf_info)
373	{
374	size_t shdr_size, i;
375
376	/*
377	* e_shnum is at most 65536 so calculating
378	* the size of the section header cannot overflow.
379	*/
380	shdr_size = sizeof(struct elf_shdr) * elf_info->ehdr->e_shnum;
381
382	elf_info->sechdrs = kzalloc(shdr_size, GFP_KERNEL);
383	if (!elf_info->sechdrs)
384	return -ENOMEM;
385
386	for (i = 0; i < elf_info->ehdr->e_shnum; i++) {
387	int ret;
388
389	ret = elf_read_shdr(buf, len, elf_info, i);
390	if (ret) {
391	kfree(elf_info->sechdrs);
392	elf_info->sechdrs = NULL;
393	return ret;
394	}
395	}
396
397	return 0;
398	}
399
400	/**
401	* elf_read_from_buffer - read ELF file and sets up ELF header and ELF info
402	* @buf: Buffer to read ELF file from.
403	* @len: Size of @buf.
404	* @ehdr: Pointer to existing struct which will be populated.
405	* @elf_info: Pointer to existing struct which will be populated.
406	*
407	* This function allows reading ELF files with different byte order than
408	* the kernel, byte-swapping the fields as needed.
409	*
410	* Return:
411	* On success returns 0, and the caller should call elf_free_info(elf_info) to
412	* free the memory allocated for the section and program headers.
413	*/
414	int elf_read_from_buffer(const char buf, size_t len, struct elfhdr ehdr,
415	struct elf_info *elf_info)
416	{
417	int ret;
418
419	ret = elf_read_ehdr(buf, len, ehdr);
420	if (ret)
421	return ret;
422
423	elf_info->buffer = buf;
424	elf_info->ehdr = ehdr;
425	if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
426	ret = elf_read_phdrs(buf, len, elf_info);
427	if (ret)
428	return ret;
429	}
430	if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
431	ret = elf_read_shdrs(buf, len, elf_info);
432	if (ret) {
433	kfree(elf_info->proghdrs);
434	return ret;
435	}
436	}
437
438	return 0;
439	}
440
441	/**
442	* elf_free_info - free memory allocated by elf_read_from_buffer
443	*/
444	void elf_free_info(struct elf_info *elf_info)
445	{
446	kfree(elf_info->proghdrs);
447	kfree(elf_info->sechdrs);
448	memset(elf_info, 0, sizeof(*elf_info));
449	}
450	/**
451	* build_elf_exec_info - read ELF executable and check that we can use it
452	*/
453	static int build_elf_exec_info(const char buf, size_t len, struct elfhdr ehdr,
454	struct elf_info *elf_info)
455	{
456	int i;
457	int ret;
458
459	ret = elf_read_from_buffer(buf, len, ehdr, elf_info);
460	if (ret)
461	return ret;
462
463	/* Big endian vmlinux has type ET_DYN. */
464	if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) {
465	pr_err("Not an ELF executable.\n");
466	goto error;
467	} else if (!elf_info->proghdrs) {
468	pr_err("No ELF program header.\n");
469	goto error;
470	}
471
472	for (i = 0; i < ehdr->e_phnum; i++) {
473	/*
474	* Kexec does not support loading interpreters.
475	* In addition this check keeps us from attempting
476	* to kexec ordinay executables.
477	*/
478	if (elf_info->proghdrs[i].p_type == PT_INTERP) {
479	pr_err("Requires an ELF interpreter.\n");
480	goto error;
481	}
482	}
483
484	return 0;
485	error:
486	elf_free_info(elf_info);
487	return -ENOEXEC;
488	}
489
490	static int elf64_probe(const char *buf, unsigned long len)
491	{
492	struct elfhdr ehdr;
493	struct elf_info elf_info;
494	int ret;
495
496	ret = build_elf_exec_info(buf, len, &ehdr, &elf_info);
497	if (ret)
498	return ret;
499
500	elf_free_info(&elf_info);
501
502	return elf_check_arch(&ehdr) ? 0 : -ENOEXEC;
503	}
504
505	/**
506	* elf_exec_load - load ELF executable image
507	* @lowest_load_addr: On return, will be the address where the first PT_LOAD
508	* section will be loaded in memory.
509	*
510	* Return:
511	* 0 on success, negative value on failure.
512	*/
513	static int elf_exec_load(struct kimage image, struct elfhdr ehdr,
514	struct elf_info *elf_info,
515	unsigned long *lowest_load_addr)
516	{
517	unsigned long base = 0, lowest_addr = UINT_MAX;
518	int ret;
519	size_t i;
520	struct kexec_buf kbuf = { .image = image, .buf_max = ppc64_rma_size,
521	.top_down = false };
522
523	/* Read in the PT_LOAD segments. */
524	for (i = 0; i < ehdr->e_phnum; i++) {
525	unsigned long load_addr;
526	size_t size;
527	const struct elf_phdr *phdr;
528
529	phdr = &elf_info->proghdrs[i];
530	if (phdr->p_type != PT_LOAD)
531	continue;
532
533	size = phdr->p_filesz;
534	if (size > phdr->p_memsz)
535	size = phdr->p_memsz;
536
537	kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
538	kbuf.bufsz = size;
539	kbuf.memsz = phdr->p_memsz;
540	kbuf.buf_align = phdr->p_align;
541	kbuf.buf_min = phdr->p_paddr + base;
8b909e35	542	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
a0458284 TJB	543	ret = kexec_add_buffer(&kbuf);
	544	if (ret)
	545	goto out;
	546	load_addr = kbuf.mem;
	547
	548	if (load_addr < lowest_addr)
	549	lowest_addr = load_addr;
	550	}
	551
	552	/* Update entry point to reflect new load address. */
	553	ehdr->e_entry += base;
	554
	555	*lowest_load_addr = lowest_addr;
	556	ret = 0;
	557	out:
	558	return ret;
	559	}
	560
	561	static void elf64_load(struct kimage image, char *kernel_buf,
	562	unsigned long kernel_len, char *initrd,
	563	unsigned long initrd_len, char *cmdline,
	564	unsigned long cmdline_len)
	565	{
	566	int ret;
	567	unsigned int fdt_size;
3be3f61d	568	unsigned long kernel_load_addr;
a0458284 TJB	569	unsigned long initrd_load_addr = 0, fdt_load_addr;
	570	void *fdt;
	571	const void *slave_code;
	572	struct elfhdr ehdr;
	573	struct elf_info elf_info;
	574	struct kexec_buf kbuf = { .image = image, .buf_min = 0,
	575	.buf_max = ppc64_rma_size };
3be3f61d	576	struct kexec_buf pbuf = { .image = image, .buf_min = 0,
8b909e35 TJB	577	.buf_max = ppc64_rma_size, .top_down = true,
8b909e35 TJB	578	.mem = KEXEC_BUF_MEM_UNKNOWN };
a0458284 TJB	579
	580	ret = build_elf_exec_info(kernel_buf, kernel_len, &ehdr, &elf_info);
	581	if (ret)
	582	goto out;
	583
	584	ret = elf_exec_load(image, &ehdr, &elf_info, &kernel_load_addr);
	585	if (ret)
	586	goto out;
	587
	588	pr_debug("Loaded the kernel at 0x%lx\n", kernel_load_addr);
	589
3be3f61d	590	ret = kexec_load_purgatory(image, &pbuf);
a0458284 TJB	591	if (ret) {
	592	pr_err("Loading purgatory failed.\n");
	593	goto out;
	594	}
	595
3be3f61d	596	pr_debug("Loaded purgatory at 0x%lx\n", pbuf.mem);
a0458284 TJB	597
	598	if (initrd != NULL) {
	599	kbuf.buffer = initrd;
	600	kbuf.bufsz = kbuf.memsz = initrd_len;
	601	kbuf.buf_align = PAGE_SIZE;
	602	kbuf.top_down = false;
8b909e35	603	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
a0458284 TJB	604	ret = kexec_add_buffer(&kbuf);
	605	if (ret)
	606	goto out;
	607	initrd_load_addr = kbuf.mem;
	608
	609	pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr);
	610	}
	611
	612	fdt_size = fdt_totalsize(initial_boot_params) * 2;
	613	fdt = kmalloc(fdt_size, GFP_KERNEL);
	614	if (!fdt) {
	615	pr_err("Not enough memory for the device tree.\n");
	616	ret = -ENOMEM;
	617	goto out;
	618	}
	619	ret = fdt_open_into(initial_boot_params, fdt, fdt_size);
	620	if (ret < 0) {
	621	pr_err("Error setting up the new device tree.\n");
	622	ret = -EINVAL;
	623	goto out;
	624	}
	625
ab6b1d1f	626	ret = setup_new_fdt(image, fdt, initrd_load_addr, initrd_len, cmdline);
a0458284 TJB	627	if (ret)
	628	goto out;
	629
	630	fdt_pack(fdt);
	631
	632	kbuf.buffer = fdt;
	633	kbuf.bufsz = kbuf.memsz = fdt_size;
	634	kbuf.buf_align = PAGE_SIZE;
	635	kbuf.top_down = true;
8b909e35	636	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
a0458284 TJB	637	ret = kexec_add_buffer(&kbuf);
	638	if (ret)
	639	goto out;
	640	fdt_load_addr = kbuf.mem;
	641
	642	pr_debug("Loaded device tree at 0x%lx\n", fdt_load_addr);
	643
	644	slave_code = elf_info.buffer + elf_info.proghdrs[0].p_offset;
	645	ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
	646	fdt_load_addr);
	647	if (ret)
	648	pr_err("Error setting up the purgatory.\n");
	649
	650	out:
	651	elf_free_info(&elf_info);
	652
	653	/* Make kimage_file_post_load_cleanup free the fdt buffer for us. */
	654	return ret ? ERR_PTR(ret) : fdt;
	655	}
	656
9ec4ecef	657	const struct kexec_file_ops kexec_elf64_ops = {
a0458284 TJB	658	.probe = elf64_probe,
	659	.load = elf64_load,
	660	};