[mirror_ubuntu-artful-kernel.git] / arch / powerpc / mm / mem.c

/*
 *  PowerPC version
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
 *    Copyright (C) 1996 Paul Mackerras
 *  PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
 *
 *  Derived from "arch/i386/mm/init.c"
 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 *
 */

#include <linux/export.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/gfp.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/highmem.h>
#include <linux/initrd.h>
#include <linux/pagemap.h>
#include <linux/suspend.h>
#include <linux/memblock.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/memremap.h>

#include <asm/pgalloc.h>
#include <asm/prom.h>
#include <asm/io.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/smp.h>
#include <asm/machdep.h>
#include <asm/btext.h>
#include <asm/tlb.h>
#include <asm/sections.h>
#include <asm/sparsemem.h>
#include <asm/vdso.h>
#include <asm/fixmap.h>
#include <asm/swiotlb.h>
#include <asm/rtas.h>

#include "mmu_decl.h"

#ifndef CPU_FTR_COHERENT_ICACHE
#define CPU_FTR_COHERENT_ICACHE	0	/* XXX for now */
#define CPU_FTR_NOEXECUTE	0
#endif

unsigned long long memory_limit;

#ifdef CONFIG_HIGHMEM
pte_t *kmap_pte;
EXPORT_SYMBOL(kmap_pte);
pgprot_t kmap_prot;
EXPORT_SYMBOL(kmap_prot);
#define TOP_ZONE ZONE_HIGHMEM

static inline pte_t *virt_to_kpte(unsigned long vaddr)
{
	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
			vaddr), vaddr), vaddr);
}
#else
#define TOP_ZONE ZONE_NORMAL
#endif

int page_is_ram(unsigned long pfn)
{
#ifndef CONFIG_PPC64	/* XXX for now */
	return pfn < max_pfn;
#else
	unsigned long paddr = (pfn << PAGE_SHIFT);
	struct memblock_region *reg;

	for_each_memblock(memory, reg)
		if (paddr >= reg->base && paddr < (reg->base + reg->size))
			return 1;
	return 0;
#endif
}

pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (ppc_md.phys_mem_access_prot)
		return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);

	if (!page_is_ram(pfn))
		vma_prot = pgprot_noncached(vma_prot);

	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

#ifdef CONFIG_MEMORY_HOTPLUG

#ifdef CONFIG_NUMA
int memory_add_physaddr_to_nid(u64 start)
{
	return hot_add_scn_to_nid(start);
}
#endif

int __weak create_section_mapping(unsigned long start, unsigned long end)
{
	return -ENODEV;
}

int __weak remove_section_mapping(unsigned long start, unsigned long end)
{
	return -ENODEV;
}

int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;
	int rc;

	resize_hpt_for_hotplug(memblock_phys_mem_size());

	start = (unsigned long)__va(start);
	rc = create_section_mapping(start, start + size);
	if (rc) {
		pr_warning(
			"Unable to create mapping for hot added memory 0x%llx..0x%llx: %d\n",
			start, start + size, rc);
		return -EFAULT;
	}

	return __add_pages(nid, start_pfn, nr_pages, want_memblock);
}

#ifdef CONFIG_MEMORY_HOTREMOVE
int arch_remove_memory(u64 start, u64 size)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;
	struct vmem_altmap *altmap;
	struct page *page;
	int ret;

	/*
	 * If we have an altmap then we need to skip over any reserved PFNs
	 * when querying the zone.
	 */
	page = pfn_to_page(start_pfn);
	altmap = to_vmem_altmap((unsigned long) page);
	if (altmap)
		page += vmem_altmap_offset(altmap);

	ret = __remove_pages(page_zone(page), start_pfn, nr_pages);
	if (ret)
		return ret;

	/* Remove htab bolted mappings for this section of memory */
	start = (unsigned long)__va(start);
	ret = remove_section_mapping(start, start + size);

	/* Ensure all vmalloc mappings are flushed in case they also
	 * hit that section of memory
	 */
	vm_unmap_aliases();

	resize_hpt_for_hotplug(memblock_phys_mem_size());

	return ret;
}
#endif
#endif /* CONFIG_MEMORY_HOTPLUG */

/*
 * walk_memory_resource() needs to make sure there is no holes in a given
 * memory range.  PPC64 does not maintain the memory layout in /proc/iomem.
 * Instead it maintains it in memblock.memory structures.  Walk through the
 * memory regions, find holes and callback for contiguous regions.
 */
int
walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
		void *arg, int (*func)(unsigned long, unsigned long, void *))
{
	struct memblock_region *reg;
	unsigned long end_pfn = start_pfn + nr_pages;
	unsigned long tstart, tend;
	int ret = -1;

	for_each_memblock(memory, reg) {
		tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
		tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
		if (tstart >= tend)
			continue;
		ret = (*func)(tstart, tend - tstart, arg);
		if (ret)
			break;
	}
	return ret;
}
EXPORT_SYMBOL_GPL(walk_system_ram_range);

#ifndef CONFIG_NEED_MULTIPLE_NODES
void __init initmem_init(void)
{
	max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
	min_low_pfn = MEMORY_START >> PAGE_SHIFT;
#ifdef CONFIG_HIGHMEM
	max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
#endif

	/* Place all memblock_regions in the same node and merge contiguous
	 * memblock_regions
	 */
	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);

	/* XXX need to clip this if using highmem? */
	sparse_memory_present_with_active_regions(0);
	sparse_init();
}

/* mark pages that don't exist as nosave */
static int __init mark_nonram_nosave(void)
{
	struct memblock_region *reg, *prev = NULL;

	for_each_memblock(memory, reg) {
		if (prev &&
		    memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
			register_nosave_region(memblock_region_memory_end_pfn(prev),
					       memblock_region_memory_base_pfn(reg));
		prev = reg;
	}
	return 0;
}
#else /* CONFIG_NEED_MULTIPLE_NODES */
static int __init mark_nonram_nosave(void)
{
	return 0;
}
#endif

static bool zone_limits_final;

/*
 * The memory zones past TOP_ZONE are managed by generic mm code.
 * These should be set to zero since that's what every other
 * architecture does.
 */
static unsigned long max_zone_pfns[MAX_NR_ZONES] = {
	[0            ... TOP_ZONE        ] = ~0UL,
	[TOP_ZONE + 1 ... MAX_NR_ZONES - 1] = 0
};

/*
 * Restrict the specified zone and all more restrictive zones
 * to be below the specified pfn.  May not be called after
 * paging_init().
 */
void __init limit_zone_pfn(enum zone_type zone, unsigned long pfn_limit)
{
	int i;

	if (WARN_ON(zone_limits_final))
		return;

	for (i = zone; i >= 0; i--) {
		if (max_zone_pfns[i] > pfn_limit)
			max_zone_pfns[i] = pfn_limit;
	}
}

/*
 * Find the least restrictive zone that is entirely below the
 * specified pfn limit.  Returns < 0 if no suitable zone is found.
 *
 * pfn_limit must be u64 because it can exceed 32 bits even on 32-bit
 * systems -- the DMA limit can be higher than any possible real pfn.
 */
int dma_pfn_limit_to_zone(u64 pfn_limit)
{
	int i;

	for (i = TOP_ZONE; i >= 0; i--) {
		if (max_zone_pfns[i] <= pfn_limit)
			return i;
	}

	return -EPERM;
}

/*
 * paging_init() sets up the page tables - in fact we've already done this.
 */
void __init paging_init(void)
{
	unsigned long long total_ram = memblock_phys_mem_size();
	phys_addr_t top_of_ram = memblock_end_of_DRAM();

#ifdef CONFIG_PPC32
	unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
	unsigned long end = __fix_to_virt(FIX_HOLE);

	for (; v < end; v += PAGE_SIZE)
		map_kernel_page(v, 0, 0); /* XXX gross */
#endif

#ifdef CONFIG_HIGHMEM
	map_kernel_page(PKMAP_BASE, 0, 0);	/* XXX gross */
	pkmap_page_table = virt_to_kpte(PKMAP_BASE);

	kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
	kmap_prot = PAGE_KERNEL;
#endif /* CONFIG_HIGHMEM */

	printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
	       (unsigned long long)top_of_ram, total_ram);
	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
	       (long int)((top_of_ram - total_ram) >> 20));

#ifdef CONFIG_HIGHMEM
	limit_zone_pfn(ZONE_NORMAL, lowmem_end_addr >> PAGE_SHIFT);
#endif
	limit_zone_pfn(TOP_ZONE, top_of_ram >> PAGE_SHIFT);
	zone_limits_final = true;
	free_area_init_nodes(max_zone_pfns);

	mark_nonram_nosave();
}

void __init mem_init(void)
{
	/*
	 * book3s is limited to 16 page sizes due to encoding this in
	 * a 4-bit field for slices.
	 */
	BUILD_BUG_ON(MMU_PAGE_COUNT > 16);

#ifdef CONFIG_SWIOTLB
	swiotlb_init(0);
#endif

	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
	set_max_mapnr(max_pfn);
	free_all_bootmem();

#ifdef CONFIG_HIGHMEM
	{
		unsigned long pfn, highmem_mapnr;

		highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
		for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
			phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
			struct page *page = pfn_to_page(pfn);
			if (!memblock_is_reserved(paddr))
				free_highmem_page(page);
		}
	}
#endif /* CONFIG_HIGHMEM */

#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
	/*
	 * If smp is enabled, next_tlbcam_idx is initialized in the cpu up
	 * functions.... do it here for the non-smp case.
	 */
	per_cpu(next_tlbcam_idx, smp_processor_id()) =
		(mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
#endif

	mem_init_print_info(NULL);
#ifdef CONFIG_PPC32
	pr_info("Kernel virtual memory layout:\n");
	pr_info("  * 0x%08lx..0x%08lx  : fixmap\n", FIXADDR_START, FIXADDR_TOP);
#ifdef CONFIG_HIGHMEM
	pr_info("  * 0x%08lx..0x%08lx  : highmem PTEs\n",
		PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
#endif /* CONFIG_HIGHMEM */
#ifdef CONFIG_NOT_COHERENT_CACHE
	pr_info("  * 0x%08lx..0x%08lx  : consistent mem\n",
		IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
#endif /* CONFIG_NOT_COHERENT_CACHE */
	pr_info("  * 0x%08lx..0x%08lx  : early ioremap\n",
		ioremap_bot, IOREMAP_TOP);
	pr_info("  * 0x%08lx..0x%08lx  : vmalloc & ioremap\n",
		VMALLOC_START, VMALLOC_END);
#endif /* CONFIG_PPC32 */
}

void free_initmem(void)
{
	ppc_md.progress = ppc_printk_progress;
	mark_initmem_nx();
	free_initmem_default(POISON_FREE_INITMEM);
}

#ifdef CONFIG_BLK_DEV_INITRD
void __init free_initrd_mem(unsigned long start, unsigned long end)
{
	free_reserved_area((void *)start, (void *)end, -1, "initrd");
}
#endif

/*
 * This is called when a page has been modified by the kernel.
 * It just marks the page as not i-cache clean.  We do the i-cache
 * flush later when the page is given to a user process, if necessary.
 */
void flush_dcache_page(struct page *page)
{
	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		return;
	/* avoid an atomic op if possible */
	if (test_bit(PG_arch_1, &page->flags))
		clear_bit(PG_arch_1, &page->flags);
}
EXPORT_SYMBOL(flush_dcache_page);

void flush_dcache_icache_page(struct page *page)
{
#ifdef CONFIG_HUGETLB_PAGE
	if (PageCompound(page)) {
		flush_dcache_icache_hugepage(page);
		return;
	}
#endif
#if defined(CONFIG_8xx) || defined(CONFIG_PPC64)
	/* On 8xx there is no need to kmap since highmem is not supported */
	__flush_dcache_icache(page_address(page));
#else
	if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) {
		void *start = kmap_atomic(page);
		__flush_dcache_icache(start);
		kunmap_atomic(start);
	} else {
		__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
	}
#endif
}
EXPORT_SYMBOL(flush_dcache_icache_page);

void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
{
	clear_page(page);

	/*
	 * We shouldn't have to do this, but some versions of glibc
	 * require it (ld.so assumes zero filled pages are icache clean)
	 * - Anton
	 */
	flush_dcache_page(pg);
}
EXPORT_SYMBOL(clear_user_page);

void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
		    struct page *pg)
{
	copy_page(vto, vfrom);

	/*
	 * We should be able to use the following optimisation, however
	 * there are two problems.
	 * Firstly a bug in some versions of binutils meant PLT sections
	 * were not marked executable.
	 * Secondly the first word in the GOT section is blrl, used
	 * to establish the GOT address. Until recently the GOT was
	 * not marked executable.
	 * - Anton
	 */
#if 0
	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
		return;
#endif

	flush_dcache_page(pg);
}

void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
			     unsigned long addr, int len)
{
	unsigned long maddr;

	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
	flush_icache_range(maddr, maddr + len);
	kunmap(page);
}
EXPORT_SYMBOL(flush_icache_user_range);

/*
 * This is called at the end of handling a user page fault, when the
 * fault has been handled by updating a PTE in the linux page tables.
 * We use it to preload an HPTE into the hash table corresponding to
 * the updated linux PTE.
 * 
 * This must always be called with the pte lock held.
 */
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
		      pte_t *ptep)
{
#ifdef CONFIG_PPC_STD_MMU
	/*
	 * We don't need to worry about _PAGE_PRESENT here because we are
	 * called with either mm->page_table_lock held or ptl lock held
	 */
	unsigned long access, trap;

	if (radix_enabled())
		return;

	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
	if (!pte_young(*ptep) || address >= TASK_SIZE)
		return;

	/* We try to figure out if we are coming from an instruction
	 * access fault and pass that down to __hash_page so we avoid
	 * double-faulting on execution of fresh text. We have to test
	 * for regs NULL since init will get here first thing at boot
	 *
	 * We also avoid filling the hash if not coming from a fault
	 */

	trap = current->thread.regs ? TRAP(current->thread.regs) : 0UL;
	switch (trap) {
	case 0x300:
		access = 0UL;
		break;
	case 0x400:
		access = _PAGE_EXEC;
		break;
	default:
		return;
	}

	hash_preload(vma->vm_mm, address, access, trap);
#endif /* CONFIG_PPC_STD_MMU */
#if (defined(CONFIG_PPC_BOOK3E_64) || defined(CONFIG_PPC_FSL_BOOK3E)) \
	&& defined(CONFIG_HUGETLB_PAGE)
	if (is_vm_hugetlb_page(vma))
		book3e_hugetlb_preload(vma, address, *ptep);
#endif
}

/*
 * System memory should not be in /proc/iomem but various tools expect it
 * (eg kdump).
 */
static int __init add_system_ram_resources(void)
{
	struct memblock_region *reg;

	for_each_memblock(memory, reg) {
		struct resource *res;
		unsigned long base = reg->base;
		unsigned long size = reg->size;

		res = kzalloc(sizeof(struct resource), GFP_KERNEL);
		WARN_ON(!res);

		if (res) {
			res->name = "System RAM";
			res->start = base;
			res->end = base + size - 1;
			res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
			WARN_ON(request_resource(&iomem_resource, res) < 0);
		}
	}

	return 0;
}
subsys_initcall(add_system_ram_resources);

#ifdef CONFIG_STRICT_DEVMEM
/*
 * devmem_is_allowed(): check to see if /dev/mem access to a certain address
 * is valid. The argument is a physical page number.
 *
 * Access has to be given to non-kernel-ram areas as well, these contain the
 * PCI mmio resources as well as potential bios/acpi data regions.
 */
int devmem_is_allowed(unsigned long pfn)
{
	if (page_is_rtas_user_buf(pfn))
		return 1;
	if (iomem_is_exclusive(PFN_PHYS(pfn)))
		return 0;
	if (!page_is_ram(pfn))
		return 1;
	return 0;
}
#endif /* CONFIG_STRICT_DEVMEM */
Commit	Line	Data
14cf11af PM	1	/*
	2	* PowerPC version
	3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	4	*
	5	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
	6	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
	7	* Copyright (C) 1996 Paul Mackerras
14cf11af PM	8	* PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
	9	*
	10	* Derived from "arch/i386/mm/init.c"
	11	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
	12	*
	13	* This program is free software; you can redistribute it and/or
	14	* modify it under the terms of the GNU General Public License
	15	* as published by the Free Software Foundation; either version
	16	* 2 of the License, or (at your option) any later version.
	17	*
	18	*/
	19
4b16f8e2	20	#include <linux/export.h>
14cf11af PM	21	#include <linux/sched.h>
	22	#include <linux/kernel.h>
	23	#include <linux/errno.h>
	24	#include <linux/string.h>
5a0e3ad6	25	#include <linux/gfp.h>
14cf11af PM	26	#include <linux/types.h>
	27	#include <linux/mm.h>
	28	#include <linux/stddef.h>
	29	#include <linux/init.h>
	30	#include <linux/bootmem.h>
	31	#include <linux/highmem.h>
	32	#include <linux/initrd.h>
	33	#include <linux/pagemap.h>
4e8ad3e8	34	#include <linux/suspend.h>
95f72d1e	35	#include <linux/memblock.h>
0895ecda	36	#include <linux/hugetlb.h>
c40dd2f7	37	#include <linux/slab.h>
16d0f5c4	38	#include <linux/vmalloc.h>
b584c254	39	#include <linux/memremap.h>
14cf11af PM	40
	41	#include <asm/pgalloc.h>
	42	#include <asm/prom.h>
	43	#include <asm/io.h>
	44	#include <asm/mmu_context.h>
	45	#include <asm/pgtable.h>
	46	#include <asm/mmu.h>
	47	#include <asm/smp.h>
	48	#include <asm/machdep.h>
	49	#include <asm/btext.h>
	50	#include <asm/tlb.h>
7c8c6b97	51	#include <asm/sections.h>
db7f37de	52	#include <asm/sparsemem.h>
ab1f9dac	53	#include <asm/vdso.h>
2c419bde	54	#include <asm/fixmap.h>
a9327296	55	#include <asm/swiotlb.h>
8a3e3d31	56	#include <asm/rtas.h>
14cf11af	57
14cf11af PM	58	#include "mmu_decl.h"
	59
	60	#ifndef CPU_FTR_COHERENT_ICACHE
	61	#define CPU_FTR_COHERENT_ICACHE 0 /* XXX for now */
	62	#define CPU_FTR_NOEXECUTE 0
	63	#endif
	64
a84fcd46	65	unsigned long long memory_limit;
7c8c6b97	66
2c419bde KG	67	#ifdef CONFIG_HIGHMEM
2c419bde KG	68	pte_t *kmap_pte;
8040bda3	69	EXPORT_SYMBOL(kmap_pte);
2c419bde	70	pgprot_t kmap_prot;
2c419bde	71	EXPORT_SYMBOL(kmap_prot);
dd0b52c4	72	#define TOP_ZONE ZONE_HIGHMEM
2c419bde KG	73
	74	static inline pte_t *virt_to_kpte(unsigned long vaddr)
	75	{
	76	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
	77	vaddr), vaddr), vaddr);
	78	}
dd0b52c4 OH	79	#else
dd0b52c4 OH	80	#define TOP_ZONE ZONE_NORMAL
2c419bde KG	81	#endif
2c419bde KG	82
14cf11af PM	83	int page_is_ram(unsigned long pfn)
14cf11af PM	84	{
14cf11af	85	#ifndef CONFIG_PPC64 /* XXX for now */
a880e762	86	return pfn < max_pfn;
14cf11af	87	#else
a880e762	88	unsigned long paddr = (pfn << PAGE_SHIFT);
28be7072	89	struct memblock_region *reg;
14cf11af	90
28be7072 BH	91	for_each_memblock(memory, reg)
28be7072 BH	92	if (paddr >= reg->base && paddr < (reg->base + reg->size))
14cf11af	93	return 1;
14cf11af PM	94	return 0;
	95	#endif
	96	}
14cf11af	97
8b150478	98	pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
14cf11af PM	99	unsigned long size, pgprot_t vma_prot)
	100	{
	101	if (ppc_md.phys_mem_access_prot)
8b150478	102	return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
14cf11af	103
8b150478	104	if (!page_is_ram(pfn))
64b3d0e8 BH	105	vma_prot = pgprot_noncached(vma_prot);
64b3d0e8 BH	106
14cf11af PM	107	return vma_prot;
	108	}
	109	EXPORT_SYMBOL(phys_mem_access_prot);
	110
23fd0775 PM	111	#ifdef CONFIG_MEMORY_HOTPLUG
23fd0775 PM	112
bc02af93 YG	113	#ifdef CONFIG_NUMA
	114	int memory_add_physaddr_to_nid(u64 start)
	115	{
	116	return hot_add_scn_to_nid(start);
	117	}
	118	#endif
	119
fecbfabe BH	120	int __weak create_section_mapping(unsigned long start, unsigned long end)
	121	{
	122	return -ENODEV;
	123	}
	124
	125	int __weak remove_section_mapping(unsigned long start, unsigned long end)
	126	{
	127	return -ENODEV;
	128	}
	129
3d79a728	130	int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
23fd0775	131	{
23fd0775 PM	132	unsigned long start_pfn = start >> PAGE_SHIFT;
23fd0775 PM	133	unsigned long nr_pages = size >> PAGE_SHIFT;
1dace6c6	134	int rc;
23fd0775	135
438cc81a DG	136	resize_hpt_for_hotplug(memblock_phys_mem_size());
438cc81a DG	137
2d0eee14	138	start = (unsigned long)__va(start);
1dace6c6 DG	139	rc = create_section_mapping(start, start + size);
	140	if (rc) {
	141	pr_warning(
	142	"Unable to create mapping for hot added memory 0x%llx..0x%llx: %d\n",
	143	start, start + size, rc);
	144	return -EFAULT;
	145	}
54b79248	146
3d79a728	147	return __add_pages(nid, start_pfn, nr_pages, want_memblock);
23fd0775	148	}
24d335ca WC	149
	150	#ifdef CONFIG_MEMORY_HOTREMOVE
	151	int arch_remove_memory(u64 start, u64 size)
	152	{
	153	unsigned long start_pfn = start >> PAGE_SHIFT;
	154	unsigned long nr_pages = size >> PAGE_SHIFT;
b584c254 OH	155	struct vmem_altmap *altmap;
b584c254 OH	156	struct page *page;
9ac8cde9	157	int ret;
24d335ca	158
b584c254 OH	159	/*
	160	* If we have an altmap then we need to skip over any reserved PFNs
	161	* when querying the zone.
	162	*/
	163	page = pfn_to_page(start_pfn);
	164	altmap = to_vmem_altmap((unsigned long) page);
	165	if (altmap)
	166	page += vmem_altmap_offset(altmap);
	167
	168	ret = __remove_pages(page_zone(page), start_pfn, nr_pages);
16d0f5c4 AB	169	if (ret)
	170	return ret;
	171
	172	/* Remove htab bolted mappings for this section of memory */
	173	start = (unsigned long)__va(start);
	174	ret = remove_section_mapping(start, start + size);
	175
	176	/* Ensure all vmalloc mappings are flushed in case they also
	177	* hit that section of memory
	178	*/
	179	vm_unmap_aliases();
9ac8cde9	180
438cc81a DG	181	resize_hpt_for_hotplug(memblock_phys_mem_size());
438cc81a DG	182
9ac8cde9	183	return ret;
24d335ca WC	184	}
24d335ca WC	185	#endif
0d579944	186	#endif /* CONFIG_MEMORY_HOTPLUG */
a99824f3 BP	187
	188	/*
	189	* walk_memory_resource() needs to make sure there is no holes in a given
9d88a2eb	190	* memory range. PPC64 does not maintain the memory layout in /proc/iomem.
95f72d1e	191	* Instead it maintains it in memblock.memory structures. Walk through the
9d88a2eb	192	* memory regions, find holes and callback for contiguous regions.
a99824f3 BP	193	*/
a99824f3 BP	194	int
908eedc6 KH	195	walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
908eedc6 KH	196	void arg, int (func)(unsigned long, unsigned long, void *))
a99824f3	197	{
28be7072 BH	198	struct memblock_region *reg;
	199	unsigned long end_pfn = start_pfn + nr_pages;
	200	unsigned long tstart, tend;
9d88a2eb BP	201	int ret = -1;
9d88a2eb BP	202
28be7072	203	for_each_memblock(memory, reg) {
c7fc2de0 YL	204	tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
c7fc2de0 YL	205	tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
28be7072 BH	206	if (tstart >= tend)
	207	continue;
	208	ret = (*func)(tstart, tend - tstart, arg);
9d88a2eb BP	209	if (ret)
9d88a2eb BP	210	break;
9d88a2eb BP	211	}
9d88a2eb BP	212	return ret;
a99824f3	213	}
908eedc6	214	EXPORT_SYMBOL_GPL(walk_system_ram_range);
a99824f3	215
7c8c6b97	216	#ifndef CONFIG_NEED_MULTIPLE_NODES
10239733	217	void __init initmem_init(void)
7c8c6b97	218	{
95f72d1e	219	max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
10239733	220	min_low_pfn = MEMORY_START >> PAGE_SHIFT;
7c8c6b97	221	#ifdef CONFIG_HIGHMEM
d7917ba7	222	max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
7c8c6b97 PM	223	#endif
7c8c6b97 PM	224
4e8309ba CS	225	/* Place all memblock_regions in the same node and merge contiguous
	226	* memblock_regions
	227	*/
e7e8de59	228	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
c67c3cb4	229
7c8c6b97	230	/* XXX need to clip this if using highmem? */
c67c3cb4	231	sparse_memory_present_with_active_regions(0);
21098b9e	232	sparse_init();
7c8c6b97 PM	233	}
7c8c6b97 PM	234
4e8ad3e8 JB	235	/* mark pages that don't exist as nosave */
	236	static int __init mark_nonram_nosave(void)
	237	{
28be7072 BH	238	struct memblock_region reg, prev = NULL;
	239
	240	for_each_memblock(memory, reg) {
	241	if (prev &&
c7fc2de0 YL	242	memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
	243	register_nosave_region(memblock_region_memory_end_pfn(prev),
	244	memblock_region_memory_base_pfn(reg));
28be7072	245	prev = reg;
4e8ad3e8	246	}
4e8ad3e8 JB	247	return 0;
4e8ad3e8 JB	248	}
6db35ad2 SW	249	#else /* CONFIG_NEED_MULTIPLE_NODES */
	250	static int __init mark_nonram_nosave(void)
	251	{
	252	return 0;
	253	}
	254	#endif
4e8ad3e8	255
1c98025c SW	256	static bool zone_limits_final;
1c98025c SW	257
3079abe5 OH	258	/*
	259	* The memory zones past TOP_ZONE are managed by generic mm code.
	260	* These should be set to zero since that's what every other
	261	* architecture does.
	262	*/
1c98025c	263	static unsigned long max_zone_pfns[MAX_NR_ZONES] = {
3079abe5 OH	264	[0 ... TOP_ZONE ] = ~0UL,
3079abe5 OH	265	[TOP_ZONE + 1 ... MAX_NR_ZONES - 1] = 0
1c98025c SW	266	};
	267
	268	/*
	269	* Restrict the specified zone and all more restrictive zones
	270	* to be below the specified pfn. May not be called after
	271	* paging_init().
	272	*/
	273	void __init limit_zone_pfn(enum zone_type zone, unsigned long pfn_limit)
	274	{
	275	int i;
	276
	277	if (WARN_ON(zone_limits_final))
	278	return;
	279
	280	for (i = zone; i >= 0; i--) {
	281	if (max_zone_pfns[i] > pfn_limit)
	282	max_zone_pfns[i] = pfn_limit;
	283	}
	284	}
	285
	286	/*
	287	* Find the least restrictive zone that is entirely below the
	288	* specified pfn limit. Returns < 0 if no suitable zone is found.
	289	*
	290	* pfn_limit must be u64 because it can exceed 32 bits even on 32-bit
	291	* systems -- the DMA limit can be higher than any possible real pfn.
	292	*/
	293	int dma_pfn_limit_to_zone(u64 pfn_limit)
	294	{
1c98025c SW	295	int i;
1c98025c SW	296
dd0b52c4	297	for (i = TOP_ZONE; i >= 0; i--) {
1c98025c SW	298	if (max_zone_pfns[i] <= pfn_limit)
	299	return i;
	300	}
	301
	302	return -EPERM;
	303	}
4e8ad3e8	304
7c8c6b97 PM	305	/*
	306	* paging_init() sets up the page tables - in fact we've already done this.
	307	*/
	308	void __init paging_init(void)
	309	{
f7ba2991	310	unsigned long long total_ram = memblock_phys_mem_size();
95f72d1e	311	phys_addr_t top_of_ram = memblock_end_of_DRAM();
7c8c6b97	312
2c419bde KG	313	#ifdef CONFIG_PPC32
	314	unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
	315	unsigned long end = __fix_to_virt(FIX_HOLE);
	316
	317	for (; v < end; v += PAGE_SIZE)
4386c096	318	map_kernel_page(v, 0, 0); /* XXX gross */
2c419bde KG	319	#endif
2c419bde KG	320
7c8c6b97	321	#ifdef CONFIG_HIGHMEM
4386c096	322	map_kernel_page(PKMAP_BASE, 0, 0); /* XXX gross */
2c419bde KG	323	pkmap_page_table = virt_to_kpte(PKMAP_BASE);
	324
	325	kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
7c8c6b97 PM	326	kmap_prot = PAGE_KERNEL;
	327	#endif /* CONFIG_HIGHMEM */
	328
f7ba2991	329	printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
fb610635	330	(unsigned long long)top_of_ram, total_ram);
e110b281	331	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
2bf3016f	332	(long int)((top_of_ram - total_ram) >> 20));
1c98025c	333
7c8c6b97	334	#ifdef CONFIG_HIGHMEM
1c98025c	335	limit_zone_pfn(ZONE_NORMAL, lowmem_end_addr >> PAGE_SHIFT);
c67c3cb4	336	#endif
dd0b52c4	337	limit_zone_pfn(TOP_ZONE, top_of_ram >> PAGE_SHIFT);
1c98025c	338	zone_limits_final = true;
c67c3cb4	339	free_area_init_nodes(max_zone_pfns);
4e8ad3e8 JB	340
4e8ad3e8 JB	341	mark_nonram_nosave();
7c8c6b97	342	}
7c8c6b97 PM	343
	344	void __init mem_init(void)
	345	{
28efc35f SW	346	/*
	347	* book3s is limited to 16 page sizes due to encoding this in
	348	* a 4-bit field for slices.
	349	*/
	350	BUILD_BUG_ON(MMU_PAGE_COUNT > 16);
	351
a9327296	352	#ifdef CONFIG_SWIOTLB
688ba1db	353	swiotlb_init(0);
a9327296 FT	354	#endif
a9327296 FT	355
7c8c6b97	356	high_memory = (void ) __va(max_low_pfn PAGE_SIZE);
602ddc70	357	set_max_mapnr(max_pfn);
0c988534	358	free_all_bootmem();
7c8c6b97 PM	359
	360	#ifdef CONFIG_HIGHMEM
	361	{
	362	unsigned long pfn, highmem_mapnr;
	363
d7917ba7	364	highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
7c8c6b97	365	for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
3d41e0f6	366	phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
7c8c6b97	367	struct page *page = pfn_to_page(pfn);
369a9d85 JL	368	if (!memblock_is_reserved(paddr))
369a9d85 JL	369	free_highmem_page(page);
7c8c6b97	370	}
7c8c6b97 PM	371	}
	372	#endif /* CONFIG_HIGHMEM */
	373
3160b097 BB	374	#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
	375	/*
	376	* If smp is enabled, next_tlbcam_idx is initialized in the cpu up
	377	* functions.... do it here for the non-smp case.
	378	*/
	379	per_cpu(next_tlbcam_idx, smp_processor_id()) =
	380	(mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
	381	#endif
	382
369a9d85	383	mem_init_print_info(NULL);
f637a49e BH	384	#ifdef CONFIG_PPC32
	385	pr_info("Kernel virtual memory layout:\n");
	386	pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP);
	387	#ifdef CONFIG_HIGHMEM
	388	pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n",
	389	PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
	390	#endif /* CONFIG_HIGHMEM */
8b31e49d BH	391	#ifdef CONFIG_NOT_COHERENT_CACHE
	392	pr_info(" * 0x%08lx..0x%08lx : consistent mem\n",
	393	IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
	394	#endif /* CONFIG_NOT_COHERENT_CACHE */
f637a49e BH	395	pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
	396	ioremap_bot, IOREMAP_TOP);
	397	pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n",
	398	VMALLOC_START, VMALLOC_END);
	399	#endif /* CONFIG_PPC32 */
7c8c6b97 PM	400	}
7c8c6b97 PM	401
2773fcc8 DC	402	void free_initmem(void)
2773fcc8 DC	403	{
a9c0f41b	404	ppc_md.progress = ppc_printk_progress;
029d9252	405	mark_initmem_nx();
5d585e5c	406	free_initmem_default(POISON_FREE_INITMEM);
2773fcc8 DC	407	}
2773fcc8 DC	408
307cfe71 BH	409	#ifdef CONFIG_BLK_DEV_INITRD
	410	void __init free_initrd_mem(unsigned long start, unsigned long end)
	411	{
dbe67df4	412	free_reserved_area((void )start, (void )end, -1, "initrd");
307cfe71 BH	413	}
	414	#endif
	415
14cf11af PM	416	/*
	417	* This is called when a page has been modified by the kernel.
	418	* It just marks the page as not i-cache clean. We do the i-cache
	419	* flush later when the page is given to a user process, if necessary.
	420	*/
	421	void flush_dcache_page(struct page *page)
	422	{
	423	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	424	return;
	425	/* avoid an atomic op if possible */
	426	if (test_bit(PG_arch_1, &page->flags))
	427	clear_bit(PG_arch_1, &page->flags);
	428	}
	429	EXPORT_SYMBOL(flush_dcache_page);
	430
	431	void flush_dcache_icache_page(struct page *page)
	432	{
0895ecda DG	433	#ifdef CONFIG_HUGETLB_PAGE
	434	if (PageCompound(page)) {
	435	flush_dcache_icache_hugepage(page);
	436	return;
	437	}
	438	#endif
2f7d2b74 SW	439	#if defined(CONFIG_8xx) \|\| defined(CONFIG_PPC64)
	440	/* On 8xx there is no need to kmap since highmem is not supported */
	441	__flush_dcache_icache(page_address(page));
	442	#else
	443	if (IS_ENABLED(CONFIG_BOOKE) \|\| sizeof(phys_addr_t) > sizeof(void *)) {
2480b208	444	void *start = kmap_atomic(page);
0895ecda	445	__flush_dcache_icache(start);
2480b208	446	kunmap_atomic(start);
2f7d2b74 SW	447	} else {
2f7d2b74 SW	448	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
0895ecda	449	}
14cf11af	450	#endif
14cf11af	451	}
249ba1ee	452	EXPORT_SYMBOL(flush_dcache_icache_page);
0895ecda	453
14cf11af PM	454	void clear_user_page(void page, unsigned long vaddr, struct page pg)
	455	{
	456	clear_page(page);
	457
14cf11af	458	/*
25985edc	459	* We shouldn't have to do this, but some versions of glibc
14cf11af PM	460	* require it (ld.so assumes zero filled pages are icache clean)
	461	* - Anton
	462	*/
09f5dc44	463	flush_dcache_page(pg);
14cf11af PM	464	}
	465	EXPORT_SYMBOL(clear_user_page);
	466
	467	void copy_user_page(void vto, void vfrom, unsigned long vaddr,
	468	struct page *pg)
	469	{
	470	copy_page(vto, vfrom);
	471
	472	/*
	473	* We should be able to use the following optimisation, however
	474	* there are two problems.
	475	* Firstly a bug in some versions of binutils meant PLT sections
	476	* were not marked executable.
	477	* Secondly the first word in the GOT section is blrl, used
	478	* to establish the GOT address. Until recently the GOT was
	479	* not marked executable.
	480	* - Anton
	481	*/
	482	#if 0
	483	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
	484	return;
	485	#endif
	486
09f5dc44	487	flush_dcache_page(pg);
14cf11af PM	488	}
	489
	490	void flush_icache_user_range(struct vm_area_struct vma, struct page page,
	491	unsigned long addr, int len)
	492	{
	493	unsigned long maddr;
	494
	495	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
	496	flush_icache_range(maddr, maddr + len);
	497	kunmap(page);
	498	}
	499	EXPORT_SYMBOL(flush_icache_user_range);
	500
	501	/*
	502	* This is called at the end of handling a user page fault, when the
	503	* fault has been handled by updating a PTE in the linux page tables.
	504	* We use it to preload an HPTE into the hash table corresponding to
	505	* the updated linux PTE.
	506	*
01edcd89	507	* This must always be called with the pte lock held.
14cf11af PM	508	*/
14cf11af PM	509	void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
4b3073e1	510	pte_t *ptep)
14cf11af	511	{
3c726f8d	512	#ifdef CONFIG_PPC_STD_MMU
0ac52dd7 AK	513	/*
	514	* We don't need to worry about _PAGE_PRESENT here because we are
	515	* called with either mm->page_table_lock held or ptl lock held
	516	*/
171cb719 GS	517	unsigned long access, trap;
171cb719 GS	518
a3dece6d AK	519	if (radix_enabled())
a3dece6d AK	520	return;
14cf11af	521
14cf11af	522	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
4b3073e1	523	if (!pte_young(*ptep) \|\| address >= TASK_SIZE)
14cf11af	524	return;
14cf11af	525
3c726f8d BH	526	/* We try to figure out if we are coming from an instruction
	527	* access fault and pass that down to __hash_page so we avoid
	528	* double-faulting on execution of fresh text. We have to test
	529	* for regs NULL since init will get here first thing at boot
	530	*
	531	* We also avoid filling the hash if not coming from a fault
	532	*/
171cb719 GS	533
	534	trap = current->thread.regs ? TRAP(current->thread.regs) : 0UL;
	535	switch (trap) {
	536	case 0x300:
	537	access = 0UL;
	538	break;
	539	case 0x400:
	540	access = _PAGE_EXEC;
	541	break;
	542	default:
3c726f8d	543	return;
171cb719 GS	544	}
171cb719 GS	545
3c726f8d BH	546	hash_preload(vma->vm_mm, address, access, trap);
3c726f8d BH	547	#endif /* CONFIG_PPC_STD_MMU */
41151e77 BB	548	#if (defined(CONFIG_PPC_BOOK3E_64) \|\| defined(CONFIG_PPC_FSL_BOOK3E)) \
	549	&& defined(CONFIG_HUGETLB_PAGE)
	550	if (is_vm_hugetlb_page(vma))
d93e4d7d	551	book3e_hugetlb_preload(vma, address, *ptep);
41151e77	552	#endif
14cf11af	553	}
c40dd2f7 AB	554
	555	/*
	556	* System memory should not be in /proc/iomem but various tools expect it
	557	* (eg kdump).
	558	*/
4f770924	559	static int __init add_system_ram_resources(void)
c40dd2f7 AB	560	{
	561	struct memblock_region *reg;
	562
	563	for_each_memblock(memory, reg) {
	564	struct resource *res;
	565	unsigned long base = reg->base;
	566	unsigned long size = reg->size;
	567
	568	res = kzalloc(sizeof(struct resource), GFP_KERNEL);
	569	WARN_ON(!res);
	570
	571	if (res) {
	572	res->name = "System RAM";
	573	res->start = base;
	574	res->end = base + size - 1;
35d98e93	575	res->flags = IORESOURCE_SYSTEM_RAM \| IORESOURCE_BUSY;
c40dd2f7 AB	576	WARN_ON(request_resource(&iomem_resource, res) < 0);
	577	}
	578	}
	579
	580	return 0;
	581	}
	582	subsys_initcall(add_system_ram_resources);
1d54cf2b	583
	584	#ifdef CONFIG_STRICT_DEVMEM
	585	/*
	586	* devmem_is_allowed(): check to see if /dev/mem access to a certain address
	587	* is valid. The argument is a physical page number.
	588	*
	589	* Access has to be given to non-kernel-ram areas as well, these contain the
	590	* PCI mmio resources as well as potential bios/acpi data regions.
	591	*/
	592	int devmem_is_allowed(unsigned long pfn)
	593	{
e256caa7 VH	594	if (page_is_rtas_user_buf(pfn))
e256caa7 VH	595	return 1;
6c0cc627	596	if (iomem_is_exclusive(PFN_PHYS(pfn)))
1d54cf2b	597	return 0;
	598	if (!page_is_ram(pfn))
	599	return 1;
	600	return 0;
	601	}
	602	#endif /* CONFIG_STRICT_DEVMEM */