[mirror_ubuntu-jammy-kernel.git] / arch / x86 / mm / ioremap.c

/*
 * Re-map IO memory to kernel address space so that we can access it.
 * This is needed for high PCI addresses that aren't mapped in the
 * 640k-1MB IO memory area on PC's
 *
 * (C) Copyright 1995 1996 Linus Torvalds
 */

#include <linux/bootmem.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

#include <asm/cacheflush.h>
#include <asm/e820.h>
#include <asm/fixmap.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>

enum ioremap_mode {
	IOR_MODE_UNCACHED,
	IOR_MODE_CACHED,
};

#ifdef CONFIG_X86_64

unsigned long __phys_addr(unsigned long x)
{
	if (x >= __START_KERNEL_map)
		return x - __START_KERNEL_map + phys_base;
	return x - PAGE_OFFSET;
}
EXPORT_SYMBOL(__phys_addr);

#endif

int page_is_ram(unsigned long pagenr)
{
	unsigned long addr, end;
	int i;

	for (i = 0; i < e820.nr_map; i++) {
		/*
		 * Not usable memory:
		 */
		if (e820.map[i].type != E820_RAM)
			continue;
		addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
		end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;

		/*
		 * Sanity check: Some BIOSen report areas as RAM that
		 * are not. Notably the 640->1Mb area, which is the
		 * PCI BIOS area.
		 */
		if (addr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
		    end < (BIOS_END >> PAGE_SHIFT))
			continue;

		if ((pagenr >= addr) && (pagenr < end))
			return 1;
	}
	return 0;
}

/*
 * Fix up the linear direct mapping of the kernel to avoid cache attribute
 * conflicts.
 */
static int ioremap_change_attr(unsigned long paddr, unsigned long size,
			       enum ioremap_mode mode)
{
	unsigned long vaddr = (unsigned long)__va(paddr);
	unsigned long nrpages = size >> PAGE_SHIFT;
	int err, level;

	/* No change for pages after the last mapping */
	if ((paddr + size - 1) >= (max_pfn_mapped << PAGE_SHIFT))
		return 0;

	/*
	 * If there is no identity map for this address,
	 * change_page_attr_addr is unnecessary
	 */
	if (!lookup_address(vaddr, &level))
		return 0;

	switch (mode) {
	case IOR_MODE_UNCACHED:
	default:
		err = set_memory_uc(vaddr, nrpages);
		break;
	case IOR_MODE_CACHED:
		err = set_memory_wb(vaddr, nrpages);
		break;
	}

	return err;
}

/*
 * Remap an arbitrary physical address space into the kernel virtual
 * address space. Needed when the kernel wants to access high addresses
 * directly.
 *
 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 * have to convert them into an offset in a page-aligned mapping, but the
 * caller shouldn't need to know that small detail.
 */
static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
			       enum ioremap_mode mode)
{
	void __iomem *addr;
	struct vm_struct *area;
	unsigned long offset, last_addr;
	pgprot_t prot;

	/* Don't allow wraparound or zero size */
	last_addr = phys_addr + size - 1;
	if (!size || last_addr < phys_addr)
		return NULL;

	/*
	 * Don't remap the low PCI/ISA area, it's always mapped..
	 */
	if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
		return (__force void __iomem *)phys_to_virt(phys_addr);

	/*
	 * Don't allow anybody to remap normal RAM that we're using..
	 */
	for (offset = phys_addr >> PAGE_SHIFT; offset < max_pfn_mapped &&
	     (offset << PAGE_SHIFT) < last_addr; offset++) {
		if (page_is_ram(offset))
			return NULL;
	}

	switch (mode) {
	case IOR_MODE_UNCACHED:
	default:
		prot = PAGE_KERNEL_NOCACHE;
		break;
	case IOR_MODE_CACHED:
		prot = PAGE_KERNEL;
		break;
	}

	/*
	 * Mappings have to be page-aligned
	 */
	offset = phys_addr & ~PAGE_MASK;
	phys_addr &= PAGE_MASK;
	size = PAGE_ALIGN(last_addr+1) - phys_addr;

	/*
	 * Ok, go for it..
	 */
	area = get_vm_area(size, VM_IOREMAP);
	if (!area)
		return NULL;
	area->phys_addr = phys_addr;
	addr = (void __iomem *) area->addr;
	if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
			       phys_addr, prot)) {
		remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr));
		return NULL;
	}

	if (ioremap_change_attr(phys_addr, size, mode) < 0) {
		vunmap(addr);
		return NULL;
	}

	return (void __iomem *) (offset + (char __iomem *)addr);
}

/**
 * ioremap_nocache     -   map bus memory into CPU space
 * @offset:    bus address of the memory
 * @size:      size of the resource to map
 *
 * ioremap_nocache performs a platform specific sequence of operations to
 * make bus memory CPU accessible via the readb/readw/readl/writeb/
 * writew/writel functions and the other mmio helpers. The returned
 * address is not guaranteed to be usable directly as a virtual
 * address.
 *
 * This version of ioremap ensures that the memory is marked uncachable
 * on the CPU as well as honouring existing caching rules from things like
 * the PCI bus. Note that there are other caches and buffers on many
 * busses. In particular driver authors should read up on PCI writes
 *
 * It's useful if some control registers are in such an area and
 * write combining or read caching is not desirable:
 *
 * Must be freed with iounmap.
 */
void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
{
	return __ioremap(phys_addr, size, IOR_MODE_UNCACHED);
}
EXPORT_SYMBOL(ioremap_nocache);

void __iomem *ioremap_cache(unsigned long phys_addr, unsigned long size)
{
	return __ioremap(phys_addr, size, IOR_MODE_CACHED);
}
EXPORT_SYMBOL(ioremap_cache);

/**
 * iounmap - Free a IO remapping
 * @addr: virtual address from ioremap_*
 *
 * Caller must ensure there is only one unmapping for the same pointer.
 */
void iounmap(volatile void __iomem *addr)
{
	struct vm_struct *p, *o;

	if ((void __force *)addr <= high_memory)
		return;

	/*
	 * __ioremap special-cases the PCI/ISA range by not instantiating a
	 * vm_area and by simply returning an address into the kernel mapping
	 * of ISA space.   So handle that here.
	 */
	if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
	    addr < phys_to_virt(ISA_END_ADDRESS))
		return;

	addr = (volatile void __iomem *)
		(PAGE_MASK & (unsigned long __force)addr);

	/* Use the vm area unlocked, assuming the caller
	   ensures there isn't another iounmap for the same address
	   in parallel. Reuse of the virtual address is prevented by
	   leaving it in the global lists until we're done with it.
	   cpa takes care of the direct mappings. */
	read_lock(&vmlist_lock);
	for (p = vmlist; p; p = p->next) {
		if (p->addr == addr)
			break;
	}
	read_unlock(&vmlist_lock);

	if (!p) {
		printk(KERN_ERR "iounmap: bad address %p\n", addr);
		dump_stack();
		return;
	}

	/* Reset the direct mapping. Can block */
	ioremap_change_attr(p->phys_addr, p->size, IOR_MODE_CACHED);

	/* Finally remove it */
	o = remove_vm_area((void *)addr);
	BUG_ON(p != o || o == NULL);
	kfree(p);
}
EXPORT_SYMBOL(iounmap);

#ifdef CONFIG_X86_32

int __initdata early_ioremap_debug;

static int __init early_ioremap_debug_setup(char *str)
{
	early_ioremap_debug = 1;

	return 0;
}
early_param("early_ioremap_debug", early_ioremap_debug_setup);

static __initdata int after_paging_init;
static __initdata unsigned long bm_pte[1024]
				__attribute__((aligned(PAGE_SIZE)));

static inline unsigned long * __init early_ioremap_pgd(unsigned long addr)
{
	return (unsigned long *)swapper_pg_dir + ((addr >> 22) & 1023);
}

static inline unsigned long * __init early_ioremap_pte(unsigned long addr)
{
	return bm_pte + ((addr >> PAGE_SHIFT) & 1023);
}

void __init early_ioremap_init(void)
{
	unsigned long *pgd;

	if (early_ioremap_debug)
		printk(KERN_INFO "early_ioremap_init()\n");

	pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
	*pgd = __pa(bm_pte) | _PAGE_TABLE;
	memset(bm_pte, 0, sizeof(bm_pte));
	/*
	 * The boot-ioremap range spans multiple pgds, for which
	 * we are not prepared:
	 */
	if (pgd != early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END))) {
		WARN_ON(1);
		printk(KERN_WARNING "pgd %p != %p\n",
		       pgd, early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END)));
		printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
		       fix_to_virt(FIX_BTMAP_BEGIN));
		printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
		       fix_to_virt(FIX_BTMAP_END));

		printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
		printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
		       FIX_BTMAP_BEGIN);
	}
}

void __init early_ioremap_clear(void)
{
	unsigned long *pgd;

	if (early_ioremap_debug)
		printk(KERN_INFO "early_ioremap_clear()\n");

	pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
	*pgd = 0;
	__flush_tlb_all();
}

void __init early_ioremap_reset(void)
{
	enum fixed_addresses idx;
	unsigned long *pte, phys, addr;

	after_paging_init = 1;
	for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
		addr = fix_to_virt(idx);
		pte = early_ioremap_pte(addr);
		if (!*pte & _PAGE_PRESENT) {
			phys = *pte & PAGE_MASK;
			set_fixmap(idx, phys);
		}
	}
}

static void __init __early_set_fixmap(enum fixed_addresses idx,
				   unsigned long phys, pgprot_t flags)
{
	unsigned long *pte, addr = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		BUG();
		return;
	}
	pte = early_ioremap_pte(addr);
	if (pgprot_val(flags))
		*pte = (phys & PAGE_MASK) | pgprot_val(flags);
	else
		*pte = 0;
	__flush_tlb_one(addr);
}

static inline void __init early_set_fixmap(enum fixed_addresses idx,
					unsigned long phys)
{
	if (after_paging_init)
		set_fixmap(idx, phys);
	else
		__early_set_fixmap(idx, phys, PAGE_KERNEL);
}

static inline void __init early_clear_fixmap(enum fixed_addresses idx)
{
	if (after_paging_init)
		clear_fixmap(idx);
	else
		__early_set_fixmap(idx, 0, __pgprot(0));
}


int __initdata early_ioremap_nested;

static int __init check_early_ioremap_leak(void)
{
	if (!early_ioremap_nested)
		return 0;

	printk(KERN_WARNING
	       "Debug warning: early ioremap leak of %d areas detected.\n",
	       early_ioremap_nested);
	printk(KERN_WARNING
	       "please boot with early_ioremap_debug and report the dmesg.\n");
	WARN_ON(1);

	return 1;
}
late_initcall(check_early_ioremap_leak);

void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
{
	unsigned long offset, last_addr;
	unsigned int nrpages, nesting;
	enum fixed_addresses idx0, idx;

	WARN_ON(system_state != SYSTEM_BOOTING);

	nesting = early_ioremap_nested;
	if (early_ioremap_debug) {
		printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
		       phys_addr, size, nesting);
		dump_stack();
	}

	/* Don't allow wraparound or zero size */
	last_addr = phys_addr + size - 1;
	if (!size || last_addr < phys_addr) {
		WARN_ON(1);
		return NULL;
	}

	if (nesting >= FIX_BTMAPS_NESTING) {
		WARN_ON(1);
		return NULL;
	}
	early_ioremap_nested++;
	/*
	 * Mappings have to be page-aligned
	 */
	offset = phys_addr & ~PAGE_MASK;
	phys_addr &= PAGE_MASK;
	size = PAGE_ALIGN(last_addr) - phys_addr;

	/*
	 * Mappings have to fit in the FIX_BTMAP area.
	 */
	nrpages = size >> PAGE_SHIFT;
	if (nrpages > NR_FIX_BTMAPS) {
		WARN_ON(1);
		return NULL;
	}

	/*
	 * Ok, go for it..
	 */
	idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
	idx = idx0;
	while (nrpages > 0) {
		early_set_fixmap(idx, phys_addr);
		phys_addr += PAGE_SIZE;
		--idx;
		--nrpages;
	}
	if (early_ioremap_debug)
		printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));

	return (void *) (offset + fix_to_virt(idx0));
}

void __init early_iounmap(void *addr, unsigned long size)
{
	unsigned long virt_addr;
	unsigned long offset;
	unsigned int nrpages;
	enum fixed_addresses idx;
	unsigned int nesting;

	nesting = --early_ioremap_nested;
	WARN_ON(nesting < 0);

	if (early_ioremap_debug) {
		printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
		       size, nesting);
		dump_stack();
	}

	virt_addr = (unsigned long)addr;
	if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
		WARN_ON(1);
		return;
	}
	offset = virt_addr & ~PAGE_MASK;
	nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;

	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
	while (nrpages > 0) {
		early_clear_fixmap(idx);
		--idx;
		--nrpages;
	}
}

void __this_fixmap_does_not_exist(void)
{
	WARN_ON(1);
}

#endif /* CONFIG_X86_32 */
Commit	Line	Data
1da177e4	1	/*
1da177e4 LT	2	* Re-map IO memory to kernel address space so that we can access it.
	3	* This is needed for high PCI addresses that aren't mapped in the
	4	* 640k-1MB IO memory area on PC's
	5	*
	6	* (C) Copyright 1995 1996 Linus Torvalds
	7	*/
	8
e9332cac	9	#include <linux/bootmem.h>
1da177e4	10	#include <linux/init.h>
a148ecfd	11	#include <linux/io.h>
3cbd09e4 TG	12	#include <linux/module.h>
	13	#include <linux/slab.h>
	14	#include <linux/vmalloc.h>
	15
1da177e4	16	#include <asm/cacheflush.h>
3cbd09e4 TG	17	#include <asm/e820.h>
3cbd09e4 TG	18	#include <asm/fixmap.h>
1da177e4	19	#include <asm/pgtable.h>
3cbd09e4	20	#include <asm/tlbflush.h>
1da177e4	21
d806e5ee TG	22	enum ioremap_mode {
	23	IOR_MODE_UNCACHED,
	24	IOR_MODE_CACHED,
	25	};
	26
240d3a7c TG	27	#ifdef CONFIG_X86_64
	28
	29	unsigned long __phys_addr(unsigned long x)
	30	{
	31	if (x >= __START_KERNEL_map)
	32	return x - __START_KERNEL_map + phys_base;
	33	return x - PAGE_OFFSET;
	34	}
	35	EXPORT_SYMBOL(__phys_addr);
	36
	37	#endif
	38
5f5192b9 TG	39	int page_is_ram(unsigned long pagenr)
	40	{
	41	unsigned long addr, end;
	42	int i;
	43
	44	for (i = 0; i < e820.nr_map; i++) {
	45	/*
	46	* Not usable memory:
	47	*/
	48	if (e820.map[i].type != E820_RAM)
	49	continue;
5f5192b9 TG	50	addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
5f5192b9 TG	51	end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
950f9d95 TG	52
	53	/*
	54	* Sanity check: Some BIOSen report areas as RAM that
	55	* are not. Notably the 640->1Mb area, which is the
	56	* PCI BIOS area.
	57	*/
	58	if (addr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
	59	end < (BIOS_END >> PAGE_SHIFT))
	60	continue;
	61
5f5192b9 TG	62	if ((pagenr >= addr) && (pagenr < end))
	63	return 1;
	64	}
	65	return 0;
	66	}
	67
e9332cac TG	68	/*
	69	* Fix up the linear direct mapping of the kernel to avoid cache attribute
	70	* conflicts.
	71	*/
d806e5ee TG	72	static int ioremap_change_attr(unsigned long paddr, unsigned long size,
d806e5ee TG	73	enum ioremap_mode mode)
e9332cac	74	{
d806e5ee TG	75	unsigned long vaddr = (unsigned long)__va(paddr);
d806e5ee TG	76	unsigned long nrpages = size >> PAGE_SHIFT;
e9332cac TG	77	int err, level;
	78
	79	/* No change for pages after the last mapping */
d806e5ee	80	if ((paddr + size - 1) >= (max_pfn_mapped << PAGE_SHIFT))
e9332cac TG	81	return 0;
e9332cac TG	82
e9332cac TG	83	/*
	84	* If there is no identity map for this address,
	85	* change_page_attr_addr is unnecessary
	86	*/
	87	if (!lookup_address(vaddr, &level))
	88	return 0;
	89
d806e5ee TG	90	switch (mode) {
	91	case IOR_MODE_UNCACHED:
	92	default:
	93	err = set_memory_uc(vaddr, nrpages);
	94	break;
	95	case IOR_MODE_CACHED:
	96	err = set_memory_wb(vaddr, nrpages);
	97	break;
	98	}
e9332cac TG	99
	100	return err;
	101	}
	102
1da177e4 LT	103	/*
	104	* Remap an arbitrary physical address space into the kernel virtual
	105	* address space. Needed when the kernel wants to access high addresses
	106	* directly.
	107	*
	108	* NOTE! We need to allow non-page-aligned mappings too: we will obviously
	109	* have to convert them into an offset in a page-aligned mapping, but the
	110	* caller shouldn't need to know that small detail.
	111	*/
5f868152	112	static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
d806e5ee	113	enum ioremap_mode mode)
1da177e4	114	{
91eebf40 TG	115	void __iomem *addr;
91eebf40 TG	116	struct vm_struct *area;
1da177e4	117	unsigned long offset, last_addr;
d806e5ee	118	pgprot_t prot;
1da177e4 LT	119
	120	/* Don't allow wraparound or zero size */
	121	last_addr = phys_addr + size - 1;
	122	if (!size \|\| last_addr < phys_addr)
	123	return NULL;
	124
	125	/*
	126	* Don't remap the low PCI/ISA area, it's always mapped..
	127	*/
	128	if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
4b40fcee	129	return (__force void __iomem *)phys_to_virt(phys_addr);
1da177e4 LT	130
	131	/*
	132	* Don't allow anybody to remap normal RAM that we're using..
	133	*/
266b9f87 TG	134	for (offset = phys_addr >> PAGE_SHIFT; offset < max_pfn_mapped &&
	135	(offset << PAGE_SHIFT) < last_addr; offset++) {
	136	if (page_is_ram(offset))
	137	return NULL;
1da177e4 LT	138	}
1da177e4 LT	139
d806e5ee TG	140	switch (mode) {
	141	case IOR_MODE_UNCACHED:
	142	default:
	143	prot = PAGE_KERNEL_NOCACHE;
	144	break;
	145	case IOR_MODE_CACHED:
	146	prot = PAGE_KERNEL;
	147	break;
	148	}
a148ecfd	149
1da177e4 LT	150	/*
	151	* Mappings have to be page-aligned
	152	*/
	153	offset = phys_addr & ~PAGE_MASK;
	154	phys_addr &= PAGE_MASK;
	155	size = PAGE_ALIGN(last_addr+1) - phys_addr;
	156
	157	/*
	158	* Ok, go for it..
	159	*/
74ff2857	160	area = get_vm_area(size, VM_IOREMAP);
1da177e4 LT	161	if (!area)
	162	return NULL;
	163	area->phys_addr = phys_addr;
	164	addr = (void __iomem *) area->addr;
e9332cac	165	if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
d806e5ee	166	phys_addr, prot)) {
e4c1b977	167	remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr));
1da177e4 LT	168	return NULL;
1da177e4 LT	169	}
e9332cac	170
d806e5ee	171	if (ioremap_change_attr(phys_addr, size, mode) < 0) {
e9332cac TG	172	vunmap(addr);
	173	return NULL;
	174	}
	175
1da177e4 LT	176	return (void __iomem ) (offset + (char __iomem )addr);
1da177e4 LT	177	}
1da177e4 LT	178
	179	/**
	180	* ioremap_nocache - map bus memory into CPU space
	181	* @offset: bus address of the memory
	182	* @size: size of the resource to map
	183	*
	184	* ioremap_nocache performs a platform specific sequence of operations to
	185	* make bus memory CPU accessible via the readb/readw/readl/writeb/
	186	* writew/writel functions and the other mmio helpers. The returned
	187	* address is not guaranteed to be usable directly as a virtual
91eebf40	188	* address.
1da177e4 LT	189	*
	190	* This version of ioremap ensures that the memory is marked uncachable
	191	* on the CPU as well as honouring existing caching rules from things like
91eebf40	192	* the PCI bus. Note that there are other caches and buffers on many
1da177e4 LT	193	* busses. In particular driver authors should read up on PCI writes
	194	*
	195	* It's useful if some control registers are in such an area and
	196	* write combining or read caching is not desirable:
91eebf40	197	*
1da177e4 LT	198	* Must be freed with iounmap.
1da177e4 LT	199	*/
91eebf40	200	void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
1da177e4	201	{
d806e5ee	202	return __ioremap(phys_addr, size, IOR_MODE_UNCACHED);
1da177e4	203	}
129f6946	204	EXPORT_SYMBOL(ioremap_nocache);
1da177e4	205
5f868152 TG	206	void __iomem *ioremap_cache(unsigned long phys_addr, unsigned long size)
5f868152 TG	207	{
d806e5ee	208	return __ioremap(phys_addr, size, IOR_MODE_CACHED);
5f868152 TG	209	}
	210	EXPORT_SYMBOL(ioremap_cache);
	211
bf5421c3 AK	212	/**
	213	* iounmap - Free a IO remapping
	214	* @addr: virtual address from ioremap_*
	215	*
	216	* Caller must ensure there is only one unmapping for the same pointer.
	217	*/
1da177e4 LT	218	void iounmap(volatile void __iomem *addr)
1da177e4 LT	219	{
bf5421c3	220	struct vm_struct p, o;
c23a4e96 AM	221
c23a4e96 AM	222	if ((void __force *)addr <= high_memory)
1da177e4 LT	223	return;
	224
	225	/*
	226	* __ioremap special-cases the PCI/ISA range by not instantiating a
	227	* vm_area and by simply returning an address into the kernel mapping
	228	* of ISA space. So handle that here.
	229	*/
	230	if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
91eebf40	231	addr < phys_to_virt(ISA_END_ADDRESS))
1da177e4 LT	232	return;
1da177e4 LT	233
91eebf40 TG	234	addr = (volatile void __iomem *)
91eebf40 TG	235	(PAGE_MASK & (unsigned long __force)addr);
bf5421c3 AK	236
	237	/* Use the vm area unlocked, assuming the caller
	238	ensures there isn't another iounmap for the same address
	239	in parallel. Reuse of the virtual address is prevented by
	240	leaving it in the global lists until we're done with it.
	241	cpa takes care of the direct mappings. */
	242	read_lock(&vmlist_lock);
	243	for (p = vmlist; p; p = p->next) {
	244	if (p->addr == addr)
	245	break;
	246	}
	247	read_unlock(&vmlist_lock);
	248
	249	if (!p) {
91eebf40	250	printk(KERN_ERR "iounmap: bad address %p\n", addr);
c23a4e96	251	dump_stack();
bf5421c3	252	return;
1da177e4 LT	253	}
1da177e4 LT	254
bf5421c3	255	/* Reset the direct mapping. Can block */
d806e5ee	256	ioremap_change_attr(p->phys_addr, p->size, IOR_MODE_CACHED);
bf5421c3 AK	257
	258	/* Finally remove it */
	259	o = remove_vm_area((void *)addr);
	260	BUG_ON(p != o \|\| o == NULL);
91eebf40	261	kfree(p);
1da177e4	262	}
129f6946	263	EXPORT_SYMBOL(iounmap);
1da177e4	264
240d3a7c	265	#ifdef CONFIG_X86_32
d18d6d65 IM	266
	267	int __initdata early_ioremap_debug;
	268
	269	static int __init early_ioremap_debug_setup(char *str)
	270	{
	271	early_ioremap_debug = 1;
	272
793b24a2	273	return 0;
d18d6d65	274	}
793b24a2	275	early_param("early_ioremap_debug", early_ioremap_debug_setup);
d18d6d65	276
0947b2f3 HY	277	static __initdata int after_paging_init;
	278	static __initdata unsigned long bm_pte[1024]
	279	__attribute__((aligned(PAGE_SIZE)));
	280
beacfaac	281	static inline unsigned long * __init early_ioremap_pgd(unsigned long addr)
0947b2f3 HY	282	{
	283	return (unsigned long *)swapper_pg_dir + ((addr >> 22) & 1023);
	284	}
	285
beacfaac	286	static inline unsigned long * __init early_ioremap_pte(unsigned long addr)
0947b2f3 HY	287	{
	288	return bm_pte + ((addr >> PAGE_SHIFT) & 1023);
	289	}
	290
beacfaac	291	void __init early_ioremap_init(void)
0947b2f3 HY	292	{
	293	unsigned long *pgd;
	294
d18d6d65	295	if (early_ioremap_debug)
adafdf6a	296	printk(KERN_INFO "early_ioremap_init()\n");
d18d6d65	297
beacfaac	298	pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
0947b2f3 HY	299	*pgd = __pa(bm_pte) \| _PAGE_TABLE;
0947b2f3 HY	300	memset(bm_pte, 0, sizeof(bm_pte));
0e3a9549 IM	301	/*
	302	* The boot-ioremap range spans multiple pgds, for which
	303	* we are not prepared:
	304	*/
	305	if (pgd != early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END))) {
	306	WARN_ON(1);
91eebf40 TG	307	printk(KERN_WARNING "pgd %p != %p\n",
	308	pgd, early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END)));
	309	printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
	310	fix_to_virt(FIX_BTMAP_BEGIN));
	311	printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
	312	fix_to_virt(FIX_BTMAP_END));
	313
	314	printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
	315	printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
	316	FIX_BTMAP_BEGIN);
0e3a9549	317	}
0947b2f3 HY	318	}
0947b2f3 HY	319
beacfaac	320	void __init early_ioremap_clear(void)
0947b2f3 HY	321	{
	322	unsigned long *pgd;
	323
d18d6d65	324	if (early_ioremap_debug)
adafdf6a	325	printk(KERN_INFO "early_ioremap_clear()\n");
d18d6d65	326
beacfaac	327	pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
0947b2f3 HY	328	*pgd = 0;
	329	__flush_tlb_all();
	330	}
	331
beacfaac	332	void __init early_ioremap_reset(void)
0947b2f3 HY	333	{
	334	enum fixed_addresses idx;
	335	unsigned long *pte, phys, addr;
	336
	337	after_paging_init = 1;
64a8f852	338	for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
0947b2f3	339	addr = fix_to_virt(idx);
beacfaac	340	pte = early_ioremap_pte(addr);
0947b2f3 HY	341	if (!*pte & _PAGE_PRESENT) {
	342	phys = *pte & PAGE_MASK;
	343	set_fixmap(idx, phys);
	344	}
	345	}
	346	}
	347
beacfaac	348	static void __init __early_set_fixmap(enum fixed_addresses idx,
0947b2f3 HY	349	unsigned long phys, pgprot_t flags)
	350	{
	351	unsigned long *pte, addr = __fix_to_virt(idx);
	352
	353	if (idx >= __end_of_fixed_addresses) {
	354	BUG();
	355	return;
	356	}
beacfaac	357	pte = early_ioremap_pte(addr);
0947b2f3 HY	358	if (pgprot_val(flags))
	359	*pte = (phys & PAGE_MASK) \| pgprot_val(flags);
	360	else
	361	*pte = 0;
	362	__flush_tlb_one(addr);
	363	}
	364
beacfaac	365	static inline void __init early_set_fixmap(enum fixed_addresses idx,
0947b2f3 HY	366	unsigned long phys)
	367	{
	368	if (after_paging_init)
	369	set_fixmap(idx, phys);
	370	else
beacfaac	371	__early_set_fixmap(idx, phys, PAGE_KERNEL);
0947b2f3 HY	372	}
0947b2f3 HY	373
beacfaac	374	static inline void __init early_clear_fixmap(enum fixed_addresses idx)
0947b2f3 HY	375	{
	376	if (after_paging_init)
	377	clear_fixmap(idx);
	378	else
beacfaac	379	__early_set_fixmap(idx, 0, __pgprot(0));
0947b2f3 HY	380	}
0947b2f3 HY	381
1b42f516 IM	382
	383	int __initdata early_ioremap_nested;
	384
d690b2af IM	385	static int __init check_early_ioremap_leak(void)
	386	{
	387	if (!early_ioremap_nested)
	388	return 0;
	389
	390	printk(KERN_WARNING
91eebf40 TG	391	"Debug warning: early ioremap leak of %d areas detected.\n",
91eebf40 TG	392	early_ioremap_nested);
d690b2af	393	printk(KERN_WARNING
91eebf40	394	"please boot with early_ioremap_debug and report the dmesg.\n");
d690b2af IM	395	WARN_ON(1);
	396
	397	return 1;
	398	}
	399	late_initcall(check_early_ioremap_leak);
	400
beacfaac	401	void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
1da177e4 LT	402	{
1da177e4 LT	403	unsigned long offset, last_addr;
1b42f516 IM	404	unsigned int nrpages, nesting;
	405	enum fixed_addresses idx0, idx;
	406
	407	WARN_ON(system_state != SYSTEM_BOOTING);
	408
	409	nesting = early_ioremap_nested;
d18d6d65	410	if (early_ioremap_debug) {
adafdf6a	411	printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
91eebf40	412	phys_addr, size, nesting);
d18d6d65 IM	413	dump_stack();
d18d6d65 IM	414	}
1da177e4 LT	415
	416	/* Don't allow wraparound or zero size */
	417	last_addr = phys_addr + size - 1;
bd796ed0 IM	418	if (!size \|\| last_addr < phys_addr) {
bd796ed0 IM	419	WARN_ON(1);
1da177e4	420	return NULL;
bd796ed0	421	}
1da177e4	422
bd796ed0 IM	423	if (nesting >= FIX_BTMAPS_NESTING) {
bd796ed0 IM	424	WARN_ON(1);
1b42f516	425	return NULL;
bd796ed0	426	}
1b42f516	427	early_ioremap_nested++;
1da177e4 LT	428	/*
	429	* Mappings have to be page-aligned
	430	*/
	431	offset = phys_addr & ~PAGE_MASK;
	432	phys_addr &= PAGE_MASK;
	433	size = PAGE_ALIGN(last_addr) - phys_addr;
	434
	435	/*
	436	* Mappings have to fit in the FIX_BTMAP area.
	437	*/
	438	nrpages = size >> PAGE_SHIFT;
bd796ed0 IM	439	if (nrpages > NR_FIX_BTMAPS) {
bd796ed0 IM	440	WARN_ON(1);
1da177e4	441	return NULL;
bd796ed0	442	}
1da177e4 LT	443
	444	/*
	445	* Ok, go for it..
	446	*/
1b42f516 IM	447	idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
1b42f516 IM	448	idx = idx0;
1da177e4	449	while (nrpages > 0) {
beacfaac	450	early_set_fixmap(idx, phys_addr);
1da177e4 LT	451	phys_addr += PAGE_SIZE;
	452	--idx;
	453	--nrpages;
	454	}
d18d6d65 IM	455	if (early_ioremap_debug)
d18d6d65 IM	456	printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
1b42f516	457
91eebf40	458	return (void *) (offset + fix_to_virt(idx0));
1da177e4 LT	459	}
1da177e4 LT	460
beacfaac	461	void __init early_iounmap(void *addr, unsigned long size)
1da177e4 LT	462	{
	463	unsigned long virt_addr;
	464	unsigned long offset;
	465	unsigned int nrpages;
	466	enum fixed_addresses idx;
1b42f516 IM	467	unsigned int nesting;
	468
	469	nesting = --early_ioremap_nested;
bd796ed0	470	WARN_ON(nesting < 0);
1da177e4	471
d18d6d65	472	if (early_ioremap_debug) {
adafdf6a	473	printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
91eebf40	474	size, nesting);
d18d6d65 IM	475	dump_stack();
	476	}
	477
1da177e4	478	virt_addr = (unsigned long)addr;
bd796ed0 IM	479	if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
bd796ed0 IM	480	WARN_ON(1);
1da177e4	481	return;
bd796ed0	482	}
1da177e4 LT	483	offset = virt_addr & ~PAGE_MASK;
	484	nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
	485
1b42f516	486	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
1da177e4	487	while (nrpages > 0) {
beacfaac	488	early_clear_fixmap(idx);
1da177e4 LT	489	--idx;
	490	--nrpages;
	491	}
	492	}
1b42f516 IM	493
	494	void __this_fixmap_does_not_exist(void)
	495	{
	496	WARN_ON(1);
	497	}
240d3a7c TG	498
240d3a7c TG	499	#endif /* CONFIG_X86_32 */