[mirror_ubuntu-bionic-kernel.git] / arch / s390 / mm / pageattr.c

/*
 * Copyright IBM Corp. 2011
 * Author(s): Jan Glauber <jang@linux.vnet.ibm.com>
 */
#include <linux/hugetlb.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <asm/cacheflush.h>
#include <asm/facility.h>
#include <asm/pgtable.h>
#include <asm/page.h>

static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
{
	asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0"
		     : [addr] "+a" (addr) : [skey] "d" (skey));
	return addr;
}

void __storage_key_init_range(unsigned long start, unsigned long end)
{
	unsigned long boundary, size;

	if (!PAGE_DEFAULT_KEY)
		return;
	while (start < end) {
		if (MACHINE_HAS_EDAT1) {
			/* set storage keys for a 1MB frame */
			size = 1UL << 20;
			boundary = (start + size) & ~(size - 1);
			if (boundary <= end) {
				do {
					start = sske_frame(start, PAGE_DEFAULT_KEY);
				} while (start < boundary);
				continue;
			}
		}
		page_set_storage_key(start, PAGE_DEFAULT_KEY, 0);
		start += PAGE_SIZE;
	}
}

#ifdef CONFIG_PROC_FS
atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];

void arch_report_meminfo(struct seq_file *m)
{
	seq_printf(m, "DirectMap4k:    %8lu kB\n",
		   atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2);
	seq_printf(m, "DirectMap1M:    %8lu kB\n",
		   atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10);
	seq_printf(m, "DirectMap2G:    %8lu kB\n",
		   atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21);
}
#endif /* CONFIG_PROC_FS */

static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr,
		    unsigned long dtt)
{
	unsigned long table, mask;

	mask = 0;
	if (MACHINE_HAS_EDAT2) {
		switch (dtt) {
		case CRDTE_DTT_REGION3:
			mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1);
			break;
		case CRDTE_DTT_SEGMENT:
			mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
			break;
		case CRDTE_DTT_PAGE:
			mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1);
			break;
		}
		table = (unsigned long)old & mask;
		crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce);
	} else if (MACHINE_HAS_IDTE) {
		cspg(old, *old, new);
	} else {
		csp((unsigned int *)old + 1, *old, new);
	}
}

struct cpa {
	unsigned int set_ro	: 1;
	unsigned int clear_ro	: 1;
};

static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end,
			  struct cpa cpa)
{
	pte_t *ptep, new;

	ptep = pte_offset(pmdp, addr);
	do {
		if (pte_none(*ptep))
			return -EINVAL;
		if (cpa.set_ro)
			new = pte_wrprotect(*ptep);
		else if (cpa.clear_ro)
			new = pte_mkwrite(pte_mkdirty(*ptep));
		pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE);
		ptep++;
		addr += PAGE_SIZE;
		cond_resched();
	} while (addr < end);
	return 0;
}

static int split_pmd_page(pmd_t *pmdp, unsigned long addr)
{
	unsigned long pte_addr, prot;
	pte_t *pt_dir, *ptep;
	pmd_t new;
	int i, ro;

	pt_dir = vmem_pte_alloc();
	if (!pt_dir)
		return -ENOMEM;
	pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT;
	ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT);
	prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
	ptep = pt_dir;
	for (i = 0; i < PTRS_PER_PTE; i++) {
		pte_val(*ptep) = pte_addr | prot;
		pte_addr += PAGE_SIZE;
		ptep++;
	}
	pmd_val(new) = __pa(pt_dir) | _SEGMENT_ENTRY;
	pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT);
	update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE);
	update_page_count(PG_DIRECT_MAP_1M, -1);
	return 0;
}

static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa)
{
	pmd_t new;

	if (cpa.set_ro)
		new = pmd_wrprotect(*pmdp);
	else if (cpa.clear_ro)
		new = pmd_mkwrite(pmd_mkdirty(*pmdp));
	pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT);
}

static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end,
			  struct cpa cpa)
{
	unsigned long next;
	pmd_t *pmdp;
	int rc = 0;

	pmdp = pmd_offset(pudp, addr);
	do {
		if (pmd_none(*pmdp))
			return -EINVAL;
		next = pmd_addr_end(addr, end);
		if (pmd_large(*pmdp)) {
			if (addr & ~PMD_MASK || addr + PMD_SIZE > next) {
				rc = split_pmd_page(pmdp, addr);
				if (rc)
					return rc;
				continue;
			}
			modify_pmd_page(pmdp, addr, cpa);
		} else {
			rc = walk_pte_level(pmdp, addr, next, cpa);
			if (rc)
				return rc;
		}
		pmdp++;
		addr = next;
		cond_resched();
	} while (addr < end);
	return rc;
}

static int split_pud_page(pud_t *pudp, unsigned long addr)
{
	unsigned long pmd_addr, prot;
	pmd_t *pm_dir, *pmdp;
	pud_t new;
	int i, ro;

	pm_dir = vmem_pmd_alloc();
	if (!pm_dir)
		return -ENOMEM;
	pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT;
	ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT);
	prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL);
	pmdp = pm_dir;
	for (i = 0; i < PTRS_PER_PMD; i++) {
		pmd_val(*pmdp) = pmd_addr | prot;
		pmd_addr += PMD_SIZE;
		pmdp++;
	}
	pud_val(new) = __pa(pm_dir) | _REGION3_ENTRY;
	pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3);
	update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD);
	update_page_count(PG_DIRECT_MAP_2G, -1);
	return 0;
}

static void modify_pud_page(pud_t *pudp, unsigned long addr, struct cpa cpa)
{
	pud_t new;

	if (cpa.set_ro)
		new = pud_wrprotect(*pudp);
	else if (cpa.clear_ro)
		new = pud_mkwrite(pud_mkdirty(*pudp));
	pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3);
}

static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end,
			  struct cpa cpa)
{
	unsigned long next;
	pud_t *pudp;
	int rc = 0;

	pudp = pud_offset(pgd, addr);
	do {
		if (pud_none(*pudp))
			return -EINVAL;
		next = pud_addr_end(addr, end);
		if (pud_large(*pudp)) {
			if (addr & ~PUD_MASK || addr + PUD_SIZE > next) {
				rc = split_pud_page(pudp, addr);
				if (rc)
					break;
				continue;
			}
			modify_pud_page(pudp, addr, cpa);
		} else {
			rc = walk_pmd_level(pudp, addr, next, cpa);
		}
		pudp++;
		addr = next;
		cond_resched();
	} while (addr < end && !rc);
	return rc;
}

static DEFINE_MUTEX(cpa_mutex);

static int change_page_attr(unsigned long addr, unsigned long end,
			    struct cpa cpa)
{
	unsigned long next;
	int rc = -EINVAL;
	pgd_t *pgdp;

	if (addr == end)
		return 0;
	if (end >= MODULES_END)
		return -EINVAL;
	mutex_lock(&cpa_mutex);
	pgdp = pgd_offset_k(addr);
	do {
		if (pgd_none(*pgdp))
			break;
		next = pgd_addr_end(addr, end);
		rc = walk_pud_level(pgdp, addr, next, cpa);
		if (rc)
			break;
		cond_resched();
	} while (pgdp++, addr = next, addr < end && !rc);
	mutex_unlock(&cpa_mutex);
	return rc;
}

int set_memory_ro(unsigned long addr, int numpages)
{
	struct cpa cpa = {
		.set_ro = 1,
	};

	addr &= PAGE_MASK;
	return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa);
}

int set_memory_rw(unsigned long addr, int numpages)
{
	struct cpa cpa = {
		.clear_ro = 1,
	};

	addr &= PAGE_MASK;
	return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa);
}

/* not possible */
int set_memory_nx(unsigned long addr, int numpages)
{
	return 0;
}

int set_memory_x(unsigned long addr, int numpages)
{
	return 0;
}

#ifdef CONFIG_DEBUG_PAGEALLOC

static void ipte_range(pte_t *pte, unsigned long address, int nr)
{
	int i;

	if (test_facility(13)) {
		__ptep_ipte_range(address, nr - 1, pte);
		return;
	}
	for (i = 0; i < nr; i++) {
		__ptep_ipte(address, pte);
		address += PAGE_SIZE;
		pte++;
	}
}

void __kernel_map_pages(struct page *page, int numpages, int enable)
{
	unsigned long address;
	int nr, i, j;
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	for (i = 0; i < numpages;) {
		address = page_to_phys(page + i);
		pgd = pgd_offset_k(address);
		pud = pud_offset(pgd, address);
		pmd = pmd_offset(pud, address);
		pte = pte_offset_kernel(pmd, address);
		nr = (unsigned long)pte >> ilog2(sizeof(long));
		nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1));
		nr = min(numpages - i, nr);
		if (enable) {
			for (j = 0; j < nr; j++) {
				pte_val(*pte) = address | pgprot_val(PAGE_KERNEL);
				address += PAGE_SIZE;
				pte++;
			}
		} else {
			ipte_range(pte, address, nr);
		}
		i += nr;
	}
}

#ifdef CONFIG_HIBERNATION
bool kernel_page_present(struct page *page)
{
	unsigned long addr;
	int cc;

	addr = page_to_phys(page);
	asm volatile(
		"	lra	%1,0(%1)\n"
		"	ipm	%0\n"
		"	srl	%0,28"
		: "=d" (cc), "+a" (addr) : : "cc");
	return cc == 0;
}
#endif /* CONFIG_HIBERNATION */

#endif /* CONFIG_DEBUG_PAGEALLOC */
Commit	Line	Data
305b1523 JG	1	/*
	2	* Copyright IBM Corp. 2011
	3	* Author(s): Jan Glauber <jang@linux.vnet.ibm.com>
	4	*/
6b70a920	5	#include <linux/hugetlb.h>
305b1523 JG	6	#include <linux/module.h>
305b1523 JG	7	#include <linux/mm.h>
638ad34a	8	#include <asm/cacheflush.h>
cfb0b241	9	#include <asm/facility.h>
305b1523	10	#include <asm/pgtable.h>
6b70a920 HC	11	#include <asm/page.h>
6b70a920 HC	12
f7f8d7e5 HC	13	static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
	14	{
	15	asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0"
	16	: [addr] "+a" (addr) : [skey] "d" (skey));
	17	return addr;
	18	}
	19
127c1fef	20	void __storage_key_init_range(unsigned long start, unsigned long end)
6b70a920	21	{
f7f8d7e5	22	unsigned long boundary, size;
6b70a920	23
de3fa841 HC	24	if (!PAGE_DEFAULT_KEY)
de3fa841 HC	25	return;
6b70a920	26	while (start < end) {
6b70a920 HC	27	if (MACHINE_HAS_EDAT1) {
6b70a920 HC	28	/* set storage keys for a 1MB frame */
6b70a920 HC	29	size = 1UL << 20;
	30	boundary = (start + size) & ~(size - 1);
	31	if (boundary <= end) {
	32	do {
f7f8d7e5	33	start = sske_frame(start, PAGE_DEFAULT_KEY);
6b70a920 HC	34	} while (start < boundary);
	35	continue;
	36	}
	37	}
	38	page_set_storage_key(start, PAGE_DEFAULT_KEY, 0);
	39	start += PAGE_SIZE;
	40	}
	41	}
305b1523	42
37cd944c HC	43	#ifdef CONFIG_PROC_FS
	44	atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];
	45
	46	void arch_report_meminfo(struct seq_file *m)
	47	{
	48	seq_printf(m, "DirectMap4k: %8lu kB\n",
	49	atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2);
	50	seq_printf(m, "DirectMap1M: %8lu kB\n",
	51	atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10);
	52	seq_printf(m, "DirectMap2G: %8lu kB\n",
	53	atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21);
	54	}
	55	#endif /* CONFIG_PROC_FS */
	56
e8a97e42 HC	57	static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr,
e8a97e42 HC	58	unsigned long dtt)
5b1ba9e3	59	{
e8a97e42 HC	60	unsigned long table, mask;
	61
	62	mask = 0;
	63	if (MACHINE_HAS_EDAT2) {
	64	switch (dtt) {
	65	case CRDTE_DTT_REGION3:
	66	mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1);
	67	break;
	68	case CRDTE_DTT_SEGMENT:
	69	mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
	70	break;
	71	case CRDTE_DTT_PAGE:
	72	mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1);
	73	break;
	74	}
	75	table = (unsigned long)old & mask;
	76	crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce);
	77	} else if (MACHINE_HAS_IDTE) {
	78	cspg(old, *old, new);
	79	} else {
	80	csp((unsigned int )old + 1, old, new);
	81	}
	82	}
	83
	84	struct cpa {
	85	unsigned int set_ro : 1;
	86	unsigned int clear_ro : 1;
	87	};
	88
	89	static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end,
	90	struct cpa cpa)
	91	{
	92	pte_t *ptep, new;
	93
	94	ptep = pte_offset(pmdp, addr);
	95	do {
	96	if (pte_none(*ptep))
	97	return -EINVAL;
	98	if (cpa.set_ro)
	99	new = pte_wrprotect(*ptep);
	100	else if (cpa.clear_ro)
	101	new = pte_mkwrite(pte_mkdirty(*ptep));
	102	pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE);
	103	ptep++;
	104	addr += PAGE_SIZE;
	105	cond_resched();
	106	} while (addr < end);
	107	return 0;
	108	}
	109
	110	static int split_pmd_page(pmd_t *pmdp, unsigned long addr)
	111	{
	112	unsigned long pte_addr, prot;
	113	pte_t pt_dir, ptep;
	114	pmd_t new;
	115	int i, ro;
	116
	117	pt_dir = vmem_pte_alloc();
	118	if (!pt_dir)
	119	return -ENOMEM;
	120	pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT;
	121	ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT);
	122	prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
	123	ptep = pt_dir;
124	for (i = 0; i < PTRS_PER_PTE; i++) {
125	pte_val(*ptep) = pte_addr \| prot;
126	pte_addr += PAGE_SIZE;
127	ptep++;
128	}
129	pmd_val(new) = __pa(pt_dir) \| _SEGMENT_ENTRY;
130	pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT);
37cd944c HC	131	update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE);
37cd944c HC	132	update_page_count(PG_DIRECT_MAP_1M, -1);
e8a97e42 HC	133	return 0;
	134	}
	135
	136	static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa)
	137	{
	138	pmd_t new;
	139
	140	if (cpa.set_ro)
	141	new = pmd_wrprotect(*pmdp);
	142	else if (cpa.clear_ro)
	143	new = pmd_mkwrite(pmd_mkdirty(*pmdp));
	144	pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT);
	145	}
	146
	147	static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end,
	148	struct cpa cpa)
	149	{
	150	unsigned long next;
5b1ba9e3	151	pmd_t *pmdp;
e8a97e42	152	int rc = 0;
5b1ba9e3	153
5b1ba9e3	154	pmdp = pmd_offset(pudp, addr);
e8a97e42 HC	155	do {
	156	if (pmd_none(*pmdp))
	157	return -EINVAL;
	158	next = pmd_addr_end(addr, end);
	159	if (pmd_large(*pmdp)) {
	160	if (addr & ~PMD_MASK \|\| addr + PMD_SIZE > next) {
	161	rc = split_pmd_page(pmdp, addr);
	162	if (rc)
	163	return rc;
	164	continue;
	165	}
	166	modify_pmd_page(pmdp, addr, cpa);
	167	} else {
	168	rc = walk_pte_level(pmdp, addr, next, cpa);
	169	if (rc)
	170	return rc;
	171	}
	172	pmdp++;
	173	addr = next;
	174	cond_resched();
	175	} while (addr < end);
	176	return rc;
5b1ba9e3 HC	177	}
5b1ba9e3 HC	178
e8a97e42	179	static int split_pud_page(pud_t *pudp, unsigned long addr)
305b1523	180	{
e8a97e42 HC	181	unsigned long pmd_addr, prot;
	182	pmd_t pm_dir, pmdp;
	183	pud_t new;
	184	int i, ro;
305b1523	185
e8a97e42 HC	186	pm_dir = vmem_pmd_alloc();
	187	if (!pm_dir)
	188	return -ENOMEM;
	189	pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT;
	190	ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT);
	191	prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL);
	192	pmdp = pm_dir;
	193	for (i = 0; i < PTRS_PER_PMD; i++) {
	194	pmd_val(*pmdp) = pmd_addr \| prot;
	195	pmd_addr += PMD_SIZE;
	196	pmdp++;
305b1523	197	}
e8a97e42 HC	198	pud_val(new) = __pa(pm_dir) \| _REGION3_ENTRY;
e8a97e42 HC	199	pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3);
37cd944c HC	200	update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD);
37cd944c HC	201	update_page_count(PG_DIRECT_MAP_2G, -1);
e8a97e42 HC	202	return 0;
	203	}
	204
	205	static void modify_pud_page(pud_t *pudp, unsigned long addr, struct cpa cpa)
	206	{
	207	pud_t new;
	208
	209	if (cpa.set_ro)
	210	new = pud_wrprotect(*pudp);
	211	else if (cpa.clear_ro)
	212	new = pud_mkwrite(pud_mkdirty(*pudp));
	213	pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3);
	214	}
	215
	216	static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end,
	217	struct cpa cpa)
	218	{
	219	unsigned long next;
	220	pud_t *pudp;
	221	int rc = 0;
	222
	223	pudp = pud_offset(pgd, addr);
	224	do {
	225	if (pud_none(*pudp))
	226	return -EINVAL;
	227	next = pud_addr_end(addr, end);
	228	if (pud_large(*pudp)) {
	229	if (addr & ~PUD_MASK \|\| addr + PUD_SIZE > next) {
	230	rc = split_pud_page(pudp, addr);
	231	if (rc)
	232	break;
	233	continue;
	234	}
	235	modify_pud_page(pudp, addr, cpa);
	236	} else {
	237	rc = walk_pmd_level(pudp, addr, next, cpa);
	238	}
	239	pudp++;
	240	addr = next;
	241	cond_resched();
	242	} while (addr < end && !rc);
	243	return rc;
	244	}
	245
	246	static DEFINE_MUTEX(cpa_mutex);
	247
	248	static int change_page_attr(unsigned long addr, unsigned long end,
	249	struct cpa cpa)
	250	{
	251	unsigned long next;
	252	int rc = -EINVAL;
	253	pgd_t *pgdp;
	254
4d81aaa5 HC	255	if (addr == end)
4d81aaa5 HC	256	return 0;
e8a97e42 HC	257	if (end >= MODULES_END)
	258	return -EINVAL;
	259	mutex_lock(&cpa_mutex);
	260	pgdp = pgd_offset_k(addr);
	261	do {
	262	if (pgd_none(*pgdp))
	263	break;
	264	next = pgd_addr_end(addr, end);
	265	rc = walk_pud_level(pgdp, addr, next, cpa);
	266	if (rc)
	267	break;
	268	cond_resched();
	269	} while (pgdp++, addr = next, addr < end && !rc);
	270	mutex_unlock(&cpa_mutex);
	271	return rc;
305b1523 JG	272	}
	273
	274	int set_memory_ro(unsigned long addr, int numpages)
	275	{
e8a97e42 HC	276	struct cpa cpa = {
	277	.set_ro = 1,
	278	};
	279
	280	addr &= PAGE_MASK;
	281	return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa);
305b1523	282	}
305b1523 JG	283
	284	int set_memory_rw(unsigned long addr, int numpages)
	285	{
e8a97e42 HC	286	struct cpa cpa = {
	287	.clear_ro = 1,
	288	};
	289
	290	addr &= PAGE_MASK;
	291	return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa);
305b1523	292	}
305b1523 JG	293
	294	/* not possible */
	295	int set_memory_nx(unsigned long addr, int numpages)
	296	{
	297	return 0;
	298	}
448694a1 JG	299
	300	int set_memory_x(unsigned long addr, int numpages)
	301	{
	302	return 0;
	303	}
0a4ccc99 HC	304
0a4ccc99 HC	305	#ifdef CONFIG_DEBUG_PAGEALLOC
cfb0b241 HC	306
	307	static void ipte_range(pte_t *pte, unsigned long address, int nr)
	308	{
	309	int i;
	310
5a79859a	311	if (test_facility(13)) {
cfb0b241 HC	312	__ptep_ipte_range(address, nr - 1, pte);
	313	return;
	314	}
	315	for (i = 0; i < nr; i++) {
	316	__ptep_ipte(address, pte);
	317	address += PAGE_SIZE;
	318	pte++;
	319	}
	320	}
	321
031bc574	322	void __kernel_map_pages(struct page *page, int numpages, int enable)
0a4ccc99 HC	323	{
0a4ccc99 HC	324	unsigned long address;
cfb0b241	325	int nr, i, j;
0a4ccc99 HC	326	pgd_t *pgd;
	327	pud_t *pud;
	328	pmd_t *pmd;
	329	pte_t *pte;
0a4ccc99	330
cfb0b241	331	for (i = 0; i < numpages;) {
0a4ccc99 HC	332	address = page_to_phys(page + i);
	333	pgd = pgd_offset_k(address);
	334	pud = pud_offset(pgd, address);
	335	pmd = pmd_offset(pud, address);
	336	pte = pte_offset_kernel(pmd, address);
cfb0b241 HC	337	nr = (unsigned long)pte >> ilog2(sizeof(long));
	338	nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1));
	339	nr = min(numpages - i, nr);
	340	if (enable) {
	341	for (j = 0; j < nr; j++) {
3e76ee99	342	pte_val(*pte) = address \| pgprot_val(PAGE_KERNEL);
cfb0b241 HC	343	address += PAGE_SIZE;
	344	pte++;
	345	}
	346	} else {
	347	ipte_range(pte, address, nr);
0a4ccc99	348	}
cfb0b241	349	i += nr;
0a4ccc99 HC	350	}
	351	}
	352
	353	#ifdef CONFIG_HIBERNATION
	354	bool kernel_page_present(struct page *page)
	355	{
	356	unsigned long addr;
	357	int cc;
	358
	359	addr = page_to_phys(page);
	360	asm volatile(
	361	" lra %1,0(%1)\n"
	362	" ipm %0\n"
	363	" srl %0,28"
	364	: "=d" (cc), "+a" (addr) : : "cc");
	365	return cc == 0;
	366	}
	367	#endif /* CONFIG_HIBERNATION */
	368
	369	#endif /* CONFIG_DEBUG_PAGEALLOC */