[mirror_ubuntu-bionic-kernel.git] / arch / powerpc / mm / hash64_64k.c

/*
 * Copyright IBM Corporation, 2015
 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU Lesser General Public License
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it would be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 */

#include <linux/mm.h>
#include <asm/machdep.h>
#include <asm/mmu.h>
/*
 * index from 0 - 15
 */
bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
{
	unsigned long g_idx;
	unsigned long ptev = pte_val(rpte.pte);

	g_idx = (ptev & H_PAGE_COMBO_VALID) >> H_PAGE_F_GIX_SHIFT;
	index = index >> 2;
	if (g_idx & (0x1 << index))
		return true;
	else
		return false;
}
/*
 * index from 0 - 15
 */
static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index)
{
	unsigned long g_idx;

	if (!(ptev & H_PAGE_COMBO))
		return ptev;
	index = index >> 2;
	g_idx = 0x1 << index;

	return ptev | (g_idx << H_PAGE_F_GIX_SHIFT);
}

int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
		   pte_t *ptep, unsigned long trap, unsigned long flags,
		   int ssize, int subpg_prot)
{
	real_pte_t rpte;
	unsigned long *hidxp;
	unsigned long hpte_group;
	unsigned int subpg_index;
	unsigned long rflags, pa, hidx;
	unsigned long old_pte, new_pte, subpg_pte;
	unsigned long vpn, hash, slot;
	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;

	/*
	 * atomically mark the linux large page PTE busy and dirty
	 */
	do {
		pte_t pte = READ_ONCE(*ptep);

		old_pte = pte_val(pte);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & H_PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(!check_pte_access(access, old_pte)))
			return 1;
		/*
		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access. Since this is 4K insert of 64K page size
		 * also add H_PAGE_COMBO
		 */
		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED | H_PAGE_COMBO;
		if (access & _PAGE_WRITE)
			new_pte |= _PAGE_DIRTY;
	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

	/*
	 * Handle the subpage protection bits
	 */
	subpg_pte = new_pte & ~subpg_prot;
	rflags = htab_convert_pte_flags(subpg_pte);

	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {

		/*
		 * No CPU has hugepages but lacks no execute, so we
		 * don't need to worry about that case
		 */
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	}

	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
	vpn  = hpt_vpn(ea, vsid, ssize);
	rpte = __real_pte(__pte(old_pte), ptep);
	/*
	 *None of the sub 4k page is hashed
	 */
	if (!(old_pte & H_PAGE_HASHPTE))
		goto htab_insert_hpte;
	/*
	 * Check if the pte was already inserted into the hash table
	 * as a 64k HW page, and invalidate the 64k HPTE if so.
	 */
	if (!(old_pte & H_PAGE_COMBO)) {
		flush_hash_page(vpn, rpte, MMU_PAGE_64K, ssize, flags);
		/*
		 * clear the old slot details from the old and new pte.
		 * On hash insert failure we use old pte value and we don't
		 * want slot information there if we have a insert failure.
		 */
		old_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND);
		new_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND);
		goto htab_insert_hpte;
	}
	/*
	 * Check for sub page valid and update
	 */
	if (__rpte_sub_valid(rpte, subpg_index)) {
		int ret;

		hash = hpt_hash(vpn, shift, ssize);
		hidx = __rpte_to_hidx(rpte, subpg_index);
		if (hidx & _PTEIDX_SECONDARY)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += hidx & _PTEIDX_GROUP_IX;

		ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
						 MMU_PAGE_4K, MMU_PAGE_4K,
						 ssize, flags);
		/*
		 *if we failed because typically the HPTE wasn't really here
		 * we try an insertion.
		 */
		if (ret == -1)
			goto htab_insert_hpte;

		*ptep = __pte(new_pte & ~H_PAGE_BUSY);
		return 0;
	}

htab_insert_hpte:
	/*
	 * handle H_PAGE_4K_PFN case
	 */
	if (old_pte & H_PAGE_4K_PFN) {
		/*
		 * All the sub 4k page have the same
		 * physical address.
		 */
		pa = pte_pfn(__pte(old_pte)) << HW_PAGE_SHIFT;
	} else {
		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
		pa += (subpg_index << shift);
	}
	hash = hpt_hash(vpn, shift, ssize);
repeat:
	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;

	/* Insert into the hash table, primary slot */
	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
					MMU_PAGE_4K, MMU_PAGE_4K, ssize);
	/*
	 * Primary is full, try the secondary
	 */
	if (unlikely(slot == -1)) {
		hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
		slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
						rflags, HPTE_V_SECONDARY,
						MMU_PAGE_4K, MMU_PAGE_4K,
						ssize);
		if (slot == -1) {
			if (mftb() & 0x1)
				hpte_group = ((hash & htab_hash_mask) *
					      HPTES_PER_GROUP) & ~0x7UL;
			mmu_hash_ops.hpte_remove(hpte_group);
			/*
			 * FIXME!! Should be try the group from which we removed ?
			 */
			goto repeat;
		}
	}
	/*
	 * Hypervisor failure. Restore old pte and return -1
	 * similar to __hash_page_*
	 */
	if (unlikely(slot == -2)) {
		*ptep = __pte(old_pte);
		hash_failure_debug(ea, access, vsid, trap, ssize,
				   MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
		return -1;
	}
	/*
	 * Insert slot number & secondary bit in PTE second half,
	 * clear H_PAGE_BUSY and set appropriate HPTE slot bit
	 * Since we have H_PAGE_BUSY set on ptep, we can be sure
	 * nobody is undating hidx.
	 */
	hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
	rpte.hidx &= ~(0xfUL << (subpg_index << 2));
	*hidxp = rpte.hidx  | (slot << (subpg_index << 2));
	new_pte = mark_subptegroup_valid(new_pte, subpg_index);
	new_pte |=  H_PAGE_HASHPTE;
	/*
	 * check __real_pte for details on matching smp_rmb()
	 */
	smp_wmb();
	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
	return 0;
}

int __hash_page_64K(unsigned long ea, unsigned long access,
		    unsigned long vsid, pte_t *ptep, unsigned long trap,
		    unsigned long flags, int ssize)
{
	unsigned long hpte_group;
	unsigned long rflags, pa;
	unsigned long old_pte, new_pte;
	unsigned long vpn, hash, slot;
	unsigned long shift = mmu_psize_defs[MMU_PAGE_64K].shift;

	/*
	 * atomically mark the linux large page PTE busy and dirty
	 */
	do {
		pte_t pte = READ_ONCE(*ptep);

		old_pte = pte_val(pte);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & H_PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(!check_pte_access(access, old_pte)))
			return 1;
		/*
		 * Check if PTE has the cache-inhibit bit set
		 * If so, bail out and refault as a 4k page
		 */
		if (!mmu_has_feature(MMU_FTR_CI_LARGE_PAGE) &&
		    unlikely(pte_ci(pte)))
			return 0;
		/*
		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access.
		 */
		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
		if (access & _PAGE_WRITE)
			new_pte |= _PAGE_DIRTY;
	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

	rflags = htab_convert_pte_flags(new_pte);

	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);

	vpn  = hpt_vpn(ea, vsid, ssize);
	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
		/*
		 * There MIGHT be an HPTE for this pte
		 */
		hash = hpt_hash(vpn, shift, ssize);
		if (old_pte & H_PAGE_F_SECOND)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;

		if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
					       MMU_PAGE_64K, ssize,
					       flags) == -1)
			old_pte &= ~_PAGE_HPTEFLAGS;
	}

	if (likely(!(old_pte & H_PAGE_HASHPTE))) {

		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
		hash = hpt_hash(vpn, shift, ssize);

repeat:
		hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;

		/* Insert into the hash table, primary slot */
		slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
						MMU_PAGE_64K, MMU_PAGE_64K,
						ssize);
		/*
		 * Primary is full, try the secondary
		 */
		if (unlikely(slot == -1)) {
			hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
			slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
							rflags,
							HPTE_V_SECONDARY,
							MMU_PAGE_64K,
							MMU_PAGE_64K, ssize);
			if (slot == -1) {
				if (mftb() & 0x1)
					hpte_group = ((hash & htab_hash_mask) *
						      HPTES_PER_GROUP) & ~0x7UL;
				mmu_hash_ops.hpte_remove(hpte_group);
				/*
				 * FIXME!! Should be try the group from which we removed ?
				 */
				goto repeat;
			}
		}
		/*
		 * Hypervisor failure. Restore old pte and return -1
		 * similar to __hash_page_*
		 */
		if (unlikely(slot == -2)) {
			*ptep = __pte(old_pte);
			hash_failure_debug(ea, access, vsid, trap, ssize,
					   MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
			return -1;
		}
		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
		new_pte |= (slot << H_PAGE_F_GIX_SHIFT) &
			(H_PAGE_F_SECOND | H_PAGE_F_GIX);
	}
	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
	return 0;
}
Commit	Line	Data
91f1da99 AK	1	/*
	2	* Copyright IBM Corporation, 2015
	3	* Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of version 2 of the GNU Lesser General Public License
	7	* as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it would be useful, but
	10	* WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	12	*
	13	*/
	14
	15	#include <linux/mm.h>
	16	#include <asm/machdep.h>
	17	#include <asm/mmu.h>
bf680d51 AK	18	/*
	19	* index from 0 - 15
	20	*/
	21	bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
	22	{
	23	unsigned long g_idx;
	24	unsigned long ptev = pte_val(rpte.pte);
	25
945537df	26	g_idx = (ptev & H_PAGE_COMBO_VALID) >> H_PAGE_F_GIX_SHIFT;
bf680d51 AK	27	index = index >> 2;
	28	if (g_idx & (0x1 << index))
	29	return true;
	30	else
	31	return false;
	32	}
	33	/*
	34	* index from 0 - 15
	35	*/
	36	static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index)
	37	{
	38	unsigned long g_idx;
	39
945537df	40	if (!(ptev & H_PAGE_COMBO))
bf680d51 AK	41	return ptev;
	42	index = index >> 2;
	43	g_idx = 0x1 << index;
	44
945537df	45	return ptev \| (g_idx << H_PAGE_F_GIX_SHIFT);
bf680d51	46	}
91f1da99 AK	47
	48	int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
	49	pte_t *ptep, unsigned long trap, unsigned long flags,
	50	int ssize, int subpg_prot)
	51	{
	52	real_pte_t rpte;
	53	unsigned long *hidxp;
	54	unsigned long hpte_group;
	55	unsigned int subpg_index;
	56	unsigned long rflags, pa, hidx;
	57	unsigned long old_pte, new_pte, subpg_pte;
	58	unsigned long vpn, hash, slot;
	59	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;
	60
	61	/*
	62	* atomically mark the linux large page PTE busy and dirty
	63	*/
	64	do {
	65	pte_t pte = READ_ONCE(*ptep);
	66
	67	old_pte = pte_val(pte);
	68	/* If PTE busy, retry the access */
945537df	69	if (unlikely(old_pte & H_PAGE_BUSY))
91f1da99 AK	70	return 0;
91f1da99 AK	71	/* If PTE permissions don't match, take page fault */
ac29c640	72	if (unlikely(!check_pte_access(access, old_pte)))
91f1da99 AK	73	return 1;
	74	/*
	75	* Try to lock the PTE, add ACCESSED and DIRTY if it was
	76	* a write access. Since this is 4K insert of 64K page size
945537df	77	* also add H_PAGE_COMBO
91f1da99	78	*/
945537df	79	new_pte = old_pte \| H_PAGE_BUSY \| _PAGE_ACCESSED \| H_PAGE_COMBO;
c7d54842	80	if (access & _PAGE_WRITE)
91f1da99	81	new_pte \|= _PAGE_DIRTY;
3910a7f4 ME	82	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
3910a7f4 ME	83
91f1da99 AK	84	/*
	85	* Handle the subpage protection bits
	86	*/
	87	subpg_pte = new_pte & ~subpg_prot;
c6a3c495	88	rflags = htab_convert_pte_flags(subpg_pte);
91f1da99	89
dd7b2f03	90	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
91f1da99 AK	91	!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
	92
	93	/*
	94	* No CPU has hugepages but lacks no execute, so we
	95	* don't need to worry about that case
	96	*/
	97	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	98	}
	99
	100	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
	101	vpn = hpt_vpn(ea, vsid, ssize);
	102	rpte = __real_pte(__pte(old_pte), ptep);
	103	/*
	104	*None of the sub 4k page is hashed
	105	*/
945537df	106	if (!(old_pte & H_PAGE_HASHPTE))
91f1da99 AK	107	goto htab_insert_hpte;
	108	/*
	109	* Check if the pte was already inserted into the hash table
	110	* as a 64k HW page, and invalidate the 64k HPTE if so.
	111	*/
945537df	112	if (!(old_pte & H_PAGE_COMBO)) {
91f1da99	113	flush_hash_page(vpn, rpte, MMU_PAGE_64K, ssize, flags);
9ab3ac23 AK	114	/*
	115	* clear the old slot details from the old and new pte.
	116	* On hash insert failure we use old pte value and we don't
	117	* want slot information there if we have a insert failure.
	118	*/
945537df AK	119	old_pte &= ~(H_PAGE_HASHPTE \| H_PAGE_F_GIX \| H_PAGE_F_SECOND);
945537df AK	120	new_pte &= ~(H_PAGE_HASHPTE \| H_PAGE_F_GIX \| H_PAGE_F_SECOND);
91f1da99 AK	121	goto htab_insert_hpte;
	122	}
	123	/*
	124	* Check for sub page valid and update
	125	*/
	126	if (__rpte_sub_valid(rpte, subpg_index)) {
	127	int ret;
	128
	129	hash = hpt_hash(vpn, shift, ssize);
	130	hidx = __rpte_to_hidx(rpte, subpg_index);
	131	if (hidx & _PTEIDX_SECONDARY)
	132	hash = ~hash;
	133	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	134	slot += hidx & _PTEIDX_GROUP_IX;
	135
7025776e BH	136	ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
	137	MMU_PAGE_4K, MMU_PAGE_4K,
	138	ssize, flags);
91f1da99 AK	139	/*
	140	*if we failed because typically the HPTE wasn't really here
	141	* we try an insertion.
	142	*/
	143	if (ret == -1)
	144	goto htab_insert_hpte;
	145
945537df	146	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
91f1da99 AK	147	return 0;
	148	}
	149
	150	htab_insert_hpte:
	151	/*
945537df	152	* handle H_PAGE_4K_PFN case
91f1da99	153	*/
945537df	154	if (old_pte & H_PAGE_4K_PFN) {
91f1da99 AK	155	/*
	156	* All the sub 4k page have the same
	157	* physical address.
	158	*/
	159	pa = pte_pfn(__pte(old_pte)) << HW_PAGE_SHIFT;
	160	} else {
	161	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	162	pa += (subpg_index << shift);
	163	}
	164	hash = hpt_hash(vpn, shift, ssize);
	165	repeat:
	166	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
	167
	168	/* Insert into the hash table, primary slot */
7025776e BH	169	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
7025776e BH	170	MMU_PAGE_4K, MMU_PAGE_4K, ssize);
91f1da99 AK	171	/*
	172	* Primary is full, try the secondary
	173	*/
	174	if (unlikely(slot == -1)) {
	175	hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
7025776e BH	176	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
	177	rflags, HPTE_V_SECONDARY,
	178	MMU_PAGE_4K, MMU_PAGE_4K,
	179	ssize);
91f1da99 AK	180	if (slot == -1) {
	181	if (mftb() & 0x1)
	182	hpte_group = ((hash & htab_hash_mask) *
	183	HPTES_PER_GROUP) & ~0x7UL;
7025776e	184	mmu_hash_ops.hpte_remove(hpte_group);
91f1da99 AK	185	/*
	186	* FIXME!! Should be try the group from which we removed ?
	187	*/
	188	goto repeat;
	189	}
	190	}
	191	/*
e9a68147	192	* Hypervisor failure. Restore old pte and return -1
91f1da99 AK	193	* similar to __hash_page_*
	194	*/
	195	if (unlikely(slot == -2)) {
	196	*ptep = __pte(old_pte);
	197	hash_failure_debug(ea, access, vsid, trap, ssize,
	198	MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
	199	return -1;
	200	}
	201	/*
	202	* Insert slot number & secondary bit in PTE second half,
945537df AK	203	* clear H_PAGE_BUSY and set appropriate HPTE slot bit
945537df AK	204	* Since we have H_PAGE_BUSY set on ptep, we can be sure
91f1da99 AK	205	* nobody is undating hidx.
	206	*/
	207	hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
91f1da99 AK	208	rpte.hidx &= ~(0xfUL << (subpg_index << 2));
91f1da99 AK	209	*hidxp = rpte.hidx \| (slot << (subpg_index << 2));
bf680d51	210	new_pte = mark_subptegroup_valid(new_pte, subpg_index);
945537df	211	new_pte \|= H_PAGE_HASHPTE;
91f1da99 AK	212	/*
	213	* check __real_pte for details on matching smp_rmb()
	214	*/
	215	smp_wmb();
945537df	216	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
91f1da99 AK	217	return 0;
91f1da99 AK	218	}
89ff7250 AK	219
	220	int __hash_page_64K(unsigned long ea, unsigned long access,
	221	unsigned long vsid, pte_t *ptep, unsigned long trap,
	222	unsigned long flags, int ssize)
	223	{
89ff7250 AK	224	unsigned long hpte_group;
	225	unsigned long rflags, pa;
	226	unsigned long old_pte, new_pte;
	227	unsigned long vpn, hash, slot;
	228	unsigned long shift = mmu_psize_defs[MMU_PAGE_64K].shift;
	229
	230	/*
	231	* atomically mark the linux large page PTE busy and dirty
	232	*/
	233	do {
	234	pte_t pte = READ_ONCE(*ptep);
	235
	236	old_pte = pte_val(pte);
	237	/* If PTE busy, retry the access */
945537df	238	if (unlikely(old_pte & H_PAGE_BUSY))
89ff7250 AK	239	return 0;
89ff7250 AK	240	/* If PTE permissions don't match, take page fault */
ac29c640	241	if (unlikely(!check_pte_access(access, old_pte)))
89ff7250 AK	242	return 1;
	243	/*
	244	* Check if PTE has the cache-inhibit bit set
	245	* If so, bail out and refault as a 4k page
	246	*/
	247	if (!mmu_has_feature(MMU_FTR_CI_LARGE_PAGE) &&
30bda41a	248	unlikely(pte_ci(pte)))
89ff7250 AK	249	return 0;
	250	/*
	251	* Try to lock the PTE, add ACCESSED and DIRTY if it was
1ec3f937	252	* a write access.
89ff7250	253	*/
945537df	254	new_pte = old_pte \| H_PAGE_BUSY \| _PAGE_ACCESSED;
c7d54842	255	if (access & _PAGE_WRITE)
89ff7250	256	new_pte \|= _PAGE_DIRTY;
3910a7f4	257	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
c6a3c495 AK	258
c6a3c495 AK	259	rflags = htab_convert_pte_flags(new_pte);
89ff7250	260
dd7b2f03	261	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
89ff7250 AK	262	!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	263	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	264
	265	vpn = hpt_vpn(ea, vsid, ssize);
945537df	266	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
89ff7250 AK	267	/*
	268	* There MIGHT be an HPTE for this pte
	269	*/
	270	hash = hpt_hash(vpn, shift, ssize);
945537df	271	if (old_pte & H_PAGE_F_SECOND)
89ff7250 AK	272	hash = ~hash;
89ff7250 AK	273	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
945537df	274	slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
89ff7250	275
7025776e BH	276	if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
	277	MMU_PAGE_64K, ssize,
	278	flags) == -1)
89ff7250 AK	279	old_pte &= ~_PAGE_HPTEFLAGS;
	280	}
	281
945537df	282	if (likely(!(old_pte & H_PAGE_HASHPTE))) {
89ff7250 AK	283
	284	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	285	hash = hpt_hash(vpn, shift, ssize);
	286
	287	repeat:
	288	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
	289
	290	/* Insert into the hash table, primary slot */
7025776e BH	291	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
	292	MMU_PAGE_64K, MMU_PAGE_64K,
	293	ssize);
89ff7250 AK	294	/*
	295	* Primary is full, try the secondary
	296	*/
	297	if (unlikely(slot == -1)) {
	298	hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
7025776e BH	299	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
	300	rflags,
	301	HPTE_V_SECONDARY,
	302	MMU_PAGE_64K,
	303	MMU_PAGE_64K, ssize);
89ff7250 AK	304	if (slot == -1) {
	305	if (mftb() & 0x1)
	306	hpte_group = ((hash & htab_hash_mask) *
	307	HPTES_PER_GROUP) & ~0x7UL;
7025776e	308	mmu_hash_ops.hpte_remove(hpte_group);
89ff7250 AK	309	/*
	310	* FIXME!! Should be try the group from which we removed ?
	311	*/
	312	goto repeat;
	313	}
	314	}
	315	/*
e9a68147	316	* Hypervisor failure. Restore old pte and return -1
89ff7250 AK	317	* similar to __hash_page_*
	318	*/
	319	if (unlikely(slot == -2)) {
	320	*ptep = __pte(old_pte);
	321	hash_failure_debug(ea, access, vsid, trap, ssize,
	322	MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
	323	return -1;
	324	}
945537df AK	325	new_pte = (new_pte & ~_PAGE_HPTEFLAGS) \| H_PAGE_HASHPTE;
	326	new_pte \|= (slot << H_PAGE_F_GIX_SHIFT) &
	327	(H_PAGE_F_SECOND \| H_PAGE_F_GIX);
89ff7250	328	}
945537df	329	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
89ff7250 AK	330	return 0;
89ff7250 AK	331	}