[mirror_ubuntu-focal-kernel.git] / arch / s390 / mm / hugetlbpage.c

/*
 *  IBM System z Huge TLB Page Support for Kernel.
 *
 *    Copyright IBM Corp. 2007,2016
 *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
 */

#define KMSG_COMPONENT "hugetlb"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/mm.h>
#include <linux/hugetlb.h>

/*
 * If the bit selected by single-bit bitmask "a" is set within "x", move
 * it to the position indicated by single-bit bitmask "b".
 */
#define move_set_bit(x, a, b)	(((x) & (a)) >> ilog2(a) << ilog2(b))

static inline unsigned long __pte_to_rste(pte_t pte)
{
	unsigned long rste;

	/*
	 * Convert encoding		  pte bits	pmd / pud bits
	 *				lIR.uswrdy.p	dy..R...I...wr
	 * empty			010.000000.0 -> 00..0...1...00
	 * prot-none, clean, old	111.000000.1 -> 00..1...1...00
	 * prot-none, clean, young	111.000001.1 -> 01..1...1...00
	 * prot-none, dirty, old	111.000010.1 -> 10..1...1...00
	 * prot-none, dirty, young	111.000011.1 -> 11..1...1...00
	 * read-only, clean, old	111.000100.1 -> 00..1...1...01
	 * read-only, clean, young	101.000101.1 -> 01..1...0...01
	 * read-only, dirty, old	111.000110.1 -> 10..1...1...01
	 * read-only, dirty, young	101.000111.1 -> 11..1...0...01
	 * read-write, clean, old	111.001100.1 -> 00..1...1...11
	 * read-write, clean, young	101.001101.1 -> 01..1...0...11
	 * read-write, dirty, old	110.001110.1 -> 10..0...1...11
	 * read-write, dirty, young	100.001111.1 -> 11..0...0...11
	 * HW-bits: R read-only, I invalid
	 * SW-bits: p present, y young, d dirty, r read, w write, s special,
	 *	    u unused, l large
	 */
	if (pte_present(pte)) {
		rste = pte_val(pte) & PAGE_MASK;
		rste |= move_set_bit(pte_val(pte), _PAGE_READ,
				     _SEGMENT_ENTRY_READ);
		rste |= move_set_bit(pte_val(pte), _PAGE_WRITE,
				     _SEGMENT_ENTRY_WRITE);
		rste |= move_set_bit(pte_val(pte), _PAGE_INVALID,
				     _SEGMENT_ENTRY_INVALID);
		rste |= move_set_bit(pte_val(pte), _PAGE_PROTECT,
				     _SEGMENT_ENTRY_PROTECT);
		rste |= move_set_bit(pte_val(pte), _PAGE_DIRTY,
				     _SEGMENT_ENTRY_DIRTY);
		rste |= move_set_bit(pte_val(pte), _PAGE_YOUNG,
				     _SEGMENT_ENTRY_YOUNG);
#ifdef CONFIG_MEM_SOFT_DIRTY
		rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
				     _SEGMENT_ENTRY_SOFT_DIRTY);
#endif
		rste |= move_set_bit(pte_val(pte), _PAGE_NOEXEC,
				     _SEGMENT_ENTRY_NOEXEC);
	} else
		rste = _SEGMENT_ENTRY_EMPTY;
	return rste;
}

static inline pte_t __rste_to_pte(unsigned long rste)
{
	int present;
	pte_t pte;

	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		present = pud_present(__pud(rste));
	else
		present = pmd_present(__pmd(rste));

	/*
	 * Convert encoding		pmd / pud bits	    pte bits
	 *				dy..R...I...wr	  lIR.uswrdy.p
	 * empty			00..0...1...00 -> 010.000000.0
	 * prot-none, clean, old	00..1...1...00 -> 111.000000.1
	 * prot-none, clean, young	01..1...1...00 -> 111.000001.1
	 * prot-none, dirty, old	10..1...1...00 -> 111.000010.1
	 * prot-none, dirty, young	11..1...1...00 -> 111.000011.1
	 * read-only, clean, old	00..1...1...01 -> 111.000100.1
	 * read-only, clean, young	01..1...0...01 -> 101.000101.1
	 * read-only, dirty, old	10..1...1...01 -> 111.000110.1
	 * read-only, dirty, young	11..1...0...01 -> 101.000111.1
	 * read-write, clean, old	00..1...1...11 -> 111.001100.1
	 * read-write, clean, young	01..1...0...11 -> 101.001101.1
	 * read-write, dirty, old	10..0...1...11 -> 110.001110.1
	 * read-write, dirty, young	11..0...0...11 -> 100.001111.1
	 * HW-bits: R read-only, I invalid
	 * SW-bits: p present, y young, d dirty, r read, w write, s special,
	 *	    u unused, l large
	 */
	if (present) {
		pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
		pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_READ,
					     _PAGE_READ);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_WRITE,
					     _PAGE_WRITE);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_INVALID,
					     _PAGE_INVALID);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT,
					     _PAGE_PROTECT);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY,
					     _PAGE_DIRTY);
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG,
					     _PAGE_YOUNG);
#ifdef CONFIG_MEM_SOFT_DIRTY
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY,
					     _PAGE_DIRTY);
#endif
		pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC,
					     _PAGE_NOEXEC);
	} else
		pte_val(pte) = _PAGE_INVALID;
	return pte;
}

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t pte)
{
	unsigned long rste;

	rste = __pte_to_rste(pte);
	if (!MACHINE_HAS_NX)
		rste &= ~_SEGMENT_ENTRY_NOEXEC;

	/* Set correct table type for 2G hugepages */
	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE;
	else
		rste |= _SEGMENT_ENTRY_LARGE;
	pte_val(*ptep) = rste;
}

pte_t huge_ptep_get(pte_t *ptep)
{
	return __rste_to_pte(pte_val(*ptep));
}

pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
			      unsigned long addr, pte_t *ptep)
{
	pte_t pte = huge_ptep_get(ptep);
	pmd_t *pmdp = (pmd_t *) ptep;
	pud_t *pudp = (pud_t *) ptep;

	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
	else
		pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
	return pte;
}

pte_t *huge_pte_alloc(struct mm_struct *mm,
			unsigned long addr, unsigned long sz)
{
	pgd_t *pgdp;
	pud_t *pudp;
	pmd_t *pmdp = NULL;

	pgdp = pgd_offset(mm, addr);
	pudp = pud_alloc(mm, pgdp, addr);
	if (pudp) {
		if (sz == PUD_SIZE)
			return (pte_t *) pudp;
		else if (sz == PMD_SIZE)
			pmdp = pmd_alloc(mm, pudp, addr);
	}
	return (pte_t *) pmdp;
}

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgdp;
	pud_t *pudp;
	pmd_t *pmdp = NULL;

	pgdp = pgd_offset(mm, addr);
	if (pgd_present(*pgdp)) {
		pudp = pud_offset(pgdp, addr);
		if (pud_present(*pudp)) {
			if (pud_large(*pudp))
				return (pte_t *) pudp;
			pmdp = pmd_offset(pudp, addr);
		}
	}
	return (pte_t *) pmdp;
}

int pmd_huge(pmd_t pmd)
{
	return pmd_large(pmd);
}

int pud_huge(pud_t pud)
{
	return pud_large(pud);
}

struct page *
follow_huge_pud(struct mm_struct *mm, unsigned long address,
		pud_t *pud, int flags)
{
	if (flags & FOLL_GET)
		return NULL;

	return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
}

static __init int setup_hugepagesz(char *opt)
{
	unsigned long size;
	char *string = opt;

	size = memparse(opt, &opt);
	if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	} else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	} else {
		hugetlb_bad_size();
		pr_err("hugepagesz= specifies an unsupported page size %s\n",
			string);
		return 0;
	}
	return 1;
}
__setup("hugepagesz=", setup_hugepagesz);
Commit	Line	Data
53492b1d GS	1	/*
	2	* IBM System z Huge TLB Page Support for Kernel.
	3	*
d08de8e2	4	* Copyright IBM Corp. 2007,2016
53492b1d GS	5	* Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
	6	*/
	7
d08de8e2 GS	8	#define KMSG_COMPONENT "hugetlb"
	9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	10
53492b1d GS	11	#include <linux/mm.h>
	12	#include <linux/hugetlb.h>
	13
bc29b7ac GS	14	/*
	15	* If the bit selected by single-bit bitmask "a" is set within "x", move
	16	* it to the position indicated by single-bit bitmask "b".
	17	*/
	18	#define move_set_bit(x, a, b) (((x) & (a)) >> ilog2(a) << ilog2(b))
	19
d08de8e2	20	static inline unsigned long __pte_to_rste(pte_t pte)
e5098611	21	{
d08de8e2	22	unsigned long rste;
e5098611 MS	23
e5098611 MS	24	/*
d08de8e2	25	* Convert encoding pte bits pmd / pud bits
a1c843b8 MS	26	* lIR.uswrdy.p dy..R...I...wr
	27	* empty 010.000000.0 -> 00..0...1...00
	28	* prot-none, clean, old 111.000000.1 -> 00..1...1...00
	29	* prot-none, clean, young 111.000001.1 -> 01..1...1...00
	30	* prot-none, dirty, old 111.000010.1 -> 10..1...1...00
	31	* prot-none, dirty, young 111.000011.1 -> 11..1...1...00
	32	* read-only, clean, old 111.000100.1 -> 00..1...1...01
	33	* read-only, clean, young 101.000101.1 -> 01..1...0...01
	34	* read-only, dirty, old 111.000110.1 -> 10..1...1...01
	35	* read-only, dirty, young 101.000111.1 -> 11..1...0...01
	36	* read-write, clean, old 111.001100.1 -> 00..1...1...11
	37	* read-write, clean, young 101.001101.1 -> 01..1...0...11
	38	* read-write, dirty, old 110.001110.1 -> 10..0...1...11
	39	* read-write, dirty, young 100.001111.1 -> 11..0...0...11
	40	* HW-bits: R read-only, I invalid
	41	* SW-bits: p present, y young, d dirty, r read, w write, s special,
	42	* u unused, l large
e5098611 MS	43	*/
e5098611 MS	44	if (pte_present(pte)) {
d08de8e2	45	rste = pte_val(pte) & PAGE_MASK;
bc29b7ac GS	46	rste \|= move_set_bit(pte_val(pte), _PAGE_READ,
	47	_SEGMENT_ENTRY_READ);
	48	rste \|= move_set_bit(pte_val(pte), _PAGE_WRITE,
	49	_SEGMENT_ENTRY_WRITE);
	50	rste \|= move_set_bit(pte_val(pte), _PAGE_INVALID,
	51	_SEGMENT_ENTRY_INVALID);
	52	rste \|= move_set_bit(pte_val(pte), _PAGE_PROTECT,
	53	_SEGMENT_ENTRY_PROTECT);
	54	rste \|= move_set_bit(pte_val(pte), _PAGE_DIRTY,
	55	_SEGMENT_ENTRY_DIRTY);
	56	rste \|= move_set_bit(pte_val(pte), _PAGE_YOUNG,
	57	_SEGMENT_ENTRY_YOUNG);
	58	#ifdef CONFIG_MEM_SOFT_DIRTY
	59	rste \|= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
	60	_SEGMENT_ENTRY_SOFT_DIRTY);
	61	#endif
57d7f939 MS	62	rste \|= move_set_bit(pte_val(pte), _PAGE_NOEXEC,
57d7f939 MS	63	_SEGMENT_ENTRY_NOEXEC);
e5098611	64	} else
54397bb0	65	rste = _SEGMENT_ENTRY_EMPTY;
d08de8e2	66	return rste;
e5098611 MS	67	}
e5098611 MS	68
d08de8e2	69	static inline pte_t __rste_to_pte(unsigned long rste)
e5098611	70	{
d08de8e2	71	int present;
e5098611 MS	72	pte_t pte;
e5098611 MS	73
d08de8e2 GS	74	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	75	present = pud_present(__pud(rste));
	76	else
	77	present = pmd_present(__pmd(rste));
	78
e5098611	79	/*
d08de8e2	80	* Convert encoding pmd / pud bits pte bits
a1c843b8 MS	81	* dy..R...I...wr lIR.uswrdy.p
	82	* empty 00..0...1...00 -> 010.000000.0
	83	* prot-none, clean, old 00..1...1...00 -> 111.000000.1
	84	* prot-none, clean, young 01..1...1...00 -> 111.000001.1
	85	* prot-none, dirty, old 10..1...1...00 -> 111.000010.1
	86	* prot-none, dirty, young 11..1...1...00 -> 111.000011.1
	87	* read-only, clean, old 00..1...1...01 -> 111.000100.1
	88	* read-only, clean, young 01..1...0...01 -> 101.000101.1
	89	* read-only, dirty, old 10..1...1...01 -> 111.000110.1
	90	* read-only, dirty, young 11..1...0...01 -> 101.000111.1
	91	* read-write, clean, old 00..1...1...11 -> 111.001100.1
	92	* read-write, clean, young 01..1...0...11 -> 101.001101.1
	93	* read-write, dirty, old 10..0...1...11 -> 110.001110.1
	94	* read-write, dirty, young 11..0...0...11 -> 100.001111.1
	95	* HW-bits: R read-only, I invalid
	96	* SW-bits: p present, y young, d dirty, r read, w write, s special,
	97	* u unused, l large
e5098611	98	*/
d08de8e2 GS	99	if (present) {
d08de8e2 GS	100	pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
152125b7	101	pte_val(pte) \|= _PAGE_LARGE \| _PAGE_PRESENT;
bc29b7ac GS	102	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_READ,
	103	_PAGE_READ);
	104	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_WRITE,
	105	_PAGE_WRITE);
	106	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_INVALID,
	107	_PAGE_INVALID);
	108	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT,
	109	_PAGE_PROTECT);
	110	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY,
	111	_PAGE_DIRTY);
	112	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG,
	113	_PAGE_YOUNG);
	114	#ifdef CONFIG_MEM_SOFT_DIRTY
	115	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY,
	116	_PAGE_DIRTY);
	117	#endif
57d7f939 MS	118	pte_val(pte) \|= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC,
57d7f939 MS	119	_PAGE_NOEXEC);
e5098611 MS	120	} else
	121	pte_val(pte) = _PAGE_INVALID;
	122	return pte;
	123	}
53492b1d GS	124
53492b1d GS	125	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
e5098611	126	pte_t *ptep, pte_t pte)
53492b1d	127	{
57d7f939 MS	128	unsigned long rste;
	129
	130	rste = __pte_to_rste(pte);
	131	if (!MACHINE_HAS_NX)
	132	rste &= ~_SEGMENT_ENTRY_NOEXEC;
d08de8e2 GS	133
	134	/* Set correct table type for 2G hugepages */
	135	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	136	rste \|= _REGION_ENTRY_TYPE_R3 \| _REGION3_ENTRY_LARGE;
	137	else
	138	rste \|= _SEGMENT_ENTRY_LARGE;
	139	pte_val(*ptep) = rste;
e5098611 MS	140	}
	141
	142	pte_t huge_ptep_get(pte_t *ptep)
	143	{
d08de8e2	144	return __rste_to_pte(pte_val(*ptep));
e5098611 MS	145	}
	146
	147	pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
	148	unsigned long addr, pte_t *ptep)
	149	{
d08de8e2	150	pte_t pte = huge_ptep_get(ptep);
e5098611	151	pmd_t pmdp = (pmd_t ) ptep;
d08de8e2	152	pud_t pudp = (pud_t ) ptep;
53492b1d	153
d08de8e2 GS	154	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	155	pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
	156	else
	157	pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
	158	return pte;
53492b1d GS	159	}
53492b1d GS	160
a5516438 AK	161	pte_t huge_pte_alloc(struct mm_struct mm,
a5516438 AK	162	unsigned long addr, unsigned long sz)
53492b1d GS	163	{
	164	pgd_t *pgdp;
	165	pud_t *pudp;
	166	pmd_t *pmdp = NULL;
	167
	168	pgdp = pgd_offset(mm, addr);
	169	pudp = pud_alloc(mm, pgdp, addr);
d08de8e2 GS	170	if (pudp) {
	171	if (sz == PUD_SIZE)
	172	return (pte_t *) pudp;
	173	else if (sz == PMD_SIZE)
	174	pmdp = pmd_alloc(mm, pudp, addr);
	175	}
53492b1d GS	176	return (pte_t *) pmdp;
	177	}
	178
	179	pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr)
	180	{
	181	pgd_t *pgdp;
	182	pud_t *pudp;
	183	pmd_t *pmdp = NULL;
	184
	185	pgdp = pgd_offset(mm, addr);
	186	if (pgd_present(*pgdp)) {
	187	pudp = pud_offset(pgdp, addr);
d08de8e2 GS	188	if (pud_present(*pudp)) {
	189	if (pud_large(*pudp))
	190	return (pte_t *) pudp;
53492b1d	191	pmdp = pmd_offset(pudp, addr);
d08de8e2	192	}
53492b1d GS	193	}
	194	return (pte_t *) pmdp;
	195	}
	196
53492b1d GS	197	int pmd_huge(pmd_t pmd)
53492b1d GS	198	{
cbd7d9c2	199	return pmd_large(pmd);
53492b1d GS	200	}
53492b1d GS	201
ceb86879 AK	202	int pud_huge(pud_t pud)
ceb86879 AK	203	{
d08de8e2 GS	204	return pud_large(pud);
	205	}
	206
	207	struct page *
	208	follow_huge_pud(struct mm_struct *mm, unsigned long address,
	209	pud_t *pud, int flags)
	210	{
	211	if (flags & FOLL_GET)
	212	return NULL;
	213
	214	return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
	215	}
	216
	217	static __init int setup_hugepagesz(char *opt)
	218	{
	219	unsigned long size;
	220	char *string = opt;
	221
	222	size = memparse(opt, &opt);
	223	if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
	224	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	225	} else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
	226	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	227	} else {
b5003b5f	228	hugetlb_bad_size();
d08de8e2 GS	229	pr_err("hugepagesz= specifies an unsupported page size %s\n",
	230	string);
	231	return 0;
	232	}
	233	return 1;
ceb86879	234	}
d08de8e2	235	__setup("hugepagesz=", setup_hugepagesz);