[mirror_ubuntu-artful-kernel.git] / arch / s390 / mm / gup.c

/*
 *  Lockless get_user_pages_fast for s390
 *
 *  Copyright IBM Corp. 2010
 *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
 */
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/vmstat.h>
#include <linux/pagemap.h>
#include <linux/rwsem.h>
#include <asm/pgtable.h>

/*
 * The performance critical leaf functions are made noinline otherwise gcc
 * inlines everything into a single function which results in too much
 * register pressure.
 */
static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
		unsigned long end, int write, struct page **pages, int *nr)
{
	struct page *head, *page;
	unsigned long mask;
	pte_t *ptep, pte;

	mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL;

	ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr);
	do {
		pte = *ptep;
		barrier();
		/* Similar to the PMD case, NUMA hinting must take slow path */
		if (pte_protnone(pte))
			return 0;
		if ((pte_val(pte) & mask) != 0)
			return 0;
		VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
		page = pte_page(pte);
		head = compound_head(page);
		if (!page_cache_get_speculative(head))
			return 0;
		if (unlikely(pte_val(pte) != pte_val(*ptep))) {
			put_page(head);
			return 0;
		}
		VM_BUG_ON_PAGE(compound_head(page) != head, page);
		pages[*nr] = page;
		(*nr)++;

	} while (ptep++, addr += PAGE_SIZE, addr != end);

	return 1;
}

static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
		unsigned long end, int write, struct page **pages, int *nr)
{
	unsigned long mask, result;
	struct page *head, *page;
	int refs;

	result = write ? 0 : _SEGMENT_ENTRY_PROTECT;
	mask = result | _SEGMENT_ENTRY_INVALID;
	if ((pmd_val(pmd) & mask) != result)
		return 0;
	VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));

	refs = 0;
	head = pmd_page(pmd);
	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
	do {
		VM_BUG_ON(compound_head(page) != head);
		pages[*nr] = page;
		(*nr)++;
		page++;
		refs++;
	} while (addr += PAGE_SIZE, addr != end);

	if (!page_cache_add_speculative(head, refs)) {
		*nr -= refs;
		return 0;
	}

	if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
		*nr -= refs;
		while (refs--)
			put_page(head);
		return 0;
	}

	return 1;
}


static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
		unsigned long end, int write, struct page **pages, int *nr)
{
	unsigned long next;
	pmd_t *pmdp, pmd;

	pmdp = (pmd_t *) pudp;
	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
		pmdp = (pmd_t *) pud_deref(pud);
	pmdp += pmd_index(addr);
	do {
		pmd = *pmdp;
		barrier();
		next = pmd_addr_end(addr, end);
		if (pmd_none(pmd))
			return 0;
		if (unlikely(pmd_large(pmd))) {
			/*
			 * NUMA hinting faults need to be handled in the GUP
			 * slowpath for accounting purposes and so that they
			 * can be serialised against THP migration.
			 */
			if (pmd_protnone(pmd))
				return 0;
			if (!gup_huge_pmd(pmdp, pmd, addr, next,
					  write, pages, nr))
				return 0;
		} else if (!gup_pte_range(pmdp, pmd, addr, next,
					  write, pages, nr))
			return 0;
	} while (pmdp++, addr = next, addr != end);

	return 1;
}

static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
		unsigned long end, int write, struct page **pages, int *nr)
{
	struct page *head, *page;
	unsigned long mask;
	int refs;

	mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID;
	if ((pud_val(pud) & mask) != 0)
		return 0;
	VM_BUG_ON(!pfn_valid(pud_pfn(pud)));

	refs = 0;
	head = pud_page(pud);
	page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
	do {
		VM_BUG_ON_PAGE(compound_head(page) != head, page);
		pages[*nr] = page;
		(*nr)++;
		page++;
		refs++;
	} while (addr += PAGE_SIZE, addr != end);

	if (!page_cache_add_speculative(head, refs)) {
		*nr -= refs;
		return 0;
	}

	if (unlikely(pud_val(pud) != pud_val(*pudp))) {
		*nr -= refs;
		while (refs--)
			put_page(head);
		return 0;
	}

	return 1;
}

static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
		unsigned long end, int write, struct page **pages, int *nr)
{
	unsigned long next;
	pud_t *pudp, pud;

	pudp = (pud_t *) pgdp;
	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
		pudp = (pud_t *) pgd_deref(pgd);
	pudp += pud_index(addr);
	do {
		pud = *pudp;
		barrier();
		next = pud_addr_end(addr, end);
		if (pud_none(pud))
			return 0;
		if (unlikely(pud_large(pud))) {
			if (!gup_huge_pud(pudp, pud, addr, next, write, pages,
					  nr))
				return 0;
		} else if (!gup_pmd_range(pudp, pud, addr, next, write, pages,
					  nr))
			return 0;
	} while (pudp++, addr = next, addr != end);

	return 1;
}

/*
 * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
 * back to the regular GUP.
 */
int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
			  struct page **pages)
{
	struct mm_struct *mm = current->mm;
	unsigned long addr, len, end;
	unsigned long next, flags;
	pgd_t *pgdp, pgd;
	int nr = 0;

	start &= PAGE_MASK;
	addr = start;
	len = (unsigned long) nr_pages << PAGE_SHIFT;
	end = start + len;
	if ((end <= start) || (end > TASK_SIZE))
		return 0;
	/*
	 * local_irq_save() doesn't prevent pagetable teardown, but does
	 * prevent the pagetables from being freed on s390.
	 *
	 * So long as we atomically load page table pointers versus teardown,
	 * we can follow the address down to the the page and take a ref on it.
	 */
	local_irq_save(flags);
	pgdp = pgd_offset(mm, addr);
	do {
		pgd = *pgdp;
		barrier();
		next = pgd_addr_end(addr, end);
		if (pgd_none(pgd))
			break;
		if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
			break;
	} while (pgdp++, addr = next, addr != end);
	local_irq_restore(flags);

	return nr;
}

/**
 * get_user_pages_fast() - pin user pages in memory
 * @start:	starting user address
 * @nr_pages:	number of pages from start to pin
 * @write:	whether pages will be written to
 * @pages:	array that receives pointers to the pages pinned.
 *		Should be at least nr_pages long.
 *
 * Attempt to pin user pages in memory without taking mm->mmap_sem.
 * If not successful, it will fall back to taking the lock and
 * calling get_user_pages().
 *
 * Returns number of pages pinned. This may be fewer than the number
 * requested. If nr_pages is 0 or negative, returns 0. If no pages
 * were pinned, returns -errno.
 */
int get_user_pages_fast(unsigned long start, int nr_pages, int write,
			struct page **pages)
{
	int nr, ret;

	might_sleep();
	start &= PAGE_MASK;
	nr = __get_user_pages_fast(start, nr_pages, write, pages);
	if (nr == nr_pages)
		return nr;

	/* Try to get the remaining pages with get_user_pages */
	start += nr << PAGE_SHIFT;
	pages += nr;
	ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
				      write ? FOLL_WRITE : 0);
	/* Have to be a bit careful with return values */
	if (nr > 0)
		ret = (ret < 0) ? nr : ret + nr;
	return ret;
}
Commit	Line	Data
80217147 MS	1	/*
	2	* Lockless get_user_pages_fast for s390
	3	*
	4	* Copyright IBM Corp. 2010
	5	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
	6	*/
	7	#include <linux/sched.h>
	8	#include <linux/mm.h>
	9	#include <linux/hugetlb.h>
	10	#include <linux/vmstat.h>
	11	#include <linux/pagemap.h>
	12	#include <linux/rwsem.h>
	13	#include <asm/pgtable.h>
	14
	15	/*
	16	* The performance critical leaf functions are made noinline otherwise gcc
	17	* inlines everything into a single function which results in too much
	18	* register pressure.
	19	*/
	20	static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
	21	unsigned long end, int write, struct page *pages, int nr)
	22	{
fc897c95	23	struct page head, page;
25591b07	24	unsigned long mask;
80217147	25	pte_t *ptep, pte;
80217147	26
e5098611	27	mask = (write ? _PAGE_PROTECT : 0) \| _PAGE_INVALID \| _PAGE_SPECIAL;
80217147 MS	28
	29	ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr);
	30	do {
	31	pte = *ptep;
	32	barrier();
ecf46abd GS	33	/* Similar to the PMD case, NUMA hinting must take slow path */
	34	if (pte_protnone(pte))
	35	return 0;
25591b07	36	if ((pte_val(pte) & mask) != 0)
80217147 MS	37	return 0;
	38	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
	39	page = pte_page(pte);
fc897c95 GS	40	head = compound_head(page);
fc897c95 GS	41	if (!page_cache_get_speculative(head))
80217147 MS	42	return 0;
80217147 MS	43	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
fc897c95	44	put_page(head);
80217147 MS	45	return 0;
80217147 MS	46	}
fc897c95	47	VM_BUG_ON_PAGE(compound_head(page) != head, page);
80217147 MS	48	pages[*nr] = page;
	49	(*nr)++;
	50
	51	} while (ptep++, addr += PAGE_SIZE, addr != end);
	52
	53	return 1;
	54	}
	55
	56	static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
	57	unsigned long end, int write, struct page *pages, int nr)
	58	{
	59	unsigned long mask, result;
ddc58f27	60	struct page head, page;
80217147 MS	61	int refs;
80217147 MS	62
e5098611 MS	63	result = write ? 0 : _SEGMENT_ENTRY_PROTECT;
e5098611 MS	64	mask = result \| _SEGMENT_ENTRY_INVALID;
80217147 MS	65	if ((pmd_val(pmd) & mask) != result)
	66	return 0;
	67	VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
	68
	69	refs = 0;
	70	head = pmd_page(pmd);
	71	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
	72	do {
	73	VM_BUG_ON(compound_head(page) != head);
	74	pages[*nr] = page;
	75	(*nr)++;
	76	page++;
	77	refs++;
	78	} while (addr += PAGE_SIZE, addr != end);
	79
	80	if (!page_cache_add_speculative(head, refs)) {
	81	*nr -= refs;
	82	return 0;
	83	}
	84
	85	if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
	86	*nr -= refs;
	87	while (refs--)
	88	put_page(head);
0693bc9c AA	89	return 0;
	90	}
	91
80217147 MS	92	return 1;
	93	}
	94
	95
	96	static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
	97	unsigned long end, int write, struct page *pages, int nr)
	98	{
	99	unsigned long next;
	100	pmd_t *pmdp, pmd;
	101
	102	pmdp = (pmd_t *) pudp;
80217147 MS	103	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
	104	pmdp = (pmd_t *) pud_deref(pud);
	105	pmdp += pmd_index(addr);
80217147 MS	106	do {
	107	pmd = *pmdp;
	108	barrier();
	109	next = pmd_addr_end(addr, end);
fecffad2	110	if (pmd_none(pmd))
80217147	111	return 0;
156152f8	112	if (unlikely(pmd_large(pmd))) {
ecf46abd GS	113	/*
	114	* NUMA hinting faults need to be handled in the GUP
	115	* slowpath for accounting purposes and so that they
	116	* can be serialised against THP migration.
	117	*/
	118	if (pmd_protnone(pmd))
	119	return 0;
80217147 MS	120	if (!gup_huge_pmd(pmdp, pmd, addr, next,
	121	write, pages, nr))
	122	return 0;
	123	} else if (!gup_pte_range(pmdp, pmd, addr, next,
	124	write, pages, nr))
	125	return 0;
	126	} while (pmdp++, addr = next, addr != end);
	127
	128	return 1;
	129	}
	130
d08de8e2 GS	131	static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
	132	unsigned long end, int write, struct page *pages, int nr)
	133	{
	134	struct page head, page;
	135	unsigned long mask;
	136	int refs;
	137
	138	mask = (write ? _REGION_ENTRY_PROTECT : 0) \| _REGION_ENTRY_INVALID;
	139	if ((pud_val(pud) & mask) != 0)
	140	return 0;
	141	VM_BUG_ON(!pfn_valid(pud_pfn(pud)));
	142
	143	refs = 0;
	144	head = pud_page(pud);
	145	page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
	146	do {
	147	VM_BUG_ON_PAGE(compound_head(page) != head, page);
	148	pages[*nr] = page;
	149	(*nr)++;
	150	page++;
	151	refs++;
	152	} while (addr += PAGE_SIZE, addr != end);
	153
	154	if (!page_cache_add_speculative(head, refs)) {
	155	*nr -= refs;
	156	return 0;
	157	}
	158
	159	if (unlikely(pud_val(pud) != pud_val(*pudp))) {
	160	*nr -= refs;
	161	while (refs--)
	162	put_page(head);
	163	return 0;
	164	}
	165
	166	return 1;
	167	}
	168
80217147 MS	169	static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
	170	unsigned long end, int write, struct page *pages, int nr)
	171	{
	172	unsigned long next;
	173	pud_t *pudp, pud;
	174
	175	pudp = (pud_t *) pgdp;
80217147 MS	176	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
	177	pudp = (pud_t *) pgd_deref(pgd);
	178	pudp += pud_index(addr);
80217147 MS	179	do {
	180	pud = *pudp;
	181	barrier();
	182	next = pud_addr_end(addr, end);
	183	if (pud_none(pud))
	184	return 0;
d08de8e2 GS	185	if (unlikely(pud_large(pud))) {
	186	if (!gup_huge_pud(pudp, pud, addr, next, write, pages,
	187	nr))
	188	return 0;
	189	} else if (!gup_pmd_range(pudp, pud, addr, next, write, pages,
	190	nr))
80217147 MS	191	return 0;
	192	} while (pudp++, addr = next, addr != end);
	193
	194	return 1;
	195	}
	196
34cda992 GS	197	/*
	198	* Like get_user_pages_fast() except its IRQ-safe in that it won't fall
	199	* back to the regular GUP.
	200	*/
	201	int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
	202	struct page **pages)
	203	{
	204	struct mm_struct *mm = current->mm;
	205	unsigned long addr, len, end;
	206	unsigned long next, flags;
	207	pgd_t *pgdp, pgd;
	208	int nr = 0;
	209
	210	start &= PAGE_MASK;
	211	addr = start;
	212	len = (unsigned long) nr_pages << PAGE_SHIFT;
	213	end = start + len;
eb0bf929	214	if ((end <= start) \|\| (end > TASK_SIZE))
34cda992	215	return 0;
01997bbc HC	216	/*
	217	* local_irq_save() doesn't prevent pagetable teardown, but does
	218	* prevent the pagetables from being freed on s390.
	219	*
	220	* So long as we atomically load page table pointers versus teardown,
	221	* we can follow the address down to the the page and take a ref on it.
	222	*/
34cda992 GS	223	local_irq_save(flags);
	224	pgdp = pgd_offset(mm, addr);
	225	do {
	226	pgd = *pgdp;
	227	barrier();
	228	next = pgd_addr_end(addr, end);
	229	if (pgd_none(pgd))
	230	break;
	231	if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
	232	break;
	233	} while (pgdp++, addr = next, addr != end);
	234	local_irq_restore(flags);
	235
	236	return nr;
	237	}
	238
80217147 MS	239	/**
	240	* get_user_pages_fast() - pin user pages in memory
	241	* @start: starting user address
	242	* @nr_pages: number of pages from start to pin
	243	* @write: whether pages will be written to
	244	* @pages: array that receives pointers to the pages pinned.
	245	* Should be at least nr_pages long.
	246	*
	247	* Attempt to pin user pages in memory without taking mm->mmap_sem.
	248	* If not successful, it will fall back to taking the lock and
	249	* calling get_user_pages().
	250	*
	251	* Returns number of pages pinned. This may be fewer than the number
	252	* requested. If nr_pages is 0 or negative, returns 0. If no pages
	253	* were pinned, returns -errno.
	254	*/
	255	int get_user_pages_fast(unsigned long start, int nr_pages, int write,
	256	struct page **pages)
	257	{
01997bbc	258	int nr, ret;
80217147	259
40612351	260	might_sleep();
80217147	261	start &= PAGE_MASK;
01997bbc HC	262	nr = __get_user_pages_fast(start, nr_pages, write, pages);
	263	if (nr == nr_pages)
	264	return nr;
	265
	266	/* Try to get the remaining pages with get_user_pages */
	267	start += nr << PAGE_SHIFT;
	268	pages += nr;
c164154f LS	269	ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
c164154f LS	270	write ? FOLL_WRITE : 0);
01997bbc HC	271	/* Have to be a bit careful with return values */
	272	if (nr > 0)
	273	ret = (ret < 0) ? nr : ret + nr;
	274	return ret;
80217147	275	}