]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/s390/mm/gup.c
ASoC: max98504: Add missing MAX98504 on SND_SOC_ALL_CODECS
[mirror_ubuntu-bionic-kernel.git] / arch / s390 / mm / gup.c
1 /*
2 * Lockless get_user_pages_fast for s390
3 *
4 * Copyright IBM Corp. 2010
5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
6 */
7 #include <linux/sched.h>
8 #include <linux/mm.h>
9 #include <linux/hugetlb.h>
10 #include <linux/vmstat.h>
11 #include <linux/pagemap.h>
12 #include <linux/rwsem.h>
13 #include <asm/pgtable.h>
14
15 /*
16 * The performance critical leaf functions are made noinline otherwise gcc
17 * inlines everything into a single function which results in too much
18 * register pressure.
19 */
20 static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
21 unsigned long end, int write, struct page **pages, int *nr)
22 {
23 struct page *head, *page;
24 unsigned long mask;
25 pte_t *ptep, pte;
26
27 mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL;
28
29 ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr);
30 do {
31 pte = *ptep;
32 barrier();
33 /* Similar to the PMD case, NUMA hinting must take slow path */
34 if (pte_protnone(pte))
35 return 0;
36 if ((pte_val(pte) & mask) != 0)
37 return 0;
38 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
39 page = pte_page(pte);
40 head = compound_head(page);
41 if (!page_cache_get_speculative(head))
42 return 0;
43 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
44 put_page(head);
45 return 0;
46 }
47 VM_BUG_ON_PAGE(compound_head(page) != head, page);
48 pages[*nr] = page;
49 (*nr)++;
50
51 } while (ptep++, addr += PAGE_SIZE, addr != end);
52
53 return 1;
54 }
55
56 static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
57 unsigned long end, int write, struct page **pages, int *nr)
58 {
59 unsigned long mask, result;
60 struct page *head, *page;
61 int refs;
62
63 result = write ? 0 : _SEGMENT_ENTRY_PROTECT;
64 mask = result | _SEGMENT_ENTRY_INVALID;
65 if ((pmd_val(pmd) & mask) != result)
66 return 0;
67 VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
68
69 refs = 0;
70 head = pmd_page(pmd);
71 page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
72 do {
73 VM_BUG_ON(compound_head(page) != head);
74 pages[*nr] = page;
75 (*nr)++;
76 page++;
77 refs++;
78 } while (addr += PAGE_SIZE, addr != end);
79
80 if (!page_cache_add_speculative(head, refs)) {
81 *nr -= refs;
82 return 0;
83 }
84
85 if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
86 *nr -= refs;
87 while (refs--)
88 put_page(head);
89 return 0;
90 }
91
92 return 1;
93 }
94
95
96 static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
97 unsigned long end, int write, struct page **pages, int *nr)
98 {
99 unsigned long next;
100 pmd_t *pmdp, pmd;
101
102 pmdp = (pmd_t *) pudp;
103 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
104 pmdp = (pmd_t *) pud_deref(pud);
105 pmdp += pmd_index(addr);
106 do {
107 pmd = *pmdp;
108 barrier();
109 next = pmd_addr_end(addr, end);
110 if (pmd_none(pmd))
111 return 0;
112 if (unlikely(pmd_large(pmd))) {
113 /*
114 * NUMA hinting faults need to be handled in the GUP
115 * slowpath for accounting purposes and so that they
116 * can be serialised against THP migration.
117 */
118 if (pmd_protnone(pmd))
119 return 0;
120 if (!gup_huge_pmd(pmdp, pmd, addr, next,
121 write, pages, nr))
122 return 0;
123 } else if (!gup_pte_range(pmdp, pmd, addr, next,
124 write, pages, nr))
125 return 0;
126 } while (pmdp++, addr = next, addr != end);
127
128 return 1;
129 }
130
131 static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
132 unsigned long end, int write, struct page **pages, int *nr)
133 {
134 struct page *head, *page;
135 unsigned long mask;
136 int refs;
137
138 mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID;
139 if ((pud_val(pud) & mask) != 0)
140 return 0;
141 VM_BUG_ON(!pfn_valid(pud_pfn(pud)));
142
143 refs = 0;
144 head = pud_page(pud);
145 page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
146 do {
147 VM_BUG_ON_PAGE(compound_head(page) != head, page);
148 pages[*nr] = page;
149 (*nr)++;
150 page++;
151 refs++;
152 } while (addr += PAGE_SIZE, addr != end);
153
154 if (!page_cache_add_speculative(head, refs)) {
155 *nr -= refs;
156 return 0;
157 }
158
159 if (unlikely(pud_val(pud) != pud_val(*pudp))) {
160 *nr -= refs;
161 while (refs--)
162 put_page(head);
163 return 0;
164 }
165
166 return 1;
167 }
168
169 static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
170 unsigned long end, int write, struct page **pages, int *nr)
171 {
172 unsigned long next;
173 pud_t *pudp, pud;
174
175 pudp = (pud_t *) pgdp;
176 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
177 pudp = (pud_t *) pgd_deref(pgd);
178 pudp += pud_index(addr);
179 do {
180 pud = *pudp;
181 barrier();
182 next = pud_addr_end(addr, end);
183 if (pud_none(pud))
184 return 0;
185 if (unlikely(pud_large(pud))) {
186 if (!gup_huge_pud(pudp, pud, addr, next, write, pages,
187 nr))
188 return 0;
189 } else if (!gup_pmd_range(pudp, pud, addr, next, write, pages,
190 nr))
191 return 0;
192 } while (pudp++, addr = next, addr != end);
193
194 return 1;
195 }
196
197 /*
198 * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
199 * back to the regular GUP.
200 */
201 int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
202 struct page **pages)
203 {
204 struct mm_struct *mm = current->mm;
205 unsigned long addr, len, end;
206 unsigned long next, flags;
207 pgd_t *pgdp, pgd;
208 int nr = 0;
209
210 start &= PAGE_MASK;
211 addr = start;
212 len = (unsigned long) nr_pages << PAGE_SHIFT;
213 end = start + len;
214 if ((end <= start) || (end > TASK_SIZE))
215 return 0;
216 /*
217 * local_irq_save() doesn't prevent pagetable teardown, but does
218 * prevent the pagetables from being freed on s390.
219 *
220 * So long as we atomically load page table pointers versus teardown,
221 * we can follow the address down to the the page and take a ref on it.
222 */
223 local_irq_save(flags);
224 pgdp = pgd_offset(mm, addr);
225 do {
226 pgd = *pgdp;
227 barrier();
228 next = pgd_addr_end(addr, end);
229 if (pgd_none(pgd))
230 break;
231 if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
232 break;
233 } while (pgdp++, addr = next, addr != end);
234 local_irq_restore(flags);
235
236 return nr;
237 }
238
239 /**
240 * get_user_pages_fast() - pin user pages in memory
241 * @start: starting user address
242 * @nr_pages: number of pages from start to pin
243 * @write: whether pages will be written to
244 * @pages: array that receives pointers to the pages pinned.
245 * Should be at least nr_pages long.
246 *
247 * Attempt to pin user pages in memory without taking mm->mmap_sem.
248 * If not successful, it will fall back to taking the lock and
249 * calling get_user_pages().
250 *
251 * Returns number of pages pinned. This may be fewer than the number
252 * requested. If nr_pages is 0 or negative, returns 0. If no pages
253 * were pinned, returns -errno.
254 */
255 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
256 struct page **pages)
257 {
258 int nr, ret;
259
260 might_sleep();
261 start &= PAGE_MASK;
262 nr = __get_user_pages_fast(start, nr_pages, write, pages);
263 if (nr == nr_pages)
264 return nr;
265
266 /* Try to get the remaining pages with get_user_pages */
267 start += nr << PAGE_SHIFT;
268 pages += nr;
269 ret = get_user_pages_unlocked(start, nr_pages - nr, write, 0, pages);
270 /* Have to be a bit careful with return values */
271 if (nr > 0)
272 ret = (ret < 0) ? nr : ret + nr;
273 return ret;
274 }