]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1da177e4 LT |
2 | /* |
3 | * TLB support routines. | |
4 | * | |
5 | * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co | |
6 | * David Mosberger-Tang <davidm@hpl.hp.com> | |
7 | * | |
8 | * 08/02/00 A. Mallick <asit.k.mallick@intel.com> | |
9 | * Modified RID allocation for SMP | |
10 | * Goutham Rao <goutham.rao@intel.com> | |
11 | * IPI based ptc implementation and A-step IPI implementation. | |
dcc17d1b PK |
12 | * Rohit Seth <rohit.seth@intel.com> |
13 | * Ken Chen <kenneth.w.chen@intel.com> | |
aec103bf | 14 | * Christophe de Dinechin <ddd@hp.com>: Avoid ptc.e on memory allocation |
2046b94e FY |
15 | * Copyright (C) 2007 Intel Corp |
16 | * Fenghua Yu <fenghua.yu@intel.com> | |
17 | * Add multiple ptc.g/ptc.ga instruction support in global tlb purge. | |
1da177e4 | 18 | */ |
1da177e4 LT |
19 | #include <linux/module.h> |
20 | #include <linux/init.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/sched.h> | |
23 | #include <linux/smp.h> | |
24 | #include <linux/mm.h> | |
57c8a661 | 25 | #include <linux/memblock.h> |
5a0e3ad6 | 26 | #include <linux/slab.h> |
1da177e4 LT |
27 | |
28 | #include <asm/delay.h> | |
29 | #include <asm/mmu_context.h> | |
1da177e4 LT |
30 | #include <asm/pal.h> |
31 | #include <asm/tlbflush.h> | |
dcc17d1b | 32 | #include <asm/dma.h> |
96651896 | 33 | #include <asm/processor.h> |
2046b94e | 34 | #include <asm/sal.h> |
96651896 | 35 | #include <asm/tlb.h> |
1da177e4 LT |
36 | |
37 | static struct { | |
e088a4ad | 38 | u64 mask; /* mask of supported purge page-sizes */ |
58cd9082 | 39 | unsigned long max_bits; /* log2 of largest supported purge page-size */ |
1da177e4 LT |
40 | } purge; |
41 | ||
42 | struct ia64_ctx ia64_ctx = { | |
8737d595 MAC |
43 | .lock = __SPIN_LOCK_UNLOCKED(ia64_ctx.lock), |
44 | .next = 1, | |
45 | .max_ctx = ~0U | |
1da177e4 LT |
46 | }; |
47 | ||
48 | DEFINE_PER_CPU(u8, ia64_need_tlb_flush); | |
96651896 XZ |
49 | DEFINE_PER_CPU(u8, ia64_tr_num); /*Number of TR slots in current processor*/ |
50 | DEFINE_PER_CPU(u8, ia64_tr_used); /*Max Slot number used by kernel*/ | |
51 | ||
6c57a332 | 52 | struct ia64_tr_entry *ia64_idtrs[NR_CPUS]; |
1da177e4 | 53 | |
dcc17d1b PK |
54 | /* |
55 | * Initializes the ia64_ctx.bitmap array based on max_ctx+1. | |
56 | * Called after cpu_init() has setup ia64_ctx.max_ctx based on | |
57 | * maximum RID that is supported by boot CPU. | |
58 | */ | |
59 | void __init | |
60 | mmu_context_init (void) | |
61 | { | |
7e1c4e27 MR |
62 | ia64_ctx.bitmap = memblock_alloc((ia64_ctx.max_ctx + 1) >> 3, |
63 | SMP_CACHE_BYTES); | |
d80db5c1 MR |
64 | if (!ia64_ctx.bitmap) |
65 | panic("%s: Failed to allocate %u bytes\n", __func__, | |
66 | (ia64_ctx.max_ctx + 1) >> 3); | |
7e1c4e27 MR |
67 | ia64_ctx.flushmap = memblock_alloc((ia64_ctx.max_ctx + 1) >> 3, |
68 | SMP_CACHE_BYTES); | |
d80db5c1 MR |
69 | if (!ia64_ctx.flushmap) |
70 | panic("%s: Failed to allocate %u bytes\n", __func__, | |
71 | (ia64_ctx.max_ctx + 1) >> 3); | |
dcc17d1b PK |
72 | } |
73 | ||
1da177e4 LT |
74 | /* |
75 | * Acquire the ia64_ctx.lock before calling this function! | |
76 | */ | |
77 | void | |
78 | wrap_mmu_context (struct mm_struct *mm) | |
79 | { | |
58cd9082 | 80 | int i, cpu; |
dcc17d1b | 81 | unsigned long flush_bit; |
1da177e4 | 82 | |
dcc17d1b PK |
83 | for (i=0; i <= ia64_ctx.max_ctx / BITS_PER_LONG; i++) { |
84 | flush_bit = xchg(&ia64_ctx.flushmap[i], 0); | |
85 | ia64_ctx.bitmap[i] ^= flush_bit; | |
1da177e4 | 86 | } |
dcc17d1b PK |
87 | |
88 | /* use offset at 300 to skip daemons */ | |
89 | ia64_ctx.next = find_next_zero_bit(ia64_ctx.bitmap, | |
90 | ia64_ctx.max_ctx, 300); | |
91 | ia64_ctx.limit = find_next_bit(ia64_ctx.bitmap, | |
92 | ia64_ctx.max_ctx, ia64_ctx.next); | |
93 | ||
58cd9082 KC |
94 | /* |
95 | * can't call flush_tlb_all() here because of race condition | |
96 | * with O(1) scheduler [EF] | |
97 | */ | |
98 | cpu = get_cpu(); /* prevent preemption/migration */ | |
99 | for_each_online_cpu(i) | |
100 | if (i != cpu) | |
101 | per_cpu(ia64_need_tlb_flush, i) = 1; | |
102 | put_cpu(); | |
1da177e4 LT |
103 | local_flush_tlb_all(); |
104 | } | |
105 | ||
2046b94e FY |
106 | /* |
107 | * Implement "spinaphores" ... like counting semaphores, but they | |
108 | * spin instead of sleeping. If there are ever any other users for | |
109 | * this primitive it can be moved up to a spinaphore.h header. | |
110 | */ | |
111 | struct spinaphore { | |
883a3acf TL |
112 | unsigned long ticket; |
113 | unsigned long serve; | |
2046b94e FY |
114 | }; |
115 | ||
116 | static inline void spinaphore_init(struct spinaphore *ss, int val) | |
117 | { | |
883a3acf TL |
118 | ss->ticket = 0; |
119 | ss->serve = val; | |
2046b94e FY |
120 | } |
121 | ||
122 | static inline void down_spin(struct spinaphore *ss) | |
123 | { | |
883a3acf TL |
124 | unsigned long t = ia64_fetchadd(1, &ss->ticket, acq), serve; |
125 | ||
126 | if (time_before(t, ss->serve)) | |
127 | return; | |
128 | ||
129 | ia64_invala(); | |
130 | ||
131 | for (;;) { | |
b70f4e85 | 132 | asm volatile ("ld8.c.nc %0=[%1]" : "=r"(serve) : "r"(&ss->serve) : "memory"); |
883a3acf TL |
133 | if (time_before(t, serve)) |
134 | return; | |
135 | cpu_relax(); | |
136 | } | |
2046b94e FY |
137 | } |
138 | ||
139 | static inline void up_spin(struct spinaphore *ss) | |
140 | { | |
883a3acf | 141 | ia64_fetchadd(1, &ss->serve, rel); |
2046b94e FY |
142 | } |
143 | ||
144 | static struct spinaphore ptcg_sem; | |
145 | static u16 nptcg = 1; | |
146 | static int need_ptcg_sem = 1; | |
147 | static int toolatetochangeptcgsem = 0; | |
148 | ||
a6c75b86 FY |
149 | /* |
150 | * Kernel parameter "nptcg=" overrides max number of concurrent global TLB | |
151 | * purges which is reported from either PAL or SAL PALO. | |
152 | * | |
153 | * We don't have sanity checking for nptcg value. It's the user's responsibility | |
154 | * for valid nptcg value on the platform. Otherwise, kernel may hang in some | |
155 | * cases. | |
156 | */ | |
157 | static int __init | |
158 | set_nptcg(char *str) | |
159 | { | |
160 | int value = 0; | |
161 | ||
162 | get_option(&str, &value); | |
163 | setup_ptcg_sem(value, NPTCG_FROM_KERNEL_PARAMETER); | |
164 | ||
165 | return 1; | |
166 | } | |
167 | ||
168 | __setup("nptcg=", set_nptcg); | |
169 | ||
2046b94e FY |
170 | /* |
171 | * Maximum number of simultaneous ptc.g purges in the system can | |
172 | * be defined by PAL_VM_SUMMARY (in which case we should take | |
173 | * the smallest value for any cpu in the system) or by the PAL | |
174 | * override table (in which case we should ignore the value from | |
175 | * PAL_VM_SUMMARY). | |
176 | * | |
a6c75b86 FY |
177 | * Kernel parameter "nptcg=" overrides maximum number of simultanesous ptc.g |
178 | * purges defined in either PAL_VM_SUMMARY or PAL override table. In this case, | |
179 | * we should ignore the value from either PAL_VM_SUMMARY or PAL override table. | |
180 | * | |
2046b94e FY |
181 | * Complicating the logic here is the fact that num_possible_cpus() |
182 | * isn't fully setup until we start bringing cpus online. | |
183 | */ | |
184 | void | |
a6c75b86 | 185 | setup_ptcg_sem(int max_purges, int nptcg_from) |
2046b94e | 186 | { |
a6c75b86 FY |
187 | static int kp_override; |
188 | static int palo_override; | |
2046b94e FY |
189 | static int firstcpu = 1; |
190 | ||
191 | if (toolatetochangeptcgsem) { | |
e617fce6 HS |
192 | if (nptcg_from == NPTCG_FROM_PAL && max_purges == 0) |
193 | BUG_ON(1 < nptcg); | |
194 | else | |
195 | BUG_ON(max_purges < nptcg); | |
2046b94e FY |
196 | return; |
197 | } | |
198 | ||
a6c75b86 FY |
199 | if (nptcg_from == NPTCG_FROM_KERNEL_PARAMETER) { |
200 | kp_override = 1; | |
201 | nptcg = max_purges; | |
202 | goto resetsema; | |
203 | } | |
204 | if (kp_override) { | |
205 | need_ptcg_sem = num_possible_cpus() > nptcg; | |
206 | return; | |
207 | } | |
208 | ||
209 | if (nptcg_from == NPTCG_FROM_PALO) { | |
210 | palo_override = 1; | |
2046b94e FY |
211 | |
212 | /* In PALO max_purges == 0 really means it! */ | |
213 | if (max_purges == 0) | |
214 | panic("Whoa! Platform does not support global TLB purges.\n"); | |
215 | nptcg = max_purges; | |
216 | if (nptcg == PALO_MAX_TLB_PURGES) { | |
217 | need_ptcg_sem = 0; | |
218 | return; | |
219 | } | |
220 | goto resetsema; | |
221 | } | |
a6c75b86 | 222 | if (palo_override) { |
2046b94e FY |
223 | if (nptcg != PALO_MAX_TLB_PURGES) |
224 | need_ptcg_sem = (num_possible_cpus() > nptcg); | |
225 | return; | |
226 | } | |
227 | ||
228 | /* In PAL_VM_SUMMARY max_purges == 0 actually means 1 */ | |
229 | if (max_purges == 0) max_purges = 1; | |
230 | ||
231 | if (firstcpu) { | |
232 | nptcg = max_purges; | |
233 | firstcpu = 0; | |
234 | } | |
235 | if (max_purges < nptcg) | |
236 | nptcg = max_purges; | |
237 | if (nptcg == PAL_MAX_PURGES) { | |
238 | need_ptcg_sem = 0; | |
239 | return; | |
240 | } else | |
241 | need_ptcg_sem = (num_possible_cpus() > nptcg); | |
242 | ||
243 | resetsema: | |
244 | spinaphore_init(&ptcg_sem, max_purges); | |
245 | } | |
246 | ||
05933aac CH |
247 | #ifdef CONFIG_SMP |
248 | static void | |
58cd9082 KC |
249 | ia64_global_tlb_purge (struct mm_struct *mm, unsigned long start, |
250 | unsigned long end, unsigned long nbits) | |
1da177e4 | 251 | { |
aec103bf DCIV |
252 | struct mm_struct *active_mm = current->active_mm; |
253 | ||
2046b94e FY |
254 | toolatetochangeptcgsem = 1; |
255 | ||
aec103bf DCIV |
256 | if (mm != active_mm) { |
257 | /* Restore region IDs for mm */ | |
258 | if (mm && active_mm) { | |
259 | activate_context(mm); | |
260 | } else { | |
261 | flush_tlb_all(); | |
262 | return; | |
263 | } | |
c1902aae DR |
264 | } |
265 | ||
2046b94e FY |
266 | if (need_ptcg_sem) |
267 | down_spin(&ptcg_sem); | |
268 | ||
269 | do { | |
270 | /* | |
271 | * Flush ALAT entries also. | |
272 | */ | |
273 | ia64_ptcga(start, (nbits << 2)); | |
274 | ia64_srlz_i(); | |
275 | start += (1UL << nbits); | |
276 | } while (start < end); | |
277 | ||
278 | if (need_ptcg_sem) | |
279 | up_spin(&ptcg_sem); | |
aec103bf DCIV |
280 | |
281 | if (mm != active_mm) { | |
282 | activate_context(active_mm); | |
283 | } | |
1da177e4 | 284 | } |
05933aac | 285 | #endif /* CONFIG_SMP */ |
1da177e4 LT |
286 | |
287 | void | |
288 | local_flush_tlb_all (void) | |
289 | { | |
290 | unsigned long i, j, flags, count0, count1, stride0, stride1, addr; | |
291 | ||
292 | addr = local_cpu_data->ptce_base; | |
293 | count0 = local_cpu_data->ptce_count[0]; | |
294 | count1 = local_cpu_data->ptce_count[1]; | |
295 | stride0 = local_cpu_data->ptce_stride[0]; | |
296 | stride1 = local_cpu_data->ptce_stride[1]; | |
297 | ||
298 | local_irq_save(flags); | |
299 | for (i = 0; i < count0; ++i) { | |
300 | for (j = 0; j < count1; ++j) { | |
301 | ia64_ptce(addr); | |
302 | addr += stride1; | |
303 | } | |
304 | addr += stride0; | |
305 | } | |
306 | local_irq_restore(flags); | |
307 | ia64_srlz_i(); /* srlz.i implies srlz.d */ | |
308 | } | |
309 | ||
e1547007 PZ |
310 | static void |
311 | __flush_tlb_range (struct vm_area_struct *vma, unsigned long start, | |
58cd9082 | 312 | unsigned long end) |
1da177e4 LT |
313 | { |
314 | struct mm_struct *mm = vma->vm_mm; | |
315 | unsigned long size = end - start; | |
316 | unsigned long nbits; | |
317 | ||
c1902aae | 318 | #ifndef CONFIG_SMP |
1da177e4 | 319 | if (mm != current->active_mm) { |
1da177e4 | 320 | mm->context = 0; |
1da177e4 LT |
321 | return; |
322 | } | |
c1902aae | 323 | #endif |
1da177e4 LT |
324 | |
325 | nbits = ia64_fls(size + 0xfff); | |
58cd9082 KC |
326 | while (unlikely (((1UL << nbits) & purge.mask) == 0) && |
327 | (nbits < purge.max_bits)) | |
1da177e4 LT |
328 | ++nbits; |
329 | if (nbits > purge.max_bits) | |
330 | nbits = purge.max_bits; | |
331 | start &= ~((1UL << nbits) - 1); | |
332 | ||
663b97f7 | 333 | preempt_disable(); |
ce9eed5a | 334 | #ifdef CONFIG_SMP |
5d8c39f6 | 335 | if (mm != current->active_mm || cpumask_weight(mm_cpumask(mm)) != 1) { |
05933aac | 336 | ia64_global_tlb_purge(mm, start, end, nbits); |
ce9eed5a KC |
337 | preempt_enable(); |
338 | return; | |
339 | } | |
340 | #endif | |
1da177e4 LT |
341 | do { |
342 | ia64_ptcl(start, (nbits<<2)); | |
343 | start += (1UL << nbits); | |
344 | } while (start < end); | |
663b97f7 | 345 | preempt_enable(); |
1da177e4 LT |
346 | ia64_srlz_i(); /* srlz.i implies srlz.d */ |
347 | } | |
e1547007 PZ |
348 | |
349 | void flush_tlb_range(struct vm_area_struct *vma, | |
350 | unsigned long start, unsigned long end) | |
351 | { | |
352 | if (unlikely(end - start >= 1024*1024*1024*1024UL | |
353 | || REGION_NUMBER(start) != REGION_NUMBER(end - 1))) { | |
354 | /* | |
355 | * If we flush more than a tera-byte or across regions, we're | |
356 | * probably better off just flushing the entire TLB(s). This | |
357 | * should be very rare and is not worth optimizing for. | |
358 | */ | |
359 | flush_tlb_all(); | |
360 | } else { | |
361 | /* flush the address range from the tlb */ | |
362 | __flush_tlb_range(vma, start, end); | |
363 | /* flush the virt. page-table area mapping the addr range */ | |
364 | __flush_tlb_range(vma, ia64_thash(start), ia64_thash(end)); | |
365 | } | |
366 | } | |
1da177e4 LT |
367 | EXPORT_SYMBOL(flush_tlb_range); |
368 | ||
5b5e76e9 | 369 | void ia64_tlb_init(void) |
1da177e4 | 370 | { |
3f649ab7 | 371 | ia64_ptce_info_t ptce_info; |
e088a4ad | 372 | u64 tr_pgbits; |
1da177e4 | 373 | long status; |
96651896 XZ |
374 | pal_vm_info_1_u_t vm_info_1; |
375 | pal_vm_info_2_u_t vm_info_2; | |
376 | int cpu = smp_processor_id(); | |
1da177e4 LT |
377 | |
378 | if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { | |
c2eeb321 | 379 | printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld; " |
1da177e4 LT |
380 | "defaulting to architected purge page-sizes.\n", status); |
381 | purge.mask = 0x115557000UL; | |
382 | } | |
383 | purge.max_bits = ia64_fls(purge.mask); | |
384 | ||
385 | ia64_get_ptce(&ptce_info); | |
386 | local_cpu_data->ptce_base = ptce_info.base; | |
387 | local_cpu_data->ptce_count[0] = ptce_info.count[0]; | |
388 | local_cpu_data->ptce_count[1] = ptce_info.count[1]; | |
389 | local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; | |
390 | local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; | |
391 | ||
58cd9082 | 392 | local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ |
96651896 XZ |
393 | status = ia64_pal_vm_summary(&vm_info_1, &vm_info_2); |
394 | ||
395 | if (status) { | |
396 | printk(KERN_ERR "ia64_pal_vm_summary=%ld\n", status); | |
397 | per_cpu(ia64_tr_num, cpu) = 8; | |
398 | return; | |
399 | } | |
400 | per_cpu(ia64_tr_num, cpu) = vm_info_1.pal_vm_info_1_s.max_itr_entry+1; | |
401 | if (per_cpu(ia64_tr_num, cpu) > | |
402 | (vm_info_1.pal_vm_info_1_s.max_dtr_entry+1)) | |
403 | per_cpu(ia64_tr_num, cpu) = | |
404 | vm_info_1.pal_vm_info_1_s.max_dtr_entry+1; | |
405 | if (per_cpu(ia64_tr_num, cpu) > IA64_TR_ALLOC_MAX) { | |
a9894a4a | 406 | static int justonce = 1; |
96651896 | 407 | per_cpu(ia64_tr_num, cpu) = IA64_TR_ALLOC_MAX; |
a9894a4a TL |
408 | if (justonce) { |
409 | justonce = 0; | |
410 | printk(KERN_DEBUG "TR register number exceeds " | |
411 | "IA64_TR_ALLOC_MAX!\n"); | |
412 | } | |
96651896 XZ |
413 | } |
414 | } | |
415 | ||
416 | /* | |
417 | * is_tr_overlap | |
418 | * | |
419 | * Check overlap with inserted TRs. | |
420 | */ | |
421 | static int is_tr_overlap(struct ia64_tr_entry *p, u64 va, u64 log_size) | |
422 | { | |
423 | u64 tr_log_size; | |
424 | u64 tr_end; | |
425 | u64 va_rr = ia64_get_rr(va); | |
426 | u64 va_rid = RR_TO_RID(va_rr); | |
427 | u64 va_end = va + (1<<log_size) - 1; | |
428 | ||
429 | if (va_rid != RR_TO_RID(p->rr)) | |
430 | return 0; | |
431 | tr_log_size = (p->itir & 0xff) >> 2; | |
432 | tr_end = p->ifa + (1<<tr_log_size) - 1; | |
433 | ||
434 | if (va > tr_end || p->ifa > va_end) | |
435 | return 0; | |
436 | return 1; | |
437 | ||
438 | } | |
439 | ||
440 | /* | |
441 | * ia64_insert_tr in virtual mode. Allocate a TR slot | |
442 | * | |
443 | * target_mask : 0x1 : itr, 0x2 : dtr, 0x3 : idtr | |
444 | * | |
445 | * va : virtual address. | |
446 | * pte : pte entries inserted. | |
447 | * log_size: range to be covered. | |
448 | * | |
449 | * Return value: <0 : error No. | |
450 | * | |
451 | * >=0 : slot number allocated for TR. | |
452 | * Must be called with preemption disabled. | |
453 | */ | |
454 | int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size) | |
455 | { | |
456 | int i, r; | |
457 | unsigned long psr; | |
458 | struct ia64_tr_entry *p; | |
459 | int cpu = smp_processor_id(); | |
460 | ||
6c57a332 | 461 | if (!ia64_idtrs[cpu]) { |
6da2ec56 KC |
462 | ia64_idtrs[cpu] = kmalloc_array(2 * IA64_TR_ALLOC_MAX, |
463 | sizeof(struct ia64_tr_entry), | |
464 | GFP_KERNEL); | |
6c57a332 TL |
465 | if (!ia64_idtrs[cpu]) |
466 | return -ENOMEM; | |
467 | } | |
96651896 XZ |
468 | r = -EINVAL; |
469 | /*Check overlap with existing TR entries*/ | |
470 | if (target_mask & 0x1) { | |
6c57a332 | 471 | p = ia64_idtrs[cpu]; |
96651896 XZ |
472 | for (i = IA64_TR_ALLOC_BASE; i <= per_cpu(ia64_tr_used, cpu); |
473 | i++, p++) { | |
474 | if (p->pte & 0x1) | |
475 | if (is_tr_overlap(p, va, log_size)) { | |
476 | printk(KERN_DEBUG "Overlapped Entry" | |
5e49e399 | 477 | "Inserted for TR Register!!\n"); |
96651896 XZ |
478 | goto out; |
479 | } | |
480 | } | |
481 | } | |
482 | if (target_mask & 0x2) { | |
6c57a332 | 483 | p = ia64_idtrs[cpu] + IA64_TR_ALLOC_MAX; |
96651896 XZ |
484 | for (i = IA64_TR_ALLOC_BASE; i <= per_cpu(ia64_tr_used, cpu); |
485 | i++, p++) { | |
486 | if (p->pte & 0x1) | |
487 | if (is_tr_overlap(p, va, log_size)) { | |
488 | printk(KERN_DEBUG "Overlapped Entry" | |
5e49e399 | 489 | "Inserted for TR Register!!\n"); |
96651896 XZ |
490 | goto out; |
491 | } | |
492 | } | |
493 | } | |
494 | ||
495 | for (i = IA64_TR_ALLOC_BASE; i < per_cpu(ia64_tr_num, cpu); i++) { | |
496 | switch (target_mask & 0x3) { | |
497 | case 1: | |
6c57a332 | 498 | if (!((ia64_idtrs[cpu] + i)->pte & 0x1)) |
96651896 XZ |
499 | goto found; |
500 | continue; | |
501 | case 2: | |
6c57a332 | 502 | if (!((ia64_idtrs[cpu] + IA64_TR_ALLOC_MAX + i)->pte & 0x1)) |
96651896 XZ |
503 | goto found; |
504 | continue; | |
505 | case 3: | |
6c57a332 TL |
506 | if (!((ia64_idtrs[cpu] + i)->pte & 0x1) && |
507 | !((ia64_idtrs[cpu] + IA64_TR_ALLOC_MAX + i)->pte & 0x1)) | |
96651896 XZ |
508 | goto found; |
509 | continue; | |
510 | default: | |
511 | r = -EINVAL; | |
512 | goto out; | |
513 | } | |
514 | } | |
515 | found: | |
516 | if (i >= per_cpu(ia64_tr_num, cpu)) | |
517 | return -EBUSY; | |
518 | ||
519 | /*Record tr info for mca hander use!*/ | |
520 | if (i > per_cpu(ia64_tr_used, cpu)) | |
521 | per_cpu(ia64_tr_used, cpu) = i; | |
522 | ||
523 | psr = ia64_clear_ic(); | |
524 | if (target_mask & 0x1) { | |
525 | ia64_itr(0x1, i, va, pte, log_size); | |
526 | ia64_srlz_i(); | |
6c57a332 | 527 | p = ia64_idtrs[cpu] + i; |
96651896 XZ |
528 | p->ifa = va; |
529 | p->pte = pte; | |
530 | p->itir = log_size << 2; | |
531 | p->rr = ia64_get_rr(va); | |
532 | } | |
533 | if (target_mask & 0x2) { | |
534 | ia64_itr(0x2, i, va, pte, log_size); | |
535 | ia64_srlz_i(); | |
6c57a332 | 536 | p = ia64_idtrs[cpu] + IA64_TR_ALLOC_MAX + i; |
96651896 XZ |
537 | p->ifa = va; |
538 | p->pte = pte; | |
539 | p->itir = log_size << 2; | |
540 | p->rr = ia64_get_rr(va); | |
541 | } | |
542 | ia64_set_psr(psr); | |
543 | r = i; | |
544 | out: | |
545 | return r; | |
546 | } | |
547 | EXPORT_SYMBOL_GPL(ia64_itr_entry); | |
548 | ||
549 | /* | |
550 | * ia64_purge_tr | |
551 | * | |
552 | * target_mask: 0x1: purge itr, 0x2 : purge dtr, 0x3 purge idtr. | |
553 | * slot: slot number to be freed. | |
554 | * | |
555 | * Must be called with preemption disabled. | |
556 | */ | |
557 | void ia64_ptr_entry(u64 target_mask, int slot) | |
558 | { | |
559 | int cpu = smp_processor_id(); | |
560 | int i; | |
561 | struct ia64_tr_entry *p; | |
562 | ||
563 | if (slot < IA64_TR_ALLOC_BASE || slot >= per_cpu(ia64_tr_num, cpu)) | |
564 | return; | |
565 | ||
566 | if (target_mask & 0x1) { | |
6c57a332 | 567 | p = ia64_idtrs[cpu] + slot; |
96651896 XZ |
568 | if ((p->pte&0x1) && is_tr_overlap(p, p->ifa, p->itir>>2)) { |
569 | p->pte = 0; | |
570 | ia64_ptr(0x1, p->ifa, p->itir>>2); | |
571 | ia64_srlz_i(); | |
572 | } | |
573 | } | |
574 | ||
575 | if (target_mask & 0x2) { | |
6c57a332 | 576 | p = ia64_idtrs[cpu] + IA64_TR_ALLOC_MAX + slot; |
96651896 XZ |
577 | if ((p->pte & 0x1) && is_tr_overlap(p, p->ifa, p->itir>>2)) { |
578 | p->pte = 0; | |
579 | ia64_ptr(0x2, p->ifa, p->itir>>2); | |
580 | ia64_srlz_i(); | |
581 | } | |
582 | } | |
583 | ||
584 | for (i = per_cpu(ia64_tr_used, cpu); i >= IA64_TR_ALLOC_BASE; i--) { | |
6c57a332 TL |
585 | if (((ia64_idtrs[cpu] + i)->pte & 0x1) || |
586 | ((ia64_idtrs[cpu] + IA64_TR_ALLOC_MAX + i)->pte & 0x1)) | |
96651896 XZ |
587 | break; |
588 | } | |
589 | per_cpu(ia64_tr_used, cpu) = i; | |
1da177e4 | 590 | } |
96651896 | 591 | EXPORT_SYMBOL_GPL(ia64_ptr_entry); |