]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/misc/cxl/fault.c
2 * Copyright 2014 IBM Corp.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
10 #include <linux/workqueue.h>
11 #include <linux/sched/signal.h>
12 #include <linux/sched/mm.h>
13 #include <linux/pid.h>
15 #include <linux/moduleparam.h>
17 #undef MODULE_PARAM_PREFIX
18 #define MODULE_PARAM_PREFIX "cxl" "."
19 #include <asm/current.h>
20 #include <asm/copro.h>
26 static bool sste_matches(struct cxl_sste
*sste
, struct copro_slb
*slb
)
28 return ((sste
->vsid_data
== cpu_to_be64(slb
->vsid
)) &&
29 (sste
->esid_data
== cpu_to_be64(slb
->esid
)));
33 * This finds a free SSTE for the given SLB, or returns NULL if it's already in
36 static struct cxl_sste
* find_free_sste(struct cxl_context
*ctx
,
37 struct copro_slb
*slb
)
39 struct cxl_sste
*primary
, *sste
, *ret
= NULL
;
40 unsigned int mask
= (ctx
->sst_size
>> 7) - 1; /* SSTP0[SegTableSize] */
44 if (slb
->vsid
& SLB_VSID_B_1T
)
45 hash
= (slb
->esid
>> SID_SHIFT_1T
) & mask
;
47 hash
= (slb
->esid
>> SID_SHIFT
) & mask
;
49 primary
= ctx
->sstp
+ (hash
<< 3);
51 for (entry
= 0, sste
= primary
; entry
< 8; entry
++, sste
++) {
52 if (!ret
&& !(be64_to_cpu(sste
->esid_data
) & SLB_ESID_V
))
54 if (sste_matches(sste
, slb
))
60 /* Nothing free, select an entry to cast out */
61 ret
= primary
+ ctx
->sst_lru
;
62 ctx
->sst_lru
= (ctx
->sst_lru
+ 1) & 0x7;
67 static void cxl_load_segment(struct cxl_context
*ctx
, struct copro_slb
*slb
)
69 /* mask is the group index, we search primary and secondary here. */
70 struct cxl_sste
*sste
;
73 spin_lock_irqsave(&ctx
->sste_lock
, flags
);
74 sste
= find_free_sste(ctx
, slb
);
78 pr_devel("CXL Populating SST[%li]: %#llx %#llx\n",
79 sste
- ctx
->sstp
, slb
->vsid
, slb
->esid
);
80 trace_cxl_ste_write(ctx
, sste
- ctx
->sstp
, slb
->esid
, slb
->vsid
);
82 sste
->vsid_data
= cpu_to_be64(slb
->vsid
);
83 sste
->esid_data
= cpu_to_be64(slb
->esid
);
85 spin_unlock_irqrestore(&ctx
->sste_lock
, flags
);
88 static int cxl_fault_segment(struct cxl_context
*ctx
, struct mm_struct
*mm
,
91 struct copro_slb slb
= {0,0};
94 if (!(rc
= copro_calculate_slb(mm
, ea
, &slb
))) {
95 cxl_load_segment(ctx
, &slb
);
101 static void cxl_ack_ae(struct cxl_context
*ctx
)
105 cxl_ops
->ack_irq(ctx
, CXL_PSL_TFC_An_AE
, 0);
107 spin_lock_irqsave(&ctx
->lock
, flags
);
108 ctx
->pending_fault
= true;
109 ctx
->fault_addr
= ctx
->dar
;
110 ctx
->fault_dsisr
= ctx
->dsisr
;
111 spin_unlock_irqrestore(&ctx
->lock
, flags
);
113 wake_up_all(&ctx
->wq
);
116 static int cxl_handle_segment_miss(struct cxl_context
*ctx
,
117 struct mm_struct
*mm
, u64 ea
)
121 pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx
->pe
, ea
);
122 trace_cxl_ste_miss(ctx
, ea
);
124 if ((rc
= cxl_fault_segment(ctx
, mm
, ea
)))
128 mb(); /* Order seg table write to TFC MMIO write */
129 cxl_ops
->ack_irq(ctx
, CXL_PSL_TFC_An_R
, 0);
135 static void cxl_handle_page_fault(struct cxl_context
*ctx
,
136 struct mm_struct
*mm
, u64 dsisr
, u64 dar
)
140 unsigned long access
, flags
, inv_flags
= 0;
142 trace_cxl_pte_miss(ctx
, dsisr
, dar
);
144 if ((result
= copro_handle_mm_fault(mm
, dar
, dsisr
, &flt
))) {
145 pr_devel("copro_handle_mm_fault failed: %#x\n", result
);
146 return cxl_ack_ae(ctx
);
150 * update_mmu_cache() will not have loaded the hash since current->trap
151 * is not a 0x400 or 0x300, so just call hash_page_mm() here.
153 access
= _PAGE_PRESENT
| _PAGE_READ
;
154 if (dsisr
& CXL_PSL_DSISR_An_S
)
155 access
|= _PAGE_WRITE
;
157 access
|= _PAGE_PRIVILEGED
;
158 if ((!ctx
->kernel
) || (REGION_ID(dar
) == USER_REGION_ID
))
159 access
&= ~_PAGE_PRIVILEGED
;
161 if (dsisr
& DSISR_NOHPTE
)
162 inv_flags
|= HPTE_NOHPTE_UPDATE
;
164 local_irq_save(flags
);
165 hash_page_mm(mm
, dar
, access
, 0x300, inv_flags
);
166 local_irq_restore(flags
);
168 pr_devel("Page fault successfully handled for pe: %i!\n", ctx
->pe
);
169 cxl_ops
->ack_irq(ctx
, CXL_PSL_TFC_An_R
, 0);
173 * Returns the mm_struct corresponding to the context ctx via ctx->pid
174 * In case the task has exited we use the task group leader accessible
175 * via ctx->glpid to find the next task in the thread group that has a
176 * valid mm_struct associated with it. If a task with valid mm_struct
177 * is found the ctx->pid is updated to use the task struct for subsequent
178 * translations. In case no valid mm_struct is found in the task group to
179 * service the fault a NULL is returned.
181 static struct mm_struct
*get_mem_context(struct cxl_context
*ctx
)
183 struct task_struct
*task
= NULL
;
184 struct mm_struct
*mm
= NULL
;
185 struct pid
*old_pid
= ctx
->pid
;
187 if (old_pid
== NULL
) {
188 pr_warn("%s: Invalid context for pe=%d\n",
193 task
= get_pid_task(old_pid
, PIDTYPE_PID
);
196 * pid_alive may look racy but this saves us from costly
197 * get_task_mm when the task is a zombie. In worst case
198 * we may think a task is alive, which is about to die
199 * but get_task_mm will return NULL.
201 if (task
!= NULL
&& pid_alive(task
))
202 mm
= get_task_mm(task
);
204 /* release the task struct that was taken earlier */
206 put_task_struct(task
);
208 pr_devel("%s: Context owning pid=%i for pe=%i dead\n",
209 __func__
, pid_nr(old_pid
), ctx
->pe
);
212 * If we couldn't find the mm context then use the group
213 * leader to iterate over the task group and find a task
214 * that gives us mm_struct.
216 if (unlikely(mm
== NULL
&& ctx
->glpid
!= NULL
)) {
219 task
= pid_task(ctx
->glpid
, PIDTYPE_PID
);
222 mm
= get_task_mm(task
);
224 ctx
->pid
= get_task_pid(task
,
228 task
= next_thread(task
);
229 } while (task
&& !thread_group_leader(task
));
232 /* check if we switched pid */
233 if (ctx
->pid
!= old_pid
) {
235 pr_devel("%s:pe=%i switch pid %i->%i\n",
236 __func__
, ctx
->pe
, pid_nr(old_pid
),
239 pr_devel("%s:Cannot find mm for pid=%i\n",
240 __func__
, pid_nr(old_pid
));
242 /* drop the reference to older pid */
252 void cxl_handle_fault(struct work_struct
*fault_work
)
254 struct cxl_context
*ctx
=
255 container_of(fault_work
, struct cxl_context
, fault_work
);
256 u64 dsisr
= ctx
->dsisr
;
258 struct mm_struct
*mm
= NULL
;
260 if (cpu_has_feature(CPU_FTR_HVMODE
)) {
261 if (cxl_p2n_read(ctx
->afu
, CXL_PSL_DSISR_An
) != dsisr
||
262 cxl_p2n_read(ctx
->afu
, CXL_PSL_DAR_An
) != dar
||
263 cxl_p2n_read(ctx
->afu
, CXL_PSL_PEHandle_An
) != ctx
->pe
) {
264 /* Most likely explanation is harmless - a dedicated
265 * process has detached and these were cleared by the
266 * PSL purge, but warn about it just in case
268 dev_notice(&ctx
->afu
->dev
, "cxl_handle_fault: Translation fault regs changed\n");
273 /* Early return if the context is being / has been detached */
274 if (ctx
->status
== CLOSED
) {
279 pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. "
280 "DSISR: %#llx DAR: %#llx\n", ctx
->pe
, dsisr
, dar
);
284 mm
= get_mem_context(ctx
);
285 /* indicates all the thread in task group have exited */
287 pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
288 __func__
, ctx
->pe
, pid_nr(ctx
->pid
));
292 pr_devel("Handling page fault for pe=%d pid=%i\n",
293 ctx
->pe
, pid_nr(ctx
->pid
));
297 if (dsisr
& CXL_PSL_DSISR_An_DS
)
298 cxl_handle_segment_miss(ctx
, mm
, dar
);
299 else if (dsisr
& CXL_PSL_DSISR_An_DM
)
300 cxl_handle_page_fault(ctx
, mm
, dsisr
, dar
);
302 WARN(1, "cxl_handle_fault has nothing to handle\n");
308 static void cxl_prefault_one(struct cxl_context
*ctx
, u64 ea
)
310 struct mm_struct
*mm
;
312 mm
= get_mem_context(ctx
);
314 pr_devel("cxl_prefault_one unable to get mm %i\n",
319 cxl_fault_segment(ctx
, mm
, ea
);
324 static u64
next_segment(u64 ea
, u64 vsid
)
326 if (vsid
& SLB_VSID_B_1T
)
327 ea
|= (1ULL << 40) - 1;
329 ea
|= (1ULL << 28) - 1;
334 static void cxl_prefault_vma(struct cxl_context
*ctx
)
336 u64 ea
, last_esid
= 0;
337 struct copro_slb slb
;
338 struct vm_area_struct
*vma
;
340 struct mm_struct
*mm
;
342 mm
= get_mem_context(ctx
);
344 pr_devel("cxl_prefault_vm unable to get mm %i\n",
349 down_read(&mm
->mmap_sem
);
350 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
) {
351 for (ea
= vma
->vm_start
; ea
< vma
->vm_end
;
352 ea
= next_segment(ea
, slb
.vsid
)) {
353 rc
= copro_calculate_slb(mm
, ea
, &slb
);
357 if (last_esid
== slb
.esid
)
360 cxl_load_segment(ctx
, &slb
);
361 last_esid
= slb
.esid
;
364 up_read(&mm
->mmap_sem
);
369 void cxl_prefault(struct cxl_context
*ctx
, u64 wed
)
371 switch (ctx
->afu
->prefault_mode
) {
372 case CXL_PREFAULT_WED
:
373 cxl_prefault_one(ctx
, wed
);
375 case CXL_PREFAULT_ALL
:
376 cxl_prefault_vma(ctx
);