]>
Commit | Line | Data |
---|---|---|
f204e0b8 IM |
1 | /* |
2 | * Copyright 2014 IBM Corp. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #include <linux/workqueue.h> | |
11 | #include <linux/sched.h> | |
12 | #include <linux/pid.h> | |
13 | #include <linux/mm.h> | |
14 | #include <linux/moduleparam.h> | |
15 | ||
16 | #undef MODULE_PARAM_PREFIX | |
17 | #define MODULE_PARAM_PREFIX "cxl" "." | |
18 | #include <asm/current.h> | |
19 | #include <asm/copro.h> | |
20 | #include <asm/mmu.h> | |
21 | ||
22 | #include "cxl.h" | |
23 | ||
24 | static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, | |
25 | bool sec_hash, | |
26 | struct cxl_sste *secondary_group, | |
27 | unsigned int *lru) | |
28 | { | |
29 | unsigned int i, entry; | |
30 | struct cxl_sste *sste, *group = primary_group; | |
31 | ||
32 | for (i = 0; i < 2; i++) { | |
33 | for (entry = 0; entry < 8; entry++) { | |
34 | sste = group + entry; | |
35 | if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) | |
36 | return sste; | |
37 | } | |
38 | if (!sec_hash) | |
39 | break; | |
40 | group = secondary_group; | |
41 | } | |
42 | /* Nothing free, select an entry to cast out */ | |
43 | if (sec_hash && (*lru & 0x8)) | |
44 | sste = secondary_group + (*lru & 0x7); | |
45 | else | |
46 | sste = primary_group + (*lru & 0x7); | |
47 | *lru = (*lru + 1) & 0xf; | |
48 | ||
49 | return sste; | |
50 | } | |
51 | ||
52 | static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) | |
53 | { | |
54 | /* mask is the group index, we search primary and secondary here. */ | |
55 | unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */ | |
56 | bool sec_hash = 1; | |
57 | struct cxl_sste *sste; | |
58 | unsigned int hash; | |
59 | unsigned long flags; | |
60 | ||
61 | ||
62 | sec_hash = !!(cxl_p1n_read(ctx->afu, CXL_PSL_SR_An) & CXL_PSL_SR_An_SC); | |
63 | ||
64 | if (slb->vsid & SLB_VSID_B_1T) | |
65 | hash = (slb->esid >> SID_SHIFT_1T) & mask; | |
66 | else /* 256M */ | |
67 | hash = (slb->esid >> SID_SHIFT) & mask; | |
68 | ||
69 | spin_lock_irqsave(&ctx->sste_lock, flags); | |
70 | sste = find_free_sste(ctx->sstp + (hash << 3), sec_hash, | |
71 | ctx->sstp + ((~hash & mask) << 3), &ctx->sst_lru); | |
72 | ||
73 | pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", | |
74 | sste - ctx->sstp, slb->vsid, slb->esid); | |
75 | ||
76 | sste->vsid_data = cpu_to_be64(slb->vsid); | |
77 | sste->esid_data = cpu_to_be64(slb->esid); | |
78 | spin_unlock_irqrestore(&ctx->sste_lock, flags); | |
79 | } | |
80 | ||
81 | static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm, | |
82 | u64 ea) | |
83 | { | |
84 | struct copro_slb slb = {0,0}; | |
85 | int rc; | |
86 | ||
87 | if (!(rc = copro_calculate_slb(mm, ea, &slb))) { | |
88 | cxl_load_segment(ctx, &slb); | |
89 | } | |
90 | ||
91 | return rc; | |
92 | } | |
93 | ||
94 | static void cxl_ack_ae(struct cxl_context *ctx) | |
95 | { | |
96 | unsigned long flags; | |
97 | ||
98 | cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0); | |
99 | ||
100 | spin_lock_irqsave(&ctx->lock, flags); | |
101 | ctx->pending_fault = true; | |
102 | ctx->fault_addr = ctx->dar; | |
103 | ctx->fault_dsisr = ctx->dsisr; | |
104 | spin_unlock_irqrestore(&ctx->lock, flags); | |
105 | ||
106 | wake_up_all(&ctx->wq); | |
107 | } | |
108 | ||
109 | static int cxl_handle_segment_miss(struct cxl_context *ctx, | |
110 | struct mm_struct *mm, u64 ea) | |
111 | { | |
112 | int rc; | |
113 | ||
114 | pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea); | |
115 | ||
116 | if ((rc = cxl_fault_segment(ctx, mm, ea))) | |
117 | cxl_ack_ae(ctx); | |
118 | else { | |
119 | ||
120 | mb(); /* Order seg table write to TFC MMIO write */ | |
121 | cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); | |
122 | } | |
123 | ||
124 | return IRQ_HANDLED; | |
125 | } | |
126 | ||
127 | static void cxl_handle_page_fault(struct cxl_context *ctx, | |
128 | struct mm_struct *mm, u64 dsisr, u64 dar) | |
129 | { | |
130 | unsigned flt = 0; | |
131 | int result; | |
132 | unsigned long access, flags; | |
133 | ||
134 | if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) { | |
135 | pr_devel("copro_handle_mm_fault failed: %#x\n", result); | |
136 | return cxl_ack_ae(ctx); | |
137 | } | |
138 | ||
139 | /* | |
140 | * update_mmu_cache() will not have loaded the hash since current->trap | |
141 | * is not a 0x400 or 0x300, so just call hash_page_mm() here. | |
142 | */ | |
143 | access = _PAGE_PRESENT; | |
144 | if (dsisr & CXL_PSL_DSISR_An_S) | |
145 | access |= _PAGE_RW; | |
146 | if ((!ctx->kernel) || ~(dar & (1ULL << 63))) | |
147 | access |= _PAGE_USER; | |
148 | local_irq_save(flags); | |
149 | hash_page_mm(mm, dar, access, 0x300); | |
150 | local_irq_restore(flags); | |
151 | ||
152 | pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe); | |
153 | cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); | |
154 | } | |
155 | ||
156 | void cxl_handle_fault(struct work_struct *fault_work) | |
157 | { | |
158 | struct cxl_context *ctx = | |
159 | container_of(fault_work, struct cxl_context, fault_work); | |
160 | u64 dsisr = ctx->dsisr; | |
161 | u64 dar = ctx->dar; | |
162 | struct task_struct *task; | |
163 | struct mm_struct *mm; | |
164 | ||
165 | if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr || | |
166 | cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar || | |
167 | cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) { | |
168 | /* Most likely explanation is harmless - a dedicated process | |
169 | * has detached and these were cleared by the PSL purge, but | |
170 | * warn about it just in case */ | |
171 | dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n"); | |
172 | return; | |
173 | } | |
174 | ||
175 | pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. " | |
176 | "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar); | |
177 | ||
178 | if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { | |
179 | pr_devel("cxl_handle_fault unable to get task %i\n", | |
180 | pid_nr(ctx->pid)); | |
181 | cxl_ack_ae(ctx); | |
182 | return; | |
183 | } | |
184 | if (!(mm = get_task_mm(task))) { | |
185 | pr_devel("cxl_handle_fault unable to get mm %i\n", | |
186 | pid_nr(ctx->pid)); | |
187 | cxl_ack_ae(ctx); | |
188 | goto out; | |
189 | } | |
190 | ||
191 | if (dsisr & CXL_PSL_DSISR_An_DS) | |
192 | cxl_handle_segment_miss(ctx, mm, dar); | |
193 | else if (dsisr & CXL_PSL_DSISR_An_DM) | |
194 | cxl_handle_page_fault(ctx, mm, dsisr, dar); | |
195 | else | |
196 | WARN(1, "cxl_handle_fault has nothing to handle\n"); | |
197 | ||
198 | mmput(mm); | |
199 | out: | |
200 | put_task_struct(task); | |
201 | } | |
202 | ||
203 | static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) | |
204 | { | |
205 | int rc; | |
206 | struct task_struct *task; | |
207 | struct mm_struct *mm; | |
208 | ||
209 | if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { | |
210 | pr_devel("cxl_prefault_one unable to get task %i\n", | |
211 | pid_nr(ctx->pid)); | |
212 | return; | |
213 | } | |
214 | if (!(mm = get_task_mm(task))) { | |
215 | pr_devel("cxl_prefault_one unable to get mm %i\n", | |
216 | pid_nr(ctx->pid)); | |
217 | put_task_struct(task); | |
218 | return; | |
219 | } | |
220 | ||
221 | rc = cxl_fault_segment(ctx, mm, ea); | |
222 | ||
223 | mmput(mm); | |
224 | put_task_struct(task); | |
225 | } | |
226 | ||
227 | static u64 next_segment(u64 ea, u64 vsid) | |
228 | { | |
229 | if (vsid & SLB_VSID_B_1T) | |
230 | ea |= (1ULL << 40) - 1; | |
231 | else | |
232 | ea |= (1ULL << 28) - 1; | |
233 | ||
234 | return ea + 1; | |
235 | } | |
236 | ||
237 | static void cxl_prefault_vma(struct cxl_context *ctx) | |
238 | { | |
239 | u64 ea, last_esid = 0; | |
240 | struct copro_slb slb; | |
241 | struct vm_area_struct *vma; | |
242 | int rc; | |
243 | struct task_struct *task; | |
244 | struct mm_struct *mm; | |
245 | ||
246 | if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { | |
247 | pr_devel("cxl_prefault_vma unable to get task %i\n", | |
248 | pid_nr(ctx->pid)); | |
249 | return; | |
250 | } | |
251 | if (!(mm = get_task_mm(task))) { | |
252 | pr_devel("cxl_prefault_vm unable to get mm %i\n", | |
253 | pid_nr(ctx->pid)); | |
254 | goto out1; | |
255 | } | |
256 | ||
257 | down_read(&mm->mmap_sem); | |
258 | for (vma = mm->mmap; vma; vma = vma->vm_next) { | |
259 | for (ea = vma->vm_start; ea < vma->vm_end; | |
260 | ea = next_segment(ea, slb.vsid)) { | |
261 | rc = copro_calculate_slb(mm, ea, &slb); | |
262 | if (rc) | |
263 | continue; | |
264 | ||
265 | if (last_esid == slb.esid) | |
266 | continue; | |
267 | ||
268 | cxl_load_segment(ctx, &slb); | |
269 | last_esid = slb.esid; | |
270 | } | |
271 | } | |
272 | up_read(&mm->mmap_sem); | |
273 | ||
274 | mmput(mm); | |
275 | out1: | |
276 | put_task_struct(task); | |
277 | } | |
278 | ||
279 | void cxl_prefault(struct cxl_context *ctx, u64 wed) | |
280 | { | |
281 | switch (ctx->afu->prefault_mode) { | |
282 | case CXL_PREFAULT_WED: | |
283 | cxl_prefault_one(ctx, wed); | |
284 | break; | |
285 | case CXL_PREFAULT_ALL: | |
286 | cxl_prefault_vma(ctx); | |
287 | break; | |
288 | default: | |
289 | break; | |
290 | } | |
291 | } |