]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
4dea2d1a SB |
2 | /* |
3 | * Copyright 2016-17 IBM Corp. | |
4dea2d1a SB |
4 | */ |
5 | ||
180fe15a SB |
6 | #define pr_fmt(fmt) "vas: " fmt |
7 | ||
4dea2d1a SB |
8 | #include <linux/types.h> |
9 | #include <linux/mutex.h> | |
180fe15a SB |
10 | #include <linux/slab.h> |
11 | #include <linux/io.h> | |
b25b33ac | 12 | #include <linux/log2.h> |
62c4eda4 SB |
13 | #include <linux/rcupdate.h> |
14 | #include <linux/cred.h> | |
6c8e6bb2 SB |
15 | #include <asm/switch_to.h> |
16 | #include <asm/ppc-opcode.h> | |
4dea2d1a | 17 | #include "vas.h" |
2392c8c8 | 18 | #include "copy-paste.h" |
4dea2d1a | 19 | |
007bb7d6 SB |
20 | #define CREATE_TRACE_POINTS |
21 | #include "vas-trace.h" | |
22 | ||
180fe15a SB |
23 | /* |
24 | * Compute the paste address region for the window @window using the | |
25 | * ->paste_base_addr and ->paste_win_id_shift we got from device tree. | |
26 | */ | |
27 | static void compute_paste_address(struct vas_window *window, u64 *addr, int *len) | |
28 | { | |
29 | int winid; | |
30 | u64 base, shift; | |
31 | ||
32 | base = window->vinst->paste_base_addr; | |
33 | shift = window->vinst->paste_win_id_shift; | |
34 | winid = window->winid; | |
35 | ||
36 | *addr = base + (winid << shift); | |
37 | if (len) | |
38 | *len = PAGE_SIZE; | |
39 | ||
40 | pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr); | |
41 | } | |
42 | ||
5676be2f SB |
43 | u64 vas_win_paste_addr(struct vas_window *win) |
44 | { | |
45 | u64 addr; | |
46 | ||
47 | compute_paste_address(win, &addr, NULL); | |
48 | ||
49 | return addr; | |
50 | } | |
51 | EXPORT_SYMBOL(vas_win_paste_addr); | |
52 | ||
180fe15a SB |
53 | static inline void get_hvwc_mmio_bar(struct vas_window *window, |
54 | u64 *start, int *len) | |
55 | { | |
56 | u64 pbaddr; | |
57 | ||
58 | pbaddr = window->vinst->hvwc_bar_start; | |
59 | *start = pbaddr + window->winid * VAS_HVWC_SIZE; | |
60 | *len = VAS_HVWC_SIZE; | |
61 | } | |
62 | ||
63 | static inline void get_uwc_mmio_bar(struct vas_window *window, | |
64 | u64 *start, int *len) | |
65 | { | |
66 | u64 pbaddr; | |
67 | ||
68 | pbaddr = window->vinst->uwc_bar_start; | |
69 | *start = pbaddr + window->winid * VAS_UWC_SIZE; | |
70 | *len = VAS_UWC_SIZE; | |
71 | } | |
72 | ||
73 | /* | |
74 | * Map the paste bus address of the given send window into kernel address | |
75 | * space. Unlike MMIO regions (map_mmio_region() below), paste region must | |
76 | * be mapped cache-able and is only applicable to send windows. | |
77 | */ | |
5239af67 | 78 | static void *map_paste_region(struct vas_window *txwin) |
180fe15a SB |
79 | { |
80 | int len; | |
81 | void *map; | |
82 | char *name; | |
83 | u64 start; | |
84 | ||
85 | name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id, | |
86 | txwin->winid); | |
87 | if (!name) | |
88 | goto free_name; | |
89 | ||
90 | txwin->paste_addr_name = name; | |
91 | compute_paste_address(txwin, &start, &len); | |
92 | ||
93 | if (!request_mem_region(start, len, name)) { | |
94 | pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", | |
95 | __func__, start, len); | |
96 | goto free_name; | |
97 | } | |
98 | ||
99 | map = ioremap_cache(start, len); | |
100 | if (!map) { | |
101 | pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__, | |
102 | start, len); | |
103 | goto free_name; | |
104 | } | |
105 | ||
106 | pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map); | |
107 | return map; | |
108 | ||
109 | free_name: | |
110 | kfree(name); | |
111 | return ERR_PTR(-ENOMEM); | |
112 | } | |
113 | ||
180fe15a SB |
114 | static void *map_mmio_region(char *name, u64 start, int len) |
115 | { | |
116 | void *map; | |
117 | ||
118 | if (!request_mem_region(start, len, name)) { | |
119 | pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n", | |
120 | __func__, start, len); | |
121 | return NULL; | |
122 | } | |
123 | ||
124 | map = ioremap(start, len); | |
125 | if (!map) { | |
126 | pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start, | |
127 | len); | |
128 | return NULL; | |
129 | } | |
130 | ||
131 | return map; | |
132 | } | |
133 | ||
134 | static void unmap_region(void *addr, u64 start, int len) | |
135 | { | |
136 | iounmap(addr); | |
137 | release_mem_region((phys_addr_t)start, len); | |
138 | } | |
139 | ||
140 | /* | |
141 | * Unmap the paste address region for a window. | |
142 | */ | |
98271d41 | 143 | static void unmap_paste_region(struct vas_window *window) |
180fe15a SB |
144 | { |
145 | int len; | |
146 | u64 busaddr_start; | |
147 | ||
148 | if (window->paste_kaddr) { | |
149 | compute_paste_address(window, &busaddr_start, &len); | |
150 | unmap_region(window->paste_kaddr, busaddr_start, len); | |
151 | window->paste_kaddr = NULL; | |
152 | kfree(window->paste_addr_name); | |
153 | window->paste_addr_name = NULL; | |
154 | } | |
155 | } | |
156 | ||
157 | /* | |
ece4e512 SB |
158 | * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't |
159 | * unmap when the window's debugfs dir is in use. This serializes close | |
160 | * of a window even on another VAS instance but since its not a critical | |
161 | * path, just minimize the time we hold the mutex for now. We can add | |
162 | * a per-instance mutex later if necessary. | |
180fe15a SB |
163 | */ |
164 | static void unmap_winctx_mmio_bars(struct vas_window *window) | |
165 | { | |
166 | int len; | |
ece4e512 SB |
167 | void *uwc_map; |
168 | void *hvwc_map; | |
180fe15a SB |
169 | u64 busaddr_start; |
170 | ||
ece4e512 SB |
171 | mutex_lock(&vas_mutex); |
172 | ||
173 | hvwc_map = window->hvwc_map; | |
174 | window->hvwc_map = NULL; | |
175 | ||
176 | uwc_map = window->uwc_map; | |
177 | window->uwc_map = NULL; | |
178 | ||
179 | mutex_unlock(&vas_mutex); | |
180 | ||
181 | if (hvwc_map) { | |
180fe15a | 182 | get_hvwc_mmio_bar(window, &busaddr_start, &len); |
ece4e512 | 183 | unmap_region(hvwc_map, busaddr_start, len); |
180fe15a SB |
184 | } |
185 | ||
ece4e512 | 186 | if (uwc_map) { |
180fe15a | 187 | get_uwc_mmio_bar(window, &busaddr_start, &len); |
ece4e512 | 188 | unmap_region(uwc_map, busaddr_start, len); |
180fe15a SB |
189 | } |
190 | } | |
191 | ||
192 | /* | |
193 | * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the | |
194 | * OS/User Window Context (UWC) MMIO Base Address Region for the given window. | |
195 | * Map these bus addresses and save the mapped kernel addresses in @window. | |
196 | */ | |
197 | int map_winctx_mmio_bars(struct vas_window *window) | |
198 | { | |
199 | int len; | |
200 | u64 start; | |
201 | ||
202 | get_hvwc_mmio_bar(window, &start, &len); | |
203 | window->hvwc_map = map_mmio_region("HVWCM_Window", start, len); | |
204 | ||
205 | get_uwc_mmio_bar(window, &start, &len); | |
206 | window->uwc_map = map_mmio_region("UWCM_Window", start, len); | |
207 | ||
208 | if (!window->hvwc_map || !window->uwc_map) { | |
209 | unmap_winctx_mmio_bars(window); | |
210 | return -1; | |
211 | } | |
212 | ||
213 | return 0; | |
214 | } | |
215 | ||
b25b33ac SB |
216 | /* |
217 | * Reset all valid registers in the HV and OS/User Window Contexts for | |
218 | * the window identified by @window. | |
219 | * | |
220 | * NOTE: We cannot really use a for loop to reset window context. Not all | |
221 | * offsets in a window context are valid registers and the valid | |
222 | * registers are not sequential. And, we can only write to offsets | |
223 | * with valid registers. | |
224 | */ | |
225 | void reset_window_regs(struct vas_window *window) | |
226 | { | |
227 | write_hvwc_reg(window, VREG(LPID), 0ULL); | |
228 | write_hvwc_reg(window, VREG(PID), 0ULL); | |
229 | write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL); | |
230 | write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL); | |
231 | write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL); | |
232 | write_hvwc_reg(window, VREG(AMR), 0ULL); | |
233 | write_hvwc_reg(window, VREG(SEIDR), 0ULL); | |
234 | write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL); | |
235 | write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); | |
236 | write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL); | |
237 | write_hvwc_reg(window, VREG(PSWID), 0ULL); | |
238 | write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL); | |
239 | write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL); | |
240 | write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL); | |
241 | write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); | |
242 | write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); | |
243 | write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); | |
244 | write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL); | |
245 | write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); | |
246 | write_hvwc_reg(window, VREG(TX_WCRED), 0ULL); | |
247 | write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); | |
248 | write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL); | |
249 | write_hvwc_reg(window, VREG(WINCTL), 0ULL); | |
250 | write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); | |
251 | write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL); | |
252 | write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL); | |
253 | write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL); | |
254 | write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL); | |
255 | write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL); | |
256 | write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL); | |
257 | write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL); | |
258 | write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL); | |
259 | write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); | |
260 | ||
261 | /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */ | |
262 | ||
263 | /* | |
264 | * The send and receive window credit adder registers are also | |
265 | * accessible from HVWC and have been initialized above. We don't | |
266 | * need to initialize from the OS/User Window Context, so skip | |
267 | * following calls: | |
268 | * | |
269 | * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); | |
270 | * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); | |
271 | */ | |
272 | } | |
273 | ||
274 | /* | |
275 | * Initialize window context registers related to Address Translation. | |
276 | * These registers are common to send/receive windows although they | |
277 | * differ for user/kernel windows. As we resolve the TODOs we may | |
278 | * want to add fields to vas_winctx and move the initialization to | |
279 | * init_vas_winctx_regs(). | |
280 | */ | |
281 | static void init_xlate_regs(struct vas_window *window, bool user_win) | |
282 | { | |
283 | u64 lpcr, val; | |
284 | ||
285 | /* | |
286 | * MSR_TA, MSR_US are false for both kernel and user. | |
287 | * MSR_DR and MSR_PR are false for kernel. | |
288 | */ | |
289 | val = 0ULL; | |
290 | val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1); | |
291 | val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1); | |
292 | if (user_win) { | |
293 | val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1); | |
294 | val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1); | |
295 | } | |
296 | write_hvwc_reg(window, VREG(XLATE_MSR), val); | |
297 | ||
298 | lpcr = mfspr(SPRN_LPCR); | |
299 | val = 0ULL; | |
300 | /* | |
301 | * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the | |
302 | * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB. | |
303 | * | |
304 | * NOTE: From Section 1.3.1, Address Translation Context of the | |
305 | * Nest MMU Workbook, LPCR_SC should be 0 for Power9. | |
306 | */ | |
307 | val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5); | |
308 | val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL); | |
309 | val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC); | |
310 | val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0); | |
311 | write_hvwc_reg(window, VREG(XLATE_LPCR), val); | |
312 | ||
313 | /* | |
314 | * Section 1.3.1 (Address translation Context) of NMMU workbook. | |
315 | * 0b00 Hashed Page Table mode | |
316 | * 0b01 Reserved | |
317 | * 0b10 Radix on HPT | |
318 | * 0b11 Radix on Radix | |
319 | */ | |
320 | val = 0ULL; | |
321 | val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2); | |
322 | write_hvwc_reg(window, VREG(XLATE_CTL), val); | |
323 | ||
324 | /* | |
325 | * TODO: Can we mfspr(AMR) even for user windows? | |
326 | */ | |
327 | val = 0ULL; | |
328 | val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR)); | |
329 | write_hvwc_reg(window, VREG(AMR), val); | |
330 | ||
331 | val = 0ULL; | |
332 | val = SET_FIELD(VAS_SEIDR, val, 0); | |
333 | write_hvwc_reg(window, VREG(SEIDR), val); | |
334 | } | |
335 | ||
336 | /* | |
337 | * Initialize Reserved Send Buffer Count for the send window. It involves | |
338 | * writing to the register, reading it back to confirm that the hardware | |
339 | * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook. | |
340 | * | |
341 | * Since we can only make a best-effort attempt to fulfill the request, | |
342 | * we don't return any errors if we cannot. | |
343 | * | |
344 | * TODO: Reserved (aka dedicated) send buffers are not supported yet. | |
345 | */ | |
346 | static void init_rsvd_tx_buf_count(struct vas_window *txwin, | |
347 | struct vas_winctx *winctx) | |
348 | { | |
349 | write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL); | |
350 | } | |
351 | ||
352 | /* | |
353 | * init_winctx_regs() | |
354 | * Initialize window context registers for a receive window. | |
355 | * Except for caching control and marking window open, the registers | |
356 | * are initialized in the order listed in Section 3.1.4 (Window Context | |
357 | * Cache Register Details) of the VAS workbook although they don't need | |
358 | * to be. | |
359 | * | |
360 | * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL | |
361 | * (so that it can get a large contiguous area) and passes that buffer | |
362 | * to kernel via device tree. We now write that buffer address to the | |
363 | * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL | |
364 | * write the per-chip RX FIFO addresses to the windows during boot-up | |
365 | * as a one-time task? That could work for NX but what about other | |
366 | * receivers? Let the receivers tell us the rx-fifo buffers for now. | |
367 | */ | |
368 | int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx) | |
369 | { | |
370 | u64 val; | |
371 | int fifo_size; | |
372 | ||
373 | reset_window_regs(window); | |
374 | ||
375 | val = 0ULL; | |
376 | val = SET_FIELD(VAS_LPID, val, winctx->lpid); | |
377 | write_hvwc_reg(window, VREG(LPID), val); | |
378 | ||
379 | val = 0ULL; | |
380 | val = SET_FIELD(VAS_PID_ID, val, winctx->pidr); | |
381 | write_hvwc_reg(window, VREG(PID), val); | |
382 | ||
383 | init_xlate_regs(window, winctx->user_win); | |
384 | ||
385 | val = 0ULL; | |
386 | val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0); | |
387 | write_hvwc_reg(window, VREG(FAULT_TX_WIN), val); | |
388 | ||
389 | /* In PowerNV, interrupts go to HV. */ | |
390 | write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); | |
391 | ||
392 | val = 0ULL; | |
393 | val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port); | |
394 | write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val); | |
395 | ||
396 | val = 0ULL; | |
397 | val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid); | |
398 | write_hvwc_reg(window, VREG(PSWID), val); | |
399 | ||
400 | write_hvwc_reg(window, VREG(SPARE1), 0ULL); | |
401 | write_hvwc_reg(window, VREG(SPARE2), 0ULL); | |
402 | write_hvwc_reg(window, VREG(SPARE3), 0ULL); | |
403 | ||
404 | /* | |
405 | * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR | |
406 | * register as is - do NOT shift the address into VAS_LFIFO_BAR | |
407 | * bit fields! Ok to set the page migration select fields - | |
408 | * VAS ignores the lower 10+ bits in the address anyway, because | |
409 | * the minimum FIFO size is 1K? | |
410 | * | |
411 | * See also: Design note in function header. | |
412 | */ | |
413 | val = __pa(winctx->rx_fifo); | |
414 | val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0); | |
415 | write_hvwc_reg(window, VREG(LFIFO_BAR), val); | |
416 | ||
417 | val = 0ULL; | |
418 | val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp); | |
419 | write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val); | |
420 | ||
421 | val = 0ULL; | |
422 | val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type); | |
423 | val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable); | |
424 | write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val); | |
425 | ||
426 | write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); | |
427 | write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); | |
428 | write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); | |
429 | ||
430 | val = 0ULL; | |
431 | val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max); | |
432 | write_hvwc_reg(window, VREG(LRX_WCRED), val); | |
433 | ||
434 | val = 0ULL; | |
435 | val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max); | |
436 | write_hvwc_reg(window, VREG(TX_WCRED), val); | |
437 | ||
438 | write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); | |
439 | write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); | |
440 | ||
441 | fifo_size = winctx->rx_fifo_size / 1024; | |
442 | ||
443 | val = 0ULL; | |
444 | val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size)); | |
445 | write_hvwc_reg(window, VREG(LFIFO_SIZE), val); | |
446 | ||
447 | /* Update window control and caching control registers last so | |
448 | * we mark the window open only after fully initializing it and | |
449 | * pushing context to cache. | |
450 | */ | |
451 | ||
452 | write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); | |
453 | ||
454 | init_rsvd_tx_buf_count(window, winctx); | |
455 | ||
456 | /* for a send window, point to the matching receive window */ | |
457 | val = 0ULL; | |
458 | val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id); | |
459 | write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val); | |
460 | ||
461 | write_hvwc_reg(window, VREG(SPARE4), 0ULL); | |
462 | ||
463 | val = 0ULL; | |
464 | val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable); | |
465 | val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable); | |
466 | val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early); | |
467 | val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg); | |
468 | write_hvwc_reg(window, VREG(LNOTIFY_CTL), val); | |
469 | ||
470 | val = 0ULL; | |
471 | val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid); | |
472 | write_hvwc_reg(window, VREG(LNOTIFY_PID), val); | |
473 | ||
474 | val = 0ULL; | |
475 | val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid); | |
476 | write_hvwc_reg(window, VREG(LNOTIFY_LPID), val); | |
477 | ||
478 | val = 0ULL; | |
479 | val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid); | |
480 | write_hvwc_reg(window, VREG(LNOTIFY_TID), val); | |
481 | ||
482 | val = 0ULL; | |
483 | val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope); | |
484 | val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope); | |
485 | write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val); | |
486 | ||
487 | /* Skip read-only registers NX_UTIL and NX_UTIL_SE */ | |
488 | ||
489 | write_hvwc_reg(window, VREG(SPARE5), 0ULL); | |
490 | write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); | |
491 | write_hvwc_reg(window, VREG(SPARE6), 0ULL); | |
492 | ||
493 | /* Finally, push window context to memory and... */ | |
494 | val = 0ULL; | |
495 | val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1); | |
496 | write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val); | |
497 | ||
498 | /* ... mark the window open for business */ | |
499 | val = 0ULL; | |
500 | val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit); | |
501 | val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win); | |
502 | val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode); | |
503 | val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode); | |
504 | val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode); | |
505 | val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode); | |
506 | val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win); | |
507 | val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win); | |
508 | val = SET_FIELD(VAS_WINCTL_OPEN, val, 1); | |
509 | write_hvwc_reg(window, VREG(WINCTL), val); | |
510 | ||
511 | return 0; | |
512 | } | |
513 | ||
bbfe59f8 SB |
514 | static void vas_release_window_id(struct ida *ida, int winid) |
515 | { | |
cd38049e | 516 | ida_free(ida, winid); |
bbfe59f8 SB |
517 | } |
518 | ||
519 | static int vas_assign_window_id(struct ida *ida) | |
520 | { | |
cd38049e | 521 | int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL); |
bbfe59f8 | 522 | |
cd38049e MW |
523 | if (winid == -ENOSPC) { |
524 | pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP); | |
bbfe59f8 SB |
525 | return -EAGAIN; |
526 | } | |
527 | ||
528 | return winid; | |
529 | } | |
530 | ||
98271d41 | 531 | static void vas_window_free(struct vas_window *window) |
bbfe59f8 SB |
532 | { |
533 | int winid = window->winid; | |
534 | struct vas_instance *vinst = window->vinst; | |
535 | ||
536 | unmap_winctx_mmio_bars(window); | |
ece4e512 SB |
537 | |
538 | vas_window_free_dbgdir(window); | |
539 | ||
bbfe59f8 SB |
540 | kfree(window); |
541 | ||
542 | vas_release_window_id(&vinst->ida, winid); | |
543 | } | |
544 | ||
62c4eda4 | 545 | static struct vas_window *vas_window_alloc(struct vas_instance *vinst) |
bbfe59f8 SB |
546 | { |
547 | int winid; | |
548 | struct vas_window *window; | |
549 | ||
550 | winid = vas_assign_window_id(&vinst->ida); | |
551 | if (winid < 0) | |
552 | return ERR_PTR(winid); | |
553 | ||
554 | window = kzalloc(sizeof(*window), GFP_KERNEL); | |
555 | if (!window) | |
556 | goto out_free; | |
557 | ||
558 | window->vinst = vinst; | |
559 | window->winid = winid; | |
560 | ||
561 | if (map_winctx_mmio_bars(window)) | |
562 | goto out_free; | |
563 | ||
ece4e512 SB |
564 | vas_window_init_dbgdir(window); |
565 | ||
bbfe59f8 SB |
566 | return window; |
567 | ||
568 | out_free: | |
569 | kfree(window); | |
570 | vas_release_window_id(&vinst->ida, winid); | |
571 | return ERR_PTR(-ENOMEM); | |
572 | } | |
573 | ||
98271d41 SB |
574 | static void put_rx_win(struct vas_window *rxwin) |
575 | { | |
576 | /* Better not be a send window! */ | |
577 | WARN_ON_ONCE(rxwin->tx_win); | |
578 | ||
579 | atomic_dec(&rxwin->num_txwins); | |
580 | } | |
581 | ||
6c8e6bb2 SB |
582 | /* |
583 | * Find the user space receive window given the @pswid. | |
584 | * - We must have a valid vasid and it must belong to this instance. | |
585 | * (so both send and receive windows are on the same VAS instance) | |
586 | * - The window must refer to an OPEN, FTW, RECEIVE window. | |
587 | * | |
588 | * NOTE: We access ->windows[] table and assume that vinst->mutex is held. | |
589 | */ | |
590 | static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid) | |
591 | { | |
592 | int vasid, winid; | |
593 | struct vas_window *rxwin; | |
594 | ||
595 | decode_pswid(pswid, &vasid, &winid); | |
596 | ||
597 | if (vinst->vas_id != vasid) | |
598 | return ERR_PTR(-EINVAL); | |
599 | ||
600 | rxwin = vinst->windows[winid]; | |
601 | ||
602 | if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW) | |
603 | return ERR_PTR(-EINVAL); | |
604 | ||
605 | return rxwin; | |
606 | } | |
607 | ||
62c4eda4 SB |
608 | /* |
609 | * Get the VAS receive window associated with NX engine identified | |
610 | * by @cop and if applicable, @pswid. | |
611 | * | |
612 | * See also function header of set_vinst_win(). | |
613 | */ | |
5239af67 | 614 | static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst, |
62c4eda4 SB |
615 | enum vas_cop_type cop, u32 pswid) |
616 | { | |
617 | struct vas_window *rxwin; | |
618 | ||
619 | mutex_lock(&vinst->mutex); | |
620 | ||
6c8e6bb2 SB |
621 | if (cop == VAS_COP_TYPE_FTW) |
622 | rxwin = get_user_rxwin(vinst, pswid); | |
62c4eda4 | 623 | else |
6c8e6bb2 | 624 | rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL); |
62c4eda4 SB |
625 | |
626 | if (!IS_ERR(rxwin)) | |
627 | atomic_inc(&rxwin->num_txwins); | |
628 | ||
629 | mutex_unlock(&vinst->mutex); | |
630 | ||
631 | return rxwin; | |
632 | } | |
633 | ||
634 | /* | |
635 | * We have two tables of windows in a VAS instance. The first one, | |
636 | * ->windows[], contains all the windows in the instance and allows | |
637 | * looking up a window by its id. It is used to look up send windows | |
638 | * during fault handling and receive windows when pairing user space | |
639 | * send/receive windows. | |
640 | * | |
641 | * The second table, ->rxwin[], contains receive windows that are | |
642 | * associated with NX engines. This table has VAS_COP_TYPE_MAX | |
643 | * entries and is used to look up a receive window by its | |
644 | * coprocessor type. | |
645 | * | |
646 | * Here, we save @window in the ->windows[] table. If it is a receive | |
647 | * window, we also save the window in the ->rxwin[] table. | |
648 | */ | |
649 | static void set_vinst_win(struct vas_instance *vinst, | |
650 | struct vas_window *window) | |
651 | { | |
652 | int id = window->winid; | |
653 | ||
654 | mutex_lock(&vinst->mutex); | |
655 | ||
656 | /* | |
657 | * There should only be one receive window for a coprocessor type | |
658 | * unless its a user (FTW) window. | |
659 | */ | |
660 | if (!window->user_win && !window->tx_win) { | |
661 | WARN_ON_ONCE(vinst->rxwin[window->cop]); | |
662 | vinst->rxwin[window->cop] = window; | |
663 | } | |
664 | ||
665 | WARN_ON_ONCE(vinst->windows[id] != NULL); | |
666 | vinst->windows[id] = window; | |
667 | ||
668 | mutex_unlock(&vinst->mutex); | |
669 | } | |
670 | ||
671 | /* | |
672 | * Clear this window from the table(s) of windows for this VAS instance. | |
673 | * See also function header of set_vinst_win(). | |
674 | */ | |
98271d41 | 675 | static void clear_vinst_win(struct vas_window *window) |
62c4eda4 SB |
676 | { |
677 | int id = window->winid; | |
678 | struct vas_instance *vinst = window->vinst; | |
679 | ||
680 | mutex_lock(&vinst->mutex); | |
681 | ||
682 | if (!window->user_win && !window->tx_win) { | |
683 | WARN_ON_ONCE(!vinst->rxwin[window->cop]); | |
684 | vinst->rxwin[window->cop] = NULL; | |
685 | } | |
686 | ||
687 | WARN_ON_ONCE(vinst->windows[id] != window); | |
688 | vinst->windows[id] = NULL; | |
689 | ||
690 | mutex_unlock(&vinst->mutex); | |
691 | } | |
692 | ||
693 | static void init_winctx_for_rxwin(struct vas_window *rxwin, | |
694 | struct vas_rx_win_attr *rxattr, | |
695 | struct vas_winctx *winctx) | |
696 | { | |
697 | /* | |
698 | * We first zero (memset()) all fields and only set non-zero fields. | |
699 | * Following fields are 0/false but maybe deserve a comment: | |
700 | * | |
701 | * ->notify_os_intr_reg In powerNV, send intrs to HV | |
702 | * ->notify_disable False for NX windows | |
703 | * ->intr_disable False for Fault Windows | |
704 | * ->xtra_write False for NX windows | |
705 | * ->notify_early NA for NX windows | |
706 | * ->rsvd_txbuf_count NA for Rx windows | |
707 | * ->lpid, ->pid, ->tid NA for Rx windows | |
708 | */ | |
709 | ||
710 | memset(winctx, 0, sizeof(struct vas_winctx)); | |
711 | ||
712 | winctx->rx_fifo = rxattr->rx_fifo; | |
713 | winctx->rx_fifo_size = rxattr->rx_fifo_size; | |
62f659e0 | 714 | winctx->wcreds_max = rxwin->wcreds_max; |
62c4eda4 SB |
715 | winctx->pin_win = rxattr->pin_win; |
716 | ||
717 | winctx->nx_win = rxattr->nx_win; | |
718 | winctx->fault_win = rxattr->fault_win; | |
e34917fb SB |
719 | winctx->user_win = rxattr->user_win; |
720 | winctx->rej_no_credit = rxattr->rej_no_credit; | |
62c4eda4 SB |
721 | winctx->rx_word_mode = rxattr->rx_win_ord_mode; |
722 | winctx->tx_word_mode = rxattr->tx_win_ord_mode; | |
723 | winctx->rx_wcred_mode = rxattr->rx_wcred_mode; | |
724 | winctx->tx_wcred_mode = rxattr->tx_wcred_mode; | |
e34917fb | 725 | winctx->notify_early = rxattr->notify_early; |
62c4eda4 SB |
726 | |
727 | if (winctx->nx_win) { | |
728 | winctx->data_stamp = true; | |
729 | winctx->intr_disable = true; | |
730 | winctx->pin_win = true; | |
731 | ||
732 | WARN_ON_ONCE(winctx->fault_win); | |
733 | WARN_ON_ONCE(!winctx->rx_word_mode); | |
734 | WARN_ON_ONCE(!winctx->tx_word_mode); | |
735 | WARN_ON_ONCE(winctx->notify_after_count); | |
736 | } else if (winctx->fault_win) { | |
737 | winctx->notify_disable = true; | |
738 | } else if (winctx->user_win) { | |
739 | /* | |
740 | * Section 1.8.1 Low Latency Core-Core Wake up of | |
741 | * the VAS workbook: | |
742 | * | |
743 | * - disable credit checks ([tr]x_wcred_mode = false) | |
744 | * - disable FIFO writes | |
745 | * - enable ASB_Notify, disable interrupt | |
746 | */ | |
747 | winctx->fifo_disable = true; | |
748 | winctx->intr_disable = true; | |
749 | winctx->rx_fifo = NULL; | |
750 | } | |
751 | ||
752 | winctx->lnotify_lpid = rxattr->lnotify_lpid; | |
753 | winctx->lnotify_pid = rxattr->lnotify_pid; | |
754 | winctx->lnotify_tid = rxattr->lnotify_tid; | |
755 | winctx->pswid = rxattr->pswid; | |
756 | winctx->dma_type = VAS_DMA_TYPE_INJECT; | |
757 | winctx->tc_mode = rxattr->tc_mode; | |
758 | ||
759 | winctx->min_scope = VAS_SCOPE_LOCAL; | |
760 | winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; | |
761 | } | |
762 | ||
763 | static bool rx_win_args_valid(enum vas_cop_type cop, | |
764 | struct vas_rx_win_attr *attr) | |
765 | { | |
0a2c2c24 SB |
766 | pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n", |
767 | attr->fault_win, attr->notify_disable, | |
768 | attr->intr_disable, attr->notify_early, | |
769 | attr->rx_fifo_size); | |
62c4eda4 SB |
770 | |
771 | if (cop >= VAS_COP_TYPE_MAX) | |
772 | return false; | |
773 | ||
774 | if (cop != VAS_COP_TYPE_FTW && | |
775 | attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN) | |
776 | return false; | |
777 | ||
778 | if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX) | |
779 | return false; | |
780 | ||
51b53712 SB |
781 | if (attr->wcreds_max > VAS_RX_WCREDS_MAX) |
782 | return false; | |
783 | ||
62c4eda4 SB |
784 | if (attr->nx_win) { |
785 | /* cannot be fault or user window if it is nx */ | |
786 | if (attr->fault_win || attr->user_win) | |
787 | return false; | |
788 | /* | |
789 | * Section 3.1.4.32: NX Windows must not disable notification, | |
790 | * and must not enable interrupts or early notification. | |
791 | */ | |
792 | if (attr->notify_disable || !attr->intr_disable || | |
793 | attr->notify_early) | |
794 | return false; | |
795 | } else if (attr->fault_win) { | |
796 | /* cannot be both fault and user window */ | |
797 | if (attr->user_win) | |
798 | return false; | |
799 | ||
800 | /* | |
801 | * Section 3.1.4.32: Fault windows must disable notification | |
802 | * but not interrupts. | |
803 | */ | |
804 | if (!attr->notify_disable || attr->intr_disable) | |
805 | return false; | |
806 | ||
807 | } else if (attr->user_win) { | |
808 | /* | |
809 | * User receive windows are only for fast-thread-wakeup | |
810 | * (FTW). They don't need a FIFO and must disable interrupts | |
811 | */ | |
812 | if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable) | |
813 | return false; | |
814 | } else { | |
815 | /* Rx window must be one of NX or Fault or User window. */ | |
816 | return false; | |
817 | } | |
818 | ||
819 | return true; | |
820 | } | |
821 | ||
822 | void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop) | |
823 | { | |
824 | memset(rxattr, 0, sizeof(*rxattr)); | |
825 | ||
826 | if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { | |
827 | rxattr->pin_win = true; | |
828 | rxattr->nx_win = true; | |
829 | rxattr->fault_win = false; | |
830 | rxattr->intr_disable = true; | |
831 | rxattr->rx_wcred_mode = true; | |
832 | rxattr->tx_wcred_mode = true; | |
833 | rxattr->rx_win_ord_mode = true; | |
834 | rxattr->tx_win_ord_mode = true; | |
835 | } else if (cop == VAS_COP_TYPE_FAULT) { | |
836 | rxattr->pin_win = true; | |
837 | rxattr->fault_win = true; | |
838 | rxattr->notify_disable = true; | |
839 | rxattr->rx_wcred_mode = true; | |
840 | rxattr->tx_wcred_mode = true; | |
841 | rxattr->rx_win_ord_mode = true; | |
842 | rxattr->tx_win_ord_mode = true; | |
843 | } else if (cop == VAS_COP_TYPE_FTW) { | |
844 | rxattr->user_win = true; | |
845 | rxattr->intr_disable = true; | |
846 | ||
847 | /* | |
848 | * As noted in the VAS Workbook we disable credit checks. | |
849 | * If we enable credit checks in the future, we must also | |
850 | * implement a mechanism to return the user credits or new | |
851 | * paste operations will fail. | |
852 | */ | |
853 | } | |
854 | } | |
855 | EXPORT_SYMBOL_GPL(vas_init_rx_win_attr); | |
856 | ||
857 | struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop, | |
858 | struct vas_rx_win_attr *rxattr) | |
859 | { | |
860 | struct vas_window *rxwin; | |
861 | struct vas_winctx winctx; | |
862 | struct vas_instance *vinst; | |
863 | ||
007bb7d6 SB |
864 | trace_vas_rx_win_open(current, vasid, cop, rxattr); |
865 | ||
62c4eda4 SB |
866 | if (!rx_win_args_valid(cop, rxattr)) |
867 | return ERR_PTR(-EINVAL); | |
868 | ||
869 | vinst = find_vas_instance(vasid); | |
870 | if (!vinst) { | |
871 | pr_devel("vasid %d not found!\n", vasid); | |
872 | return ERR_PTR(-EINVAL); | |
873 | } | |
874 | pr_devel("Found instance %d\n", vasid); | |
875 | ||
876 | rxwin = vas_window_alloc(vinst); | |
877 | if (IS_ERR(rxwin)) { | |
878 | pr_devel("Unable to allocate memory for Rx window\n"); | |
879 | return rxwin; | |
880 | } | |
881 | ||
882 | rxwin->tx_win = false; | |
883 | rxwin->nx_win = rxattr->nx_win; | |
884 | rxwin->user_win = rxattr->user_win; | |
885 | rxwin->cop = cop; | |
62f659e0 | 886 | rxwin->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT; |
62c4eda4 SB |
887 | if (rxattr->user_win) |
888 | rxwin->pid = task_pid_vnr(current); | |
889 | ||
890 | init_winctx_for_rxwin(rxwin, rxattr, &winctx); | |
891 | init_winctx_regs(rxwin, &winctx); | |
892 | ||
893 | set_vinst_win(vinst, rxwin); | |
894 | ||
895 | return rxwin; | |
896 | } | |
897 | EXPORT_SYMBOL_GPL(vas_rx_win_open); | |
898 | ||
5239af67 SB |
899 | void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop) |
900 | { | |
901 | memset(txattr, 0, sizeof(*txattr)); | |
902 | ||
903 | if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) { | |
904 | txattr->rej_no_credit = false; | |
905 | txattr->rx_wcred_mode = true; | |
906 | txattr->tx_wcred_mode = true; | |
907 | txattr->rx_win_ord_mode = true; | |
908 | txattr->tx_win_ord_mode = true; | |
909 | } else if (cop == VAS_COP_TYPE_FTW) { | |
910 | txattr->user_win = true; | |
911 | } | |
912 | } | |
913 | EXPORT_SYMBOL_GPL(vas_init_tx_win_attr); | |
914 | ||
915 | static void init_winctx_for_txwin(struct vas_window *txwin, | |
916 | struct vas_tx_win_attr *txattr, | |
917 | struct vas_winctx *winctx) | |
918 | { | |
919 | /* | |
920 | * We first zero all fields and only set non-zero ones. Following | |
921 | * are some fields set to 0/false for the stated reason: | |
922 | * | |
923 | * ->notify_os_intr_reg In powernv, send intrs to HV | |
924 | * ->rsvd_txbuf_count Not supported yet. | |
925 | * ->notify_disable False for NX windows | |
926 | * ->xtra_write False for NX windows | |
927 | * ->notify_early NA for NX windows | |
928 | * ->lnotify_lpid NA for Tx windows | |
929 | * ->lnotify_pid NA for Tx windows | |
930 | * ->lnotify_tid NA for Tx windows | |
931 | * ->tx_win_cred_mode Ignore for now for NX windows | |
932 | * ->rx_win_cred_mode Ignore for now for NX windows | |
933 | */ | |
934 | memset(winctx, 0, sizeof(struct vas_winctx)); | |
935 | ||
62f659e0 | 936 | winctx->wcreds_max = txwin->wcreds_max; |
5239af67 SB |
937 | |
938 | winctx->user_win = txattr->user_win; | |
939 | winctx->nx_win = txwin->rxwin->nx_win; | |
940 | winctx->pin_win = txattr->pin_win; | |
e34917fb SB |
941 | winctx->rej_no_credit = txattr->rej_no_credit; |
942 | winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable; | |
5239af67 SB |
943 | |
944 | winctx->rx_wcred_mode = txattr->rx_wcred_mode; | |
945 | winctx->tx_wcred_mode = txattr->tx_wcred_mode; | |
946 | winctx->rx_word_mode = txattr->rx_win_ord_mode; | |
947 | winctx->tx_word_mode = txattr->tx_win_ord_mode; | |
e34917fb | 948 | winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count; |
5239af67 | 949 | |
6c8e6bb2 SB |
950 | winctx->intr_disable = true; |
951 | if (winctx->nx_win) | |
5239af67 | 952 | winctx->data_stamp = true; |
5239af67 SB |
953 | |
954 | winctx->lpid = txattr->lpid; | |
955 | winctx->pidr = txattr->pidr; | |
956 | winctx->rx_win_id = txwin->rxwin->winid; | |
957 | ||
958 | winctx->dma_type = VAS_DMA_TYPE_INJECT; | |
959 | winctx->tc_mode = txattr->tc_mode; | |
960 | winctx->min_scope = VAS_SCOPE_LOCAL; | |
961 | winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; | |
962 | ||
963 | winctx->pswid = 0; | |
964 | } | |
965 | ||
966 | static bool tx_win_args_valid(enum vas_cop_type cop, | |
967 | struct vas_tx_win_attr *attr) | |
968 | { | |
969 | if (attr->tc_mode != VAS_THRESH_DISABLED) | |
970 | return false; | |
971 | ||
972 | if (cop > VAS_COP_TYPE_MAX) | |
973 | return false; | |
974 | ||
51b53712 SB |
975 | if (attr->wcreds_max > VAS_TX_WCREDS_MAX) |
976 | return false; | |
977 | ||
5239af67 SB |
978 | if (attr->user_win && |
979 | (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count)) | |
980 | return false; | |
981 | ||
982 | return true; | |
983 | } | |
984 | ||
985 | struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop, | |
986 | struct vas_tx_win_attr *attr) | |
987 | { | |
988 | int rc; | |
989 | struct vas_window *txwin; | |
990 | struct vas_window *rxwin; | |
991 | struct vas_winctx winctx; | |
992 | struct vas_instance *vinst; | |
993 | ||
007bb7d6 SB |
994 | trace_vas_tx_win_open(current, vasid, cop, attr); |
995 | ||
5239af67 SB |
996 | if (!tx_win_args_valid(cop, attr)) |
997 | return ERR_PTR(-EINVAL); | |
998 | ||
6c8e6bb2 SB |
999 | /* |
1000 | * If caller did not specify a vasid but specified the PSWID of a | |
1001 | * receive window (applicable only to FTW windows), use the vasid | |
1002 | * from that receive window. | |
1003 | */ | |
1004 | if (vasid == -1 && attr->pswid) | |
1005 | decode_pswid(attr->pswid, &vasid, NULL); | |
1006 | ||
5239af67 SB |
1007 | vinst = find_vas_instance(vasid); |
1008 | if (!vinst) { | |
1009 | pr_devel("vasid %d not found!\n", vasid); | |
1010 | return ERR_PTR(-EINVAL); | |
1011 | } | |
1012 | ||
1013 | rxwin = get_vinst_rxwin(vinst, cop, attr->pswid); | |
1014 | if (IS_ERR(rxwin)) { | |
1015 | pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop); | |
1016 | return rxwin; | |
1017 | } | |
1018 | ||
1019 | txwin = vas_window_alloc(vinst); | |
1020 | if (IS_ERR(txwin)) { | |
1021 | rc = PTR_ERR(txwin); | |
1022 | goto put_rxwin; | |
1023 | } | |
1024 | ||
ece4e512 | 1025 | txwin->cop = cop; |
5239af67 SB |
1026 | txwin->tx_win = 1; |
1027 | txwin->rxwin = rxwin; | |
1028 | txwin->nx_win = txwin->rxwin->nx_win; | |
1029 | txwin->pid = attr->pid; | |
1030 | txwin->user_win = attr->user_win; | |
62f659e0 | 1031 | txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT; |
5239af67 SB |
1032 | |
1033 | init_winctx_for_txwin(txwin, attr, &winctx); | |
1034 | ||
1035 | init_winctx_regs(txwin, &winctx); | |
1036 | ||
1037 | /* | |
1038 | * If its a kernel send window, map the window address into the | |
1039 | * kernel's address space. For user windows, user must issue an | |
1040 | * mmap() to map the window into their address space. | |
1041 | * | |
1042 | * NOTE: If kernel ever resubmits a user CRB after handling a page | |
1043 | * fault, we will need to map this into kernel as well. | |
1044 | */ | |
1045 | if (!txwin->user_win) { | |
1046 | txwin->paste_kaddr = map_paste_region(txwin); | |
1047 | if (IS_ERR(txwin->paste_kaddr)) { | |
1048 | rc = PTR_ERR(txwin->paste_kaddr); | |
1049 | goto free_window; | |
1050 | } | |
b00b6289 NP |
1051 | } else { |
1052 | /* | |
1053 | * A user mapping must ensure that context switch issues | |
1054 | * CP_ABORT for this thread. | |
1055 | */ | |
1056 | rc = set_thread_uses_vas(); | |
1057 | if (rc) | |
1058 | goto free_window; | |
5239af67 SB |
1059 | } |
1060 | ||
1061 | set_vinst_win(vinst, txwin); | |
1062 | ||
1063 | return txwin; | |
1064 | ||
1065 | free_window: | |
1066 | vas_window_free(txwin); | |
1067 | ||
1068 | put_rxwin: | |
1069 | put_rx_win(rxwin); | |
1070 | return ERR_PTR(rc); | |
1071 | ||
1072 | } | |
1073 | EXPORT_SYMBOL_GPL(vas_tx_win_open); | |
1074 | ||
2392c8c8 SB |
1075 | int vas_copy_crb(void *crb, int offset) |
1076 | { | |
1077 | return vas_copy(crb, offset); | |
1078 | } | |
1079 | EXPORT_SYMBOL_GPL(vas_copy_crb); | |
1080 | ||
1081 | #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53) | |
1082 | int vas_paste_crb(struct vas_window *txwin, int offset, bool re) | |
1083 | { | |
1084 | int rc; | |
1085 | void *addr; | |
1086 | uint64_t val; | |
1087 | ||
007bb7d6 SB |
1088 | trace_vas_paste_crb(current, txwin); |
1089 | ||
2392c8c8 SB |
1090 | /* |
1091 | * Only NX windows are supported for now and hardware assumes | |
1092 | * report-enable flag is set for NX windows. Ensure software | |
1093 | * complies too. | |
1094 | */ | |
1095 | WARN_ON_ONCE(txwin->nx_win && !re); | |
1096 | ||
1097 | addr = txwin->paste_kaddr; | |
1098 | if (re) { | |
1099 | /* | |
1100 | * Set the REPORT_ENABLE bit (equivalent to writing | |
1101 | * to 1K offset of the paste address) | |
1102 | */ | |
1103 | val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1); | |
1104 | addr += val; | |
1105 | } | |
1106 | ||
1107 | /* | |
1108 | * Map the raw CR value from vas_paste() to an error code (there | |
1109 | * is just pass or fail for now though). | |
1110 | */ | |
1111 | rc = vas_paste(addr, offset); | |
1112 | if (rc == 2) | |
1113 | rc = 0; | |
1114 | else | |
1115 | rc = -EINVAL; | |
1116 | ||
0a2c2c24 SB |
1117 | pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid, |
1118 | read_hvwc_reg(txwin, VREG(LRFIFO_PUSH))); | |
2392c8c8 SB |
1119 | |
1120 | return rc; | |
1121 | } | |
1122 | EXPORT_SYMBOL_GPL(vas_paste_crb); | |
1123 | ||
6fccac16 SB |
1124 | /* |
1125 | * If credit checking is enabled for this window, poll for the return | |
1126 | * of window credits (i.e for NX engines to process any outstanding CRBs). | |
1127 | * Since NX-842 waits for the CRBs to be processed before closing the | |
1128 | * window, we should not have to wait for too long. | |
1129 | * | |
1130 | * TODO: We retry in 10ms intervals now. We could/should probably peek at | |
1131 | * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending | |
1132 | * CRBs on the FIFO and compute the delay dynamically on each retry. | |
1133 | * But that is not really needed until we support NX-GZIP access from | |
1134 | * user space. (NX-842 driver waits for CSB and Fast thread-wakeup | |
1135 | * doesn't use credit checking). | |
1136 | */ | |
1137 | static void poll_window_credits(struct vas_window *window) | |
1138 | { | |
1139 | u64 val; | |
1140 | int creds, mode; | |
1141 | ||
1142 | val = read_hvwc_reg(window, VREG(WINCTL)); | |
1143 | if (window->tx_win) | |
1144 | mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val); | |
1145 | else | |
1146 | mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val); | |
1147 | ||
1148 | if (!mode) | |
1149 | return; | |
1150 | retry: | |
1151 | if (window->tx_win) { | |
1152 | val = read_hvwc_reg(window, VREG(TX_WCRED)); | |
1153 | creds = GET_FIELD(VAS_TX_WCRED, val); | |
1154 | } else { | |
1155 | val = read_hvwc_reg(window, VREG(LRX_WCRED)); | |
1156 | creds = GET_FIELD(VAS_LRX_WCRED, val); | |
1157 | } | |
1158 | ||
1159 | if (creds < window->wcreds_max) { | |
1160 | val = 0; | |
1161 | set_current_state(TASK_UNINTERRUPTIBLE); | |
1162 | schedule_timeout(msecs_to_jiffies(10)); | |
1163 | goto retry; | |
1164 | } | |
1165 | } | |
1166 | ||
dfe954e4 SB |
1167 | /* |
1168 | * Wait for the window to go to "not-busy" state. It should only take a | |
1169 | * short time to queue a CRB, so window should not be busy for too long. | |
1170 | * Trying 5ms intervals. | |
1171 | */ | |
98271d41 SB |
1172 | static void poll_window_busy_state(struct vas_window *window) |
1173 | { | |
1174 | int busy; | |
1175 | u64 val; | |
1176 | ||
1177 | retry: | |
98271d41 SB |
1178 | val = read_hvwc_reg(window, VREG(WIN_STATUS)); |
1179 | busy = GET_FIELD(VAS_WIN_BUSY, val); | |
1180 | if (busy) { | |
1181 | val = 0; | |
1182 | set_current_state(TASK_UNINTERRUPTIBLE); | |
dfe954e4 | 1183 | schedule_timeout(msecs_to_jiffies(5)); |
98271d41 SB |
1184 | goto retry; |
1185 | } | |
1186 | } | |
1187 | ||
4963ac36 SB |
1188 | /* |
1189 | * Have the hardware cast a window out of cache and wait for it to | |
1190 | * be completed. | |
1191 | * | |
1192 | * NOTE: It can take a relatively long time to cast the window context | |
1193 | * out of the cache. It is not strictly necessary to cast out if: | |
1194 | * | |
1195 | * - we clear the "Pin Window" bit (so hardware is free to evict) | |
1196 | * | |
1197 | * - we re-initialize the window context when it is reassigned. | |
1198 | * | |
1199 | * We do the former in vas_win_close() and latter in vas_win_open(). | |
1200 | * So, ignoring the cast-out for now. We can add it as needed. If | |
1201 | * casting out becomes necessary we should consider offloading the | |
1202 | * job to a worker thread, so the window close can proceed quickly. | |
1203 | */ | |
98271d41 SB |
1204 | static void poll_window_castout(struct vas_window *window) |
1205 | { | |
4963ac36 | 1206 | /* stub for now */ |
98271d41 SB |
1207 | } |
1208 | ||
36a288fe SB |
1209 | /* |
1210 | * Unpin and close a window so no new requests are accepted and the | |
1211 | * hardware can evict this window from cache if necessary. | |
1212 | */ | |
1213 | static void unpin_close_window(struct vas_window *window) | |
1214 | { | |
1215 | u64 val; | |
1216 | ||
1217 | val = read_hvwc_reg(window, VREG(WINCTL)); | |
1218 | val = SET_FIELD(VAS_WINCTL_PIN, val, 0); | |
1219 | val = SET_FIELD(VAS_WINCTL_OPEN, val, 0); | |
1220 | write_hvwc_reg(window, VREG(WINCTL), val); | |
1221 | } | |
1222 | ||
98271d41 SB |
1223 | /* |
1224 | * Close a window. | |
1225 | * | |
1226 | * See Section 1.12.1 of VAS workbook v1.05 for details on closing window: | |
1227 | * - Disable new paste operations (unmap paste address) | |
1228 | * - Poll for the "Window Busy" bit to be cleared | |
1229 | * - Clear the Open/Enable bit for the Window. | |
1230 | * - Poll for return of window Credits (implies FIFO empty for Rx win?) | |
1231 | * - Unpin and cast window context out of cache | |
1232 | * | |
1233 | * Besides the hardware, kernel has some bookkeeping of course. | |
1234 | */ | |
4dea2d1a SB |
1235 | int vas_win_close(struct vas_window *window) |
1236 | { | |
98271d41 SB |
1237 | if (!window) |
1238 | return 0; | |
1239 | ||
1240 | if (!window->tx_win && atomic_read(&window->num_txwins) != 0) { | |
1241 | pr_devel("Attempting to close an active Rx window!\n"); | |
1242 | WARN_ON_ONCE(1); | |
1243 | return -EBUSY; | |
1244 | } | |
1245 | ||
1246 | unmap_paste_region(window); | |
1247 | ||
1248 | clear_vinst_win(window); | |
1249 | ||
1250 | poll_window_busy_state(window); | |
1251 | ||
36a288fe | 1252 | unpin_close_window(window); |
98271d41 | 1253 | |
6fccac16 SB |
1254 | poll_window_credits(window); |
1255 | ||
98271d41 SB |
1256 | poll_window_castout(window); |
1257 | ||
1258 | /* if send window, drop reference to matching receive window */ | |
1259 | if (window->tx_win) | |
1260 | put_rx_win(window->rxwin); | |
1261 | ||
1262 | vas_window_free(window); | |
1263 | ||
1264 | return 0; | |
4dea2d1a | 1265 | } |
98271d41 | 1266 | EXPORT_SYMBOL_GPL(vas_win_close); |
61f3cca8 SB |
1267 | |
1268 | /* | |
1269 | * Return a system-wide unique window id for the window @win. | |
1270 | */ | |
1271 | u32 vas_win_id(struct vas_window *win) | |
1272 | { | |
1273 | return encode_pswid(win->vinst->vas_id, win->winid); | |
1274 | } | |
1275 | EXPORT_SYMBOL_GPL(vas_win_id); |