2 * Copyright 2016-17 IBM Corp.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
10 #define pr_fmt(fmt) "vas: " fmt
12 #include <linux/types.h>
13 #include <linux/mutex.h>
14 #include <linux/slab.h>
16 #include <linux/log2.h>
17 #include <linux/rcupdate.h>
18 #include <linux/cred.h>
21 #include "copy-paste.h"
24 * Compute the paste address region for the window @window using the
25 * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
27 static void compute_paste_address(struct vas_window
*window
, u64
*addr
, int *len
)
32 base
= window
->vinst
->paste_base_addr
;
33 shift
= window
->vinst
->paste_win_id_shift
;
34 winid
= window
->winid
;
36 *addr
= base
+ (winid
<< shift
);
40 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid
, *addr
);
43 static inline void get_hvwc_mmio_bar(struct vas_window
*window
,
48 pbaddr
= window
->vinst
->hvwc_bar_start
;
49 *start
= pbaddr
+ window
->winid
* VAS_HVWC_SIZE
;
53 static inline void get_uwc_mmio_bar(struct vas_window
*window
,
58 pbaddr
= window
->vinst
->uwc_bar_start
;
59 *start
= pbaddr
+ window
->winid
* VAS_UWC_SIZE
;
64 * Map the paste bus address of the given send window into kernel address
65 * space. Unlike MMIO regions (map_mmio_region() below), paste region must
66 * be mapped cache-able and is only applicable to send windows.
68 static void *map_paste_region(struct vas_window
*txwin
)
75 name
= kasprintf(GFP_KERNEL
, "window-v%d-w%d", txwin
->vinst
->vas_id
,
80 txwin
->paste_addr_name
= name
;
81 compute_paste_address(txwin
, &start
, &len
);
83 if (!request_mem_region(start
, len
, name
)) {
84 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
85 __func__
, start
, len
);
89 map
= ioremap_cache(start
, len
);
91 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__
,
96 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start
, map
);
101 return ERR_PTR(-ENOMEM
);
104 static void *map_mmio_region(char *name
, u64 start
, int len
)
108 if (!request_mem_region(start
, len
, name
)) {
109 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
110 __func__
, start
, len
);
114 map
= ioremap(start
, len
);
116 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__
, start
,
124 static void unmap_region(void *addr
, u64 start
, int len
)
127 release_mem_region((phys_addr_t
)start
, len
);
131 * Unmap the paste address region for a window.
133 static void unmap_paste_region(struct vas_window
*window
)
138 if (window
->paste_kaddr
) {
139 compute_paste_address(window
, &busaddr_start
, &len
);
140 unmap_region(window
->paste_kaddr
, busaddr_start
, len
);
141 window
->paste_kaddr
= NULL
;
142 kfree(window
->paste_addr_name
);
143 window
->paste_addr_name
= NULL
;
148 * Unmap the MMIO regions for a window.
150 static void unmap_winctx_mmio_bars(struct vas_window
*window
)
155 if (window
->hvwc_map
) {
156 get_hvwc_mmio_bar(window
, &busaddr_start
, &len
);
157 unmap_region(window
->hvwc_map
, busaddr_start
, len
);
158 window
->hvwc_map
= NULL
;
161 if (window
->uwc_map
) {
162 get_uwc_mmio_bar(window
, &busaddr_start
, &len
);
163 unmap_region(window
->uwc_map
, busaddr_start
, len
);
164 window
->uwc_map
= NULL
;
169 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
170 * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
171 * Map these bus addresses and save the mapped kernel addresses in @window.
173 int map_winctx_mmio_bars(struct vas_window
*window
)
178 get_hvwc_mmio_bar(window
, &start
, &len
);
179 window
->hvwc_map
= map_mmio_region("HVWCM_Window", start
, len
);
181 get_uwc_mmio_bar(window
, &start
, &len
);
182 window
->uwc_map
= map_mmio_region("UWCM_Window", start
, len
);
184 if (!window
->hvwc_map
|| !window
->uwc_map
) {
185 unmap_winctx_mmio_bars(window
);
193 * Reset all valid registers in the HV and OS/User Window Contexts for
194 * the window identified by @window.
196 * NOTE: We cannot really use a for loop to reset window context. Not all
197 * offsets in a window context are valid registers and the valid
198 * registers are not sequential. And, we can only write to offsets
199 * with valid registers.
201 void reset_window_regs(struct vas_window
*window
)
203 write_hvwc_reg(window
, VREG(LPID
), 0ULL);
204 write_hvwc_reg(window
, VREG(PID
), 0ULL);
205 write_hvwc_reg(window
, VREG(XLATE_MSR
), 0ULL);
206 write_hvwc_reg(window
, VREG(XLATE_LPCR
), 0ULL);
207 write_hvwc_reg(window
, VREG(XLATE_CTL
), 0ULL);
208 write_hvwc_reg(window
, VREG(AMR
), 0ULL);
209 write_hvwc_reg(window
, VREG(SEIDR
), 0ULL);
210 write_hvwc_reg(window
, VREG(FAULT_TX_WIN
), 0ULL);
211 write_hvwc_reg(window
, VREG(OSU_INTR_SRC_RA
), 0ULL);
212 write_hvwc_reg(window
, VREG(HV_INTR_SRC_RA
), 0ULL);
213 write_hvwc_reg(window
, VREG(PSWID
), 0ULL);
214 write_hvwc_reg(window
, VREG(LFIFO_BAR
), 0ULL);
215 write_hvwc_reg(window
, VREG(LDATA_STAMP_CTL
), 0ULL);
216 write_hvwc_reg(window
, VREG(LDMA_CACHE_CTL
), 0ULL);
217 write_hvwc_reg(window
, VREG(LRFIFO_PUSH
), 0ULL);
218 write_hvwc_reg(window
, VREG(CURR_MSG_COUNT
), 0ULL);
219 write_hvwc_reg(window
, VREG(LNOTIFY_AFTER_COUNT
), 0ULL);
220 write_hvwc_reg(window
, VREG(LRX_WCRED
), 0ULL);
221 write_hvwc_reg(window
, VREG(LRX_WCRED_ADDER
), 0ULL);
222 write_hvwc_reg(window
, VREG(TX_WCRED
), 0ULL);
223 write_hvwc_reg(window
, VREG(TX_WCRED_ADDER
), 0ULL);
224 write_hvwc_reg(window
, VREG(LFIFO_SIZE
), 0ULL);
225 write_hvwc_reg(window
, VREG(WINCTL
), 0ULL);
226 write_hvwc_reg(window
, VREG(WIN_STATUS
), 0ULL);
227 write_hvwc_reg(window
, VREG(WIN_CTX_CACHING_CTL
), 0ULL);
228 write_hvwc_reg(window
, VREG(TX_RSVD_BUF_COUNT
), 0ULL);
229 write_hvwc_reg(window
, VREG(LRFIFO_WIN_PTR
), 0ULL);
230 write_hvwc_reg(window
, VREG(LNOTIFY_CTL
), 0ULL);
231 write_hvwc_reg(window
, VREG(LNOTIFY_PID
), 0ULL);
232 write_hvwc_reg(window
, VREG(LNOTIFY_LPID
), 0ULL);
233 write_hvwc_reg(window
, VREG(LNOTIFY_TID
), 0ULL);
234 write_hvwc_reg(window
, VREG(LNOTIFY_SCOPE
), 0ULL);
235 write_hvwc_reg(window
, VREG(NX_UTIL_ADDER
), 0ULL);
237 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
240 * The send and receive window credit adder registers are also
241 * accessible from HVWC and have been initialized above. We don't
242 * need to initialize from the OS/User Window Context, so skip
245 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
246 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
251 * Initialize window context registers related to Address Translation.
252 * These registers are common to send/receive windows although they
253 * differ for user/kernel windows. As we resolve the TODOs we may
254 * want to add fields to vas_winctx and move the initialization to
255 * init_vas_winctx_regs().
257 static void init_xlate_regs(struct vas_window
*window
, bool user_win
)
262 * MSR_TA, MSR_US are false for both kernel and user.
263 * MSR_DR and MSR_PR are false for kernel.
266 val
= SET_FIELD(VAS_XLATE_MSR_HV
, val
, 1);
267 val
= SET_FIELD(VAS_XLATE_MSR_SF
, val
, 1);
269 val
= SET_FIELD(VAS_XLATE_MSR_DR
, val
, 1);
270 val
= SET_FIELD(VAS_XLATE_MSR_PR
, val
, 1);
272 write_hvwc_reg(window
, VREG(XLATE_MSR
), val
);
274 lpcr
= mfspr(SPRN_LPCR
);
277 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
278 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
280 * NOTE: From Section 1.3.1, Address Translation Context of the
281 * Nest MMU Workbook, LPCR_SC should be 0 for Power9.
283 val
= SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE
, val
, 5);
284 val
= SET_FIELD(VAS_XLATE_LPCR_ISL
, val
, lpcr
& LPCR_ISL
);
285 val
= SET_FIELD(VAS_XLATE_LPCR_TC
, val
, lpcr
& LPCR_TC
);
286 val
= SET_FIELD(VAS_XLATE_LPCR_SC
, val
, 0);
287 write_hvwc_reg(window
, VREG(XLATE_LPCR
), val
);
290 * Section 1.3.1 (Address translation Context) of NMMU workbook.
291 * 0b00 Hashed Page Table mode
294 * 0b11 Radix on Radix
297 val
= SET_FIELD(VAS_XLATE_MODE
, val
, radix_enabled() ? 3 : 2);
298 write_hvwc_reg(window
, VREG(XLATE_CTL
), val
);
301 * TODO: Can we mfspr(AMR) even for user windows?
304 val
= SET_FIELD(VAS_AMR
, val
, mfspr(SPRN_AMR
));
305 write_hvwc_reg(window
, VREG(AMR
), val
);
308 val
= SET_FIELD(VAS_SEIDR
, val
, 0);
309 write_hvwc_reg(window
, VREG(SEIDR
), val
);
313 * Initialize Reserved Send Buffer Count for the send window. It involves
314 * writing to the register, reading it back to confirm that the hardware
315 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
317 * Since we can only make a best-effort attempt to fulfill the request,
318 * we don't return any errors if we cannot.
320 * TODO: Reserved (aka dedicated) send buffers are not supported yet.
322 static void init_rsvd_tx_buf_count(struct vas_window
*txwin
,
323 struct vas_winctx
*winctx
)
325 write_hvwc_reg(txwin
, VREG(TX_RSVD_BUF_COUNT
), 0ULL);
330 * Initialize window context registers for a receive window.
331 * Except for caching control and marking window open, the registers
332 * are initialized in the order listed in Section 3.1.4 (Window Context
333 * Cache Register Details) of the VAS workbook although they don't need
336 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
337 * (so that it can get a large contiguous area) and passes that buffer
338 * to kernel via device tree. We now write that buffer address to the
339 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
340 * write the per-chip RX FIFO addresses to the windows during boot-up
341 * as a one-time task? That could work for NX but what about other
342 * receivers? Let the receivers tell us the rx-fifo buffers for now.
344 int init_winctx_regs(struct vas_window
*window
, struct vas_winctx
*winctx
)
349 reset_window_regs(window
);
352 val
= SET_FIELD(VAS_LPID
, val
, winctx
->lpid
);
353 write_hvwc_reg(window
, VREG(LPID
), val
);
356 val
= SET_FIELD(VAS_PID_ID
, val
, winctx
->pidr
);
357 write_hvwc_reg(window
, VREG(PID
), val
);
359 init_xlate_regs(window
, winctx
->user_win
);
362 val
= SET_FIELD(VAS_FAULT_TX_WIN
, val
, 0);
363 write_hvwc_reg(window
, VREG(FAULT_TX_WIN
), val
);
365 /* In PowerNV, interrupts go to HV. */
366 write_hvwc_reg(window
, VREG(OSU_INTR_SRC_RA
), 0ULL);
369 val
= SET_FIELD(VAS_HV_INTR_SRC_RA
, val
, winctx
->irq_port
);
370 write_hvwc_reg(window
, VREG(HV_INTR_SRC_RA
), val
);
373 val
= SET_FIELD(VAS_PSWID_EA_HANDLE
, val
, winctx
->pswid
);
374 write_hvwc_reg(window
, VREG(PSWID
), val
);
376 write_hvwc_reg(window
, VREG(SPARE1
), 0ULL);
377 write_hvwc_reg(window
, VREG(SPARE2
), 0ULL);
378 write_hvwc_reg(window
, VREG(SPARE3
), 0ULL);
381 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
382 * register as is - do NOT shift the address into VAS_LFIFO_BAR
383 * bit fields! Ok to set the page migration select fields -
384 * VAS ignores the lower 10+ bits in the address anyway, because
385 * the minimum FIFO size is 1K?
387 * See also: Design note in function header.
389 val
= __pa(winctx
->rx_fifo
);
390 val
= SET_FIELD(VAS_PAGE_MIGRATION_SELECT
, val
, 0);
391 write_hvwc_reg(window
, VREG(LFIFO_BAR
), val
);
394 val
= SET_FIELD(VAS_LDATA_STAMP
, val
, winctx
->data_stamp
);
395 write_hvwc_reg(window
, VREG(LDATA_STAMP_CTL
), val
);
398 val
= SET_FIELD(VAS_LDMA_TYPE
, val
, winctx
->dma_type
);
399 val
= SET_FIELD(VAS_LDMA_FIFO_DISABLE
, val
, winctx
->fifo_disable
);
400 write_hvwc_reg(window
, VREG(LDMA_CACHE_CTL
), val
);
402 write_hvwc_reg(window
, VREG(LRFIFO_PUSH
), 0ULL);
403 write_hvwc_reg(window
, VREG(CURR_MSG_COUNT
), 0ULL);
404 write_hvwc_reg(window
, VREG(LNOTIFY_AFTER_COUNT
), 0ULL);
407 val
= SET_FIELD(VAS_LRX_WCRED
, val
, winctx
->wcreds_max
);
408 write_hvwc_reg(window
, VREG(LRX_WCRED
), val
);
411 val
= SET_FIELD(VAS_TX_WCRED
, val
, winctx
->wcreds_max
);
412 write_hvwc_reg(window
, VREG(TX_WCRED
), val
);
414 write_hvwc_reg(window
, VREG(LRX_WCRED_ADDER
), 0ULL);
415 write_hvwc_reg(window
, VREG(TX_WCRED_ADDER
), 0ULL);
417 fifo_size
= winctx
->rx_fifo_size
/ 1024;
420 val
= SET_FIELD(VAS_LFIFO_SIZE
, val
, ilog2(fifo_size
));
421 write_hvwc_reg(window
, VREG(LFIFO_SIZE
), val
);
423 /* Update window control and caching control registers last so
424 * we mark the window open only after fully initializing it and
425 * pushing context to cache.
428 write_hvwc_reg(window
, VREG(WIN_STATUS
), 0ULL);
430 init_rsvd_tx_buf_count(window
, winctx
);
432 /* for a send window, point to the matching receive window */
434 val
= SET_FIELD(VAS_LRX_WIN_ID
, val
, winctx
->rx_win_id
);
435 write_hvwc_reg(window
, VREG(LRFIFO_WIN_PTR
), val
);
437 write_hvwc_reg(window
, VREG(SPARE4
), 0ULL);
440 val
= SET_FIELD(VAS_NOTIFY_DISABLE
, val
, winctx
->notify_disable
);
441 val
= SET_FIELD(VAS_INTR_DISABLE
, val
, winctx
->intr_disable
);
442 val
= SET_FIELD(VAS_NOTIFY_EARLY
, val
, winctx
->notify_early
);
443 val
= SET_FIELD(VAS_NOTIFY_OSU_INTR
, val
, winctx
->notify_os_intr_reg
);
444 write_hvwc_reg(window
, VREG(LNOTIFY_CTL
), val
);
447 val
= SET_FIELD(VAS_LNOTIFY_PID
, val
, winctx
->lnotify_pid
);
448 write_hvwc_reg(window
, VREG(LNOTIFY_PID
), val
);
451 val
= SET_FIELD(VAS_LNOTIFY_LPID
, val
, winctx
->lnotify_lpid
);
452 write_hvwc_reg(window
, VREG(LNOTIFY_LPID
), val
);
455 val
= SET_FIELD(VAS_LNOTIFY_TID
, val
, winctx
->lnotify_tid
);
456 write_hvwc_reg(window
, VREG(LNOTIFY_TID
), val
);
459 val
= SET_FIELD(VAS_LNOTIFY_MIN_SCOPE
, val
, winctx
->min_scope
);
460 val
= SET_FIELD(VAS_LNOTIFY_MAX_SCOPE
, val
, winctx
->max_scope
);
461 write_hvwc_reg(window
, VREG(LNOTIFY_SCOPE
), val
);
463 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */
465 write_hvwc_reg(window
, VREG(SPARE5
), 0ULL);
466 write_hvwc_reg(window
, VREG(NX_UTIL_ADDER
), 0ULL);
467 write_hvwc_reg(window
, VREG(SPARE6
), 0ULL);
469 /* Finally, push window context to memory and... */
471 val
= SET_FIELD(VAS_PUSH_TO_MEM
, val
, 1);
472 write_hvwc_reg(window
, VREG(WIN_CTX_CACHING_CTL
), val
);
474 /* ... mark the window open for business */
476 val
= SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT
, val
, winctx
->rej_no_credit
);
477 val
= SET_FIELD(VAS_WINCTL_PIN
, val
, winctx
->pin_win
);
478 val
= SET_FIELD(VAS_WINCTL_TX_WCRED_MODE
, val
, winctx
->tx_wcred_mode
);
479 val
= SET_FIELD(VAS_WINCTL_RX_WCRED_MODE
, val
, winctx
->rx_wcred_mode
);
480 val
= SET_FIELD(VAS_WINCTL_TX_WORD_MODE
, val
, winctx
->tx_word_mode
);
481 val
= SET_FIELD(VAS_WINCTL_RX_WORD_MODE
, val
, winctx
->rx_word_mode
);
482 val
= SET_FIELD(VAS_WINCTL_FAULT_WIN
, val
, winctx
->fault_win
);
483 val
= SET_FIELD(VAS_WINCTL_NX_WIN
, val
, winctx
->nx_win
);
484 val
= SET_FIELD(VAS_WINCTL_OPEN
, val
, 1);
485 write_hvwc_reg(window
, VREG(WINCTL
), val
);
490 static DEFINE_SPINLOCK(vas_ida_lock
);
492 static void vas_release_window_id(struct ida
*ida
, int winid
)
494 spin_lock(&vas_ida_lock
);
495 ida_remove(ida
, winid
);
496 spin_unlock(&vas_ida_lock
);
499 static int vas_assign_window_id(struct ida
*ida
)
504 rc
= ida_pre_get(ida
, GFP_KERNEL
);
508 spin_lock(&vas_ida_lock
);
509 rc
= ida_get_new(ida
, &winid
);
510 spin_unlock(&vas_ida_lock
);
511 } while (rc
== -EAGAIN
);
516 if (winid
> VAS_WINDOWS_PER_CHIP
) {
517 pr_err("Too many (%d) open windows\n", winid
);
518 vas_release_window_id(ida
, winid
);
525 static void vas_window_free(struct vas_window
*window
)
527 int winid
= window
->winid
;
528 struct vas_instance
*vinst
= window
->vinst
;
530 unmap_winctx_mmio_bars(window
);
533 vas_release_window_id(&vinst
->ida
, winid
);
536 static struct vas_window
*vas_window_alloc(struct vas_instance
*vinst
)
539 struct vas_window
*window
;
541 winid
= vas_assign_window_id(&vinst
->ida
);
543 return ERR_PTR(winid
);
545 window
= kzalloc(sizeof(*window
), GFP_KERNEL
);
549 window
->vinst
= vinst
;
550 window
->winid
= winid
;
552 if (map_winctx_mmio_bars(window
))
559 vas_release_window_id(&vinst
->ida
, winid
);
560 return ERR_PTR(-ENOMEM
);
563 static void put_rx_win(struct vas_window
*rxwin
)
565 /* Better not be a send window! */
566 WARN_ON_ONCE(rxwin
->tx_win
);
568 atomic_dec(&rxwin
->num_txwins
);
572 * Get the VAS receive window associated with NX engine identified
573 * by @cop and if applicable, @pswid.
575 * See also function header of set_vinst_win().
577 static struct vas_window
*get_vinst_rxwin(struct vas_instance
*vinst
,
578 enum vas_cop_type cop
, u32 pswid
)
580 struct vas_window
*rxwin
;
582 mutex_lock(&vinst
->mutex
);
584 if (cop
== VAS_COP_TYPE_842
|| cop
== VAS_COP_TYPE_842_HIPRI
)
585 rxwin
= vinst
->rxwin
[cop
] ?: ERR_PTR(-EINVAL
);
587 rxwin
= ERR_PTR(-EINVAL
);
590 atomic_inc(&rxwin
->num_txwins
);
592 mutex_unlock(&vinst
->mutex
);
598 * We have two tables of windows in a VAS instance. The first one,
599 * ->windows[], contains all the windows in the instance and allows
600 * looking up a window by its id. It is used to look up send windows
601 * during fault handling and receive windows when pairing user space
602 * send/receive windows.
604 * The second table, ->rxwin[], contains receive windows that are
605 * associated with NX engines. This table has VAS_COP_TYPE_MAX
606 * entries and is used to look up a receive window by its
609 * Here, we save @window in the ->windows[] table. If it is a receive
610 * window, we also save the window in the ->rxwin[] table.
612 static void set_vinst_win(struct vas_instance
*vinst
,
613 struct vas_window
*window
)
615 int id
= window
->winid
;
617 mutex_lock(&vinst
->mutex
);
620 * There should only be one receive window for a coprocessor type
621 * unless its a user (FTW) window.
623 if (!window
->user_win
&& !window
->tx_win
) {
624 WARN_ON_ONCE(vinst
->rxwin
[window
->cop
]);
625 vinst
->rxwin
[window
->cop
] = window
;
628 WARN_ON_ONCE(vinst
->windows
[id
] != NULL
);
629 vinst
->windows
[id
] = window
;
631 mutex_unlock(&vinst
->mutex
);
635 * Clear this window from the table(s) of windows for this VAS instance.
636 * See also function header of set_vinst_win().
638 static void clear_vinst_win(struct vas_window
*window
)
640 int id
= window
->winid
;
641 struct vas_instance
*vinst
= window
->vinst
;
643 mutex_lock(&vinst
->mutex
);
645 if (!window
->user_win
&& !window
->tx_win
) {
646 WARN_ON_ONCE(!vinst
->rxwin
[window
->cop
]);
647 vinst
->rxwin
[window
->cop
] = NULL
;
650 WARN_ON_ONCE(vinst
->windows
[id
] != window
);
651 vinst
->windows
[id
] = NULL
;
653 mutex_unlock(&vinst
->mutex
);
656 static void init_winctx_for_rxwin(struct vas_window
*rxwin
,
657 struct vas_rx_win_attr
*rxattr
,
658 struct vas_winctx
*winctx
)
661 * We first zero (memset()) all fields and only set non-zero fields.
662 * Following fields are 0/false but maybe deserve a comment:
664 * ->notify_os_intr_reg In powerNV, send intrs to HV
665 * ->notify_disable False for NX windows
666 * ->intr_disable False for Fault Windows
667 * ->xtra_write False for NX windows
668 * ->notify_early NA for NX windows
669 * ->rsvd_txbuf_count NA for Rx windows
670 * ->lpid, ->pid, ->tid NA for Rx windows
673 memset(winctx
, 0, sizeof(struct vas_winctx
));
675 winctx
->rx_fifo
= rxattr
->rx_fifo
;
676 winctx
->rx_fifo_size
= rxattr
->rx_fifo_size
;
677 winctx
->wcreds_max
= rxattr
->wcreds_max
?: VAS_WCREDS_DEFAULT
;
678 winctx
->pin_win
= rxattr
->pin_win
;
680 winctx
->nx_win
= rxattr
->nx_win
;
681 winctx
->fault_win
= rxattr
->fault_win
;
682 winctx
->rx_word_mode
= rxattr
->rx_win_ord_mode
;
683 winctx
->tx_word_mode
= rxattr
->tx_win_ord_mode
;
684 winctx
->rx_wcred_mode
= rxattr
->rx_wcred_mode
;
685 winctx
->tx_wcred_mode
= rxattr
->tx_wcred_mode
;
687 if (winctx
->nx_win
) {
688 winctx
->data_stamp
= true;
689 winctx
->intr_disable
= true;
690 winctx
->pin_win
= true;
692 WARN_ON_ONCE(winctx
->fault_win
);
693 WARN_ON_ONCE(!winctx
->rx_word_mode
);
694 WARN_ON_ONCE(!winctx
->tx_word_mode
);
695 WARN_ON_ONCE(winctx
->notify_after_count
);
696 } else if (winctx
->fault_win
) {
697 winctx
->notify_disable
= true;
698 } else if (winctx
->user_win
) {
700 * Section 1.8.1 Low Latency Core-Core Wake up of
703 * - disable credit checks ([tr]x_wcred_mode = false)
704 * - disable FIFO writes
705 * - enable ASB_Notify, disable interrupt
707 winctx
->fifo_disable
= true;
708 winctx
->intr_disable
= true;
709 winctx
->rx_fifo
= NULL
;
712 winctx
->lnotify_lpid
= rxattr
->lnotify_lpid
;
713 winctx
->lnotify_pid
= rxattr
->lnotify_pid
;
714 winctx
->lnotify_tid
= rxattr
->lnotify_tid
;
715 winctx
->pswid
= rxattr
->pswid
;
716 winctx
->dma_type
= VAS_DMA_TYPE_INJECT
;
717 winctx
->tc_mode
= rxattr
->tc_mode
;
719 winctx
->min_scope
= VAS_SCOPE_LOCAL
;
720 winctx
->max_scope
= VAS_SCOPE_VECTORED_GROUP
;
723 static bool rx_win_args_valid(enum vas_cop_type cop
,
724 struct vas_rx_win_attr
*attr
)
726 dump_rx_win_attr(attr
);
728 if (cop
>= VAS_COP_TYPE_MAX
)
731 if (cop
!= VAS_COP_TYPE_FTW
&&
732 attr
->rx_fifo_size
< VAS_RX_FIFO_SIZE_MIN
)
735 if (attr
->rx_fifo_size
> VAS_RX_FIFO_SIZE_MAX
)
739 /* cannot be fault or user window if it is nx */
740 if (attr
->fault_win
|| attr
->user_win
)
743 * Section 3.1.4.32: NX Windows must not disable notification,
744 * and must not enable interrupts or early notification.
746 if (attr
->notify_disable
|| !attr
->intr_disable
||
749 } else if (attr
->fault_win
) {
750 /* cannot be both fault and user window */
755 * Section 3.1.4.32: Fault windows must disable notification
756 * but not interrupts.
758 if (!attr
->notify_disable
|| attr
->intr_disable
)
761 } else if (attr
->user_win
) {
763 * User receive windows are only for fast-thread-wakeup
764 * (FTW). They don't need a FIFO and must disable interrupts
766 if (attr
->rx_fifo
|| attr
->rx_fifo_size
|| !attr
->intr_disable
)
769 /* Rx window must be one of NX or Fault or User window. */
776 void vas_init_rx_win_attr(struct vas_rx_win_attr
*rxattr
, enum vas_cop_type cop
)
778 memset(rxattr
, 0, sizeof(*rxattr
));
780 if (cop
== VAS_COP_TYPE_842
|| cop
== VAS_COP_TYPE_842_HIPRI
) {
781 rxattr
->pin_win
= true;
782 rxattr
->nx_win
= true;
783 rxattr
->fault_win
= false;
784 rxattr
->intr_disable
= true;
785 rxattr
->rx_wcred_mode
= true;
786 rxattr
->tx_wcred_mode
= true;
787 rxattr
->rx_win_ord_mode
= true;
788 rxattr
->tx_win_ord_mode
= true;
789 } else if (cop
== VAS_COP_TYPE_FAULT
) {
790 rxattr
->pin_win
= true;
791 rxattr
->fault_win
= true;
792 rxattr
->notify_disable
= true;
793 rxattr
->rx_wcred_mode
= true;
794 rxattr
->tx_wcred_mode
= true;
795 rxattr
->rx_win_ord_mode
= true;
796 rxattr
->tx_win_ord_mode
= true;
797 } else if (cop
== VAS_COP_TYPE_FTW
) {
798 rxattr
->user_win
= true;
799 rxattr
->intr_disable
= true;
802 * As noted in the VAS Workbook we disable credit checks.
803 * If we enable credit checks in the future, we must also
804 * implement a mechanism to return the user credits or new
805 * paste operations will fail.
809 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr
);
811 struct vas_window
*vas_rx_win_open(int vasid
, enum vas_cop_type cop
,
812 struct vas_rx_win_attr
*rxattr
)
814 struct vas_window
*rxwin
;
815 struct vas_winctx winctx
;
816 struct vas_instance
*vinst
;
818 if (!rx_win_args_valid(cop
, rxattr
))
819 return ERR_PTR(-EINVAL
);
821 vinst
= find_vas_instance(vasid
);
823 pr_devel("vasid %d not found!\n", vasid
);
824 return ERR_PTR(-EINVAL
);
826 pr_devel("Found instance %d\n", vasid
);
828 rxwin
= vas_window_alloc(vinst
);
830 pr_devel("Unable to allocate memory for Rx window\n");
834 rxwin
->tx_win
= false;
835 rxwin
->nx_win
= rxattr
->nx_win
;
836 rxwin
->user_win
= rxattr
->user_win
;
838 if (rxattr
->user_win
)
839 rxwin
->pid
= task_pid_vnr(current
);
841 init_winctx_for_rxwin(rxwin
, rxattr
, &winctx
);
842 init_winctx_regs(rxwin
, &winctx
);
844 set_vinst_win(vinst
, rxwin
);
848 EXPORT_SYMBOL_GPL(vas_rx_win_open
);
850 void vas_init_tx_win_attr(struct vas_tx_win_attr
*txattr
, enum vas_cop_type cop
)
852 memset(txattr
, 0, sizeof(*txattr
));
854 if (cop
== VAS_COP_TYPE_842
|| cop
== VAS_COP_TYPE_842_HIPRI
) {
855 txattr
->rej_no_credit
= false;
856 txattr
->rx_wcred_mode
= true;
857 txattr
->tx_wcred_mode
= true;
858 txattr
->rx_win_ord_mode
= true;
859 txattr
->tx_win_ord_mode
= true;
860 } else if (cop
== VAS_COP_TYPE_FTW
) {
861 txattr
->user_win
= true;
864 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr
);
866 static void init_winctx_for_txwin(struct vas_window
*txwin
,
867 struct vas_tx_win_attr
*txattr
,
868 struct vas_winctx
*winctx
)
871 * We first zero all fields and only set non-zero ones. Following
872 * are some fields set to 0/false for the stated reason:
874 * ->notify_os_intr_reg In powernv, send intrs to HV
875 * ->rsvd_txbuf_count Not supported yet.
876 * ->notify_disable False for NX windows
877 * ->xtra_write False for NX windows
878 * ->notify_early NA for NX windows
879 * ->lnotify_lpid NA for Tx windows
880 * ->lnotify_pid NA for Tx windows
881 * ->lnotify_tid NA for Tx windows
882 * ->tx_win_cred_mode Ignore for now for NX windows
883 * ->rx_win_cred_mode Ignore for now for NX windows
885 memset(winctx
, 0, sizeof(struct vas_winctx
));
887 winctx
->wcreds_max
= txattr
->wcreds_max
?: VAS_WCREDS_DEFAULT
;
889 winctx
->user_win
= txattr
->user_win
;
890 winctx
->nx_win
= txwin
->rxwin
->nx_win
;
891 winctx
->pin_win
= txattr
->pin_win
;
893 winctx
->rx_wcred_mode
= txattr
->rx_wcred_mode
;
894 winctx
->tx_wcred_mode
= txattr
->tx_wcred_mode
;
895 winctx
->rx_word_mode
= txattr
->rx_win_ord_mode
;
896 winctx
->tx_word_mode
= txattr
->tx_win_ord_mode
;
898 if (winctx
->nx_win
) {
899 winctx
->data_stamp
= true;
900 winctx
->intr_disable
= true;
903 winctx
->lpid
= txattr
->lpid
;
904 winctx
->pidr
= txattr
->pidr
;
905 winctx
->rx_win_id
= txwin
->rxwin
->winid
;
907 winctx
->dma_type
= VAS_DMA_TYPE_INJECT
;
908 winctx
->tc_mode
= txattr
->tc_mode
;
909 winctx
->min_scope
= VAS_SCOPE_LOCAL
;
910 winctx
->max_scope
= VAS_SCOPE_VECTORED_GROUP
;
915 static bool tx_win_args_valid(enum vas_cop_type cop
,
916 struct vas_tx_win_attr
*attr
)
918 if (attr
->tc_mode
!= VAS_THRESH_DISABLED
)
921 if (cop
> VAS_COP_TYPE_MAX
)
924 if (attr
->user_win
&&
925 (cop
!= VAS_COP_TYPE_FTW
|| attr
->rsvd_txbuf_count
))
931 struct vas_window
*vas_tx_win_open(int vasid
, enum vas_cop_type cop
,
932 struct vas_tx_win_attr
*attr
)
935 struct vas_window
*txwin
;
936 struct vas_window
*rxwin
;
937 struct vas_winctx winctx
;
938 struct vas_instance
*vinst
;
940 if (!tx_win_args_valid(cop
, attr
))
941 return ERR_PTR(-EINVAL
);
943 vinst
= find_vas_instance(vasid
);
945 pr_devel("vasid %d not found!\n", vasid
);
946 return ERR_PTR(-EINVAL
);
949 rxwin
= get_vinst_rxwin(vinst
, cop
, attr
->pswid
);
951 pr_devel("No RxWin for vasid %d, cop %d\n", vasid
, cop
);
955 txwin
= vas_window_alloc(vinst
);
962 txwin
->rxwin
= rxwin
;
963 txwin
->nx_win
= txwin
->rxwin
->nx_win
;
964 txwin
->pid
= attr
->pid
;
965 txwin
->user_win
= attr
->user_win
;
967 init_winctx_for_txwin(txwin
, attr
, &winctx
);
969 init_winctx_regs(txwin
, &winctx
);
972 * If its a kernel send window, map the window address into the
973 * kernel's address space. For user windows, user must issue an
974 * mmap() to map the window into their address space.
976 * NOTE: If kernel ever resubmits a user CRB after handling a page
977 * fault, we will need to map this into kernel as well.
979 if (!txwin
->user_win
) {
980 txwin
->paste_kaddr
= map_paste_region(txwin
);
981 if (IS_ERR(txwin
->paste_kaddr
)) {
982 rc
= PTR_ERR(txwin
->paste_kaddr
);
987 set_vinst_win(vinst
, txwin
);
992 vas_window_free(txwin
);
999 EXPORT_SYMBOL_GPL(vas_tx_win_open
);
1001 int vas_copy_crb(void *crb
, int offset
)
1003 return vas_copy(crb
, offset
);
1005 EXPORT_SYMBOL_GPL(vas_copy_crb
);
1007 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1008 int vas_paste_crb(struct vas_window
*txwin
, int offset
, bool re
)
1015 * Only NX windows are supported for now and hardware assumes
1016 * report-enable flag is set for NX windows. Ensure software
1019 WARN_ON_ONCE(txwin
->nx_win
&& !re
);
1021 addr
= txwin
->paste_kaddr
;
1024 * Set the REPORT_ENABLE bit (equivalent to writing
1025 * to 1K offset of the paste address)
1027 val
= SET_FIELD(RMA_LSMP_REPORT_ENABLE
, 0ULL, 1);
1032 * Map the raw CR value from vas_paste() to an error code (there
1033 * is just pass or fail for now though).
1035 rc
= vas_paste(addr
, offset
);
1041 print_fifo_msg_count(txwin
);
1045 EXPORT_SYMBOL_GPL(vas_paste_crb
);
1047 static void poll_window_busy_state(struct vas_window
*window
)
1054 * Poll Window Busy flag
1056 val
= read_hvwc_reg(window
, VREG(WIN_STATUS
));
1057 busy
= GET_FIELD(VAS_WIN_BUSY
, val
);
1060 set_current_state(TASK_UNINTERRUPTIBLE
);
1061 schedule_timeout(HZ
);
1066 static void poll_window_castout(struct vas_window
*window
)
1071 /* Cast window context out of the cache */
1073 val
= read_hvwc_reg(window
, VREG(WIN_CTX_CACHING_CTL
));
1074 cached
= GET_FIELD(VAS_WIN_CACHE_STATUS
, val
);
1077 val
= SET_FIELD(VAS_CASTOUT_REQ
, val
, 1);
1078 val
= SET_FIELD(VAS_PUSH_TO_MEM
, val
, 0);
1079 write_hvwc_reg(window
, VREG(WIN_CTX_CACHING_CTL
), val
);
1081 set_current_state(TASK_UNINTERRUPTIBLE
);
1082 schedule_timeout(HZ
);
1090 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1091 * - Disable new paste operations (unmap paste address)
1092 * - Poll for the "Window Busy" bit to be cleared
1093 * - Clear the Open/Enable bit for the Window.
1094 * - Poll for return of window Credits (implies FIFO empty for Rx win?)
1095 * - Unpin and cast window context out of cache
1097 * Besides the hardware, kernel has some bookkeeping of course.
1099 int vas_win_close(struct vas_window
*window
)
1106 if (!window
->tx_win
&& atomic_read(&window
->num_txwins
) != 0) {
1107 pr_devel("Attempting to close an active Rx window!\n");
1112 unmap_paste_region(window
);
1114 clear_vinst_win(window
);
1116 poll_window_busy_state(window
);
1118 /* Unpin window from cache and close it */
1119 val
= read_hvwc_reg(window
, VREG(WINCTL
));
1120 val
= SET_FIELD(VAS_WINCTL_PIN
, val
, 0);
1121 val
= SET_FIELD(VAS_WINCTL_OPEN
, val
, 0);
1122 write_hvwc_reg(window
, VREG(WINCTL
), val
);
1124 poll_window_castout(window
);
1126 /* if send window, drop reference to matching receive window */
1128 put_rx_win(window
->rxwin
);
1130 vas_window_free(window
);
1134 EXPORT_SYMBOL_GPL(vas_win_close
);