2 * Copyright 2014 IBM Corp.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
10 #include <linux/spinlock.h>
11 #include <linux/sched.h>
12 #include <linux/sched/clock.h>
13 #include <linux/slab.h>
14 #include <linux/mutex.h>
16 #include <linux/uaccess.h>
17 #include <linux/delay.h>
18 #include <asm/synch.h>
19 #include <misc/cxl-base.h>
24 static int afu_control(struct cxl_afu
*afu
, u64 command
, u64 clear
,
25 u64 result
, u64 mask
, bool enabled
)
28 unsigned long timeout
= jiffies
+ (HZ
* CXL_TIMEOUT
);
31 spin_lock(&afu
->afu_cntl_lock
);
32 pr_devel("AFU command starting: %llx\n", command
);
34 trace_cxl_afu_ctrl(afu
, command
);
36 AFU_Cntl
= cxl_p2n_read(afu
, CXL_AFU_Cntl_An
);
37 cxl_p2n_write(afu
, CXL_AFU_Cntl_An
, (AFU_Cntl
& ~clear
) | command
);
39 AFU_Cntl
= cxl_p2n_read(afu
, CXL_AFU_Cntl_An
);
40 while ((AFU_Cntl
& mask
) != result
) {
41 if (time_after_eq(jiffies
, timeout
)) {
42 dev_warn(&afu
->dev
, "WARNING: AFU control timed out!\n");
47 if (!cxl_ops
->link_ok(afu
->adapter
, afu
)) {
48 afu
->enabled
= enabled
;
53 pr_devel_ratelimited("AFU control... (0x%016llx)\n",
56 AFU_Cntl
= cxl_p2n_read(afu
, CXL_AFU_Cntl_An
);
59 if (AFU_Cntl
& CXL_AFU_Cntl_An_RA
) {
61 * Workaround for a bug in the XSL used in the Mellanox CX4
62 * that fails to clear the RA bit after an AFU reset,
63 * preventing subsequent AFU resets from working.
65 cxl_p2n_write(afu
, CXL_AFU_Cntl_An
, AFU_Cntl
& ~CXL_AFU_Cntl_An_RA
);
68 pr_devel("AFU command complete: %llx\n", command
);
69 afu
->enabled
= enabled
;
71 trace_cxl_afu_ctrl_done(afu
, command
, rc
);
72 spin_unlock(&afu
->afu_cntl_lock
);
77 static int afu_enable(struct cxl_afu
*afu
)
79 pr_devel("AFU enable request\n");
81 return afu_control(afu
, CXL_AFU_Cntl_An_E
, 0,
82 CXL_AFU_Cntl_An_ES_Enabled
,
83 CXL_AFU_Cntl_An_ES_MASK
, true);
86 int cxl_afu_disable(struct cxl_afu
*afu
)
88 pr_devel("AFU disable request\n");
90 return afu_control(afu
, 0, CXL_AFU_Cntl_An_E
,
91 CXL_AFU_Cntl_An_ES_Disabled
,
92 CXL_AFU_Cntl_An_ES_MASK
, false);
95 /* This will disable as well as reset */
96 static int native_afu_reset(struct cxl_afu
*afu
)
101 pr_devel("AFU reset request\n");
103 rc
= afu_control(afu
, CXL_AFU_Cntl_An_RA
, 0,
104 CXL_AFU_Cntl_An_RS_Complete
| CXL_AFU_Cntl_An_ES_Disabled
,
105 CXL_AFU_Cntl_An_RS_MASK
| CXL_AFU_Cntl_An_ES_MASK
,
108 /* Re-enable any masked interrupts */
109 serr
= cxl_p1n_read(afu
, CXL_PSL_SERR_An
);
110 serr
&= ~CXL_PSL_SERR_An_IRQ_MASKS
;
111 cxl_p1n_write(afu
, CXL_PSL_SERR_An
, serr
);
117 static int native_afu_check_and_enable(struct cxl_afu
*afu
)
119 if (!cxl_ops
->link_ok(afu
->adapter
, afu
)) {
120 WARN(1, "Refusing to enable afu while link down!\n");
125 return afu_enable(afu
);
128 int cxl_psl_purge(struct cxl_afu
*afu
)
130 u64 PSL_CNTL
= cxl_p1n_read(afu
, CXL_PSL_SCNTL_An
);
131 u64 AFU_Cntl
= cxl_p2n_read(afu
, CXL_AFU_Cntl_An
);
134 u64 trans_fault
= 0x0ULL
;
135 unsigned long timeout
= jiffies
+ (HZ
* CXL_TIMEOUT
);
138 trace_cxl_psl_ctrl(afu
, CXL_PSL_SCNTL_An_Pc
);
140 pr_devel("PSL purge request\n");
142 if (cxl_is_psl8(afu
))
143 trans_fault
= CXL_PSL_DSISR_TRANS
;
144 if (cxl_is_psl9(afu
))
145 trans_fault
= CXL_PSL9_DSISR_An_TF
;
147 if (!cxl_ops
->link_ok(afu
->adapter
, afu
)) {
148 dev_warn(&afu
->dev
, "PSL Purge called with link down, ignoring\n");
153 if ((AFU_Cntl
& CXL_AFU_Cntl_An_ES_MASK
) != CXL_AFU_Cntl_An_ES_Disabled
) {
154 WARN(1, "psl_purge request while AFU not disabled!\n");
155 cxl_afu_disable(afu
);
158 cxl_p1n_write(afu
, CXL_PSL_SCNTL_An
,
159 PSL_CNTL
| CXL_PSL_SCNTL_An_Pc
);
160 start
= local_clock();
161 PSL_CNTL
= cxl_p1n_read(afu
, CXL_PSL_SCNTL_An
);
162 while ((PSL_CNTL
& CXL_PSL_SCNTL_An_Ps_MASK
)
163 == CXL_PSL_SCNTL_An_Ps_Pending
) {
164 if (time_after_eq(jiffies
, timeout
)) {
165 dev_warn(&afu
->dev
, "WARNING: PSL Purge timed out!\n");
169 if (!cxl_ops
->link_ok(afu
->adapter
, afu
)) {
174 dsisr
= cxl_p2n_read(afu
, CXL_PSL_DSISR_An
);
175 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx PSL_DSISR: 0x%016llx\n",
178 if (dsisr
& trans_fault
) {
179 dar
= cxl_p2n_read(afu
, CXL_PSL_DAR_An
);
180 dev_notice(&afu
->dev
, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n",
182 cxl_p2n_write(afu
, CXL_PSL_TFC_An
, CXL_PSL_TFC_An_AE
);
184 dev_notice(&afu
->dev
, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n",
186 cxl_p2n_write(afu
, CXL_PSL_TFC_An
, CXL_PSL_TFC_An_A
);
190 PSL_CNTL
= cxl_p1n_read(afu
, CXL_PSL_SCNTL_An
);
193 pr_devel("PSL purged in %lld ns\n", end
- start
);
195 cxl_p1n_write(afu
, CXL_PSL_SCNTL_An
,
196 PSL_CNTL
& ~CXL_PSL_SCNTL_An_Pc
);
198 trace_cxl_psl_ctrl_done(afu
, CXL_PSL_SCNTL_An_Pc
, rc
);
202 static int spa_max_procs(int spa_size
)
206 * end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
207 * Most of that junk is really just an overly-complicated way of saying
208 * the last 256 bytes are __aligned(128), so it's really:
209 * end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
211 * end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
213 * sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
214 * Ignore the alignment (which is safe in this case as long as we are
215 * careful with our rounding) and solve for n:
217 return ((spa_size
/ 8) - 96) / 17;
220 static int cxl_alloc_spa(struct cxl_afu
*afu
, int mode
)
224 /* Work out how many pages to allocate */
225 afu
->native
->spa_order
= -1;
227 afu
->native
->spa_order
++;
228 spa_size
= (1 << afu
->native
->spa_order
) * PAGE_SIZE
;
230 if (spa_size
> 0x100000) {
231 dev_warn(&afu
->dev
, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n",
232 afu
->native
->spa_max_procs
, afu
->native
->spa_size
);
233 if (mode
!= CXL_MODE_DEDICATED
)
234 afu
->num_procs
= afu
->native
->spa_max_procs
;
238 afu
->native
->spa_size
= spa_size
;
239 afu
->native
->spa_max_procs
= spa_max_procs(afu
->native
->spa_size
);
240 } while (afu
->native
->spa_max_procs
< afu
->num_procs
);
242 if (!(afu
->native
->spa
= (struct cxl_process_element
*)
243 __get_free_pages(GFP_KERNEL
| __GFP_ZERO
, afu
->native
->spa_order
))) {
244 pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
247 pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n",
248 1<<afu
->native
->spa_order
, afu
->native
->spa_max_procs
, afu
->num_procs
);
253 static void attach_spa(struct cxl_afu
*afu
)
257 afu
->native
->sw_command_status
= (__be64
*)((char *)afu
->native
->spa
+
258 ((afu
->native
->spa_max_procs
+ 3) * 128));
260 spap
= virt_to_phys(afu
->native
->spa
) & CXL_PSL_SPAP_Addr
;
261 spap
|= ((afu
->native
->spa_size
>> (12 - CXL_PSL_SPAP_Size_Shift
)) - 1) & CXL_PSL_SPAP_Size
;
262 spap
|= CXL_PSL_SPAP_V
;
263 pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n",
264 afu
->native
->spa
, afu
->native
->spa_max_procs
,
265 afu
->native
->sw_command_status
, spap
);
266 cxl_p1n_write(afu
, CXL_PSL_SPAP_An
, spap
);
269 static inline void detach_spa(struct cxl_afu
*afu
)
271 cxl_p1n_write(afu
, CXL_PSL_SPAP_An
, 0);
274 void cxl_release_spa(struct cxl_afu
*afu
)
276 if (afu
->native
->spa
) {
277 free_pages((unsigned long) afu
->native
->spa
,
278 afu
->native
->spa_order
);
279 afu
->native
->spa
= NULL
;
284 * Invalidation of all ERAT entries is no longer required by CAIA2. Use
287 int cxl_invalidate_all_psl9(struct cxl
*adapter
)
289 unsigned long timeout
= jiffies
+ (HZ
* CXL_TIMEOUT
);
292 pr_devel("CXL adapter - invalidation of all ERAT entries\n");
294 /* Invalidates all ERAT entries for Radix or HPT */
295 ierat
= CXL_XSL9_IERAT_IALL
;
297 ierat
|= CXL_XSL9_IERAT_INVR
;
298 cxl_p1_write(adapter
, CXL_XSL9_IERAT
, ierat
);
300 while (cxl_p1_read(adapter
, CXL_XSL9_IERAT
) & CXL_XSL9_IERAT_IINPROG
) {
301 if (time_after_eq(jiffies
, timeout
)) {
302 dev_warn(&adapter
->dev
,
303 "WARNING: CXL adapter invalidation of all ERAT entries timed out!\n");
306 if (!cxl_ops
->link_ok(adapter
, NULL
))
313 int cxl_invalidate_all_psl8(struct cxl
*adapter
)
315 unsigned long timeout
= jiffies
+ (HZ
* CXL_TIMEOUT
);
317 pr_devel("CXL adapter wide TLBIA & SLBIA\n");
319 cxl_p1_write(adapter
, CXL_PSL_AFUSEL
, CXL_PSL_AFUSEL_A
);
321 cxl_p1_write(adapter
, CXL_PSL_TLBIA
, CXL_TLB_SLB_IQ_ALL
);
322 while (cxl_p1_read(adapter
, CXL_PSL_TLBIA
) & CXL_TLB_SLB_P
) {
323 if (time_after_eq(jiffies
, timeout
)) {
324 dev_warn(&adapter
->dev
, "WARNING: CXL adapter wide TLBIA timed out!\n");
327 if (!cxl_ops
->link_ok(adapter
, NULL
))
332 cxl_p1_write(adapter
, CXL_PSL_SLBIA
, CXL_TLB_SLB_IQ_ALL
);
333 while (cxl_p1_read(adapter
, CXL_PSL_SLBIA
) & CXL_TLB_SLB_P
) {
334 if (time_after_eq(jiffies
, timeout
)) {
335 dev_warn(&adapter
->dev
, "WARNING: CXL adapter wide SLBIA timed out!\n");
338 if (!cxl_ops
->link_ok(adapter
, NULL
))
345 int cxl_data_cache_flush(struct cxl
*adapter
)
348 unsigned long timeout
= jiffies
+ (HZ
* CXL_TIMEOUT
);
350 pr_devel("Flushing data cache\n");
352 reg
= cxl_p1_read(adapter
, CXL_PSL_Control
);
353 reg
|= CXL_PSL_Control_Fr
;
354 cxl_p1_write(adapter
, CXL_PSL_Control
, reg
);
356 reg
= cxl_p1_read(adapter
, CXL_PSL_Control
);
357 while ((reg
& CXL_PSL_Control_Fs_MASK
) != CXL_PSL_Control_Fs_Complete
) {
358 if (time_after_eq(jiffies
, timeout
)) {
359 dev_warn(&adapter
->dev
, "WARNING: cache flush timed out!\n");
363 if (!cxl_ops
->link_ok(adapter
, NULL
)) {
364 dev_warn(&adapter
->dev
, "WARNING: link down when flushing cache\n");
368 reg
= cxl_p1_read(adapter
, CXL_PSL_Control
);
371 reg
&= ~CXL_PSL_Control_Fr
;
372 cxl_p1_write(adapter
, CXL_PSL_Control
, reg
);
376 static int cxl_write_sstp(struct cxl_afu
*afu
, u64 sstp0
, u64 sstp1
)
380 /* 1. Disable SSTP by writing 0 to SSTP1[V] */
381 cxl_p2n_write(afu
, CXL_SSTP1_An
, 0);
383 /* 2. Invalidate all SLB entries */
384 if ((rc
= cxl_afu_slbia(afu
)))
387 /* 3. Set SSTP0_An */
388 cxl_p2n_write(afu
, CXL_SSTP0_An
, sstp0
);
390 /* 4. Set SSTP1_An */
391 cxl_p2n_write(afu
, CXL_SSTP1_An
, sstp1
);
396 /* Using per slice version may improve performance here. (ie. SLBIA_An) */
397 static void slb_invalid(struct cxl_context
*ctx
)
399 struct cxl
*adapter
= ctx
->afu
->adapter
;
402 WARN_ON(!mutex_is_locked(&ctx
->afu
->native
->spa_mutex
));
404 cxl_p1_write(adapter
, CXL_PSL_LBISEL
,
405 ((u64
)be32_to_cpu(ctx
->elem
->common
.pid
) << 32) |
406 be32_to_cpu(ctx
->elem
->lpid
));
407 cxl_p1_write(adapter
, CXL_PSL_SLBIA
, CXL_TLB_SLB_IQ_LPIDPID
);
410 if (!cxl_ops
->link_ok(adapter
, NULL
))
412 slbia
= cxl_p1_read(adapter
, CXL_PSL_SLBIA
);
413 if (!(slbia
& CXL_TLB_SLB_P
))
419 static int do_process_element_cmd(struct cxl_context
*ctx
,
420 u64 cmd
, u64 pe_state
)
423 unsigned long timeout
= jiffies
+ (HZ
* CXL_TIMEOUT
);
426 trace_cxl_llcmd(ctx
, cmd
);
428 WARN_ON(!ctx
->afu
->enabled
);
430 ctx
->elem
->software_state
= cpu_to_be32(pe_state
);
432 *(ctx
->afu
->native
->sw_command_status
) = cpu_to_be64(cmd
| 0 | ctx
->pe
);
434 cxl_p1n_write(ctx
->afu
, CXL_PSL_LLCMD_An
, cmd
| ctx
->pe
);
436 if (time_after_eq(jiffies
, timeout
)) {
437 dev_warn(&ctx
->afu
->dev
, "WARNING: Process Element Command timed out!\n");
441 if (!cxl_ops
->link_ok(ctx
->afu
->adapter
, ctx
->afu
)) {
442 dev_warn(&ctx
->afu
->dev
, "WARNING: Device link down, aborting Process Element Command!\n");
446 state
= be64_to_cpup(ctx
->afu
->native
->sw_command_status
);
447 if (state
== ~0ULL) {
448 pr_err("cxl: Error adding process element to AFU\n");
452 if ((state
& (CXL_SPA_SW_CMD_MASK
| CXL_SPA_SW_STATE_MASK
| CXL_SPA_SW_LINK_MASK
)) ==
453 (cmd
| (cmd
>> 16) | ctx
->pe
))
456 * The command won't finish in the PSL if there are
457 * outstanding DSIs. Hence we need to yield here in
458 * case there are outstanding DSIs that we need to
459 * service. Tuning possiblity: we could wait for a
466 trace_cxl_llcmd_done(ctx
, cmd
, rc
);
470 static int add_process_element(struct cxl_context
*ctx
)
474 mutex_lock(&ctx
->afu
->native
->spa_mutex
);
475 pr_devel("%s Adding pe: %i started\n", __func__
, ctx
->pe
);
476 if (!(rc
= do_process_element_cmd(ctx
, CXL_SPA_SW_CMD_ADD
, CXL_PE_SOFTWARE_STATE_V
)))
477 ctx
->pe_inserted
= true;
478 pr_devel("%s Adding pe: %i finished\n", __func__
, ctx
->pe
);
479 mutex_unlock(&ctx
->afu
->native
->spa_mutex
);
483 static int terminate_process_element(struct cxl_context
*ctx
)
487 /* fast path terminate if it's already invalid */
488 if (!(ctx
->elem
->software_state
& cpu_to_be32(CXL_PE_SOFTWARE_STATE_V
)))
491 mutex_lock(&ctx
->afu
->native
->spa_mutex
);
492 pr_devel("%s Terminate pe: %i started\n", __func__
, ctx
->pe
);
493 /* We could be asked to terminate when the hw is down. That
494 * should always succeed: it's not running if the hw has gone
495 * away and is being reset.
497 if (cxl_ops
->link_ok(ctx
->afu
->adapter
, ctx
->afu
))
498 rc
= do_process_element_cmd(ctx
, CXL_SPA_SW_CMD_TERMINATE
,
499 CXL_PE_SOFTWARE_STATE_V
| CXL_PE_SOFTWARE_STATE_T
);
500 ctx
->elem
->software_state
= 0; /* Remove Valid bit */
501 pr_devel("%s Terminate pe: %i finished\n", __func__
, ctx
->pe
);
502 mutex_unlock(&ctx
->afu
->native
->spa_mutex
);
506 static int remove_process_element(struct cxl_context
*ctx
)
510 mutex_lock(&ctx
->afu
->native
->spa_mutex
);
511 pr_devel("%s Remove pe: %i started\n", __func__
, ctx
->pe
);
513 /* We could be asked to remove when the hw is down. Again, if
514 * the hw is down, the PE is gone, so we succeed.
516 if (cxl_ops
->link_ok(ctx
->afu
->adapter
, ctx
->afu
))
517 rc
= do_process_element_cmd(ctx
, CXL_SPA_SW_CMD_REMOVE
, 0);
520 ctx
->pe_inserted
= false;
523 pr_devel("%s Remove pe: %i finished\n", __func__
, ctx
->pe
);
524 mutex_unlock(&ctx
->afu
->native
->spa_mutex
);
529 void cxl_assign_psn_space(struct cxl_context
*ctx
)
531 if (!ctx
->afu
->pp_size
|| ctx
->master
) {
532 ctx
->psn_phys
= ctx
->afu
->psn_phys
;
533 ctx
->psn_size
= ctx
->afu
->adapter
->ps_size
;
535 ctx
->psn_phys
= ctx
->afu
->psn_phys
+
536 (ctx
->afu
->native
->pp_offset
+ ctx
->afu
->pp_size
* ctx
->pe
);
537 ctx
->psn_size
= ctx
->afu
->pp_size
;
541 static int activate_afu_directed(struct cxl_afu
*afu
)
545 dev_info(&afu
->dev
, "Activating AFU directed mode\n");
547 afu
->num_procs
= afu
->max_procs_virtualised
;
548 if (afu
->native
->spa
== NULL
) {
549 if (cxl_alloc_spa(afu
, CXL_MODE_DIRECTED
))
554 cxl_p1n_write(afu
, CXL_PSL_SCNTL_An
, CXL_PSL_SCNTL_An_PM_AFU
);
556 cxl_p1n_write(afu
, CXL_PSL_AMOR_An
, 0xFFFFFFFFFFFFFFFFULL
);
557 cxl_p1n_write(afu
, CXL_PSL_ID_An
, CXL_PSL_ID_An_F
| CXL_PSL_ID_An_L
);
559 afu
->current_mode
= CXL_MODE_DIRECTED
;
561 if ((rc
= cxl_chardev_m_afu_add(afu
)))
564 if ((rc
= cxl_sysfs_afu_m_add(afu
)))
567 if ((rc
= cxl_chardev_s_afu_add(afu
)))
572 cxl_sysfs_afu_m_remove(afu
);
574 cxl_chardev_afu_remove(afu
);
578 #ifdef CONFIG_CPU_LITTLE_ENDIAN
579 #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
581 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
584 static u64
calculate_sr(struct cxl_context
*ctx
)
590 sr
|= CXL_PSL_SR_An_MP
;
591 if (mfspr(SPRN_LPCR
) & LPCR_TC
)
592 sr
|= CXL_PSL_SR_An_TC
;
595 sr
|= CXL_PSL_SR_An_R
;
596 sr
|= (mfmsr() & MSR_SF
) | CXL_PSL_SR_An_HV
;
598 sr
|= CXL_PSL_SR_An_PR
| CXL_PSL_SR_An_R
;
600 sr
|= CXL_PSL_SR_An_HV
;
602 sr
&= ~(CXL_PSL_SR_An_HV
);
603 if (!test_tsk_thread_flag(current
, TIF_32BIT
))
604 sr
|= CXL_PSL_SR_An_SF
;
606 if (cxl_is_psl9(ctx
->afu
)) {
608 sr
|= CXL_PSL_SR_An_XLAT_ror
;
610 sr
|= CXL_PSL_SR_An_XLAT_hpt
;
615 static void update_ivtes_directed(struct cxl_context
*ctx
)
617 bool need_update
= (ctx
->status
== STARTED
);
621 WARN_ON(terminate_process_element(ctx
));
622 WARN_ON(remove_process_element(ctx
));
625 for (r
= 0; r
< CXL_IRQ_RANGES
; r
++) {
626 ctx
->elem
->ivte_offsets
[r
] = cpu_to_be16(ctx
->irqs
.offset
[r
]);
627 ctx
->elem
->ivte_ranges
[r
] = cpu_to_be16(ctx
->irqs
.range
[r
]);
631 * Theoretically we could use the update llcmd, instead of a
632 * terminate/remove/add (or if an atomic update was required we could
633 * do a suspend/update/resume), however it seems there might be issues
634 * with the update llcmd on some cards (including those using an XSL on
635 * an ASIC) so for now it's safest to go with the commands that are
636 * known to work. In the future if we come across a situation where the
637 * card may be performing transactions using the same PE while we are
638 * doing this update we might need to revisit this.
641 WARN_ON(add_process_element(ctx
));
644 static int process_element_entry_psl9(struct cxl_context
*ctx
, u64 wed
, u64 amr
)
648 cxl_assign_psn_space(ctx
);
650 ctx
->elem
->ctxtime
= 0; /* disable */
651 ctx
->elem
->lpid
= cpu_to_be32(mfspr(SPRN_LPID
));
652 ctx
->elem
->haurp
= 0; /* disable */
657 if (ctx
->mm
== NULL
) {
658 pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
659 __func__
, ctx
->pe
, pid_nr(ctx
->pid
));
662 pid
= ctx
->mm
->context
.id
;
665 ctx
->elem
->common
.tid
= 0;
666 ctx
->elem
->common
.pid
= cpu_to_be32(pid
);
668 ctx
->elem
->sr
= cpu_to_be64(calculate_sr(ctx
));
670 ctx
->elem
->common
.csrp
= 0; /* disable */
672 cxl_prefault(ctx
, wed
);
675 * Ensure we have the multiplexed PSL interrupt set up to take faults
676 * for kernel contexts that may not have allocated any AFU IRQs at all:
678 if (ctx
->irqs
.range
[0] == 0) {
679 ctx
->irqs
.offset
[0] = ctx
->afu
->native
->psl_hwirq
;
680 ctx
->irqs
.range
[0] = 1;
683 ctx
->elem
->common
.amr
= cpu_to_be64(amr
);
684 ctx
->elem
->common
.wed
= cpu_to_be64(wed
);
689 int cxl_attach_afu_directed_psl9(struct cxl_context
*ctx
, u64 wed
, u64 amr
)
693 /* fill the process element entry */
694 result
= process_element_entry_psl9(ctx
, wed
, amr
);
698 update_ivtes_directed(ctx
);
700 /* first guy needs to enable */
701 result
= cxl_ops
->afu_check_and_enable(ctx
->afu
);
705 return add_process_element(ctx
);
708 int cxl_attach_afu_directed_psl8(struct cxl_context
*ctx
, u64 wed
, u64 amr
)
713 cxl_assign_psn_space(ctx
);
715 ctx
->elem
->ctxtime
= 0; /* disable */
716 ctx
->elem
->lpid
= cpu_to_be32(mfspr(SPRN_LPID
));
717 ctx
->elem
->haurp
= 0; /* disable */
718 ctx
->elem
->u
.sdr
= cpu_to_be64(mfspr(SPRN_SDR1
));
723 ctx
->elem
->common
.tid
= 0;
724 ctx
->elem
->common
.pid
= cpu_to_be32(pid
);
726 ctx
->elem
->sr
= cpu_to_be64(calculate_sr(ctx
));
728 ctx
->elem
->common
.csrp
= 0; /* disable */
729 ctx
->elem
->common
.u
.psl8
.aurp0
= 0; /* disable */
730 ctx
->elem
->common
.u
.psl8
.aurp1
= 0; /* disable */
732 cxl_prefault(ctx
, wed
);
734 ctx
->elem
->common
.u
.psl8
.sstp0
= cpu_to_be64(ctx
->sstp0
);
735 ctx
->elem
->common
.u
.psl8
.sstp1
= cpu_to_be64(ctx
->sstp1
);
738 * Ensure we have the multiplexed PSL interrupt set up to take faults
739 * for kernel contexts that may not have allocated any AFU IRQs at all:
741 if (ctx
->irqs
.range
[0] == 0) {
742 ctx
->irqs
.offset
[0] = ctx
->afu
->native
->psl_hwirq
;
743 ctx
->irqs
.range
[0] = 1;
746 update_ivtes_directed(ctx
);
748 ctx
->elem
->common
.amr
= cpu_to_be64(amr
);
749 ctx
->elem
->common
.wed
= cpu_to_be64(wed
);
751 /* first guy needs to enable */
752 if ((result
= cxl_ops
->afu_check_and_enable(ctx
->afu
)))
755 return add_process_element(ctx
);
758 static int deactivate_afu_directed(struct cxl_afu
*afu
)
760 dev_info(&afu
->dev
, "Deactivating AFU directed mode\n");
762 afu
->current_mode
= 0;
765 cxl_sysfs_afu_m_remove(afu
);
766 cxl_chardev_afu_remove(afu
);
769 * The CAIA section 2.2.1 indicates that the procedure for starting and
770 * stopping an AFU in AFU directed mode is AFU specific, which is not
771 * ideal since this code is generic and with one exception has no
772 * knowledge of the AFU. This is in contrast to the procedure for
773 * disabling a dedicated process AFU, which is documented to just
774 * require a reset. The architecture does indicate that both an AFU
775 * reset and an AFU disable should result in the AFU being disabled and
776 * we do both followed by a PSL purge for safety.
778 * Notably we used to have some issues with the disable sequence on PSL
779 * cards, which is why we ended up using this heavy weight procedure in
780 * the first place, however a bug was discovered that had rendered the
781 * disable operation ineffective, so it is conceivable that was the
782 * sole explanation for those difficulties. Careful regression testing
783 * is recommended if anyone attempts to remove or reorder these
786 * The XSL on the Mellanox CX4 behaves a little differently from the
787 * PSL based cards and will time out an AFU reset if the AFU is still
788 * enabled. That card is special in that we do have a means to identify
789 * it from this code, so in that case we skip the reset and just use a
790 * disable/purge to avoid the timeout and corresponding noise in the
793 if (afu
->adapter
->native
->sl_ops
->needs_reset_before_disable
)
794 cxl_ops
->afu_reset(afu
);
795 cxl_afu_disable(afu
);
801 int cxl_activate_dedicated_process_psl9(struct cxl_afu
*afu
)
803 dev_info(&afu
->dev
, "Activating dedicated process mode\n");
806 * If XSL is set to dedicated mode (Set in PSL_SCNTL reg), the
807 * XSL and AFU are programmed to work with a single context.
808 * The context information should be configured in the SPA area
809 * index 0 (so PSL_SPAP must be configured before enabling the
813 if (afu
->native
->spa
== NULL
) {
814 if (cxl_alloc_spa(afu
, CXL_MODE_DEDICATED
))
819 cxl_p1n_write(afu
, CXL_PSL_SCNTL_An
, CXL_PSL_SCNTL_An_PM_Process
);
820 cxl_p1n_write(afu
, CXL_PSL_ID_An
, CXL_PSL_ID_An_F
| CXL_PSL_ID_An_L
);
822 afu
->current_mode
= CXL_MODE_DEDICATED
;
824 return cxl_chardev_d_afu_add(afu
);
827 int cxl_activate_dedicated_process_psl8(struct cxl_afu
*afu
)
829 dev_info(&afu
->dev
, "Activating dedicated process mode\n");
831 cxl_p1n_write(afu
, CXL_PSL_SCNTL_An
, CXL_PSL_SCNTL_An_PM_Process
);
833 cxl_p1n_write(afu
, CXL_PSL_CtxTime_An
, 0); /* disable */
834 cxl_p1n_write(afu
, CXL_PSL_SPAP_An
, 0); /* disable */
835 cxl_p1n_write(afu
, CXL_PSL_AMOR_An
, 0xFFFFFFFFFFFFFFFFULL
);
836 cxl_p1n_write(afu
, CXL_PSL_LPID_An
, mfspr(SPRN_LPID
));
837 cxl_p1n_write(afu
, CXL_HAURP_An
, 0); /* disable */
838 cxl_p1n_write(afu
, CXL_PSL_SDR_An
, mfspr(SPRN_SDR1
));
840 cxl_p2n_write(afu
, CXL_CSRP_An
, 0); /* disable */
841 cxl_p2n_write(afu
, CXL_AURP0_An
, 0); /* disable */
842 cxl_p2n_write(afu
, CXL_AURP1_An
, 0); /* disable */
844 afu
->current_mode
= CXL_MODE_DEDICATED
;
847 return cxl_chardev_d_afu_add(afu
);
850 void cxl_update_dedicated_ivtes_psl9(struct cxl_context
*ctx
)
854 for (r
= 0; r
< CXL_IRQ_RANGES
; r
++) {
855 ctx
->elem
->ivte_offsets
[r
] = cpu_to_be16(ctx
->irqs
.offset
[r
]);
856 ctx
->elem
->ivte_ranges
[r
] = cpu_to_be16(ctx
->irqs
.range
[r
]);
860 void cxl_update_dedicated_ivtes_psl8(struct cxl_context
*ctx
)
862 struct cxl_afu
*afu
= ctx
->afu
;
864 cxl_p1n_write(afu
, CXL_PSL_IVTE_Offset_An
,
865 (((u64
)ctx
->irqs
.offset
[0] & 0xffff) << 48) |
866 (((u64
)ctx
->irqs
.offset
[1] & 0xffff) << 32) |
867 (((u64
)ctx
->irqs
.offset
[2] & 0xffff) << 16) |
868 ((u64
)ctx
->irqs
.offset
[3] & 0xffff));
869 cxl_p1n_write(afu
, CXL_PSL_IVTE_Limit_An
, (u64
)
870 (((u64
)ctx
->irqs
.range
[0] & 0xffff) << 48) |
871 (((u64
)ctx
->irqs
.range
[1] & 0xffff) << 32) |
872 (((u64
)ctx
->irqs
.range
[2] & 0xffff) << 16) |
873 ((u64
)ctx
->irqs
.range
[3] & 0xffff));
876 int cxl_attach_dedicated_process_psl9(struct cxl_context
*ctx
, u64 wed
, u64 amr
)
878 struct cxl_afu
*afu
= ctx
->afu
;
881 /* fill the process element entry */
882 result
= process_element_entry_psl9(ctx
, wed
, amr
);
886 if (ctx
->afu
->adapter
->native
->sl_ops
->update_dedicated_ivtes
)
887 afu
->adapter
->native
->sl_ops
->update_dedicated_ivtes(ctx
);
889 result
= cxl_ops
->afu_reset(afu
);
893 return afu_enable(afu
);
896 int cxl_attach_dedicated_process_psl8(struct cxl_context
*ctx
, u64 wed
, u64 amr
)
898 struct cxl_afu
*afu
= ctx
->afu
;
902 pid
= (u64
)current
->pid
<< 32;
905 cxl_p2n_write(afu
, CXL_PSL_PID_TID_An
, pid
);
907 cxl_p1n_write(afu
, CXL_PSL_SR_An
, calculate_sr(ctx
));
909 if ((rc
= cxl_write_sstp(afu
, ctx
->sstp0
, ctx
->sstp1
)))
912 cxl_prefault(ctx
, wed
);
914 if (ctx
->afu
->adapter
->native
->sl_ops
->update_dedicated_ivtes
)
915 afu
->adapter
->native
->sl_ops
->update_dedicated_ivtes(ctx
);
917 cxl_p2n_write(afu
, CXL_PSL_AMR_An
, amr
);
919 /* master only context for dedicated */
920 cxl_assign_psn_space(ctx
);
922 if ((rc
= cxl_ops
->afu_reset(afu
)))
925 cxl_p2n_write(afu
, CXL_PSL_WED_An
, wed
);
927 return afu_enable(afu
);
930 static int deactivate_dedicated_process(struct cxl_afu
*afu
)
932 dev_info(&afu
->dev
, "Deactivating dedicated process mode\n");
934 afu
->current_mode
= 0;
937 cxl_chardev_afu_remove(afu
);
942 static int native_afu_deactivate_mode(struct cxl_afu
*afu
, int mode
)
944 if (mode
== CXL_MODE_DIRECTED
)
945 return deactivate_afu_directed(afu
);
946 if (mode
== CXL_MODE_DEDICATED
)
947 return deactivate_dedicated_process(afu
);
951 static int native_afu_activate_mode(struct cxl_afu
*afu
, int mode
)
955 if (!(mode
& afu
->modes_supported
))
958 if (!cxl_ops
->link_ok(afu
->adapter
, afu
)) {
959 WARN(1, "Device link is down, refusing to activate!\n");
963 if (mode
== CXL_MODE_DIRECTED
)
964 return activate_afu_directed(afu
);
965 if ((mode
== CXL_MODE_DEDICATED
) &&
966 (afu
->adapter
->native
->sl_ops
->activate_dedicated_process
))
967 return afu
->adapter
->native
->sl_ops
->activate_dedicated_process(afu
);
972 static int native_attach_process(struct cxl_context
*ctx
, bool kernel
,
975 if (!cxl_ops
->link_ok(ctx
->afu
->adapter
, ctx
->afu
)) {
976 WARN(1, "Device link is down, refusing to attach process!\n");
980 ctx
->kernel
= kernel
;
981 if ((ctx
->afu
->current_mode
== CXL_MODE_DIRECTED
) &&
982 (ctx
->afu
->adapter
->native
->sl_ops
->attach_afu_directed
))
983 return ctx
->afu
->adapter
->native
->sl_ops
->attach_afu_directed(ctx
, wed
, amr
);
985 if ((ctx
->afu
->current_mode
== CXL_MODE_DEDICATED
) &&
986 (ctx
->afu
->adapter
->native
->sl_ops
->attach_dedicated_process
))
987 return ctx
->afu
->adapter
->native
->sl_ops
->attach_dedicated_process(ctx
, wed
, amr
);
992 static inline int detach_process_native_dedicated(struct cxl_context
*ctx
)
995 * The CAIA section 2.1.1 indicates that we need to do an AFU reset to
996 * stop the AFU in dedicated mode (we therefore do not make that
997 * optional like we do in the afu directed path). It does not indicate
998 * that we need to do an explicit disable (which should occur
999 * implicitly as part of the reset) or purge, but we do these as well
1000 * to be on the safe side.
1002 * Notably we used to have some issues with the disable sequence
1003 * (before the sequence was spelled out in the architecture) which is
1004 * why we were so heavy weight in the first place, however a bug was
1005 * discovered that had rendered the disable operation ineffective, so
1006 * it is conceivable that was the sole explanation for those
1007 * difficulties. Point is, we should be careful and do some regression
1008 * testing if we ever attempt to remove any part of this procedure.
1010 cxl_ops
->afu_reset(ctx
->afu
);
1011 cxl_afu_disable(ctx
->afu
);
1012 cxl_psl_purge(ctx
->afu
);
1016 static void native_update_ivtes(struct cxl_context
*ctx
)
1018 if (ctx
->afu
->current_mode
== CXL_MODE_DIRECTED
)
1019 return update_ivtes_directed(ctx
);
1020 if ((ctx
->afu
->current_mode
== CXL_MODE_DEDICATED
) &&
1021 (ctx
->afu
->adapter
->native
->sl_ops
->update_dedicated_ivtes
))
1022 return ctx
->afu
->adapter
->native
->sl_ops
->update_dedicated_ivtes(ctx
);
1023 WARN(1, "native_update_ivtes: Bad mode\n");
1026 static inline int detach_process_native_afu_directed(struct cxl_context
*ctx
)
1028 if (!ctx
->pe_inserted
)
1030 if (terminate_process_element(ctx
))
1032 if (remove_process_element(ctx
))
1038 static int native_detach_process(struct cxl_context
*ctx
)
1040 trace_cxl_detach(ctx
);
1042 if (ctx
->afu
->current_mode
== CXL_MODE_DEDICATED
)
1043 return detach_process_native_dedicated(ctx
);
1045 return detach_process_native_afu_directed(ctx
);
1048 static int native_get_irq_info(struct cxl_afu
*afu
, struct cxl_irq_info
*info
)
1050 /* If the adapter has gone away, we can't get any meaningful
1053 if (!cxl_ops
->link_ok(afu
->adapter
, afu
))
1056 info
->dsisr
= cxl_p2n_read(afu
, CXL_PSL_DSISR_An
);
1057 info
->dar
= cxl_p2n_read(afu
, CXL_PSL_DAR_An
);
1058 if (cxl_is_power8())
1059 info
->dsr
= cxl_p2n_read(afu
, CXL_PSL_DSR_An
);
1060 info
->afu_err
= cxl_p2n_read(afu
, CXL_AFU_ERR_An
);
1061 info
->errstat
= cxl_p2n_read(afu
, CXL_PSL_ErrStat_An
);
1062 info
->proc_handle
= 0;
1067 void cxl_native_irq_dump_regs_psl9(struct cxl_context
*ctx
)
1069 u64 fir1
, fir2
, serr
;
1071 fir1
= cxl_p1_read(ctx
->afu
->adapter
, CXL_PSL9_FIR1
);
1072 fir2
= cxl_p1_read(ctx
->afu
->adapter
, CXL_PSL9_FIR2
);
1074 dev_crit(&ctx
->afu
->dev
, "PSL_FIR1: 0x%016llx\n", fir1
);
1075 dev_crit(&ctx
->afu
->dev
, "PSL_FIR2: 0x%016llx\n", fir2
);
1076 if (ctx
->afu
->adapter
->native
->sl_ops
->register_serr_irq
) {
1077 serr
= cxl_p1n_read(ctx
->afu
, CXL_PSL_SERR_An
);
1078 cxl_afu_decode_psl_serr(ctx
->afu
, serr
);
1082 void cxl_native_irq_dump_regs_psl8(struct cxl_context
*ctx
)
1084 u64 fir1
, fir2
, fir_slice
, serr
, afu_debug
;
1086 fir1
= cxl_p1_read(ctx
->afu
->adapter
, CXL_PSL_FIR1
);
1087 fir2
= cxl_p1_read(ctx
->afu
->adapter
, CXL_PSL_FIR2
);
1088 fir_slice
= cxl_p1n_read(ctx
->afu
, CXL_PSL_FIR_SLICE_An
);
1089 afu_debug
= cxl_p1n_read(ctx
->afu
, CXL_AFU_DEBUG_An
);
1091 dev_crit(&ctx
->afu
->dev
, "PSL_FIR1: 0x%016llx\n", fir1
);
1092 dev_crit(&ctx
->afu
->dev
, "PSL_FIR2: 0x%016llx\n", fir2
);
1093 if (ctx
->afu
->adapter
->native
->sl_ops
->register_serr_irq
) {
1094 serr
= cxl_p1n_read(ctx
->afu
, CXL_PSL_SERR_An
);
1095 cxl_afu_decode_psl_serr(ctx
->afu
, serr
);
1097 dev_crit(&ctx
->afu
->dev
, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice
);
1098 dev_crit(&ctx
->afu
->dev
, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug
);
1101 static irqreturn_t
native_handle_psl_slice_error(struct cxl_context
*ctx
,
1102 u64 dsisr
, u64 errstat
)
1105 dev_crit(&ctx
->afu
->dev
, "PSL ERROR STATUS: 0x%016llx\n", errstat
);
1107 if (ctx
->afu
->adapter
->native
->sl_ops
->psl_irq_dump_registers
)
1108 ctx
->afu
->adapter
->native
->sl_ops
->psl_irq_dump_registers(ctx
);
1110 if (ctx
->afu
->adapter
->native
->sl_ops
->debugfs_stop_trace
) {
1111 dev_crit(&ctx
->afu
->dev
, "STOPPING CXL TRACE\n");
1112 ctx
->afu
->adapter
->native
->sl_ops
->debugfs_stop_trace(ctx
->afu
->adapter
);
1115 return cxl_ops
->ack_irq(ctx
, 0, errstat
);
1118 static bool cxl_is_translation_fault(struct cxl_afu
*afu
, u64 dsisr
)
1120 if ((cxl_is_psl8(afu
)) && (dsisr
& CXL_PSL_DSISR_TRANS
))
1123 if ((cxl_is_psl9(afu
)) && (dsisr
& CXL_PSL9_DSISR_An_TF
))
1129 irqreturn_t
cxl_fail_irq_psl(struct cxl_afu
*afu
, struct cxl_irq_info
*irq_info
)
1131 if (cxl_is_translation_fault(afu
, irq_info
->dsisr
))
1132 cxl_p2n_write(afu
, CXL_PSL_TFC_An
, CXL_PSL_TFC_An_AE
);
1134 cxl_p2n_write(afu
, CXL_PSL_TFC_An
, CXL_PSL_TFC_An_A
);
1139 static irqreturn_t
native_irq_multiplexed(int irq
, void *data
)
1141 struct cxl_afu
*afu
= data
;
1142 struct cxl_context
*ctx
;
1143 struct cxl_irq_info irq_info
;
1144 u64 phreg
= cxl_p2n_read(afu
, CXL_PSL_PEHandle_An
);
1145 int ph
, ret
= IRQ_HANDLED
, res
;
1147 /* check if eeh kicked in while the interrupt was in flight */
1148 if (unlikely(phreg
== ~0ULL)) {
1150 "Ignoring slice interrupt(%d) due to fenced card",
1154 /* Mask the pe-handle from register value */
1155 ph
= phreg
& 0xffff;
1156 if ((res
= native_get_irq_info(afu
, &irq_info
))) {
1157 WARN(1, "Unable to get CXL IRQ Info: %i\n", res
);
1158 if (afu
->adapter
->native
->sl_ops
->fail_irq
)
1159 return afu
->adapter
->native
->sl_ops
->fail_irq(afu
, &irq_info
);
1164 ctx
= idr_find(&afu
->contexts_idr
, ph
);
1166 if (afu
->adapter
->native
->sl_ops
->handle_interrupt
)
1167 ret
= afu
->adapter
->native
->sl_ops
->handle_interrupt(irq
, ctx
, &irq_info
);
1173 WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR"
1174 " %016llx\n(Possible AFU HW issue - was a term/remove acked"
1175 " with outstanding transactions?)\n", ph
, irq_info
.dsisr
,
1177 if (afu
->adapter
->native
->sl_ops
->fail_irq
)
1178 ret
= afu
->adapter
->native
->sl_ops
->fail_irq(afu
, &irq_info
);
1182 static void native_irq_wait(struct cxl_context
*ctx
)
1189 * Wait until no further interrupts are presented by the PSL
1193 ph
= cxl_p2n_read(ctx
->afu
, CXL_PSL_PEHandle_An
) & 0xffff;
1196 dsisr
= cxl_p2n_read(ctx
->afu
, CXL_PSL_DSISR_An
);
1197 if (cxl_is_psl8(ctx
->afu
) &&
1198 ((dsisr
& CXL_PSL_DSISR_PENDING
) == 0))
1200 if (cxl_is_psl9(ctx
->afu
) &&
1201 ((dsisr
& CXL_PSL9_DSISR_PENDING
) == 0))
1204 * We are waiting for the workqueue to process our
1205 * irq, so need to let that run here.
1210 dev_warn(&ctx
->afu
->dev
, "WARNING: waiting on DSI for PE %i"
1211 " DSISR %016llx!\n", ph
, dsisr
);
1215 static irqreturn_t
native_slice_irq_err(int irq
, void *data
)
1217 struct cxl_afu
*afu
= data
;
1218 u64 errstat
, serr
, afu_error
, dsisr
;
1219 u64 fir_slice
, afu_debug
, irq_mask
;
1222 * slice err interrupt is only used with full PSL (no XSL)
1224 serr
= cxl_p1n_read(afu
, CXL_PSL_SERR_An
);
1225 errstat
= cxl_p2n_read(afu
, CXL_PSL_ErrStat_An
);
1226 afu_error
= cxl_p2n_read(afu
, CXL_AFU_ERR_An
);
1227 dsisr
= cxl_p2n_read(afu
, CXL_PSL_DSISR_An
);
1228 cxl_afu_decode_psl_serr(afu
, serr
);
1230 if (cxl_is_power8()) {
1231 fir_slice
= cxl_p1n_read(afu
, CXL_PSL_FIR_SLICE_An
);
1232 afu_debug
= cxl_p1n_read(afu
, CXL_AFU_DEBUG_An
);
1233 dev_crit(&afu
->dev
, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice
);
1234 dev_crit(&afu
->dev
, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug
);
1236 dev_crit(&afu
->dev
, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat
);
1237 dev_crit(&afu
->dev
, "AFU_ERR_An: 0x%.16llx\n", afu_error
);
1238 dev_crit(&afu
->dev
, "PSL_DSISR_An: 0x%.16llx\n", dsisr
);
1240 /* mask off the IRQ so it won't retrigger until the AFU is reset */
1241 irq_mask
= (serr
& CXL_PSL_SERR_An_IRQS
) >> 32;
1243 cxl_p1n_write(afu
, CXL_PSL_SERR_An
, serr
);
1244 dev_info(&afu
->dev
, "Further such interrupts will be masked until the AFU is reset\n");
1249 void cxl_native_err_irq_dump_regs(struct cxl
*adapter
)
1253 fir1
= cxl_p1_read(adapter
, CXL_PSL_FIR1
);
1254 fir2
= cxl_p1_read(adapter
, CXL_PSL_FIR2
);
1256 dev_crit(&adapter
->dev
, "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", fir1
, fir2
);
1259 static irqreturn_t
native_irq_err(int irq
, void *data
)
1261 struct cxl
*adapter
= data
;
1264 WARN(1, "CXL ERROR interrupt %i\n", irq
);
1266 err_ivte
= cxl_p1_read(adapter
, CXL_PSL_ErrIVTE
);
1267 dev_crit(&adapter
->dev
, "PSL_ErrIVTE: 0x%016llx\n", err_ivte
);
1269 if (adapter
->native
->sl_ops
->debugfs_stop_trace
) {
1270 dev_crit(&adapter
->dev
, "STOPPING CXL TRACE\n");
1271 adapter
->native
->sl_ops
->debugfs_stop_trace(adapter
);
1274 if (adapter
->native
->sl_ops
->err_irq_dump_registers
)
1275 adapter
->native
->sl_ops
->err_irq_dump_registers(adapter
);
1280 int cxl_native_register_psl_err_irq(struct cxl
*adapter
)
1284 adapter
->irq_name
= kasprintf(GFP_KERNEL
, "cxl-%s-err",
1285 dev_name(&adapter
->dev
));
1286 if (!adapter
->irq_name
)
1289 if ((rc
= cxl_register_one_irq(adapter
, native_irq_err
, adapter
,
1290 &adapter
->native
->err_hwirq
,
1291 &adapter
->native
->err_virq
,
1292 adapter
->irq_name
))) {
1293 kfree(adapter
->irq_name
);
1294 adapter
->irq_name
= NULL
;
1298 cxl_p1_write(adapter
, CXL_PSL_ErrIVTE
, adapter
->native
->err_hwirq
& 0xffff);
1303 void cxl_native_release_psl_err_irq(struct cxl
*adapter
)
1305 if (adapter
->native
->err_virq
!= irq_find_mapping(NULL
, adapter
->native
->err_hwirq
))
1308 cxl_p1_write(adapter
, CXL_PSL_ErrIVTE
, 0x0000000000000000);
1309 cxl_unmap_irq(adapter
->native
->err_virq
, adapter
);
1310 cxl_ops
->release_one_irq(adapter
, adapter
->native
->err_hwirq
);
1311 kfree(adapter
->irq_name
);
1314 int cxl_native_register_serr_irq(struct cxl_afu
*afu
)
1319 afu
->err_irq_name
= kasprintf(GFP_KERNEL
, "cxl-%s-err",
1320 dev_name(&afu
->dev
));
1321 if (!afu
->err_irq_name
)
1324 if ((rc
= cxl_register_one_irq(afu
->adapter
, native_slice_irq_err
, afu
,
1326 &afu
->serr_virq
, afu
->err_irq_name
))) {
1327 kfree(afu
->err_irq_name
);
1328 afu
->err_irq_name
= NULL
;
1332 serr
= cxl_p1n_read(afu
, CXL_PSL_SERR_An
);
1333 if (cxl_is_power8())
1334 serr
= (serr
& 0x00ffffffffff0000ULL
) | (afu
->serr_hwirq
& 0xffff);
1335 if (cxl_is_power9()) {
1337 * By default, all errors are masked. So don't set all masks.
1338 * Slice errors will be transfered.
1340 serr
= (serr
& ~0xff0000007fffffffULL
) | (afu
->serr_hwirq
& 0xffff);
1342 cxl_p1n_write(afu
, CXL_PSL_SERR_An
, serr
);
1347 void cxl_native_release_serr_irq(struct cxl_afu
*afu
)
1349 if (afu
->serr_virq
!= irq_find_mapping(NULL
, afu
->serr_hwirq
))
1352 cxl_p1n_write(afu
, CXL_PSL_SERR_An
, 0x0000000000000000);
1353 cxl_unmap_irq(afu
->serr_virq
, afu
);
1354 cxl_ops
->release_one_irq(afu
->adapter
, afu
->serr_hwirq
);
1355 kfree(afu
->err_irq_name
);
1358 int cxl_native_register_psl_irq(struct cxl_afu
*afu
)
1362 afu
->psl_irq_name
= kasprintf(GFP_KERNEL
, "cxl-%s",
1363 dev_name(&afu
->dev
));
1364 if (!afu
->psl_irq_name
)
1367 if ((rc
= cxl_register_one_irq(afu
->adapter
, native_irq_multiplexed
,
1368 afu
, &afu
->native
->psl_hwirq
, &afu
->native
->psl_virq
,
1369 afu
->psl_irq_name
))) {
1370 kfree(afu
->psl_irq_name
);
1371 afu
->psl_irq_name
= NULL
;
1376 void cxl_native_release_psl_irq(struct cxl_afu
*afu
)
1378 if (afu
->native
->psl_virq
!= irq_find_mapping(NULL
, afu
->native
->psl_hwirq
))
1381 cxl_unmap_irq(afu
->native
->psl_virq
, afu
);
1382 cxl_ops
->release_one_irq(afu
->adapter
, afu
->native
->psl_hwirq
);
1383 kfree(afu
->psl_irq_name
);
1386 static void recover_psl_err(struct cxl_afu
*afu
, u64 errstat
)
1390 pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat
);
1392 /* Clear PSL_DSISR[PE] */
1393 dsisr
= cxl_p2n_read(afu
, CXL_PSL_DSISR_An
);
1394 cxl_p2n_write(afu
, CXL_PSL_DSISR_An
, dsisr
& ~CXL_PSL_DSISR_An_PE
);
1396 /* Write 1s to clear error status bits */
1397 cxl_p2n_write(afu
, CXL_PSL_ErrStat_An
, errstat
);
1400 static int native_ack_irq(struct cxl_context
*ctx
, u64 tfc
, u64 psl_reset_mask
)
1402 trace_cxl_psl_irq_ack(ctx
, tfc
);
1404 cxl_p2n_write(ctx
->afu
, CXL_PSL_TFC_An
, tfc
);
1406 recover_psl_err(ctx
->afu
, psl_reset_mask
);
1411 int cxl_check_error(struct cxl_afu
*afu
)
1413 return (cxl_p1n_read(afu
, CXL_PSL_SCNTL_An
) == ~0ULL);
1416 static bool native_support_attributes(const char *attr_name
,
1417 enum cxl_attrs type
)
1422 static int native_afu_cr_read64(struct cxl_afu
*afu
, int cr
, u64 off
, u64
*out
)
1424 if (unlikely(!cxl_ops
->link_ok(afu
->adapter
, afu
)))
1426 if (unlikely(off
>= afu
->crs_len
))
1428 *out
= in_le64(afu
->native
->afu_desc_mmio
+ afu
->crs_offset
+
1429 (cr
* afu
->crs_len
) + off
);
1433 static int native_afu_cr_read32(struct cxl_afu
*afu
, int cr
, u64 off
, u32
*out
)
1435 if (unlikely(!cxl_ops
->link_ok(afu
->adapter
, afu
)))
1437 if (unlikely(off
>= afu
->crs_len
))
1439 *out
= in_le32(afu
->native
->afu_desc_mmio
+ afu
->crs_offset
+
1440 (cr
* afu
->crs_len
) + off
);
1444 static int native_afu_cr_read16(struct cxl_afu
*afu
, int cr
, u64 off
, u16
*out
)
1446 u64 aligned_off
= off
& ~0x3L
;
1450 rc
= native_afu_cr_read32(afu
, cr
, aligned_off
, &val
);
1452 *out
= (val
>> ((off
& 0x3) * 8)) & 0xffff;
1456 static int native_afu_cr_read8(struct cxl_afu
*afu
, int cr
, u64 off
, u8
*out
)
1458 u64 aligned_off
= off
& ~0x3L
;
1462 rc
= native_afu_cr_read32(afu
, cr
, aligned_off
, &val
);
1464 *out
= (val
>> ((off
& 0x3) * 8)) & 0xff;
1468 static int native_afu_cr_write32(struct cxl_afu
*afu
, int cr
, u64 off
, u32 in
)
1470 if (unlikely(!cxl_ops
->link_ok(afu
->adapter
, afu
)))
1472 if (unlikely(off
>= afu
->crs_len
))
1474 out_le32(afu
->native
->afu_desc_mmio
+ afu
->crs_offset
+
1475 (cr
* afu
->crs_len
) + off
, in
);
1479 static int native_afu_cr_write16(struct cxl_afu
*afu
, int cr
, u64 off
, u16 in
)
1481 u64 aligned_off
= off
& ~0x3L
;
1482 u32 val32
, mask
, shift
;
1485 rc
= native_afu_cr_read32(afu
, cr
, aligned_off
, &val32
);
1488 shift
= (off
& 0x3) * 8;
1489 WARN_ON(shift
== 24);
1490 mask
= 0xffff << shift
;
1491 val32
= (val32
& ~mask
) | (in
<< shift
);
1493 rc
= native_afu_cr_write32(afu
, cr
, aligned_off
, val32
);
1497 static int native_afu_cr_write8(struct cxl_afu
*afu
, int cr
, u64 off
, u8 in
)
1499 u64 aligned_off
= off
& ~0x3L
;
1500 u32 val32
, mask
, shift
;
1503 rc
= native_afu_cr_read32(afu
, cr
, aligned_off
, &val32
);
1506 shift
= (off
& 0x3) * 8;
1507 mask
= 0xff << shift
;
1508 val32
= (val32
& ~mask
) | (in
<< shift
);
1510 rc
= native_afu_cr_write32(afu
, cr
, aligned_off
, val32
);
1514 const struct cxl_backend_ops cxl_native_ops
= {
1515 .module
= THIS_MODULE
,
1516 .adapter_reset
= cxl_pci_reset
,
1517 .alloc_one_irq
= cxl_pci_alloc_one_irq
,
1518 .release_one_irq
= cxl_pci_release_one_irq
,
1519 .alloc_irq_ranges
= cxl_pci_alloc_irq_ranges
,
1520 .release_irq_ranges
= cxl_pci_release_irq_ranges
,
1521 .setup_irq
= cxl_pci_setup_irq
,
1522 .handle_psl_slice_error
= native_handle_psl_slice_error
,
1523 .psl_interrupt
= NULL
,
1524 .ack_irq
= native_ack_irq
,
1525 .irq_wait
= native_irq_wait
,
1526 .attach_process
= native_attach_process
,
1527 .detach_process
= native_detach_process
,
1528 .update_ivtes
= native_update_ivtes
,
1529 .support_attributes
= native_support_attributes
,
1530 .link_ok
= cxl_adapter_link_ok
,
1531 .release_afu
= cxl_pci_release_afu
,
1532 .afu_read_err_buffer
= cxl_pci_afu_read_err_buffer
,
1533 .afu_check_and_enable
= native_afu_check_and_enable
,
1534 .afu_activate_mode
= native_afu_activate_mode
,
1535 .afu_deactivate_mode
= native_afu_deactivate_mode
,
1536 .afu_reset
= native_afu_reset
,
1537 .afu_cr_read8
= native_afu_cr_read8
,
1538 .afu_cr_read16
= native_afu_cr_read16
,
1539 .afu_cr_read32
= native_afu_cr_read32
,
1540 .afu_cr_read64
= native_afu_cr_read64
,
1541 .afu_cr_write8
= native_afu_cr_write8
,
1542 .afu_cr_write16
= native_afu_cr_write16
,
1543 .afu_cr_write32
= native_afu_cr_write32
,
1544 .read_adapter_vpd
= cxl_pci_read_adapter_vpd
,