]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/infiniband/hw/hfi1/chip.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/hfi1: Add VL XmitDiscards counters to the opapmaquery
[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / hw / hfi1 / chip.c
CommitLineData
77241056 1/*
05d6ac1d 2 * Copyright(c) 2015, 2016 Intel Corporation.
77241056
MM		 3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
77241056
MM		 9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
77241056
MM		 20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48/*
49 * This file contains all of the code that is specific to the HFI chip
50 */
51
52#include <linux/pci.h>
53#include <linux/delay.h>
54#include <linux/interrupt.h>
55#include <linux/module.h>
56
57#include "hfi.h"
58#include "trace.h"
59#include "mad.h"
60#include "pio.h"
61#include "sdma.h"
62#include "eprom.h"
5d9157aa 63#include "efivar.h"
8ebd4cf1 64#include "platform.h"
affa48de 65#include "aspm.h"
77241056
MM		 66
67#define NUM_IB_PORTS 1
68
69uint kdeth_qp;
70module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
71MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
72
73uint num_vls = HFI1_MAX_VLS_SUPPORTED;
74module_param(num_vls, uint, S_IRUGO);
75MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
76
77/*
78 * Default time to aggregate two 10K packets from the idle state
79 * (timer not running). The timer starts at the end of the first packet,
80 * so only the time for one 10K packet and header plus a bit extra is needed.
81 * 10 * 1024 + 64 header byte = 10304 byte
82 * 10304 byte / 12.5 GB/s = 824.32ns
83 */
84uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
85module_param(rcv_intr_timeout, uint, S_IRUGO);
86MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
87
88uint rcv_intr_count = 16; /* same as qib */
89module_param(rcv_intr_count, uint, S_IRUGO);
90MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
91
92ushort link_crc_mask = SUPPORTED_CRCS;
93module_param(link_crc_mask, ushort, S_IRUGO);
94MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
95
96uint loopback;
97module_param_named(loopback, loopback, uint, S_IRUGO);
98MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
99
100/* Other driver tunables */
101uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
102static ushort crc_14b_sideband = 1;
103static uint use_flr = 1;
104uint quick_linkup; /* skip LNI */
105
106struct flag_table {
107 u64 flag; /* the flag */
108 char *str; /* description string */
109 u16 extra; /* extra information */
110 u16 unused0;
111 u32 unused1;
112};
113
114/* str must be a string constant */
115#define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
116#define FLAG_ENTRY0(str, flag) {flag, str, 0}
117
118/* Send Error Consequences */
119#define SEC_WRITE_DROPPED 0x1
120#define SEC_PACKET_DROPPED 0x2
121#define SEC_SC_HALTED 0x4 /* per-context only */
122#define SEC_SPC_FREEZE 0x8 /* per-HFI only */
123
77241056 124#define MIN_KERNEL_KCTXTS 2
82c2611d 125#define FIRST_KERNEL_KCTXT 1
372cc85a
DL		 126/* sizes for both the QP and RSM map tables */
127#define NUM_MAP_ENTRIES 256
77241056
MM		 128#define NUM_MAP_REGS 32
129
130/* Bit offset into the GUID which carries HFI id information */
131#define GUID_HFI_INDEX_SHIFT 39
132
133/* extract the emulation revision */
134#define emulator_rev(dd) ((dd)->irev >> 8)
135/* parallel and serial emulation versions are 3 and 4 respectively */
136#define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
137#define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
138
139/* RSM fields */
140
141/* packet type */
142#define IB_PACKET_TYPE 2ull
143#define QW_SHIFT 6ull
144/* QPN[7..1] */
145#define QPN_WIDTH 7ull
146
147/* LRH.BTH: QW 0, OFFSET 48 - for match */
148#define LRH_BTH_QW 0ull
149#define LRH_BTH_BIT_OFFSET 48ull
150#define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
151#define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
152#define LRH_BTH_SELECT
153#define LRH_BTH_MASK 3ull
154#define LRH_BTH_VALUE 2ull
155
156/* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
157#define LRH_SC_QW 0ull
158#define LRH_SC_BIT_OFFSET 56ull
159#define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
160#define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
161#define LRH_SC_MASK 128ull
162#define LRH_SC_VALUE 0ull
163
164/* SC[n..0] QW 0, OFFSET 60 - for select */
165#define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
166
167/* QPN[m+n:1] QW 1, OFFSET 1 */
168#define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
169
170/* defines to build power on SC2VL table */
171#define SC2VL_VAL( \
172 num, \
173 sc0, sc0val, \
174 sc1, sc1val, \
175 sc2, sc2val, \
176 sc3, sc3val, \
177 sc4, sc4val, \
178 sc5, sc5val, \
179 sc6, sc6val, \
180 sc7, sc7val) \
181( \
182 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
183 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
184 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
185 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
186 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
187 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
188 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
189 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
190)
191
192#define DC_SC_VL_VAL( \
193 range, \
194 e0, e0val, \
195 e1, e1val, \
196 e2, e2val, \
197 e3, e3val, \
198 e4, e4val, \
199 e5, e5val, \
200 e6, e6val, \
201 e7, e7val, \
202 e8, e8val, \
203 e9, e9val, \
204 e10, e10val, \
205 e11, e11val, \
206 e12, e12val, \
207 e13, e13val, \
208 e14, e14val, \
209 e15, e15val) \
210( \
211 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
212 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
213 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
214 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
215 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
216 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
217 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
218 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
219 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
220 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
221 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
222 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
223 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
224 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
225 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
226 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
227)
228
229/* all CceStatus sub-block freeze bits */
230#define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
231 | CCE_STATUS_RXE_FROZE_SMASK \
232 | CCE_STATUS_TXE_FROZE_SMASK \
233 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
234/* all CceStatus sub-block TXE pause bits */
235#define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
236 | CCE_STATUS_TXE_PAUSED_SMASK \
237 | CCE_STATUS_SDMA_PAUSED_SMASK)
238/* all CceStatus sub-block RXE pause bits */
239#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
240
241/*
242 * CCE Error flags.
243 */
244static struct flag_table cce_err_status_flags[] = {
245/* 0*/ FLAG_ENTRY0("CceCsrParityErr",
246 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
247/* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr",
248 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
249/* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr",
250 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
251/* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
252 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
253/* 4*/ FLAG_ENTRY0("CceTrgtAccessErr",
254 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
255/* 5*/ FLAG_ENTRY0("CceRspdDataParityErr",
256 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
257/* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
258 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
259/* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr",
260 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
261/* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
262 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
263/* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
264 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
265/*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
266 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
267/*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
268 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
269/*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
270 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
271/*13*/ FLAG_ENTRY0("PcicRetryMemCorErr",
272 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
273/*14*/ FLAG_ENTRY0("PcicRetryMemCorErr",
274 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
275/*15*/ FLAG_ENTRY0("PcicPostHdQCorErr",
276 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
277/*16*/ FLAG_ENTRY0("PcicPostHdQCorErr",
278 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
279/*17*/ FLAG_ENTRY0("PcicPostHdQCorErr",
280 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
281/*18*/ FLAG_ENTRY0("PcicCplDatQCorErr",
282 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
283/*19*/ FLAG_ENTRY0("PcicNPostHQParityErr",
284 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
285/*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr",
286 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
287/*21*/ FLAG_ENTRY0("PcicRetryMemUncErr",
288 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
289/*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr",
290 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
291/*23*/ FLAG_ENTRY0("PcicPostHdQUncErr",
292 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
293/*24*/ FLAG_ENTRY0("PcicPostDatQUncErr",
294 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
295/*25*/ FLAG_ENTRY0("PcicCplHdQUncErr",
296 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
297/*26*/ FLAG_ENTRY0("PcicCplDatQUncErr",
298 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
299/*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr",
300 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
301/*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr",
302 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
303/*29*/ FLAG_ENTRY0("PcicReceiveParityErr",
304 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
305/*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr",
306 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
307/*31*/ FLAG_ENTRY0("LATriggered",
308 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
309/*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr",
310 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
311/*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr",
312 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
313/*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
314 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
315/*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
316 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
317/*36*/ FLAG_ENTRY0("CceMsixTableCorErr",
318 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
319/*37*/ FLAG_ENTRY0("CceMsixTableUncErr",
320 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
321/*38*/ FLAG_ENTRY0("CceIntMapCorErr",
322 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
323/*39*/ FLAG_ENTRY0("CceIntMapUncErr",
324 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
325/*40*/ FLAG_ENTRY0("CceMsixCsrParityErr",
326 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
327/*41-63 reserved*/
328};
329
330/*
331 * Misc Error flags
332 */
333#define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
334static struct flag_table misc_err_status_flags[] = {
335/* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
336/* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
337/* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
338/* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
339/* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
340/* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
341/* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
342/* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
343/* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
344/* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
345/*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
346/*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
347/*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
348};
349
350/*
351 * TXE PIO Error flags and consequences
352 */
353static struct flag_table pio_err_status_flags[] = {
354/* 0*/ FLAG_ENTRY("PioWriteBadCtxt",
355 SEC_WRITE_DROPPED,
356 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
357/* 1*/ FLAG_ENTRY("PioWriteAddrParity",
358 SEC_SPC_FREEZE,
359 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
360/* 2*/ FLAG_ENTRY("PioCsrParity",
361 SEC_SPC_FREEZE,
362 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
363/* 3*/ FLAG_ENTRY("PioSbMemFifo0",
364 SEC_SPC_FREEZE,
365 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
366/* 4*/ FLAG_ENTRY("PioSbMemFifo1",
367 SEC_SPC_FREEZE,
368 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
369/* 5*/ FLAG_ENTRY("PioPccFifoParity",
370 SEC_SPC_FREEZE,
371 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
372/* 6*/ FLAG_ENTRY("PioPecFifoParity",
373 SEC_SPC_FREEZE,
374 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
375/* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity",
376 SEC_SPC_FREEZE,
377 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
378/* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
379 SEC_SPC_FREEZE,
380 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
381/* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr",
382 SEC_SPC_FREEZE,
383 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
384/*10*/ FLAG_ENTRY("PioSmPktResetParity",
385 SEC_SPC_FREEZE,
386 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
387/*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc",
388 SEC_SPC_FREEZE,
389 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
390/*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc",
391 SEC_SPC_FREEZE,
392 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
393/*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor",
394 0,
395 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
396/*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor",
397 0,
398 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
399/*15*/ FLAG_ENTRY("PioCreditRetFifoParity",
400 SEC_SPC_FREEZE,
401 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
402/*16*/ FLAG_ENTRY("PioPpmcPblFifo",
403 SEC_SPC_FREEZE,
404 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
405/*17*/ FLAG_ENTRY("PioInitSmIn",
406 0,
407 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
408/*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm",
409 SEC_SPC_FREEZE,
410 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
411/*19*/ FLAG_ENTRY("PioHostAddrMemUnc",
412 SEC_SPC_FREEZE,
413 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
414/*20*/ FLAG_ENTRY("PioHostAddrMemCor",
415 0,
416 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
417/*21*/ FLAG_ENTRY("PioWriteDataParity",
418 SEC_SPC_FREEZE,
419 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
420/*22*/ FLAG_ENTRY("PioStateMachine",
421 SEC_SPC_FREEZE,
422 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
423/*23*/ FLAG_ENTRY("PioWriteQwValidParity",
8638b77f 424 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
77241056
MM		 425 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
426/*24*/ FLAG_ENTRY("PioBlockQwCountParity",
8638b77f 427 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
77241056
MM		 428 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
429/*25*/ FLAG_ENTRY("PioVlfVlLenParity",
430 SEC_SPC_FREEZE,
431 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
432/*26*/ FLAG_ENTRY("PioVlfSopParity",
433 SEC_SPC_FREEZE,
434 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
435/*27*/ FLAG_ENTRY("PioVlFifoParity",
436 SEC_SPC_FREEZE,
437 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
438/*28*/ FLAG_ENTRY("PioPpmcBqcMemParity",
439 SEC_SPC_FREEZE,
440 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
441/*29*/ FLAG_ENTRY("PioPpmcSopLen",
442 SEC_SPC_FREEZE,
443 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
444/*30-31 reserved*/
445/*32*/ FLAG_ENTRY("PioCurrentFreeCntParity",
446 SEC_SPC_FREEZE,
447 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
448/*33*/ FLAG_ENTRY("PioLastReturnedCntParity",
449 SEC_SPC_FREEZE,
450 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
451/*34*/ FLAG_ENTRY("PioPccSopHeadParity",
452 SEC_SPC_FREEZE,
453 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
454/*35*/ FLAG_ENTRY("PioPecSopHeadParityErr",
455 SEC_SPC_FREEZE,
456 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
457/*36-63 reserved*/
458};
459
460/* TXE PIO errors that cause an SPC freeze */
461#define ALL_PIO_FREEZE_ERR \
462 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
463 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
464 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
465 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
466 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
467 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
468 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
469 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
470 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
471 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
472 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
473 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
474 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
475 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
476 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
477 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
478 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
479 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
480 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
481 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
482 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
483 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
484 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
485 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
486 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
487 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
488 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
489 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
490 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
491
492/*
493 * TXE SDMA Error flags
494 */
495static struct flag_table sdma_err_status_flags[] = {
496/* 0*/ FLAG_ENTRY0("SDmaRpyTagErr",
497 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
498/* 1*/ FLAG_ENTRY0("SDmaCsrParityErr",
499 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
500/* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
501 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
502/* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
503 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
504/*04-63 reserved*/
505};
506
507/* TXE SDMA errors that cause an SPC freeze */
508#define ALL_SDMA_FREEZE_ERR \
509 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
510 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
511 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
512
69a00b8e
MM		 513/* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
514#define PORT_DISCARD_EGRESS_ERRS \
515 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
516 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
517 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
518
77241056
MM		 519/*
520 * TXE Egress Error flags
521 */
522#define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
523static struct flag_table egress_err_status_flags[] = {
524/* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
525/* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
526/* 2 reserved */
527/* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
528 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
529/* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
530/* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
531/* 6 reserved */
532/* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr",
533 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
534/* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
535 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
536/* 9-10 reserved */
537/*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
538 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
539/*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
540/*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
541/*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
542/*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
543/*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
544 SEES(TX_SDMA0_DISALLOWED_PACKET)),
545/*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
546 SEES(TX_SDMA1_DISALLOWED_PACKET)),
547/*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
548 SEES(TX_SDMA2_DISALLOWED_PACKET)),
549/*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
550 SEES(TX_SDMA3_DISALLOWED_PACKET)),
551/*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
552 SEES(TX_SDMA4_DISALLOWED_PACKET)),
553/*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
554 SEES(TX_SDMA5_DISALLOWED_PACKET)),
555/*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
556 SEES(TX_SDMA6_DISALLOWED_PACKET)),
557/*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
558 SEES(TX_SDMA7_DISALLOWED_PACKET)),
559/*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
560 SEES(TX_SDMA8_DISALLOWED_PACKET)),
561/*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
562 SEES(TX_SDMA9_DISALLOWED_PACKET)),
563/*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
564 SEES(TX_SDMA10_DISALLOWED_PACKET)),
565/*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
566 SEES(TX_SDMA11_DISALLOWED_PACKET)),
567/*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
568 SEES(TX_SDMA12_DISALLOWED_PACKET)),
569/*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
570 SEES(TX_SDMA13_DISALLOWED_PACKET)),
571/*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
572 SEES(TX_SDMA14_DISALLOWED_PACKET)),
573/*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
574 SEES(TX_SDMA15_DISALLOWED_PACKET)),
575/*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
576 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
577/*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
578 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
579/*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
580 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
581/*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
582 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
583/*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
584 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
585/*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
586 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
587/*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
588 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
589/*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
590 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
591/*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
592 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
593/*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
594/*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
595/*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
596/*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
597/*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
598/*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
599/*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
600/*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
601/*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
602/*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
603/*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
604/*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
605/*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
606/*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
607/*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
608/*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
609/*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
610/*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
611/*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
612/*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
613/*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
614/*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
615 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
616/*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
617 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
618};
619
620/*
621 * TXE Egress Error Info flags
622 */
623#define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
624static struct flag_table egress_err_info_flags[] = {
625/* 0*/ FLAG_ENTRY0("Reserved", 0ull),
626/* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)),
627/* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
628/* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
629/* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
630/* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
631/* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
632/* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
633/* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)),
634/* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
635/*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)),
636/*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
637/*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
638/*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
639/*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
640/*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
641/*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
642/*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
643/*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
644/*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
645/*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
646/*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
647};
648
649/* TXE Egress errors that cause an SPC freeze */
650#define ALL_TXE_EGRESS_FREEZE_ERR \
651 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
652 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
653 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
654 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
655 | SEES(TX_LAUNCH_CSR_PARITY) \
656 | SEES(TX_SBRD_CTL_CSR_PARITY) \
657 | SEES(TX_CONFIG_PARITY) \
658 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
659 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
660 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
661 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
662 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
663 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
664 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
665 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
666 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
667 | SEES(TX_CREDIT_RETURN_PARITY))
668
669/*
670 * TXE Send error flags
671 */
672#define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
673static struct flag_table send_err_status_flags[] = {
2c5b521a 674/* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
77241056
MM		 675/* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
676/* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
677};
678
679/*
680 * TXE Send Context Error flags and consequences
681 */
682static struct flag_table sc_err_status_flags[] = {
683/* 0*/ FLAG_ENTRY("InconsistentSop",
684 SEC_PACKET_DROPPED | SEC_SC_HALTED,
685 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
686/* 1*/ FLAG_ENTRY("DisallowedPacket",
687 SEC_PACKET_DROPPED | SEC_SC_HALTED,
688 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
689/* 2*/ FLAG_ENTRY("WriteCrossesBoundary",
690 SEC_WRITE_DROPPED | SEC_SC_HALTED,
691 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
692/* 3*/ FLAG_ENTRY("WriteOverflow",
693 SEC_WRITE_DROPPED | SEC_SC_HALTED,
694 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
695/* 4*/ FLAG_ENTRY("WriteOutOfBounds",
696 SEC_WRITE_DROPPED | SEC_SC_HALTED,
697 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
698/* 5-63 reserved*/
699};
700
701/*
702 * RXE Receive Error flags
703 */
704#define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
705static struct flag_table rxe_err_status_flags[] = {
706/* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
707/* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
708/* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
709/* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
710/* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
711/* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
712/* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
713/* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
714/* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
715/* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
716/*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
717/*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
718/*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
719/*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
720/*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
721/*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
722/*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
723 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
724/*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
725/*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
726/*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr",
727 RXES(RBUF_BLOCK_LIST_READ_UNC)),
728/*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr",
729 RXES(RBUF_BLOCK_LIST_READ_COR)),
730/*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
731 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
732/*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
733 RXES(RBUF_CSR_QENT_CNT_PARITY)),
734/*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
735 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
736/*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
737 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
738/*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
739/*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
740/*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
741 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
742/*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
743/*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
744/*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
745/*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
746/*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
747/*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
748/*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
749/*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
750 RXES(RBUF_FL_INITDONE_PARITY)),
751/*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
752 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
753/*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
754/*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
755/*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
756/*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
757 RXES(LOOKUP_DES_PART1_UNC_COR)),
758/*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr",
759 RXES(LOOKUP_DES_PART2_PARITY)),
760/*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
761/*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
762/*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
763/*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
764/*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
765/*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
766/*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
767/*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
768/*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
769/*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
770/*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
771/*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
772/*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
773/*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
774/*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
775/*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
776/*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
777/*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
778/*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
779/*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
780/*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
781/*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
782};
783
784/* RXE errors that will trigger an SPC freeze */
785#define ALL_RXE_FREEZE_ERR \
786 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
787 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
788 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
789 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
790 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
791 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
792 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
793 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
794 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
795 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
796 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
797 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
798 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
799 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
800 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
801 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
802 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
803 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
804 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
805 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
806 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
807 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
808 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
809 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
810 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
811 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
812 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
813 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
814 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
815 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
816 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
817 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
818 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
819 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
822 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
823 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
824 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
825 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
826 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
827 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
828 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
829 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
830
831#define RXE_FREEZE_ABORT_MASK \
832 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
833 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
834 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
835
836/*
837 * DCC Error Flags
838 */
839#define DCCE(name) DCC_ERR_FLG_##name##_SMASK
840static struct flag_table dcc_err_flags[] = {
841 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
842 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
843 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
844 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
845 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
846 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
847 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
848 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
849 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
850 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
851 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
852 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
853 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
854 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
855 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
856 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
857 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
858 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
859 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
860 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
861 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
862 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
863 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
864 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
865 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
866 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
867 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
868 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
869 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
870 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
871 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
872 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
873 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
874 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
875 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
876 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
877 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
878 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
879 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
880 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
881 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
882 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
883 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
884 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
885 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
886 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
887};
888
889/*
890 * LCB error flags
891 */
892#define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
893static struct flag_table lcb_err_flags[] = {
894/* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
895/* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
896/* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
897/* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
898 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
899/* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
900/* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
901/* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
902/* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
903/* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
904/* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
905/*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
906/*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
907/*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
908/*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
909 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
910/*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
911/*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
912/*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
913/*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
914/*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
915/*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
916 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
917/*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
918/*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
919/*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
920/*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
921/*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
922/*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
923/*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
924 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
925/*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
926/*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
927 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
928/*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
929 LCBE(REDUNDANT_FLIT_PARITY_ERR))
930};
931
932/*
933 * DC8051 Error Flags
934 */
935#define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
936static struct flag_table dc8051_err_flags[] = {
937 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
938 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
939 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
940 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
941 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
942 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
943 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
944 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
945 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
17fb4f29 946 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
77241056
MM		 947 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
948};
949
950/*
951 * DC8051 Information Error flags
952 *
953 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
954 */
955static struct flag_table dc8051_info_err_flags[] = {
956 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
957 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
958 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
959 FLAG_ENTRY0("Serdes internal loopback failure",
17fb4f29 960 FAILED_SERDES_INTERNAL_LOOPBACK),
77241056
MM		 961 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
962 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
963 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
964 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
965 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
966 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
967 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
8fefef12
JJ		 968 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT),
969 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT)
77241056
MM		 970};
971
972/*
973 * DC8051 Information Host Information flags
974 *
975 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
976 */
977static struct flag_table dc8051_info_host_msg_flags[] = {
978 FLAG_ENTRY0("Host request done", 0x0001),
979 FLAG_ENTRY0("BC SMA message", 0x0002),
980 FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
981 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
982 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
983 FLAG_ENTRY0("External device config request", 0x0020),
984 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
985 FLAG_ENTRY0("LinkUp achieved", 0x0080),
986 FLAG_ENTRY0("Link going down", 0x0100),
987};
988
77241056
MM		 989static u32 encoded_size(u32 size);
990static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
991static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
992static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
993 u8 *continuous);
994static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
995 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
996static void read_vc_remote_link_width(struct hfi1_devdata *dd,
997 u8 *remote_tx_rate, u16 *link_widths);
998static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
999 u8 *flag_bits, u16 *link_widths);
1000static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1001 u8 *device_rev);
1002static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed);
1003static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1004static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1005 u8 *tx_polarity_inversion,
1006 u8 *rx_polarity_inversion, u8 *max_rate);
1007static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1008 unsigned int context, u64 err_status);
1009static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1010static void handle_dcc_err(struct hfi1_devdata *dd,
1011 unsigned int context, u64 err_status);
1012static void handle_lcb_err(struct hfi1_devdata *dd,
1013 unsigned int context, u64 err_status);
1014static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1015static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1016static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1017static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1018static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1019static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1020static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1021static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1022static void set_partition_keys(struct hfi1_pportdata *);
1023static const char *link_state_name(u32 state);
1024static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1025 u32 state);
1026static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1027 u64 *out_data);
1028static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1029static int thermal_init(struct hfi1_devdata *dd);
1030
1031static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1032 int msecs);
1033static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
feb831dd 1034static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
77241056
MM		 1035static void handle_temp_err(struct hfi1_devdata *);
1036static void dc_shutdown(struct hfi1_devdata *);
1037static void dc_start(struct hfi1_devdata *);
8f000f7f
DL		 1038static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1039 unsigned int *np);
3ec5fa28 1040static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
77241056
MM		 1041
1042/*
1043 * Error interrupt table entry. This is used as input to the interrupt
1044 * "clear down" routine used for all second tier error interrupt register.
1045 * Second tier interrupt registers have a single bit representing them
1046 * in the top-level CceIntStatus.
1047 */
1048struct err_reg_info {
1049 u32 status; /* status CSR offset */
1050 u32 clear; /* clear CSR offset */
1051 u32 mask; /* mask CSR offset */
1052 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1053 const char *desc;
1054};
1055
1056#define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START)
1057#define NUM_DC_ERRS (IS_DC_END - IS_DC_START)
1058#define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START)
1059
1060/*
1061 * Helpers for building HFI and DC error interrupt table entries. Different
1062 * helpers are needed because of inconsistent register names.
1063 */
1064#define EE(reg, handler, desc) \
1065 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1066 handler, desc }
1067#define DC_EE1(reg, handler, desc) \
1068 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1069#define DC_EE2(reg, handler, desc) \
1070 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1071
1072/*
1073 * Table of the "misc" grouping of error interrupts. Each entry refers to
1074 * another register containing more information.
1075 */
1076static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1077/* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"),
1078/* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1079/* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"),
1080/* 3*/ { 0, 0, 0, NULL }, /* reserved */
1081/* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1082/* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1083/* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1084/* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr")
1085 /* the rest are reserved */
1086};
1087
1088/*
1089 * Index into the Various section of the interrupt sources
1090 * corresponding to the Critical Temperature interrupt.
1091 */
1092#define TCRIT_INT_SOURCE 4
1093
1094/*
1095 * SDMA error interrupt entry - refers to another register containing more
1096 * information.
1097 */
1098static const struct err_reg_info sdma_eng_err =
1099 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1100
1101static const struct err_reg_info various_err[NUM_VARIOUS] = {
1102/* 0*/ { 0, 0, 0, NULL }, /* PbcInt */
1103/* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */
1104/* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1105/* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1106/* 4*/ { 0, 0, 0, NULL }, /* TCritInt */
1107 /* rest are reserved */
1108};
1109
1110/*
1111 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1112 * register can not be derived from the MTU value because 10K is not
1113 * a power of 2. Therefore, we need a constant. Everything else can
1114 * be calculated.
1115 */
1116#define DCC_CFG_PORT_MTU_CAP_10240 7
1117
1118/*
1119 * Table of the DC grouping of error interrupts. Each entry refers to
1120 * another register containing more information.
1121 */
1122static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1123/* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1124/* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1125/* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1126/* 3*/ /* dc_lbm_int - special, see is_dc_int() */
1127 /* the rest are reserved */
1128};
1129
1130struct cntr_entry {
1131 /*
1132 * counter name
1133 */
1134 char *name;
1135
1136 /*
1137 * csr to read for name (if applicable)
1138 */
1139 u64 csr;
1140
1141 /*
1142 * offset into dd or ppd to store the counter's value
1143 */
1144 int offset;
1145
1146 /*
1147 * flags
1148 */
1149 u8 flags;
1150
1151 /*
1152 * accessor for stat element, context either dd or ppd
1153 */
17fb4f29
JJ		 1154 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1155 int mode, u64 data);
77241056
MM		 1156};
1157
1158#define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1159#define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1160
1161#define CNTR_ELEM(name, csr, offset, flags, accessor) \
1162{ \
1163 name, \
1164 csr, \
1165 offset, \
1166 flags, \
1167 accessor \
1168}
1169
1170/* 32bit RXE */
1171#define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1172CNTR_ELEM(#name, \
1173 (counter * 8 + RCV_COUNTER_ARRAY32), \
1174 0, flags | CNTR_32BIT, \
1175 port_access_u32_csr)
1176
1177#define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1178CNTR_ELEM(#name, \
1179 (counter * 8 + RCV_COUNTER_ARRAY32), \
1180 0, flags | CNTR_32BIT, \
1181 dev_access_u32_csr)
1182
1183/* 64bit RXE */
1184#define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1185CNTR_ELEM(#name, \
1186 (counter * 8 + RCV_COUNTER_ARRAY64), \
1187 0, flags, \
1188 port_access_u64_csr)
1189
1190#define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1191CNTR_ELEM(#name, \
1192 (counter * 8 + RCV_COUNTER_ARRAY64), \
1193 0, flags, \
1194 dev_access_u64_csr)
1195
1196#define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1197#define OVR_ELM(ctx) \
1198CNTR_ELEM("RcvHdrOvr" #ctx, \
8638b77f 1199 (RCV_HDR_OVFL_CNT + ctx * 0x100), \
77241056
MM		 1200 0, CNTR_NORMAL, port_access_u64_csr)
1201
1202/* 32bit TXE */
1203#define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1204CNTR_ELEM(#name, \
1205 (counter * 8 + SEND_COUNTER_ARRAY32), \
1206 0, flags | CNTR_32BIT, \
1207 port_access_u32_csr)
1208
1209/* 64bit TXE */
1210#define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1211CNTR_ELEM(#name, \
1212 (counter * 8 + SEND_COUNTER_ARRAY64), \
1213 0, flags, \
1214 port_access_u64_csr)
1215
1216# define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1217CNTR_ELEM(#name,\
1218 counter * 8 + SEND_COUNTER_ARRAY64, \
1219 0, \
1220 flags, \
1221 dev_access_u64_csr)
1222
1223/* CCE */
1224#define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1225CNTR_ELEM(#name, \
1226 (counter * 8 + CCE_COUNTER_ARRAY32), \
1227 0, flags | CNTR_32BIT, \
1228 dev_access_u32_csr)
1229
1230#define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1231CNTR_ELEM(#name, \
1232 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1233 0, flags | CNTR_32BIT, \
1234 dev_access_u32_csr)
1235
1236/* DC */
1237#define DC_PERF_CNTR(name, counter, flags) \
1238CNTR_ELEM(#name, \
1239 counter, \
1240 0, \
1241 flags, \
1242 dev_access_u64_csr)
1243
1244#define DC_PERF_CNTR_LCB(name, counter, flags) \
1245CNTR_ELEM(#name, \
1246 counter, \
1247 0, \
1248 flags, \
1249 dc_access_lcb_cntr)
1250
1251/* ibp counters */
1252#define SW_IBP_CNTR(name, cntr) \
1253CNTR_ELEM(#name, \
1254 0, \
1255 0, \
1256 CNTR_SYNTH, \
1257 access_ibp_##cntr)
1258
1259u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1260{
77241056 1261 if (dd->flags & HFI1_PRESENT) {
6d210eef 1262 return readq((void __iomem *)dd->kregbase + offset);
77241056
MM		 1263 }
1264 return -1;
1265}
1266
1267void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1268{
1269 if (dd->flags & HFI1_PRESENT)
1270 writeq(value, (void __iomem *)dd->kregbase + offset);
1271}
1272
1273void __iomem *get_csr_addr(
1274 struct hfi1_devdata *dd,
1275 u32 offset)
1276{
1277 return (void __iomem *)dd->kregbase + offset;
1278}
1279
1280static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1281 int mode, u64 value)
1282{
1283 u64 ret;
1284
77241056
MM		 1285 if (mode == CNTR_MODE_R) {
1286 ret = read_csr(dd, csr);
1287 } else if (mode == CNTR_MODE_W) {
1288 write_csr(dd, csr, value);
1289 ret = value;
1290 } else {
1291 dd_dev_err(dd, "Invalid cntr register access mode");
1292 return 0;
1293 }
1294
1295 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1296 return ret;
1297}
1298
1299/* Dev Access */
1300static u64 dev_access_u32_csr(const struct cntr_entry *entry,
17fb4f29 1301 void *context, int vl, int mode, u64 data)
77241056 1302{
a787bde8 1303 struct hfi1_devdata *dd = context;
a699c6c2 1304 u64 csr = entry->csr;
77241056 1305
a699c6c2
VM		 1306 if (entry->flags & CNTR_SDMA) {
1307 if (vl == CNTR_INVALID_VL)
1308 return 0;
1309 csr += 0x100 * vl;
1310 } else {
1311 if (vl != CNTR_INVALID_VL)
1312 return 0;
1313 }
1314 return read_write_csr(dd, csr, mode, data);
1315}
1316
1317static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1318 void *context, int idx, int mode, u64 data)
1319{
1320 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1321
1322 if (dd->per_sdma && idx < dd->num_sdma)
1323 return dd->per_sdma[idx].err_cnt;
1324 return 0;
1325}
1326
1327static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1328 void *context, int idx, int mode, u64 data)
1329{
1330 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1331
1332 if (dd->per_sdma && idx < dd->num_sdma)
1333 return dd->per_sdma[idx].sdma_int_cnt;
1334 return 0;
1335}
1336
1337static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1338 void *context, int idx, int mode, u64 data)
1339{
1340 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1341
1342 if (dd->per_sdma && idx < dd->num_sdma)
1343 return dd->per_sdma[idx].idle_int_cnt;
1344 return 0;
1345}
1346
1347static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1348 void *context, int idx, int mode,
1349 u64 data)
1350{
1351 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1352
1353 if (dd->per_sdma && idx < dd->num_sdma)
1354 return dd->per_sdma[idx].progress_int_cnt;
1355 return 0;
77241056
MM		 1356}
1357
1358static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
17fb4f29 1359 int vl, int mode, u64 data)
77241056 1360{
a787bde8 1361 struct hfi1_devdata *dd = context;
77241056
MM		 1362
1363 u64 val = 0;
1364 u64 csr = entry->csr;
1365
1366 if (entry->flags & CNTR_VL) {
1367 if (vl == CNTR_INVALID_VL)
1368 return 0;
1369 csr += 8 * vl;
1370 } else {
1371 if (vl != CNTR_INVALID_VL)
1372 return 0;
1373 }
1374
1375 val = read_write_csr(dd, csr, mode, data);
1376 return val;
1377}
1378
1379static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
17fb4f29 1380 int vl, int mode, u64 data)
77241056 1381{
a787bde8 1382 struct hfi1_devdata *dd = context;
77241056
MM		 1383 u32 csr = entry->csr;
1384 int ret = 0;
1385
1386 if (vl != CNTR_INVALID_VL)
1387 return 0;
1388 if (mode == CNTR_MODE_R)
1389 ret = read_lcb_csr(dd, csr, &data);
1390 else if (mode == CNTR_MODE_W)
1391 ret = write_lcb_csr(dd, csr, data);
1392
1393 if (ret) {
1394 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1395 return 0;
1396 }
1397
1398 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1399 return data;
1400}
1401
1402/* Port Access */
1403static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
17fb4f29 1404 int vl, int mode, u64 data)
77241056 1405{
a787bde8 1406 struct hfi1_pportdata *ppd = context;
77241056
MM		 1407
1408 if (vl != CNTR_INVALID_VL)
1409 return 0;
1410 return read_write_csr(ppd->dd, entry->csr, mode, data);
1411}
1412
1413static u64 port_access_u64_csr(const struct cntr_entry *entry,
17fb4f29 1414 void *context, int vl, int mode, u64 data)
77241056 1415{
a787bde8 1416 struct hfi1_pportdata *ppd = context;
77241056
MM		 1417 u64 val;
1418 u64 csr = entry->csr;
1419
1420 if (entry->flags & CNTR_VL) {
1421 if (vl == CNTR_INVALID_VL)
1422 return 0;
1423 csr += 8 * vl;
1424 } else {
1425 if (vl != CNTR_INVALID_VL)
1426 return 0;
1427 }
1428 val = read_write_csr(ppd->dd, csr, mode, data);
1429 return val;
1430}
1431
1432/* Software defined */
1433static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1434 u64 data)
1435{
1436 u64 ret;
1437
1438 if (mode == CNTR_MODE_R) {
1439 ret = *cntr;
1440 } else if (mode == CNTR_MODE_W) {
1441 *cntr = data;
1442 ret = data;
1443 } else {
1444 dd_dev_err(dd, "Invalid cntr sw access mode");
1445 return 0;
1446 }
1447
1448 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1449
1450 return ret;
1451}
1452
1453static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
17fb4f29 1454 int vl, int mode, u64 data)
77241056 1455{
a787bde8 1456 struct hfi1_pportdata *ppd = context;
77241056
MM		 1457
1458 if (vl != CNTR_INVALID_VL)
1459 return 0;
1460 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1461}
1462
1463static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
17fb4f29 1464 int vl, int mode, u64 data)
77241056 1465{
a787bde8 1466 struct hfi1_pportdata *ppd = context;
77241056
MM		 1467
1468 if (vl != CNTR_INVALID_VL)
1469 return 0;
1470 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1471}
1472
6d014530
DL		 1473static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1474 void *context, int vl, int mode,
1475 u64 data)
1476{
1477 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1478
1479 if (vl != CNTR_INVALID_VL)
1480 return 0;
1481 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1482}
1483
77241056 1484static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
17fb4f29 1485 void *context, int vl, int mode, u64 data)
77241056 1486{
69a00b8e
MM		 1487 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1488 u64 zero = 0;
1489 u64 *counter;
77241056 1490
69a00b8e
MM		 1491 if (vl == CNTR_INVALID_VL)
1492 counter = &ppd->port_xmit_discards;
1493 else if (vl >= 0 && vl < C_VL_COUNT)
1494 counter = &ppd->port_xmit_discards_vl[vl];
1495 else
1496 counter = &zero;
77241056 1497
69a00b8e 1498 return read_write_sw(ppd->dd, counter, mode, data);
77241056
MM		 1499}
1500
1501static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
17fb4f29
JJ		 1502 void *context, int vl, int mode,
1503 u64 data)
77241056 1504{
a787bde8 1505 struct hfi1_pportdata *ppd = context;
77241056
MM		 1506
1507 if (vl != CNTR_INVALID_VL)
1508 return 0;
1509
1510 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1511 mode, data);
1512}
1513
1514static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
17fb4f29 1515 void *context, int vl, int mode, u64 data)
77241056 1516{
a787bde8 1517 struct hfi1_pportdata *ppd = context;
77241056
MM		 1518
1519 if (vl != CNTR_INVALID_VL)
1520 return 0;
1521
1522 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1523 mode, data);
1524}
1525
1526u64 get_all_cpu_total(u64 __percpu *cntr)
1527{
1528 int cpu;
1529 u64 counter = 0;
1530
1531 for_each_possible_cpu(cpu)
1532 counter += *per_cpu_ptr(cntr, cpu);
1533 return counter;
1534}
1535
1536static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1537 u64 __percpu *cntr,
1538 int vl, int mode, u64 data)
1539{
77241056
MM		 1540 u64 ret = 0;
1541
1542 if (vl != CNTR_INVALID_VL)
1543 return 0;
1544
1545 if (mode == CNTR_MODE_R) {
1546 ret = get_all_cpu_total(cntr) - *z_val;
1547 } else if (mode == CNTR_MODE_W) {
1548 /* A write can only zero the counter */
1549 if (data == 0)
1550 *z_val = get_all_cpu_total(cntr);
1551 else
1552 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1553 } else {
1554 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1555 return 0;
1556 }
1557
1558 return ret;
1559}
1560
1561static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1562 void *context, int vl, int mode, u64 data)
1563{
a787bde8 1564 struct hfi1_devdata *dd = context;
77241056
MM		 1565
1566 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1567 mode, data);
1568}
1569
1570static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
17fb4f29 1571 void *context, int vl, int mode, u64 data)
77241056 1572{
a787bde8 1573 struct hfi1_devdata *dd = context;
77241056
MM		 1574
1575 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1576 mode, data);
1577}
1578
1579static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1580 void *context, int vl, int mode, u64 data)
1581{
a787bde8 1582 struct hfi1_devdata *dd = context;
77241056
MM		 1583
1584 return dd->verbs_dev.n_piowait;
1585}
1586
14553ca1
MM		 1587static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1588 void *context, int vl, int mode, u64 data)
1589{
1590 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1591
1592 return dd->verbs_dev.n_piodrain;
1593}
1594
77241056
MM		 1595static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1596 void *context, int vl, int mode, u64 data)
1597{
a787bde8 1598 struct hfi1_devdata *dd = context;
77241056
MM		 1599
1600 return dd->verbs_dev.n_txwait;
1601}
1602
1603static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1604 void *context, int vl, int mode, u64 data)
1605{
a787bde8 1606 struct hfi1_devdata *dd = context;
77241056
MM		 1607
1608 return dd->verbs_dev.n_kmem_wait;
1609}
1610
b421922e 1611static u64 access_sw_send_schedule(const struct cntr_entry *entry,
17fb4f29 1612 void *context, int vl, int mode, u64 data)
b421922e
DL		 1613{
1614 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1615
89abfc8d
VM		 1616 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1617 mode, data);
b421922e
DL		 1618}
1619
2c5b521a
JR		 1620/* Software counters for the error status bits within MISC_ERR_STATUS */
1621static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1622 void *context, int vl, int mode,
1623 u64 data)
1624{
1625 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1626
1627 return dd->misc_err_status_cnt[12];
1628}
1629
1630static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1631 void *context, int vl, int mode,
1632 u64 data)
1633{
1634 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1635
1636 return dd->misc_err_status_cnt[11];
1637}
1638
1639static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1640 void *context, int vl, int mode,
1641 u64 data)
1642{
1643 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1644
1645 return dd->misc_err_status_cnt[10];
1646}
1647
1648static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1649 void *context, int vl,
1650 int mode, u64 data)
1651{
1652 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1653
1654 return dd->misc_err_status_cnt[9];
1655}
1656
1657static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1658 void *context, int vl, int mode,
1659 u64 data)
1660{
1661 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1662
1663 return dd->misc_err_status_cnt[8];
1664}
1665
1666static u64 access_misc_efuse_read_bad_addr_err_cnt(
1667 const struct cntr_entry *entry,
1668 void *context, int vl, int mode, u64 data)
1669{
1670 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1671
1672 return dd->misc_err_status_cnt[7];
1673}
1674
1675static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1676 void *context, int vl,
1677 int mode, u64 data)
1678{
1679 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1680
1681 return dd->misc_err_status_cnt[6];
1682}
1683
1684static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1685 void *context, int vl, int mode,
1686 u64 data)
1687{
1688 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1689
1690 return dd->misc_err_status_cnt[5];
1691}
1692
1693static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1694 void *context, int vl, int mode,
1695 u64 data)
1696{
1697 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1698
1699 return dd->misc_err_status_cnt[4];
1700}
1701
1702static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1703 void *context, int vl,
1704 int mode, u64 data)
1705{
1706 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1707
1708 return dd->misc_err_status_cnt[3];
1709}
1710
1711static u64 access_misc_csr_write_bad_addr_err_cnt(
1712 const struct cntr_entry *entry,
1713 void *context, int vl, int mode, u64 data)
1714{
1715 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1716
1717 return dd->misc_err_status_cnt[2];
1718}
1719
1720static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1721 void *context, int vl,
1722 int mode, u64 data)
1723{
1724 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1725
1726 return dd->misc_err_status_cnt[1];
1727}
1728
1729static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1730 void *context, int vl, int mode,
1731 u64 data)
1732{
1733 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1734
1735 return dd->misc_err_status_cnt[0];
1736}
1737
1738/*
1739 * Software counter for the aggregate of
1740 * individual CceErrStatus counters
1741 */
1742static u64 access_sw_cce_err_status_aggregated_cnt(
1743 const struct cntr_entry *entry,
1744 void *context, int vl, int mode, u64 data)
1745{
1746 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1747
1748 return dd->sw_cce_err_status_aggregate;
1749}
1750
1751/*
1752 * Software counters corresponding to each of the
1753 * error status bits within CceErrStatus
1754 */
1755static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1756 void *context, int vl, int mode,
1757 u64 data)
1758{
1759 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1760
1761 return dd->cce_err_status_cnt[40];
1762}
1763
1764static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1765 void *context, int vl, int mode,
1766 u64 data)
1767{
1768 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1769
1770 return dd->cce_err_status_cnt[39];
1771}
1772
1773static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1774 void *context, int vl, int mode,
1775 u64 data)
1776{
1777 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1778
1779 return dd->cce_err_status_cnt[38];
1780}
1781
1782static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1783 void *context, int vl, int mode,
1784 u64 data)
1785{
1786 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1787
1788 return dd->cce_err_status_cnt[37];
1789}
1790
1791static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1792 void *context, int vl, int mode,
1793 u64 data)
1794{
1795 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1796
1797 return dd->cce_err_status_cnt[36];
1798}
1799
1800static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1801 const struct cntr_entry *entry,
1802 void *context, int vl, int mode, u64 data)
1803{
1804 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1805
1806 return dd->cce_err_status_cnt[35];
1807}
1808
1809static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1810 const struct cntr_entry *entry,
1811 void *context, int vl, int mode, u64 data)
1812{
1813 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1814
1815 return dd->cce_err_status_cnt[34];
1816}
1817
1818static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1819 void *context, int vl,
1820 int mode, u64 data)
1821{
1822 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1823
1824 return dd->cce_err_status_cnt[33];
1825}
1826
1827static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1828 void *context, int vl, int mode,
1829 u64 data)
1830{
1831 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1832
1833 return dd->cce_err_status_cnt[32];
1834}
1835
1836static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1837 void *context, int vl, int mode, u64 data)
1838{
1839 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1840
1841 return dd->cce_err_status_cnt[31];
1842}
1843
1844static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1845 void *context, int vl, int mode,
1846 u64 data)
1847{
1848 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1849
1850 return dd->cce_err_status_cnt[30];
1851}
1852
1853static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1854 void *context, int vl, int mode,
1855 u64 data)
1856{
1857 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1858
1859 return dd->cce_err_status_cnt[29];
1860}
1861
1862static u64 access_pcic_transmit_back_parity_err_cnt(
1863 const struct cntr_entry *entry,
1864 void *context, int vl, int mode, u64 data)
1865{
1866 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1867
1868 return dd->cce_err_status_cnt[28];
1869}
1870
1871static u64 access_pcic_transmit_front_parity_err_cnt(
1872 const struct cntr_entry *entry,
1873 void *context, int vl, int mode, u64 data)
1874{
1875 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1876
1877 return dd->cce_err_status_cnt[27];
1878}
1879
1880static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1881 void *context, int vl, int mode,
1882 u64 data)
1883{
1884 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1885
1886 return dd->cce_err_status_cnt[26];
1887}
1888
1889static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1890 void *context, int vl, int mode,
1891 u64 data)
1892{
1893 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1894
1895 return dd->cce_err_status_cnt[25];
1896}
1897
1898static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1899 void *context, int vl, int mode,
1900 u64 data)
1901{
1902 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1903
1904 return dd->cce_err_status_cnt[24];
1905}
1906
1907static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1908 void *context, int vl, int mode,
1909 u64 data)
1910{
1911 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1912
1913 return dd->cce_err_status_cnt[23];
1914}
1915
1916static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
1917 void *context, int vl,
1918 int mode, u64 data)
1919{
1920 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1921
1922 return dd->cce_err_status_cnt[22];
1923}
1924
1925static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
1926 void *context, int vl, int mode,
1927 u64 data)
1928{
1929 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1930
1931 return dd->cce_err_status_cnt[21];
1932}
1933
1934static u64 access_pcic_n_post_dat_q_parity_err_cnt(
1935 const struct cntr_entry *entry,
1936 void *context, int vl, int mode, u64 data)
1937{
1938 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1939
1940 return dd->cce_err_status_cnt[20];
1941}
1942
1943static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
1944 void *context, int vl,
1945 int mode, u64 data)
1946{
1947 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1948
1949 return dd->cce_err_status_cnt[19];
1950}
1951
1952static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1953 void *context, int vl, int mode,
1954 u64 data)
1955{
1956 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1957
1958 return dd->cce_err_status_cnt[18];
1959}
1960
1961static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1962 void *context, int vl, int mode,
1963 u64 data)
1964{
1965 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1966
1967 return dd->cce_err_status_cnt[17];
1968}
1969
1970static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1971 void *context, int vl, int mode,
1972 u64 data)
1973{
1974 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1975
1976 return dd->cce_err_status_cnt[16];
1977}
1978
1979static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1980 void *context, int vl, int mode,
1981 u64 data)
1982{
1983 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1984
1985 return dd->cce_err_status_cnt[15];
1986}
1987
1988static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
1989 void *context, int vl,
1990 int mode, u64 data)
1991{
1992 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1993
1994 return dd->cce_err_status_cnt[14];
1995}
1996
1997static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
1998 void *context, int vl, int mode,
1999 u64 data)
2000{
2001 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2002
2003 return dd->cce_err_status_cnt[13];
2004}
2005
2006static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2007 const struct cntr_entry *entry,
2008 void *context, int vl, int mode, u64 data)
2009{
2010 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2011
2012 return dd->cce_err_status_cnt[12];
2013}
2014
2015static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2016 const struct cntr_entry *entry,
2017 void *context, int vl, int mode, u64 data)
2018{
2019 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2020
2021 return dd->cce_err_status_cnt[11];
2022}
2023
2024static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2025 const struct cntr_entry *entry,
2026 void *context, int vl, int mode, u64 data)
2027{
2028 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2029
2030 return dd->cce_err_status_cnt[10];
2031}
2032
2033static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2034 const struct cntr_entry *entry,
2035 void *context, int vl, int mode, u64 data)
2036{
2037 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2038
2039 return dd->cce_err_status_cnt[9];
2040}
2041
2042static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2043 const struct cntr_entry *entry,
2044 void *context, int vl, int mode, u64 data)
2045{
2046 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2047
2048 return dd->cce_err_status_cnt[8];
2049}
2050
2051static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2052 void *context, int vl,
2053 int mode, u64 data)
2054{
2055 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2056
2057 return dd->cce_err_status_cnt[7];
2058}
2059
2060static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2061 const struct cntr_entry *entry,
2062 void *context, int vl, int mode, u64 data)
2063{
2064 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2065
2066 return dd->cce_err_status_cnt[6];
2067}
2068
2069static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2070 void *context, int vl, int mode,
2071 u64 data)
2072{
2073 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2074
2075 return dd->cce_err_status_cnt[5];
2076}
2077
2078static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2079 void *context, int vl, int mode,
2080 u64 data)
2081{
2082 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2083
2084 return dd->cce_err_status_cnt[4];
2085}
2086
2087static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2088 const struct cntr_entry *entry,
2089 void *context, int vl, int mode, u64 data)
2090{
2091 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2092
2093 return dd->cce_err_status_cnt[3];
2094}
2095
2096static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2097 void *context, int vl,
2098 int mode, u64 data)
2099{
2100 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2101
2102 return dd->cce_err_status_cnt[2];
2103}
2104
2105static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2106 void *context, int vl,
2107 int mode, u64 data)
2108{
2109 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2110
2111 return dd->cce_err_status_cnt[1];
2112}
2113
2114static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2115 void *context, int vl, int mode,
2116 u64 data)
2117{
2118 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2119
2120 return dd->cce_err_status_cnt[0];
2121}
2122
2123/*
2124 * Software counters corresponding to each of the
2125 * error status bits within RcvErrStatus
2126 */
2127static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2128 void *context, int vl, int mode,
2129 u64 data)
2130{
2131 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2132
2133 return dd->rcv_err_status_cnt[63];
2134}
2135
2136static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2137 void *context, int vl,
2138 int mode, u64 data)
2139{
2140 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2141
2142 return dd->rcv_err_status_cnt[62];
2143}
2144
2145static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2146 void *context, int vl, int mode,
2147 u64 data)
2148{
2149 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2150
2151 return dd->rcv_err_status_cnt[61];
2152}
2153
2154static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2155 void *context, int vl, int mode,
2156 u64 data)
2157{
2158 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2159
2160 return dd->rcv_err_status_cnt[60];
2161}
2162
2163static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2164 void *context, int vl,
2165 int mode, u64 data)
2166{
2167 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2168
2169 return dd->rcv_err_status_cnt[59];
2170}
2171
2172static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2173 void *context, int vl,
2174 int mode, u64 data)
2175{
2176 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2177
2178 return dd->rcv_err_status_cnt[58];
2179}
2180
2181static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2182 void *context, int vl, int mode,
2183 u64 data)
2184{
2185 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2186
2187 return dd->rcv_err_status_cnt[57];
2188}
2189
2190static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2191 void *context, int vl, int mode,
2192 u64 data)
2193{
2194 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2195
2196 return dd->rcv_err_status_cnt[56];
2197}
2198
2199static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2200 void *context, int vl, int mode,
2201 u64 data)
2202{
2203 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2204
2205 return dd->rcv_err_status_cnt[55];
2206}
2207
2208static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2209 const struct cntr_entry *entry,
2210 void *context, int vl, int mode, u64 data)
2211{
2212 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2213
2214 return dd->rcv_err_status_cnt[54];
2215}
2216
2217static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2218 const struct cntr_entry *entry,
2219 void *context, int vl, int mode, u64 data)
2220{
2221 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2222
2223 return dd->rcv_err_status_cnt[53];
2224}
2225
2226static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2227 void *context, int vl,
2228 int mode, u64 data)
2229{
2230 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2231
2232 return dd->rcv_err_status_cnt[52];
2233}
2234
2235static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2236 void *context, int vl,
2237 int mode, u64 data)
2238{
2239 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2240
2241 return dd->rcv_err_status_cnt[51];
2242}
2243
2244static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2245 void *context, int vl,
2246 int mode, u64 data)
2247{
2248 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2249
2250 return dd->rcv_err_status_cnt[50];
2251}
2252
2253static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2254 void *context, int vl,
2255 int mode, u64 data)
2256{
2257 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2258
2259 return dd->rcv_err_status_cnt[49];
2260}
2261
2262static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2263 void *context, int vl,
2264 int mode, u64 data)
2265{
2266 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2267
2268 return dd->rcv_err_status_cnt[48];
2269}
2270
2271static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2272 void *context, int vl,
2273 int mode, u64 data)
2274{
2275 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2276
2277 return dd->rcv_err_status_cnt[47];
2278}
2279
2280static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2281 void *context, int vl, int mode,
2282 u64 data)
2283{
2284 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2285
2286 return dd->rcv_err_status_cnt[46];
2287}
2288
2289static u64 access_rx_hq_intr_csr_parity_err_cnt(
2290 const struct cntr_entry *entry,
2291 void *context, int vl, int mode, u64 data)
2292{
2293 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2294
2295 return dd->rcv_err_status_cnt[45];
2296}
2297
2298static u64 access_rx_lookup_csr_parity_err_cnt(
2299 const struct cntr_entry *entry,
2300 void *context, int vl, int mode, u64 data)
2301{
2302 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2303
2304 return dd->rcv_err_status_cnt[44];
2305}
2306
2307static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2308 const struct cntr_entry *entry,
2309 void *context, int vl, int mode, u64 data)
2310{
2311 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2312
2313 return dd->rcv_err_status_cnt[43];
2314}
2315
2316static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2317 const struct cntr_entry *entry,
2318 void *context, int vl, int mode, u64 data)
2319{
2320 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2321
2322 return dd->rcv_err_status_cnt[42];
2323}
2324
2325static u64 access_rx_lookup_des_part2_parity_err_cnt(
2326 const struct cntr_entry *entry,
2327 void *context, int vl, int mode, u64 data)
2328{
2329 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2330
2331 return dd->rcv_err_status_cnt[41];
2332}
2333
2334static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2335 const struct cntr_entry *entry,
2336 void *context, int vl, int mode, u64 data)
2337{
2338 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2339
2340 return dd->rcv_err_status_cnt[40];
2341}
2342
2343static u64 access_rx_lookup_des_part1_unc_err_cnt(
2344 const struct cntr_entry *entry,
2345 void *context, int vl, int mode, u64 data)
2346{
2347 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2348
2349 return dd->rcv_err_status_cnt[39];
2350}
2351
2352static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2353 const struct cntr_entry *entry,
2354 void *context, int vl, int mode, u64 data)
2355{
2356 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2357
2358 return dd->rcv_err_status_cnt[38];
2359}
2360
2361static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2362 const struct cntr_entry *entry,
2363 void *context, int vl, int mode, u64 data)
2364{
2365 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2366
2367 return dd->rcv_err_status_cnt[37];
2368}
2369
2370static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2371 const struct cntr_entry *entry,
2372 void *context, int vl, int mode, u64 data)
2373{
2374 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2375
2376 return dd->rcv_err_status_cnt[36];
2377}
2378
2379static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2380 const struct cntr_entry *entry,
2381 void *context, int vl, int mode, u64 data)
2382{
2383 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2384
2385 return dd->rcv_err_status_cnt[35];
2386}
2387
2388static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2389 const struct cntr_entry *entry,
2390 void *context, int vl, int mode, u64 data)
2391{
2392 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2393
2394 return dd->rcv_err_status_cnt[34];
2395}
2396
2397static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2398 const struct cntr_entry *entry,
2399 void *context, int vl, int mode, u64 data)
2400{
2401 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2402
2403 return dd->rcv_err_status_cnt[33];
2404}
2405
2406static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2407 void *context, int vl, int mode,
2408 u64 data)
2409{
2410 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2411
2412 return dd->rcv_err_status_cnt[32];
2413}
2414
2415static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2416 void *context, int vl, int mode,
2417 u64 data)
2418{
2419 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2420
2421 return dd->rcv_err_status_cnt[31];
2422}
2423
2424static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2425 void *context, int vl, int mode,
2426 u64 data)
2427{
2428 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2429
2430 return dd->rcv_err_status_cnt[30];
2431}
2432
2433static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2434 void *context, int vl, int mode,
2435 u64 data)
2436{
2437 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2438
2439 return dd->rcv_err_status_cnt[29];
2440}
2441
2442static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2443 void *context, int vl,
2444 int mode, u64 data)
2445{
2446 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2447
2448 return dd->rcv_err_status_cnt[28];
2449}
2450
2451static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2452 const struct cntr_entry *entry,
2453 void *context, int vl, int mode, u64 data)
2454{
2455 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2456
2457 return dd->rcv_err_status_cnt[27];
2458}
2459
2460static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2461 const struct cntr_entry *entry,
2462 void *context, int vl, int mode, u64 data)
2463{
2464 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2465
2466 return dd->rcv_err_status_cnt[26];
2467}
2468
2469static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2470 const struct cntr_entry *entry,
2471 void *context, int vl, int mode, u64 data)
2472{
2473 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2474
2475 return dd->rcv_err_status_cnt[25];
2476}
2477
2478static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2479 const struct cntr_entry *entry,
2480 void *context, int vl, int mode, u64 data)
2481{
2482 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2483
2484 return dd->rcv_err_status_cnt[24];
2485}
2486
2487static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2488 const struct cntr_entry *entry,
2489 void *context, int vl, int mode, u64 data)
2490{
2491 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2492
2493 return dd->rcv_err_status_cnt[23];
2494}
2495
2496static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2497 const struct cntr_entry *entry,
2498 void *context, int vl, int mode, u64 data)
2499{
2500 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2501
2502 return dd->rcv_err_status_cnt[22];
2503}
2504
2505static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2506 const struct cntr_entry *entry,
2507 void *context, int vl, int mode, u64 data)
2508{
2509 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2510
2511 return dd->rcv_err_status_cnt[21];
2512}
2513
2514static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2515 const struct cntr_entry *entry,
2516 void *context, int vl, int mode, u64 data)
2517{
2518 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2519
2520 return dd->rcv_err_status_cnt[20];
2521}
2522
2523static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2524 const struct cntr_entry *entry,
2525 void *context, int vl, int mode, u64 data)
2526{
2527 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2528
2529 return dd->rcv_err_status_cnt[19];
2530}
2531
2532static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2533 void *context, int vl,
2534 int mode, u64 data)
2535{
2536 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2537
2538 return dd->rcv_err_status_cnt[18];
2539}
2540
2541static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2542 void *context, int vl,
2543 int mode, u64 data)
2544{
2545 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2546
2547 return dd->rcv_err_status_cnt[17];
2548}
2549
2550static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2551 const struct cntr_entry *entry,
2552 void *context, int vl, int mode, u64 data)
2553{
2554 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2555
2556 return dd->rcv_err_status_cnt[16];
2557}
2558
2559static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2560 const struct cntr_entry *entry,
2561 void *context, int vl, int mode, u64 data)
2562{
2563 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2564
2565 return dd->rcv_err_status_cnt[15];
2566}
2567
2568static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2569 void *context, int vl,
2570 int mode, u64 data)
2571{
2572 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2573
2574 return dd->rcv_err_status_cnt[14];
2575}
2576
2577static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2578 void *context, int vl,
2579 int mode, u64 data)
2580{
2581 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2582
2583 return dd->rcv_err_status_cnt[13];
2584}
2585
2586static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2587 void *context, int vl, int mode,
2588 u64 data)
2589{
2590 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2591
2592 return dd->rcv_err_status_cnt[12];
2593}
2594
2595static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2596 void *context, int vl, int mode,
2597 u64 data)
2598{
2599 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2600
2601 return dd->rcv_err_status_cnt[11];
2602}
2603
2604static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2605 void *context, int vl, int mode,
2606 u64 data)
2607{
2608 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2609
2610 return dd->rcv_err_status_cnt[10];
2611}
2612
2613static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2614 void *context, int vl, int mode,
2615 u64 data)
2616{
2617 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2618
2619 return dd->rcv_err_status_cnt[9];
2620}
2621
2622static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2623 void *context, int vl, int mode,
2624 u64 data)
2625{
2626 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2627
2628 return dd->rcv_err_status_cnt[8];
2629}
2630
2631static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2632 const struct cntr_entry *entry,
2633 void *context, int vl, int mode, u64 data)
2634{
2635 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2636
2637 return dd->rcv_err_status_cnt[7];
2638}
2639
2640static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2641 const struct cntr_entry *entry,
2642 void *context, int vl, int mode, u64 data)
2643{
2644 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2645
2646 return dd->rcv_err_status_cnt[6];
2647}
2648
2649static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2650 void *context, int vl, int mode,
2651 u64 data)
2652{
2653 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2654
2655 return dd->rcv_err_status_cnt[5];
2656}
2657
2658static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2659 void *context, int vl, int mode,
2660 u64 data)
2661{
2662 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2663
2664 return dd->rcv_err_status_cnt[4];
2665}
2666
2667static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2668 void *context, int vl, int mode,
2669 u64 data)
2670{
2671 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2672
2673 return dd->rcv_err_status_cnt[3];
2674}
2675
2676static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2677 void *context, int vl, int mode,
2678 u64 data)
2679{
2680 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2681
2682 return dd->rcv_err_status_cnt[2];
2683}
2684
2685static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2686 void *context, int vl, int mode,
2687 u64 data)
2688{
2689 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2690
2691 return dd->rcv_err_status_cnt[1];
2692}
2693
2694static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2695 void *context, int vl, int mode,
2696 u64 data)
2697{
2698 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2699
2700 return dd->rcv_err_status_cnt[0];
2701}
2702
2703/*
2704 * Software counters corresponding to each of the
2705 * error status bits within SendPioErrStatus
2706 */
2707static u64 access_pio_pec_sop_head_parity_err_cnt(
2708 const struct cntr_entry *entry,
2709 void *context, int vl, int mode, u64 data)
2710{
2711 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2712
2713 return dd->send_pio_err_status_cnt[35];
2714}
2715
2716static u64 access_pio_pcc_sop_head_parity_err_cnt(
2717 const struct cntr_entry *entry,
2718 void *context, int vl, int mode, u64 data)
2719{
2720 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2721
2722 return dd->send_pio_err_status_cnt[34];
2723}
2724
2725static u64 access_pio_last_returned_cnt_parity_err_cnt(
2726 const struct cntr_entry *entry,
2727 void *context, int vl, int mode, u64 data)
2728{
2729 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2730
2731 return dd->send_pio_err_status_cnt[33];
2732}
2733
2734static u64 access_pio_current_free_cnt_parity_err_cnt(
2735 const struct cntr_entry *entry,
2736 void *context, int vl, int mode, u64 data)
2737{
2738 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2739
2740 return dd->send_pio_err_status_cnt[32];
2741}
2742
2743static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2744 void *context, int vl, int mode,
2745 u64 data)
2746{
2747 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2748
2749 return dd->send_pio_err_status_cnt[31];
2750}
2751
2752static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2753 void *context, int vl, int mode,
2754 u64 data)
2755{
2756 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2757
2758 return dd->send_pio_err_status_cnt[30];
2759}
2760
2761static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2762 void *context, int vl, int mode,
2763 u64 data)
2764{
2765 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2766
2767 return dd->send_pio_err_status_cnt[29];
2768}
2769
2770static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2771 const struct cntr_entry *entry,
2772 void *context, int vl, int mode, u64 data)
2773{
2774 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2775
2776 return dd->send_pio_err_status_cnt[28];
2777}
2778
2779static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2780 void *context, int vl, int mode,
2781 u64 data)
2782{
2783 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2784
2785 return dd->send_pio_err_status_cnt[27];
2786}
2787
2788static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2789 void *context, int vl, int mode,
2790 u64 data)
2791{
2792 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2793
2794 return dd->send_pio_err_status_cnt[26];
2795}
2796
2797static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2798 void *context, int vl,
2799 int mode, u64 data)
2800{
2801 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2802
2803 return dd->send_pio_err_status_cnt[25];
2804}
2805
2806static u64 access_pio_block_qw_count_parity_err_cnt(
2807 const struct cntr_entry *entry,
2808 void *context, int vl, int mode, u64 data)
2809{
2810 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2811
2812 return dd->send_pio_err_status_cnt[24];
2813}
2814
2815static u64 access_pio_write_qw_valid_parity_err_cnt(
2816 const struct cntr_entry *entry,
2817 void *context, int vl, int mode, u64 data)
2818{
2819 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2820
2821 return dd->send_pio_err_status_cnt[23];
2822}
2823
2824static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2825 void *context, int vl, int mode,
2826 u64 data)
2827{
2828 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2829
2830 return dd->send_pio_err_status_cnt[22];
2831}
2832
2833static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2834 void *context, int vl,
2835 int mode, u64 data)
2836{
2837 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2838
2839 return dd->send_pio_err_status_cnt[21];
2840}
2841
2842static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2843 void *context, int vl,
2844 int mode, u64 data)
2845{
2846 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2847
2848 return dd->send_pio_err_status_cnt[20];
2849}
2850
2851static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2852 void *context, int vl,
2853 int mode, u64 data)
2854{
2855 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2856
2857 return dd->send_pio_err_status_cnt[19];
2858}
2859
2860static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2861 const struct cntr_entry *entry,
2862 void *context, int vl, int mode, u64 data)
2863{
2864 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2865
2866 return dd->send_pio_err_status_cnt[18];
2867}
2868
2869static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2870 void *context, int vl, int mode,
2871 u64 data)
2872{
2873 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2874
2875 return dd->send_pio_err_status_cnt[17];
2876}
2877
2878static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2879 void *context, int vl, int mode,
2880 u64 data)
2881{
2882 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2883
2884 return dd->send_pio_err_status_cnt[16];
2885}
2886
2887static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2888 const struct cntr_entry *entry,
2889 void *context, int vl, int mode, u64 data)
2890{
2891 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2892
2893 return dd->send_pio_err_status_cnt[15];
2894}
2895
2896static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2897 const struct cntr_entry *entry,
2898 void *context, int vl, int mode, u64 data)
2899{
2900 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2901
2902 return dd->send_pio_err_status_cnt[14];
2903}
2904
2905static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2906 const struct cntr_entry *entry,
2907 void *context, int vl, int mode, u64 data)
2908{
2909 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2910
2911 return dd->send_pio_err_status_cnt[13];
2912}
2913
2914static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
2915 const struct cntr_entry *entry,
2916 void *context, int vl, int mode, u64 data)
2917{
2918 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2919
2920 return dd->send_pio_err_status_cnt[12];
2921}
2922
2923static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
2924 const struct cntr_entry *entry,
2925 void *context, int vl, int mode, u64 data)
2926{
2927 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2928
2929 return dd->send_pio_err_status_cnt[11];
2930}
2931
2932static u64 access_pio_sm_pkt_reset_parity_err_cnt(
2933 const struct cntr_entry *entry,
2934 void *context, int vl, int mode, u64 data)
2935{
2936 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2937
2938 return dd->send_pio_err_status_cnt[10];
2939}
2940
2941static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
2942 const struct cntr_entry *entry,
2943 void *context, int vl, int mode, u64 data)
2944{
2945 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2946
2947 return dd->send_pio_err_status_cnt[9];
2948}
2949
2950static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
2951 const struct cntr_entry *entry,
2952 void *context, int vl, int mode, u64 data)
2953{
2954 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2955
2956 return dd->send_pio_err_status_cnt[8];
2957}
2958
2959static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
2960 const struct cntr_entry *entry,
2961 void *context, int vl, int mode, u64 data)
2962{
2963 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2964
2965 return dd->send_pio_err_status_cnt[7];
2966}
2967
2968static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
2969 void *context, int vl, int mode,
2970 u64 data)
2971{
2972 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2973
2974 return dd->send_pio_err_status_cnt[6];
2975}
2976
2977static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
2978 void *context, int vl, int mode,
2979 u64 data)
2980{
2981 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2982
2983 return dd->send_pio_err_status_cnt[5];
2984}
2985
2986static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
2987 void *context, int vl, int mode,
2988 u64 data)
2989{
2990 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2991
2992 return dd->send_pio_err_status_cnt[4];
2993}
2994
2995static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
2996 void *context, int vl, int mode,
2997 u64 data)
2998{
2999 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3000
3001 return dd->send_pio_err_status_cnt[3];
3002}
3003
3004static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3005 void *context, int vl, int mode,
3006 u64 data)
3007{
3008 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3009
3010 return dd->send_pio_err_status_cnt[2];
3011}
3012
3013static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3014 void *context, int vl,
3015 int mode, u64 data)
3016{
3017 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3018
3019 return dd->send_pio_err_status_cnt[1];
3020}
3021
3022static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3023 void *context, int vl, int mode,
3024 u64 data)
3025{
3026 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3027
3028 return dd->send_pio_err_status_cnt[0];
3029}
3030
3031/*
3032 * Software counters corresponding to each of the
3033 * error status bits within SendDmaErrStatus
3034 */
3035static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3036 const struct cntr_entry *entry,
3037 void *context, int vl, int mode, u64 data)
3038{
3039 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3040
3041 return dd->send_dma_err_status_cnt[3];
3042}
3043
3044static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3045 const struct cntr_entry *entry,
3046 void *context, int vl, int mode, u64 data)
3047{
3048 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3049
3050 return dd->send_dma_err_status_cnt[2];
3051}
3052
3053static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3054 void *context, int vl, int mode,
3055 u64 data)
3056{
3057 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3058
3059 return dd->send_dma_err_status_cnt[1];
3060}
3061
3062static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3063 void *context, int vl, int mode,
3064 u64 data)
3065{
3066 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3067
3068 return dd->send_dma_err_status_cnt[0];
3069}
3070
3071/*
3072 * Software counters corresponding to each of the
3073 * error status bits within SendEgressErrStatus
3074 */
3075static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3076 const struct cntr_entry *entry,
3077 void *context, int vl, int mode, u64 data)
3078{
3079 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3080
3081 return dd->send_egress_err_status_cnt[63];
3082}
3083
3084static u64 access_tx_read_sdma_memory_csr_err_cnt(
3085 const struct cntr_entry *entry,
3086 void *context, int vl, int mode, u64 data)
3087{
3088 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3089
3090 return dd->send_egress_err_status_cnt[62];
3091}
3092
3093static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3094 void *context, int vl, int mode,
3095 u64 data)
3096{
3097 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3098
3099 return dd->send_egress_err_status_cnt[61];
3100}
3101
3102static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3103 void *context, int vl,
3104 int mode, u64 data)
3105{
3106 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3107
3108 return dd->send_egress_err_status_cnt[60];
3109}
3110
3111static u64 access_tx_read_sdma_memory_cor_err_cnt(
3112 const struct cntr_entry *entry,
3113 void *context, int vl, int mode, u64 data)
3114{
3115 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3116
3117 return dd->send_egress_err_status_cnt[59];
3118}
3119
3120static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3121 void *context, int vl, int mode,
3122 u64 data)
3123{
3124 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3125
3126 return dd->send_egress_err_status_cnt[58];
3127}
3128
3129static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3130 void *context, int vl, int mode,
3131 u64 data)
3132{
3133 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3134
3135 return dd->send_egress_err_status_cnt[57];
3136}
3137
3138static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3139 void *context, int vl, int mode,
3140 u64 data)
3141{
3142 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3143
3144 return dd->send_egress_err_status_cnt[56];
3145}
3146
3147static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3148 void *context, int vl, int mode,
3149 u64 data)
3150{
3151 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3152
3153 return dd->send_egress_err_status_cnt[55];
3154}
3155
3156static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3157 void *context, int vl, int mode,
3158 u64 data)
3159{
3160 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3161
3162 return dd->send_egress_err_status_cnt[54];
3163}
3164
3165static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3166 void *context, int vl, int mode,
3167 u64 data)
3168{
3169 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3170
3171 return dd->send_egress_err_status_cnt[53];
3172}
3173
3174static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3175 void *context, int vl, int mode,
3176 u64 data)
3177{
3178 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3179
3180 return dd->send_egress_err_status_cnt[52];
3181}
3182
3183static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3184 void *context, int vl, int mode,
3185 u64 data)
3186{
3187 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3188
3189 return dd->send_egress_err_status_cnt[51];
3190}
3191
3192static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3193 void *context, int vl, int mode,
3194 u64 data)
3195{
3196 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3197
3198 return dd->send_egress_err_status_cnt[50];
3199}
3200
3201static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3202 void *context, int vl, int mode,
3203 u64 data)
3204{
3205 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3206
3207 return dd->send_egress_err_status_cnt[49];
3208}
3209
3210static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3211 void *context, int vl, int mode,
3212 u64 data)
3213{
3214 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3215
3216 return dd->send_egress_err_status_cnt[48];
3217}
3218
3219static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3220 void *context, int vl, int mode,
3221 u64 data)
3222{
3223 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3224
3225 return dd->send_egress_err_status_cnt[47];
3226}
3227
3228static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3229 void *context, int vl, int mode,
3230 u64 data)
3231{
3232 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3233
3234 return dd->send_egress_err_status_cnt[46];
3235}
3236
3237static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3238 void *context, int vl, int mode,
3239 u64 data)
3240{
3241 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3242
3243 return dd->send_egress_err_status_cnt[45];
3244}
3245
3246static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3247 void *context, int vl,
3248 int mode, u64 data)
3249{
3250 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3251
3252 return dd->send_egress_err_status_cnt[44];
3253}
3254
3255static u64 access_tx_read_sdma_memory_unc_err_cnt(
3256 const struct cntr_entry *entry,
3257 void *context, int vl, int mode, u64 data)
3258{
3259 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3260
3261 return dd->send_egress_err_status_cnt[43];
3262}
3263
3264static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3265 void *context, int vl, int mode,
3266 u64 data)
3267{
3268 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3269
3270 return dd->send_egress_err_status_cnt[42];
3271}
3272
3273static u64 access_tx_credit_return_partiy_err_cnt(
3274 const struct cntr_entry *entry,
3275 void *context, int vl, int mode, u64 data)
3276{
3277 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3278
3279 return dd->send_egress_err_status_cnt[41];
3280}
3281
3282static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3283 const struct cntr_entry *entry,
3284 void *context, int vl, int mode, u64 data)
3285{
3286 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3287
3288 return dd->send_egress_err_status_cnt[40];
3289}
3290
3291static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3292 const struct cntr_entry *entry,
3293 void *context, int vl, int mode, u64 data)
3294{
3295 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3296
3297 return dd->send_egress_err_status_cnt[39];
3298}
3299
3300static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3301 const struct cntr_entry *entry,
3302 void *context, int vl, int mode, u64 data)
3303{
3304 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3305
3306 return dd->send_egress_err_status_cnt[38];
3307}
3308
3309static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3310 const struct cntr_entry *entry,
3311 void *context, int vl, int mode, u64 data)
3312{
3313 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3314
3315 return dd->send_egress_err_status_cnt[37];
3316}
3317
3318static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3319 const struct cntr_entry *entry,
3320 void *context, int vl, int mode, u64 data)
3321{
3322 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3323
3324 return dd->send_egress_err_status_cnt[36];
3325}
3326
3327static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3328 const struct cntr_entry *entry,
3329 void *context, int vl, int mode, u64 data)
3330{
3331 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3332
3333 return dd->send_egress_err_status_cnt[35];
3334}
3335
3336static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3337 const struct cntr_entry *entry,
3338 void *context, int vl, int mode, u64 data)
3339{
3340 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3341
3342 return dd->send_egress_err_status_cnt[34];
3343}
3344
3345static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3346 const struct cntr_entry *entry,
3347 void *context, int vl, int mode, u64 data)
3348{
3349 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3350
3351 return dd->send_egress_err_status_cnt[33];
3352}
3353
3354static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3355 const struct cntr_entry *entry,
3356 void *context, int vl, int mode, u64 data)
3357{
3358 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3359
3360 return dd->send_egress_err_status_cnt[32];
3361}
3362
3363static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3364 const struct cntr_entry *entry,
3365 void *context, int vl, int mode, u64 data)
3366{
3367 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3368
3369 return dd->send_egress_err_status_cnt[31];
3370}
3371
3372static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3373 const struct cntr_entry *entry,
3374 void *context, int vl, int mode, u64 data)
3375{
3376 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3377
3378 return dd->send_egress_err_status_cnt[30];
3379}
3380
3381static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3382 const struct cntr_entry *entry,
3383 void *context, int vl, int mode, u64 data)
3384{
3385 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3386
3387 return dd->send_egress_err_status_cnt[29];
3388}
3389
3390static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3391 const struct cntr_entry *entry,
3392 void *context, int vl, int mode, u64 data)
3393{
3394 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3395
3396 return dd->send_egress_err_status_cnt[28];
3397}
3398
3399static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3400 const struct cntr_entry *entry,
3401 void *context, int vl, int mode, u64 data)
3402{
3403 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3404
3405 return dd->send_egress_err_status_cnt[27];
3406}
3407
3408static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3409 const struct cntr_entry *entry,
3410 void *context, int vl, int mode, u64 data)
3411{
3412 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3413
3414 return dd->send_egress_err_status_cnt[26];
3415}
3416
3417static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3418 const struct cntr_entry *entry,
3419 void *context, int vl, int mode, u64 data)
3420{
3421 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3422
3423 return dd->send_egress_err_status_cnt[25];
3424}
3425
3426static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3427 const struct cntr_entry *entry,
3428 void *context, int vl, int mode, u64 data)
3429{
3430 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3431
3432 return dd->send_egress_err_status_cnt[24];
3433}
3434
3435static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3436 const struct cntr_entry *entry,
3437 void *context, int vl, int mode, u64 data)
3438{
3439 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3440
3441 return dd->send_egress_err_status_cnt[23];
3442}
3443
3444static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3445 const struct cntr_entry *entry,
3446 void *context, int vl, int mode, u64 data)
3447{
3448 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3449
3450 return dd->send_egress_err_status_cnt[22];
3451}
3452
3453static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3454 const struct cntr_entry *entry,
3455 void *context, int vl, int mode, u64 data)
3456{
3457 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3458
3459 return dd->send_egress_err_status_cnt[21];
3460}
3461
3462static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3463 const struct cntr_entry *entry,
3464 void *context, int vl, int mode, u64 data)
3465{
3466 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3467
3468 return dd->send_egress_err_status_cnt[20];
3469}
3470
3471static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3472 const struct cntr_entry *entry,
3473 void *context, int vl, int mode, u64 data)
3474{
3475 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3476
3477 return dd->send_egress_err_status_cnt[19];
3478}
3479
3480static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3481 const struct cntr_entry *entry,
3482 void *context, int vl, int mode, u64 data)
3483{
3484 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3485
3486 return dd->send_egress_err_status_cnt[18];
3487}
3488
3489static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3490 const struct cntr_entry *entry,
3491 void *context, int vl, int mode, u64 data)
3492{
3493 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3494
3495 return dd->send_egress_err_status_cnt[17];
3496}
3497
3498static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3499 const struct cntr_entry *entry,
3500 void *context, int vl, int mode, u64 data)
3501{
3502 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3503
3504 return dd->send_egress_err_status_cnt[16];
3505}
3506
3507static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3508 void *context, int vl, int mode,
3509 u64 data)
3510{
3511 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3512
3513 return dd->send_egress_err_status_cnt[15];
3514}
3515
3516static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3517 void *context, int vl,
3518 int mode, u64 data)
3519{
3520 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3521
3522 return dd->send_egress_err_status_cnt[14];
3523}
3524
3525static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3526 void *context, int vl, int mode,
3527 u64 data)
3528{
3529 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3530
3531 return dd->send_egress_err_status_cnt[13];
3532}
3533
3534static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3535 void *context, int vl, int mode,
3536 u64 data)
3537{
3538 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3539
3540 return dd->send_egress_err_status_cnt[12];
3541}
3542
3543static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3544 const struct cntr_entry *entry,
3545 void *context, int vl, int mode, u64 data)
3546{
3547 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3548
3549 return dd->send_egress_err_status_cnt[11];
3550}
3551
3552static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3553 void *context, int vl, int mode,
3554 u64 data)
3555{
3556 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3557
3558 return dd->send_egress_err_status_cnt[10];
3559}
3560
3561static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3562 void *context, int vl, int mode,
3563 u64 data)
3564{
3565 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3566
3567 return dd->send_egress_err_status_cnt[9];
3568}
3569
3570static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3571 const struct cntr_entry *entry,
3572 void *context, int vl, int mode, u64 data)
3573{
3574 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3575
3576 return dd->send_egress_err_status_cnt[8];
3577}
3578
3579static u64 access_tx_pio_launch_intf_parity_err_cnt(
3580 const struct cntr_entry *entry,
3581 void *context, int vl, int mode, u64 data)
3582{
3583 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3584
3585 return dd->send_egress_err_status_cnt[7];
3586}
3587
3588static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3589 void *context, int vl, int mode,
3590 u64 data)
3591{
3592 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3593
3594 return dd->send_egress_err_status_cnt[6];
3595}
3596
3597static u64 access_tx_incorrect_link_state_err_cnt(
3598 const struct cntr_entry *entry,
3599 void *context, int vl, int mode, u64 data)
3600{
3601 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3602
3603 return dd->send_egress_err_status_cnt[5];
3604}
3605
3606static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3607 void *context, int vl, int mode,
3608 u64 data)
3609{
3610 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3611
3612 return dd->send_egress_err_status_cnt[4];
3613}
3614
3615static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3616 const struct cntr_entry *entry,
3617 void *context, int vl, int mode, u64 data)
3618{
3619 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3620
3621 return dd->send_egress_err_status_cnt[3];
3622}
3623
3624static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3625 void *context, int vl, int mode,
3626 u64 data)
3627{
3628 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3629
3630 return dd->send_egress_err_status_cnt[2];
3631}
3632
3633static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3634 const struct cntr_entry *entry,
3635 void *context, int vl, int mode, u64 data)
3636{
3637 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3638
3639 return dd->send_egress_err_status_cnt[1];
3640}
3641
3642static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3643 const struct cntr_entry *entry,
3644 void *context, int vl, int mode, u64 data)
3645{
3646 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3647
3648 return dd->send_egress_err_status_cnt[0];
3649}
3650
3651/*
3652 * Software counters corresponding to each of the
3653 * error status bits within SendErrStatus
3654 */
3655static u64 access_send_csr_write_bad_addr_err_cnt(
3656 const struct cntr_entry *entry,
3657 void *context, int vl, int mode, u64 data)
3658{
3659 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3660
3661 return dd->send_err_status_cnt[2];
3662}
3663
3664static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3665 void *context, int vl,
3666 int mode, u64 data)
3667{
3668 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3669
3670 return dd->send_err_status_cnt[1];
3671}
3672
3673static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3674 void *context, int vl, int mode,
3675 u64 data)
3676{
3677 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3678
3679 return dd->send_err_status_cnt[0];
3680}
3681
3682/*
3683 * Software counters corresponding to each of the
3684 * error status bits within SendCtxtErrStatus
3685 */
3686static u64 access_pio_write_out_of_bounds_err_cnt(
3687 const struct cntr_entry *entry,
3688 void *context, int vl, int mode, u64 data)
3689{
3690 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3691
3692 return dd->sw_ctxt_err_status_cnt[4];
3693}
3694
3695static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3696 void *context, int vl, int mode,
3697 u64 data)
3698{
3699 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3700
3701 return dd->sw_ctxt_err_status_cnt[3];
3702}
3703
3704static u64 access_pio_write_crosses_boundary_err_cnt(
3705 const struct cntr_entry *entry,
3706 void *context, int vl, int mode, u64 data)
3707{
3708 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3709
3710 return dd->sw_ctxt_err_status_cnt[2];
3711}
3712
3713static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3714 void *context, int vl,
3715 int mode, u64 data)
3716{
3717 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3718
3719 return dd->sw_ctxt_err_status_cnt[1];
3720}
3721
3722static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3723 void *context, int vl, int mode,
3724 u64 data)
3725{
3726 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3727
3728 return dd->sw_ctxt_err_status_cnt[0];
3729}
3730
3731/*
3732 * Software counters corresponding to each of the
3733 * error status bits within SendDmaEngErrStatus
3734 */
3735static u64 access_sdma_header_request_fifo_cor_err_cnt(
3736 const struct cntr_entry *entry,
3737 void *context, int vl, int mode, u64 data)
3738{
3739 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3740
3741 return dd->sw_send_dma_eng_err_status_cnt[23];
3742}
3743
3744static u64 access_sdma_header_storage_cor_err_cnt(
3745 const struct cntr_entry *entry,
3746 void *context, int vl, int mode, u64 data)
3747{
3748 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3749
3750 return dd->sw_send_dma_eng_err_status_cnt[22];
3751}
3752
3753static u64 access_sdma_packet_tracking_cor_err_cnt(
3754 const struct cntr_entry *entry,
3755 void *context, int vl, int mode, u64 data)
3756{
3757 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3758
3759 return dd->sw_send_dma_eng_err_status_cnt[21];
3760}
3761
3762static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3763 void *context, int vl, int mode,
3764 u64 data)
3765{
3766 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3767
3768 return dd->sw_send_dma_eng_err_status_cnt[20];
3769}
3770
3771static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3772 void *context, int vl, int mode,
3773 u64 data)
3774{
3775 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3776
3777 return dd->sw_send_dma_eng_err_status_cnt[19];
3778}
3779
3780static u64 access_sdma_header_request_fifo_unc_err_cnt(
3781 const struct cntr_entry *entry,
3782 void *context, int vl, int mode, u64 data)
3783{
3784 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3785
3786 return dd->sw_send_dma_eng_err_status_cnt[18];
3787}
3788
3789static u64 access_sdma_header_storage_unc_err_cnt(
3790 const struct cntr_entry *entry,
3791 void *context, int vl, int mode, u64 data)
3792{
3793 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3794
3795 return dd->sw_send_dma_eng_err_status_cnt[17];
3796}
3797
3798static u64 access_sdma_packet_tracking_unc_err_cnt(
3799 const struct cntr_entry *entry,
3800 void *context, int vl, int mode, u64 data)
3801{
3802 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3803
3804 return dd->sw_send_dma_eng_err_status_cnt[16];
3805}
3806
3807static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3808 void *context, int vl, int mode,
3809 u64 data)
3810{
3811 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3812
3813 return dd->sw_send_dma_eng_err_status_cnt[15];
3814}
3815
3816static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3817 void *context, int vl, int mode,
3818 u64 data)
3819{
3820 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3821
3822 return dd->sw_send_dma_eng_err_status_cnt[14];
3823}
3824
3825static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3826 void *context, int vl, int mode,
3827 u64 data)
3828{
3829 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3830
3831 return dd->sw_send_dma_eng_err_status_cnt[13];
3832}
3833
3834static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3835 void *context, int vl, int mode,
3836 u64 data)
3837{
3838 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3839
3840 return dd->sw_send_dma_eng_err_status_cnt[12];
3841}
3842
3843static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3844 void *context, int vl, int mode,
3845 u64 data)
3846{
3847 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3848
3849 return dd->sw_send_dma_eng_err_status_cnt[11];
3850}
3851
3852static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3853 void *context, int vl, int mode,
3854 u64 data)
3855{
3856 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3857
3858 return dd->sw_send_dma_eng_err_status_cnt[10];
3859}
3860
3861static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3862 void *context, int vl, int mode,
3863 u64 data)
3864{
3865 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3866
3867 return dd->sw_send_dma_eng_err_status_cnt[9];
3868}
3869
3870static u64 access_sdma_packet_desc_overflow_err_cnt(
3871 const struct cntr_entry *entry,
3872 void *context, int vl, int mode, u64 data)
3873{
3874 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3875
3876 return dd->sw_send_dma_eng_err_status_cnt[8];
3877}
3878
3879static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3880 void *context, int vl,
3881 int mode, u64 data)
3882{
3883 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3884
3885 return dd->sw_send_dma_eng_err_status_cnt[7];
3886}
3887
3888static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3889 void *context, int vl, int mode, u64 data)
3890{
3891 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3892
3893 return dd->sw_send_dma_eng_err_status_cnt[6];
3894}
3895
3896static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3897 void *context, int vl, int mode,
3898 u64 data)
3899{
3900 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3901
3902 return dd->sw_send_dma_eng_err_status_cnt[5];
3903}
3904
3905static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3906 void *context, int vl, int mode,
3907 u64 data)
3908{
3909 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3910
3911 return dd->sw_send_dma_eng_err_status_cnt[4];
3912}
3913
3914static u64 access_sdma_tail_out_of_bounds_err_cnt(
3915 const struct cntr_entry *entry,
3916 void *context, int vl, int mode, u64 data)
3917{
3918 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3919
3920 return dd->sw_send_dma_eng_err_status_cnt[3];
3921}
3922
3923static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
3924 void *context, int vl, int mode,
3925 u64 data)
3926{
3927 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3928
3929 return dd->sw_send_dma_eng_err_status_cnt[2];
3930}
3931
3932static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
3933 void *context, int vl, int mode,
3934 u64 data)
3935{
3936 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3937
3938 return dd->sw_send_dma_eng_err_status_cnt[1];
3939}
3940
3941static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
3942 void *context, int vl, int mode,
3943 u64 data)
3944{
3945 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3946
3947 return dd->sw_send_dma_eng_err_status_cnt[0];
3948}
3949
77241056
MM		 3950#define def_access_sw_cpu(cntr) \
3951static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
3952 void *context, int vl, int mode, u64 data) \
3953{ \
3954 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4eb06882
DD		 3955 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \
3956 ppd->ibport_data.rvp.cntr, vl, \
77241056
MM		 3957 mode, data); \
3958}
3959
3960def_access_sw_cpu(rc_acks);
3961def_access_sw_cpu(rc_qacks);
3962def_access_sw_cpu(rc_delayed_comp);
3963
3964#define def_access_ibp_counter(cntr) \
3965static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
3966 void *context, int vl, int mode, u64 data) \
3967{ \
3968 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
3969 \
3970 if (vl != CNTR_INVALID_VL) \
3971 return 0; \
3972 \
4eb06882 3973 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \
77241056
MM		 3974 mode, data); \
3975}
3976
3977def_access_ibp_counter(loop_pkts);
3978def_access_ibp_counter(rc_resends);
3979def_access_ibp_counter(rnr_naks);
3980def_access_ibp_counter(other_naks);
3981def_access_ibp_counter(rc_timeouts);
3982def_access_ibp_counter(pkt_drops);
3983def_access_ibp_counter(dmawait);
3984def_access_ibp_counter(rc_seqnak);
3985def_access_ibp_counter(rc_dupreq);
3986def_access_ibp_counter(rdma_seq);
3987def_access_ibp_counter(unaligned);
3988def_access_ibp_counter(seq_naks);
3989
3990static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
3991[C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
3992[C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
3993 CNTR_NORMAL),
3994[C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
3995 CNTR_NORMAL),
3996[C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
3997 RCV_TID_FLOW_GEN_MISMATCH_CNT,
3998 CNTR_NORMAL),
77241056
MM		 3999[C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4000 CNTR_NORMAL),
4001[C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4002 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4003[C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4004 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4005[C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4006 CNTR_NORMAL),
4007[C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4008 CNTR_NORMAL),
4009[C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4010 CNTR_NORMAL),
4011[C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4012 CNTR_NORMAL),
4013[C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4014 CNTR_NORMAL),
4015[C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4016 CNTR_NORMAL),
4017[C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4018 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4019[C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4020 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4021[C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4022 CNTR_SYNTH),
4023[C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
4024[C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4025 CNTR_SYNTH),
4026[C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4027 CNTR_SYNTH),
4028[C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4029 CNTR_SYNTH),
4030[C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4031 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4032[C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4033 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4034 CNTR_SYNTH),
4035[C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4036 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4037[C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4038 CNTR_SYNTH),
4039[C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4040 CNTR_SYNTH),
4041[C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4042 CNTR_SYNTH),
4043[C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4044 CNTR_SYNTH),
4045[C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4046 CNTR_SYNTH),
4047[C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4048 CNTR_SYNTH),
4049[C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4050 CNTR_SYNTH),
4051[C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4052 CNTR_SYNTH | CNTR_VL),
4053[C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4054 CNTR_SYNTH | CNTR_VL),
4055[C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4056[C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4057 CNTR_SYNTH | CNTR_VL),
4058[C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4059[C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4060 CNTR_SYNTH | CNTR_VL),
4061[C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4062 CNTR_SYNTH),
4063[C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4064 CNTR_SYNTH | CNTR_VL),
4065[C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4066 CNTR_SYNTH),
4067[C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4068 CNTR_SYNTH | CNTR_VL),
4069[C_DC_TOTAL_CRC] =
4070 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4071 CNTR_SYNTH),
4072[C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4073 CNTR_SYNTH),
4074[C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4075 CNTR_SYNTH),
4076[C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4077 CNTR_SYNTH),
4078[C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4079 CNTR_SYNTH),
4080[C_DC_CRC_MULT_LN] =
4081 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4082 CNTR_SYNTH),
4083[C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4084 CNTR_SYNTH),
4085[C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4086 CNTR_SYNTH),
4087[C_DC_SEQ_CRC_CNT] =
4088 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4089 CNTR_SYNTH),
4090[C_DC_ESC0_ONLY_CNT] =
4091 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4092 CNTR_SYNTH),
4093[C_DC_ESC0_PLUS1_CNT] =
4094 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4095 CNTR_SYNTH),
4096[C_DC_ESC0_PLUS2_CNT] =
4097 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4098 CNTR_SYNTH),
4099[C_DC_REINIT_FROM_PEER_CNT] =
4100 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4101 CNTR_SYNTH),
4102[C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4103 CNTR_SYNTH),
4104[C_DC_MISC_FLG_CNT] =
4105 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4106 CNTR_SYNTH),
4107[C_DC_PRF_GOOD_LTP_CNT] =
4108 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4109[C_DC_PRF_ACCEPTED_LTP_CNT] =
4110 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4111 CNTR_SYNTH),
4112[C_DC_PRF_RX_FLIT_CNT] =
4113 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4114[C_DC_PRF_TX_FLIT_CNT] =
4115 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4116[C_DC_PRF_CLK_CNTR] =
4117 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4118[C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4119 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4120[C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4121 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4122 CNTR_SYNTH),
4123[C_DC_PG_STS_TX_SBE_CNT] =
4124 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4125[C_DC_PG_STS_TX_MBE_CNT] =
4126 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4127 CNTR_SYNTH),
4128[C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4129 access_sw_cpu_intr),
4130[C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4131 access_sw_cpu_rcv_limit),
4132[C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4133 access_sw_vtx_wait),
4134[C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4135 access_sw_pio_wait),
14553ca1
MM		 4136[C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4137 access_sw_pio_drain),
77241056
MM		 4138[C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4139 access_sw_kmem_wait),
b421922e
DL		 4140[C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4141 access_sw_send_schedule),
a699c6c2
VM		 4142[C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4143 SEND_DMA_DESC_FETCHED_CNT, 0,
4144 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4145 dev_access_u32_csr),
4146[C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4147 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4148 access_sde_int_cnt),
4149[C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4150 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4151 access_sde_err_cnt),
4152[C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4153 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4154 access_sde_idle_int_cnt),
4155[C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4156 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4157 access_sde_progress_int_cnt),
2c5b521a
JR		 4158/* MISC_ERR_STATUS */
4159[C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4160 CNTR_NORMAL,
4161 access_misc_pll_lock_fail_err_cnt),
4162[C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4163 CNTR_NORMAL,
4164 access_misc_mbist_fail_err_cnt),
4165[C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4166 CNTR_NORMAL,
4167 access_misc_invalid_eep_cmd_err_cnt),
4168[C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4169 CNTR_NORMAL,
4170 access_misc_efuse_done_parity_err_cnt),
4171[C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4172 CNTR_NORMAL,
4173 access_misc_efuse_write_err_cnt),
4174[C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4175 0, CNTR_NORMAL,
4176 access_misc_efuse_read_bad_addr_err_cnt),
4177[C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4178 CNTR_NORMAL,
4179 access_misc_efuse_csr_parity_err_cnt),
4180[C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4181 CNTR_NORMAL,
4182 access_misc_fw_auth_failed_err_cnt),
4183[C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4184 CNTR_NORMAL,
4185 access_misc_key_mismatch_err_cnt),
4186[C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4187 CNTR_NORMAL,
4188 access_misc_sbus_write_failed_err_cnt),
4189[C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4190 CNTR_NORMAL,
4191 access_misc_csr_write_bad_addr_err_cnt),
4192[C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4193 CNTR_NORMAL,
4194 access_misc_csr_read_bad_addr_err_cnt),
4195[C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4196 CNTR_NORMAL,
4197 access_misc_csr_parity_err_cnt),
4198/* CceErrStatus */
4199[C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4200 CNTR_NORMAL,
4201 access_sw_cce_err_status_aggregated_cnt),
4202[C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4203 CNTR_NORMAL,
4204 access_cce_msix_csr_parity_err_cnt),
4205[C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4206 CNTR_NORMAL,
4207 access_cce_int_map_unc_err_cnt),
4208[C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4209 CNTR_NORMAL,
4210 access_cce_int_map_cor_err_cnt),
4211[C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4212 CNTR_NORMAL,
4213 access_cce_msix_table_unc_err_cnt),
4214[C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4215 CNTR_NORMAL,
4216 access_cce_msix_table_cor_err_cnt),
4217[C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4218 0, CNTR_NORMAL,
4219 access_cce_rxdma_conv_fifo_parity_err_cnt),
4220[C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4221 0, CNTR_NORMAL,
4222 access_cce_rcpl_async_fifo_parity_err_cnt),
4223[C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4224 CNTR_NORMAL,
4225 access_cce_seg_write_bad_addr_err_cnt),
4226[C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4227 CNTR_NORMAL,
4228 access_cce_seg_read_bad_addr_err_cnt),
4229[C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4230 CNTR_NORMAL,
4231 access_la_triggered_cnt),
4232[C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4233 CNTR_NORMAL,
4234 access_cce_trgt_cpl_timeout_err_cnt),
4235[C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4236 CNTR_NORMAL,
4237 access_pcic_receive_parity_err_cnt),
4238[C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4239 CNTR_NORMAL,
4240 access_pcic_transmit_back_parity_err_cnt),
4241[C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4242 0, CNTR_NORMAL,
4243 access_pcic_transmit_front_parity_err_cnt),
4244[C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4245 CNTR_NORMAL,
4246 access_pcic_cpl_dat_q_unc_err_cnt),
4247[C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4248 CNTR_NORMAL,
4249 access_pcic_cpl_hd_q_unc_err_cnt),
4250[C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4251 CNTR_NORMAL,
4252 access_pcic_post_dat_q_unc_err_cnt),
4253[C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4254 CNTR_NORMAL,
4255 access_pcic_post_hd_q_unc_err_cnt),
4256[C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4257 CNTR_NORMAL,
4258 access_pcic_retry_sot_mem_unc_err_cnt),
4259[C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4260 CNTR_NORMAL,
4261 access_pcic_retry_mem_unc_err),
4262[C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4263 CNTR_NORMAL,
4264 access_pcic_n_post_dat_q_parity_err_cnt),
4265[C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4266 CNTR_NORMAL,
4267 access_pcic_n_post_h_q_parity_err_cnt),
4268[C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4269 CNTR_NORMAL,
4270 access_pcic_cpl_dat_q_cor_err_cnt),
4271[C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4272 CNTR_NORMAL,
4273 access_pcic_cpl_hd_q_cor_err_cnt),
4274[C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4275 CNTR_NORMAL,
4276 access_pcic_post_dat_q_cor_err_cnt),
4277[C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4278 CNTR_NORMAL,
4279 access_pcic_post_hd_q_cor_err_cnt),
4280[C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4281 CNTR_NORMAL,
4282 access_pcic_retry_sot_mem_cor_err_cnt),
4283[C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4284 CNTR_NORMAL,
4285 access_pcic_retry_mem_cor_err_cnt),
4286[C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4287 "CceCli1AsyncFifoDbgParityError", 0, 0,
4288 CNTR_NORMAL,
4289 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4290[C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4291 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4292 CNTR_NORMAL,
4293 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4294 ),
4295[C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4296 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4297 CNTR_NORMAL,
4298 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4299[C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4300 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4301 CNTR_NORMAL,
4302 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4303[C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4304 0, CNTR_NORMAL,
4305 access_cce_cli2_async_fifo_parity_err_cnt),
4306[C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4307 CNTR_NORMAL,
4308 access_cce_csr_cfg_bus_parity_err_cnt),
4309[C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4310 0, CNTR_NORMAL,
4311 access_cce_cli0_async_fifo_parity_err_cnt),
4312[C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4313 CNTR_NORMAL,
4314 access_cce_rspd_data_parity_err_cnt),
4315[C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4316 CNTR_NORMAL,
4317 access_cce_trgt_access_err_cnt),
4318[C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4319 0, CNTR_NORMAL,
4320 access_cce_trgt_async_fifo_parity_err_cnt),
4321[C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4322 CNTR_NORMAL,
4323 access_cce_csr_write_bad_addr_err_cnt),
4324[C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4325 CNTR_NORMAL,
4326 access_cce_csr_read_bad_addr_err_cnt),
4327[C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4328 CNTR_NORMAL,
4329 access_ccs_csr_parity_err_cnt),
4330
4331/* RcvErrStatus */
4332[C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4333 CNTR_NORMAL,
4334 access_rx_csr_parity_err_cnt),
4335[C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4336 CNTR_NORMAL,
4337 access_rx_csr_write_bad_addr_err_cnt),
4338[C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4339 CNTR_NORMAL,
4340 access_rx_csr_read_bad_addr_err_cnt),
4341[C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4342 CNTR_NORMAL,
4343 access_rx_dma_csr_unc_err_cnt),
4344[C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4345 CNTR_NORMAL,
4346 access_rx_dma_dq_fsm_encoding_err_cnt),
4347[C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4348 CNTR_NORMAL,
4349 access_rx_dma_eq_fsm_encoding_err_cnt),
4350[C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4351 CNTR_NORMAL,
4352 access_rx_dma_csr_parity_err_cnt),
4353[C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4354 CNTR_NORMAL,
4355 access_rx_rbuf_data_cor_err_cnt),
4356[C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4357 CNTR_NORMAL,
4358 access_rx_rbuf_data_unc_err_cnt),
4359[C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4360 CNTR_NORMAL,
4361 access_rx_dma_data_fifo_rd_cor_err_cnt),
4362[C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4363 CNTR_NORMAL,
4364 access_rx_dma_data_fifo_rd_unc_err_cnt),
4365[C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4366 CNTR_NORMAL,
4367 access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4368[C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4369 CNTR_NORMAL,
4370 access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4371[C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4372 CNTR_NORMAL,
4373 access_rx_rbuf_desc_part2_cor_err_cnt),
4374[C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4375 CNTR_NORMAL,
4376 access_rx_rbuf_desc_part2_unc_err_cnt),
4377[C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4378 CNTR_NORMAL,
4379 access_rx_rbuf_desc_part1_cor_err_cnt),
4380[C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4381 CNTR_NORMAL,
4382 access_rx_rbuf_desc_part1_unc_err_cnt),
4383[C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4384 CNTR_NORMAL,
4385 access_rx_hq_intr_fsm_err_cnt),
4386[C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4387 CNTR_NORMAL,
4388 access_rx_hq_intr_csr_parity_err_cnt),
4389[C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4390 CNTR_NORMAL,
4391 access_rx_lookup_csr_parity_err_cnt),
4392[C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4393 CNTR_NORMAL,
4394 access_rx_lookup_rcv_array_cor_err_cnt),
4395[C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4396 CNTR_NORMAL,
4397 access_rx_lookup_rcv_array_unc_err_cnt),
4398[C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4399 0, CNTR_NORMAL,
4400 access_rx_lookup_des_part2_parity_err_cnt),
4401[C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4402 0, CNTR_NORMAL,
4403 access_rx_lookup_des_part1_unc_cor_err_cnt),
4404[C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4405 CNTR_NORMAL,
4406 access_rx_lookup_des_part1_unc_err_cnt),
4407[C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4408 CNTR_NORMAL,
4409 access_rx_rbuf_next_free_buf_cor_err_cnt),
4410[C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4411 CNTR_NORMAL,
4412 access_rx_rbuf_next_free_buf_unc_err_cnt),
4413[C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4414 "RxRbufFlInitWrAddrParityErr", 0, 0,
4415 CNTR_NORMAL,
4416 access_rbuf_fl_init_wr_addr_parity_err_cnt),
4417[C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4418 0, CNTR_NORMAL,
4419 access_rx_rbuf_fl_initdone_parity_err_cnt),
4420[C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4421 0, CNTR_NORMAL,
4422 access_rx_rbuf_fl_write_addr_parity_err_cnt),
4423[C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4424 CNTR_NORMAL,
4425 access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4426[C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4427 CNTR_NORMAL,
4428 access_rx_rbuf_empty_err_cnt),
4429[C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4430 CNTR_NORMAL,
4431 access_rx_rbuf_full_err_cnt),
4432[C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4433 CNTR_NORMAL,
4434 access_rbuf_bad_lookup_err_cnt),
4435[C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4436 CNTR_NORMAL,
4437 access_rbuf_ctx_id_parity_err_cnt),
4438[C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4439 CNTR_NORMAL,
4440 access_rbuf_csr_qeopdw_parity_err_cnt),
4441[C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4442 "RxRbufCsrQNumOfPktParityErr", 0, 0,
4443 CNTR_NORMAL,
4444 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4445[C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4446 "RxRbufCsrQTlPtrParityErr", 0, 0,
4447 CNTR_NORMAL,
4448 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4449[C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4450 0, CNTR_NORMAL,
4451 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4452[C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4453 0, CNTR_NORMAL,
4454 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4455[C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4456 0, 0, CNTR_NORMAL,
4457 access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4458[C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4459 0, CNTR_NORMAL,
4460 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4461[C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4462 "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4463 CNTR_NORMAL,
4464 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4465[C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4466 0, CNTR_NORMAL,
4467 access_rx_rbuf_block_list_read_cor_err_cnt),
4468[C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4469 0, CNTR_NORMAL,
4470 access_rx_rbuf_block_list_read_unc_err_cnt),
4471[C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4472 CNTR_NORMAL,
4473 access_rx_rbuf_lookup_des_cor_err_cnt),
4474[C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4475 CNTR_NORMAL,
4476 access_rx_rbuf_lookup_des_unc_err_cnt),
4477[C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4478 "RxRbufLookupDesRegUncCorErr", 0, 0,
4479 CNTR_NORMAL,
4480 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4481[C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4482 CNTR_NORMAL,
4483 access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4484[C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4485 CNTR_NORMAL,
4486 access_rx_rbuf_free_list_cor_err_cnt),
4487[C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4488 CNTR_NORMAL,
4489 access_rx_rbuf_free_list_unc_err_cnt),
4490[C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4491 CNTR_NORMAL,
4492 access_rx_rcv_fsm_encoding_err_cnt),
4493[C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4494 CNTR_NORMAL,
4495 access_rx_dma_flag_cor_err_cnt),
4496[C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4497 CNTR_NORMAL,
4498 access_rx_dma_flag_unc_err_cnt),
4499[C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4500 CNTR_NORMAL,
4501 access_rx_dc_sop_eop_parity_err_cnt),
4502[C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4503 CNTR_NORMAL,
4504 access_rx_rcv_csr_parity_err_cnt),
4505[C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4506 CNTR_NORMAL,
4507 access_rx_rcv_qp_map_table_cor_err_cnt),
4508[C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4509 CNTR_NORMAL,
4510 access_rx_rcv_qp_map_table_unc_err_cnt),
4511[C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4512 CNTR_NORMAL,
4513 access_rx_rcv_data_cor_err_cnt),
4514[C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4515 CNTR_NORMAL,
4516 access_rx_rcv_data_unc_err_cnt),
4517[C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4518 CNTR_NORMAL,
4519 access_rx_rcv_hdr_cor_err_cnt),
4520[C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4521 CNTR_NORMAL,
4522 access_rx_rcv_hdr_unc_err_cnt),
4523[C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4524 CNTR_NORMAL,
4525 access_rx_dc_intf_parity_err_cnt),
4526[C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4527 CNTR_NORMAL,
4528 access_rx_dma_csr_cor_err_cnt),
4529/* SendPioErrStatus */
4530[C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4531 CNTR_NORMAL,
4532 access_pio_pec_sop_head_parity_err_cnt),
4533[C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4534 CNTR_NORMAL,
4535 access_pio_pcc_sop_head_parity_err_cnt),
4536[C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4537 0, 0, CNTR_NORMAL,
4538 access_pio_last_returned_cnt_parity_err_cnt),
4539[C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4540 0, CNTR_NORMAL,
4541 access_pio_current_free_cnt_parity_err_cnt),
4542[C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4543 CNTR_NORMAL,
4544 access_pio_reserved_31_err_cnt),
4545[C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4546 CNTR_NORMAL,
4547 access_pio_reserved_30_err_cnt),
4548[C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4549 CNTR_NORMAL,
4550 access_pio_ppmc_sop_len_err_cnt),
4551[C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4552 CNTR_NORMAL,
4553 access_pio_ppmc_bqc_mem_parity_err_cnt),
4554[C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4555 CNTR_NORMAL,
4556 access_pio_vl_fifo_parity_err_cnt),
4557[C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4558 CNTR_NORMAL,
4559 access_pio_vlf_sop_parity_err_cnt),
4560[C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4561 CNTR_NORMAL,
4562 access_pio_vlf_v1_len_parity_err_cnt),
4563[C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4564 CNTR_NORMAL,
4565 access_pio_block_qw_count_parity_err_cnt),
4566[C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4567 CNTR_NORMAL,
4568 access_pio_write_qw_valid_parity_err_cnt),
4569[C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4570 CNTR_NORMAL,
4571 access_pio_state_machine_err_cnt),
4572[C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4573 CNTR_NORMAL,
4574 access_pio_write_data_parity_err_cnt),
4575[C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4576 CNTR_NORMAL,
4577 access_pio_host_addr_mem_cor_err_cnt),
4578[C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4579 CNTR_NORMAL,
4580 access_pio_host_addr_mem_unc_err_cnt),
4581[C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4582 CNTR_NORMAL,
4583 access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4584[C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4585 CNTR_NORMAL,
4586 access_pio_init_sm_in_err_cnt),
4587[C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4588 CNTR_NORMAL,
4589 access_pio_ppmc_pbl_fifo_err_cnt),
4590[C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4591 0, CNTR_NORMAL,
4592 access_pio_credit_ret_fifo_parity_err_cnt),
4593[C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4594 CNTR_NORMAL,
4595 access_pio_v1_len_mem_bank1_cor_err_cnt),
4596[C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4597 CNTR_NORMAL,
4598 access_pio_v1_len_mem_bank0_cor_err_cnt),
4599[C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4600 CNTR_NORMAL,
4601 access_pio_v1_len_mem_bank1_unc_err_cnt),
4602[C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4603 CNTR_NORMAL,
4604 access_pio_v1_len_mem_bank0_unc_err_cnt),
4605[C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4606 CNTR_NORMAL,
4607 access_pio_sm_pkt_reset_parity_err_cnt),
4608[C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4609 CNTR_NORMAL,
4610 access_pio_pkt_evict_fifo_parity_err_cnt),
4611[C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4612 "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4613 CNTR_NORMAL,
4614 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4615[C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4616 CNTR_NORMAL,
4617 access_pio_sbrdctl_crrel_parity_err_cnt),
4618[C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4619 CNTR_NORMAL,
4620 access_pio_pec_fifo_parity_err_cnt),
4621[C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4622 CNTR_NORMAL,
4623 access_pio_pcc_fifo_parity_err_cnt),
4624[C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4625 CNTR_NORMAL,
4626 access_pio_sb_mem_fifo1_err_cnt),
4627[C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4628 CNTR_NORMAL,
4629 access_pio_sb_mem_fifo0_err_cnt),
4630[C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4631 CNTR_NORMAL,
4632 access_pio_csr_parity_err_cnt),
4633[C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4634 CNTR_NORMAL,
4635 access_pio_write_addr_parity_err_cnt),
4636[C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4637 CNTR_NORMAL,
4638 access_pio_write_bad_ctxt_err_cnt),
4639/* SendDmaErrStatus */
4640[C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4641 0, CNTR_NORMAL,
4642 access_sdma_pcie_req_tracking_cor_err_cnt),
4643[C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4644 0, CNTR_NORMAL,
4645 access_sdma_pcie_req_tracking_unc_err_cnt),
4646[C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4647 CNTR_NORMAL,
4648 access_sdma_csr_parity_err_cnt),
4649[C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4650 CNTR_NORMAL,
4651 access_sdma_rpy_tag_err_cnt),
4652/* SendEgressErrStatus */
4653[C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4654 CNTR_NORMAL,
4655 access_tx_read_pio_memory_csr_unc_err_cnt),
4656[C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4657 0, CNTR_NORMAL,
4658 access_tx_read_sdma_memory_csr_err_cnt),
4659[C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4660 CNTR_NORMAL,
4661 access_tx_egress_fifo_cor_err_cnt),
4662[C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4663 CNTR_NORMAL,
4664 access_tx_read_pio_memory_cor_err_cnt),
4665[C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4666 CNTR_NORMAL,
4667 access_tx_read_sdma_memory_cor_err_cnt),
4668[C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4669 CNTR_NORMAL,
4670 access_tx_sb_hdr_cor_err_cnt),
4671[C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4672 CNTR_NORMAL,
4673 access_tx_credit_overrun_err_cnt),
4674[C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4675 CNTR_NORMAL,
4676 access_tx_launch_fifo8_cor_err_cnt),
4677[C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4678 CNTR_NORMAL,
4679 access_tx_launch_fifo7_cor_err_cnt),
4680[C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4681 CNTR_NORMAL,
4682 access_tx_launch_fifo6_cor_err_cnt),
4683[C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4684 CNTR_NORMAL,
4685 access_tx_launch_fifo5_cor_err_cnt),
4686[C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4687 CNTR_NORMAL,
4688 access_tx_launch_fifo4_cor_err_cnt),
4689[C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4690 CNTR_NORMAL,
4691 access_tx_launch_fifo3_cor_err_cnt),
4692[C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4693 CNTR_NORMAL,
4694 access_tx_launch_fifo2_cor_err_cnt),
4695[C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4696 CNTR_NORMAL,
4697 access_tx_launch_fifo1_cor_err_cnt),
4698[C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4699 CNTR_NORMAL,
4700 access_tx_launch_fifo0_cor_err_cnt),
4701[C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4702 CNTR_NORMAL,
4703 access_tx_credit_return_vl_err_cnt),
4704[C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4705 CNTR_NORMAL,
4706 access_tx_hcrc_insertion_err_cnt),
4707[C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4708 CNTR_NORMAL,
4709 access_tx_egress_fifo_unc_err_cnt),
4710[C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4711 CNTR_NORMAL,
4712 access_tx_read_pio_memory_unc_err_cnt),
4713[C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4714 CNTR_NORMAL,
4715 access_tx_read_sdma_memory_unc_err_cnt),
4716[C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4717 CNTR_NORMAL,
4718 access_tx_sb_hdr_unc_err_cnt),
4719[C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4720 CNTR_NORMAL,
4721 access_tx_credit_return_partiy_err_cnt),
4722[C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4723 0, 0, CNTR_NORMAL,
4724 access_tx_launch_fifo8_unc_or_parity_err_cnt),
4725[C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4726 0, 0, CNTR_NORMAL,
4727 access_tx_launch_fifo7_unc_or_parity_err_cnt),
4728[C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4729 0, 0, CNTR_NORMAL,
4730 access_tx_launch_fifo6_unc_or_parity_err_cnt),
4731[C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4732 0, 0, CNTR_NORMAL,
4733 access_tx_launch_fifo5_unc_or_parity_err_cnt),
4734[C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4735 0, 0, CNTR_NORMAL,
4736 access_tx_launch_fifo4_unc_or_parity_err_cnt),
4737[C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4738 0, 0, CNTR_NORMAL,
4739 access_tx_launch_fifo3_unc_or_parity_err_cnt),
4740[C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4741 0, 0, CNTR_NORMAL,
4742 access_tx_launch_fifo2_unc_or_parity_err_cnt),
4743[C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4744 0, 0, CNTR_NORMAL,
4745 access_tx_launch_fifo1_unc_or_parity_err_cnt),
4746[C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4747 0, 0, CNTR_NORMAL,
4748 access_tx_launch_fifo0_unc_or_parity_err_cnt),
4749[C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4750 0, 0, CNTR_NORMAL,
4751 access_tx_sdma15_disallowed_packet_err_cnt),
4752[C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4753 0, 0, CNTR_NORMAL,
4754 access_tx_sdma14_disallowed_packet_err_cnt),
4755[C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4756 0, 0, CNTR_NORMAL,
4757 access_tx_sdma13_disallowed_packet_err_cnt),
4758[C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4759 0, 0, CNTR_NORMAL,
4760 access_tx_sdma12_disallowed_packet_err_cnt),
4761[C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4762 0, 0, CNTR_NORMAL,
4763 access_tx_sdma11_disallowed_packet_err_cnt),
4764[C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4765 0, 0, CNTR_NORMAL,
4766 access_tx_sdma10_disallowed_packet_err_cnt),
4767[C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4768 0, 0, CNTR_NORMAL,
4769 access_tx_sdma9_disallowed_packet_err_cnt),
4770[C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4771 0, 0, CNTR_NORMAL,
4772 access_tx_sdma8_disallowed_packet_err_cnt),
4773[C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4774 0, 0, CNTR_NORMAL,
4775 access_tx_sdma7_disallowed_packet_err_cnt),
4776[C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4777 0, 0, CNTR_NORMAL,
4778 access_tx_sdma6_disallowed_packet_err_cnt),
4779[C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4780 0, 0, CNTR_NORMAL,
4781 access_tx_sdma5_disallowed_packet_err_cnt),
4782[C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4783 0, 0, CNTR_NORMAL,
4784 access_tx_sdma4_disallowed_packet_err_cnt),
4785[C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4786 0, 0, CNTR_NORMAL,
4787 access_tx_sdma3_disallowed_packet_err_cnt),
4788[C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4789 0, 0, CNTR_NORMAL,
4790 access_tx_sdma2_disallowed_packet_err_cnt),
4791[C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4792 0, 0, CNTR_NORMAL,
4793 access_tx_sdma1_disallowed_packet_err_cnt),
4794[C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4795 0, 0, CNTR_NORMAL,
4796 access_tx_sdma0_disallowed_packet_err_cnt),
4797[C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4798 CNTR_NORMAL,
4799 access_tx_config_parity_err_cnt),
4800[C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4801 CNTR_NORMAL,
4802 access_tx_sbrd_ctl_csr_parity_err_cnt),
4803[C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4804 CNTR_NORMAL,
4805 access_tx_launch_csr_parity_err_cnt),
4806[C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4807 CNTR_NORMAL,
4808 access_tx_illegal_vl_err_cnt),
4809[C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4810 "TxSbrdCtlStateMachineParityErr", 0, 0,
4811 CNTR_NORMAL,
4812 access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4813[C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4814 CNTR_NORMAL,
4815 access_egress_reserved_10_err_cnt),
4816[C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4817 CNTR_NORMAL,
4818 access_egress_reserved_9_err_cnt),
4819[C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4820 0, 0, CNTR_NORMAL,
4821 access_tx_sdma_launch_intf_parity_err_cnt),
4822[C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4823 CNTR_NORMAL,
4824 access_tx_pio_launch_intf_parity_err_cnt),
4825[C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4826 CNTR_NORMAL,
4827 access_egress_reserved_6_err_cnt),
4828[C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4829 CNTR_NORMAL,
4830 access_tx_incorrect_link_state_err_cnt),
4831[C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4832 CNTR_NORMAL,
4833 access_tx_linkdown_err_cnt),
4834[C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4835 "EgressFifoUnderrunOrParityErr", 0, 0,
4836 CNTR_NORMAL,
4837 access_tx_egress_fifi_underrun_or_parity_err_cnt),
4838[C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4839 CNTR_NORMAL,
4840 access_egress_reserved_2_err_cnt),
4841[C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4842 CNTR_NORMAL,
4843 access_tx_pkt_integrity_mem_unc_err_cnt),
4844[C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4845 CNTR_NORMAL,
4846 access_tx_pkt_integrity_mem_cor_err_cnt),
4847/* SendErrStatus */
4848[C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4849 CNTR_NORMAL,
4850 access_send_csr_write_bad_addr_err_cnt),
4851[C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4852 CNTR_NORMAL,
4853 access_send_csr_read_bad_addr_err_cnt),
4854[C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4855 CNTR_NORMAL,
4856 access_send_csr_parity_cnt),
4857/* SendCtxtErrStatus */
4858[C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4859 CNTR_NORMAL,
4860 access_pio_write_out_of_bounds_err_cnt),
4861[C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4862 CNTR_NORMAL,
4863 access_pio_write_overflow_err_cnt),
4864[C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4865 0, 0, CNTR_NORMAL,
4866 access_pio_write_crosses_boundary_err_cnt),
4867[C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4868 CNTR_NORMAL,
4869 access_pio_disallowed_packet_err_cnt),
4870[C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4871 CNTR_NORMAL,
4872 access_pio_inconsistent_sop_err_cnt),
4873/* SendDmaEngErrStatus */
4874[C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4875 0, 0, CNTR_NORMAL,
4876 access_sdma_header_request_fifo_cor_err_cnt),
4877[C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4878 CNTR_NORMAL,
4879 access_sdma_header_storage_cor_err_cnt),
4880[C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4881 CNTR_NORMAL,
4882 access_sdma_packet_tracking_cor_err_cnt),
4883[C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4884 CNTR_NORMAL,
4885 access_sdma_assembly_cor_err_cnt),
4886[C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
4887 CNTR_NORMAL,
4888 access_sdma_desc_table_cor_err_cnt),
4889[C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
4890 0, 0, CNTR_NORMAL,
4891 access_sdma_header_request_fifo_unc_err_cnt),
4892[C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
4893 CNTR_NORMAL,
4894 access_sdma_header_storage_unc_err_cnt),
4895[C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
4896 CNTR_NORMAL,
4897 access_sdma_packet_tracking_unc_err_cnt),
4898[C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
4899 CNTR_NORMAL,
4900 access_sdma_assembly_unc_err_cnt),
4901[C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
4902 CNTR_NORMAL,
4903 access_sdma_desc_table_unc_err_cnt),
4904[C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
4905 CNTR_NORMAL,
4906 access_sdma_timeout_err_cnt),
4907[C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
4908 CNTR_NORMAL,
4909 access_sdma_header_length_err_cnt),
4910[C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
4911 CNTR_NORMAL,
4912 access_sdma_header_address_err_cnt),
4913[C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
4914 CNTR_NORMAL,
4915 access_sdma_header_select_err_cnt),
4916[C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
4917 CNTR_NORMAL,
4918 access_sdma_reserved_9_err_cnt),
4919[C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
4920 CNTR_NORMAL,
4921 access_sdma_packet_desc_overflow_err_cnt),
4922[C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
4923 CNTR_NORMAL,
4924 access_sdma_length_mismatch_err_cnt),
4925[C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
4926 CNTR_NORMAL,
4927 access_sdma_halt_err_cnt),
4928[C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
4929 CNTR_NORMAL,
4930 access_sdma_mem_read_err_cnt),
4931[C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
4932 CNTR_NORMAL,
4933 access_sdma_first_desc_err_cnt),
4934[C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
4935 CNTR_NORMAL,
4936 access_sdma_tail_out_of_bounds_err_cnt),
4937[C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
4938 CNTR_NORMAL,
4939 access_sdma_too_long_err_cnt),
4940[C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
4941 CNTR_NORMAL,
4942 access_sdma_gen_mismatch_err_cnt),
4943[C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
4944 CNTR_NORMAL,
4945 access_sdma_wrong_dw_err_cnt),
77241056
MM		 4946};
4947
4948static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
4949[C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
4950 CNTR_NORMAL),
4951[C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
4952 CNTR_NORMAL),
4953[C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
4954 CNTR_NORMAL),
4955[C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
4956 CNTR_NORMAL),
4957[C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
4958 CNTR_NORMAL),
4959[C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
4960 CNTR_NORMAL),
4961[C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
4962 CNTR_NORMAL),
4963[C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
4964[C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
4965[C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
4966[C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
17fb4f29 4967 CNTR_SYNTH | CNTR_VL),
77241056 4968[C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
17fb4f29 4969 CNTR_SYNTH | CNTR_VL),
77241056 4970[C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
17fb4f29 4971 CNTR_SYNTH | CNTR_VL),
77241056
MM		 4972[C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
4973[C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
4974[C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
17fb4f29 4975 access_sw_link_dn_cnt),
77241056 4976[C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
17fb4f29 4977 access_sw_link_up_cnt),
6d014530
DL		 4978[C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
4979 access_sw_unknown_frame_cnt),
77241056 4980[C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
17fb4f29 4981 access_sw_xmit_discards),
77241056 4982[C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
17fb4f29
JJ		 4983 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
4984 access_sw_xmit_discards),
77241056 4985[C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
17fb4f29 4986 access_xmit_constraint_errs),
77241056 4987[C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
17fb4f29 4988 access_rcv_constraint_errs),
77241056
MM		 4989[C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
4990[C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
4991[C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
4992[C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
4993[C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
4994[C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
4995[C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
4996[C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
4997[C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
4998[C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
4999[C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5000[C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5001[C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5002 access_sw_cpu_rc_acks),
5003[C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
17fb4f29 5004 access_sw_cpu_rc_qacks),
77241056 5005[C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
17fb4f29 5006 access_sw_cpu_rc_delayed_comp),
77241056
MM		 5007[OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5008[OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5009[OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5010[OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5011[OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5012[OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5013[OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5014[OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5015[OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5016[OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5017[OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5018[OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5019[OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5020[OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5021[OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5022[OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5023[OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5024[OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5025[OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5026[OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5027[OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5028[OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5029[OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5030[OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5031[OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5032[OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5033[OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5034[OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5035[OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5036[OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5037[OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5038[OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5039[OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5040[OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5041[OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5042[OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5043[OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5044[OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5045[OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5046[OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5047[OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5048[OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5049[OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5050[OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5051[OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5052[OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5053[OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5054[OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5055[OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5056[OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5057[OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5058[OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5059[OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5060[OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5061[OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5062[OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5063[OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5064[OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5065[OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5066[OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5067[OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5068[OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5069[OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5070[OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5071[OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5072[OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5073[OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5074[OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5075[OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5076[OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5077[OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5078[OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5079[OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5080[OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5081[OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5082[OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5083[OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5084[OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5085[OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5086[OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5087};
5088
5089/* ======================================================================== */
5090
77241056
MM		 5091/* return true if this is chip revision revision a */
5092int is_ax(struct hfi1_devdata *dd)
5093{
5094 u8 chip_rev_minor =
5095 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5096 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5097 return (chip_rev_minor & 0xf0) == 0;
5098}
5099
5100/* return true if this is chip revision revision b */
5101int is_bx(struct hfi1_devdata *dd)
5102{
5103 u8 chip_rev_minor =
5104 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5105 & CCE_REVISION_CHIP_REV_MINOR_MASK;
995deafa 5106 return (chip_rev_minor & 0xF0) == 0x10;
77241056
MM		 5107}
5108
5109/*
5110 * Append string s to buffer buf. Arguments curp and len are the current
5111 * position and remaining length, respectively.
5112 *
5113 * return 0 on success, 1 on out of room
5114 */
5115static int append_str(char *buf, char **curp, int *lenp, const char *s)
5116{
5117 char *p = *curp;
5118 int len = *lenp;
5119 int result = 0; /* success */
5120 char c;
5121
5122 /* add a comma, if first in the buffer */
5123 if (p != buf) {
5124 if (len == 0) {
5125 result = 1; /* out of room */
5126 goto done;
5127 }
5128 *p++ = ',';
5129 len--;
5130 }
5131
5132 /* copy the string */
5133 while ((c = *s++) != 0) {
5134 if (len == 0) {
5135 result = 1; /* out of room */
5136 goto done;
5137 }
5138 *p++ = c;
5139 len--;
5140 }
5141
5142done:
5143 /* write return values */
5144 *curp = p;
5145 *lenp = len;
5146
5147 return result;
5148}
5149
5150/*
5151 * Using the given flag table, print a comma separated string into
5152 * the buffer. End in '*' if the buffer is too short.
5153 */
5154static char *flag_string(char *buf, int buf_len, u64 flags,
17fb4f29 5155 struct flag_table *table, int table_size)
77241056
MM		 5156{
5157 char extra[32];
5158 char *p = buf;
5159 int len = buf_len;
5160 int no_room = 0;
5161 int i;
5162
5163 /* make sure there is at least 2 so we can form "*" */
5164 if (len < 2)
5165 return "";
5166
5167 len--; /* leave room for a nul */
5168 for (i = 0; i < table_size; i++) {
5169 if (flags & table[i].flag) {
5170 no_room = append_str(buf, &p, &len, table[i].str);
5171 if (no_room)
5172 break;
5173 flags &= ~table[i].flag;
5174 }
5175 }
5176
5177 /* any undocumented bits left? */
5178 if (!no_room && flags) {
5179 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5180 no_room = append_str(buf, &p, &len, extra);
5181 }
5182
5183 /* add * if ran out of room */
5184 if (no_room) {
5185 /* may need to back up to add space for a '*' */
5186 if (len == 0)
5187 --p;
5188 *p++ = '*';
5189 }
5190
5191 /* add final nul - space already allocated above */
5192 *p = 0;
5193 return buf;
5194}
5195
5196/* first 8 CCE error interrupt source names */
5197static const char * const cce_misc_names[] = {
5198 "CceErrInt", /* 0 */
5199 "RxeErrInt", /* 1 */
5200 "MiscErrInt", /* 2 */
5201 "Reserved3", /* 3 */
5202 "PioErrInt", /* 4 */
5203 "SDmaErrInt", /* 5 */
5204 "EgressErrInt", /* 6 */
5205 "TxeErrInt" /* 7 */
5206};
5207
5208/*
5209 * Return the miscellaneous error interrupt name.
5210 */
5211static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5212{
5213 if (source < ARRAY_SIZE(cce_misc_names))
5214 strncpy(buf, cce_misc_names[source], bsize);
5215 else
17fb4f29
JJ		 5216 snprintf(buf, bsize, "Reserved%u",
5217 source + IS_GENERAL_ERR_START);
77241056
MM		 5218
5219 return buf;
5220}
5221
5222/*
5223 * Return the SDMA engine error interrupt name.
5224 */
5225static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5226{
5227 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5228 return buf;
5229}
5230
5231/*
5232 * Return the send context error interrupt name.
5233 */
5234static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5235{
5236 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5237 return buf;
5238}
5239
5240static const char * const various_names[] = {
5241 "PbcInt",
5242 "GpioAssertInt",
5243 "Qsfp1Int",
5244 "Qsfp2Int",
5245 "TCritInt"
5246};
5247
5248/*
5249 * Return the various interrupt name.
5250 */
5251static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5252{
5253 if (source < ARRAY_SIZE(various_names))
5254 strncpy(buf, various_names[source], bsize);
5255 else
8638b77f 5256 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
77241056
MM		 5257 return buf;
5258}
5259
5260/*
5261 * Return the DC interrupt name.
5262 */
5263static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5264{
5265 static const char * const dc_int_names[] = {
5266 "common",
5267 "lcb",
5268 "8051",
5269 "lbm" /* local block merge */
5270 };
5271
5272 if (source < ARRAY_SIZE(dc_int_names))
5273 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5274 else
5275 snprintf(buf, bsize, "DCInt%u", source);
5276 return buf;
5277}
5278
5279static const char * const sdma_int_names[] = {
5280 "SDmaInt",
5281 "SdmaIdleInt",
5282 "SdmaProgressInt",
5283};
5284
5285/*
5286 * Return the SDMA engine interrupt name.
5287 */
5288static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5289{
5290 /* what interrupt */
5291 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
5292 /* which engine */
5293 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5294
5295 if (likely(what < 3))
5296 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5297 else
5298 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5299 return buf;
5300}
5301
5302/*
5303 * Return the receive available interrupt name.
5304 */
5305static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5306{
5307 snprintf(buf, bsize, "RcvAvailInt%u", source);
5308 return buf;
5309}
5310
5311/*
5312 * Return the receive urgent interrupt name.
5313 */
5314static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5315{
5316 snprintf(buf, bsize, "RcvUrgentInt%u", source);
5317 return buf;
5318}
5319
5320/*
5321 * Return the send credit interrupt name.
5322 */
5323static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5324{
5325 snprintf(buf, bsize, "SendCreditInt%u", source);
5326 return buf;
5327}
5328
5329/*
5330 * Return the reserved interrupt name.
5331 */
5332static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5333{
5334 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5335 return buf;
5336}
5337
5338static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5339{
5340 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5341 cce_err_status_flags,
5342 ARRAY_SIZE(cce_err_status_flags));
77241056
MM		 5343}
5344
5345static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5346{
5347 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5348 rxe_err_status_flags,
5349 ARRAY_SIZE(rxe_err_status_flags));
77241056
MM		 5350}
5351
5352static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5353{
5354 return flag_string(buf, buf_len, flags, misc_err_status_flags,
17fb4f29 5355 ARRAY_SIZE(misc_err_status_flags));
77241056
MM		 5356}
5357
5358static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5359{
5360 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5361 pio_err_status_flags,
5362 ARRAY_SIZE(pio_err_status_flags));
77241056
MM		 5363}
5364
5365static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5366{
5367 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5368 sdma_err_status_flags,
5369 ARRAY_SIZE(sdma_err_status_flags));
77241056
MM		 5370}
5371
5372static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5373{
5374 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5375 egress_err_status_flags,
5376 ARRAY_SIZE(egress_err_status_flags));
77241056
MM		 5377}
5378
5379static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5380{
5381 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5382 egress_err_info_flags,
5383 ARRAY_SIZE(egress_err_info_flags));
77241056
MM		 5384}
5385
5386static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5387{
5388 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5389 send_err_status_flags,
5390 ARRAY_SIZE(send_err_status_flags));
77241056
MM		 5391}
5392
5393static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5394{
5395 char buf[96];
2c5b521a 5396 int i = 0;
77241056
MM		 5397
5398 /*
5399 * For most these errors, there is nothing that can be done except
5400 * report or record it.
5401 */
5402 dd_dev_info(dd, "CCE Error: %s\n",
17fb4f29 5403 cce_err_status_string(buf, sizeof(buf), reg));
77241056 5404
995deafa
MM		 5405 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5406 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
77241056
MM		 5407 /* this error requires a manual drop into SPC freeze mode */
5408 /* then a fix up */
5409 start_freeze_handling(dd->pport, FREEZE_SELF);
5410 }
2c5b521a
JR		 5411
5412 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5413 if (reg & (1ull << i)) {
5414 incr_cntr64(&dd->cce_err_status_cnt[i]);
5415 /* maintain a counter over all cce_err_status errors */
5416 incr_cntr64(&dd->sw_cce_err_status_aggregate);
5417 }
5418 }
77241056
MM		 5419}
5420
5421/*
5422 * Check counters for receive errors that do not have an interrupt
5423 * associated with them.
5424 */
5425#define RCVERR_CHECK_TIME 10
5426static void update_rcverr_timer(unsigned long opaque)
5427{
5428 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
5429 struct hfi1_pportdata *ppd = dd->pport;
5430 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5431
5432 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
17fb4f29 5433 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
77241056 5434 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
17fb4f29
JJ		 5435 set_link_down_reason(
5436 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5437 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
77241056
MM		 5438 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
5439 }
50e5dcbe 5440 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
77241056
MM		 5441
5442 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5443}
5444
5445static int init_rcverr(struct hfi1_devdata *dd)
5446{
24523a94 5447 setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd);
77241056
MM		 5448 /* Assume the hardware counter has been reset */
5449 dd->rcv_ovfl_cnt = 0;
5450 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5451}
5452
5453static void free_rcverr(struct hfi1_devdata *dd)
5454{
5455 if (dd->rcverr_timer.data)
5456 del_timer_sync(&dd->rcverr_timer);
5457 dd->rcverr_timer.data = 0;
5458}
5459
5460static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5461{
5462 char buf[96];
2c5b521a 5463 int i = 0;
77241056
MM		 5464
5465 dd_dev_info(dd, "Receive Error: %s\n",
17fb4f29 5466 rxe_err_status_string(buf, sizeof(buf), reg));
77241056
MM		 5467
5468 if (reg & ALL_RXE_FREEZE_ERR) {
5469 int flags = 0;
5470
5471 /*
5472 * Freeze mode recovery is disabled for the errors
5473 * in RXE_FREEZE_ABORT_MASK
5474 */
995deafa 5475 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
77241056
MM		 5476 flags = FREEZE_ABORT;
5477
5478 start_freeze_handling(dd->pport, flags);
5479 }
2c5b521a
JR		 5480
5481 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5482 if (reg & (1ull << i))
5483 incr_cntr64(&dd->rcv_err_status_cnt[i]);
5484 }
77241056
MM		 5485}
5486
5487static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5488{
5489 char buf[96];
2c5b521a 5490 int i = 0;
77241056
MM		 5491
5492 dd_dev_info(dd, "Misc Error: %s",
17fb4f29 5493 misc_err_status_string(buf, sizeof(buf), reg));
2c5b521a
JR		 5494 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5495 if (reg & (1ull << i))
5496 incr_cntr64(&dd->misc_err_status_cnt[i]);
5497 }
77241056
MM		 5498}
5499
5500static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5501{
5502 char buf[96];
2c5b521a 5503 int i = 0;
77241056
MM		 5504
5505 dd_dev_info(dd, "PIO Error: %s\n",
17fb4f29 5506 pio_err_status_string(buf, sizeof(buf), reg));
77241056
MM		 5507
5508 if (reg & ALL_PIO_FREEZE_ERR)
5509 start_freeze_handling(dd->pport, 0);
2c5b521a
JR		 5510
5511 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5512 if (reg & (1ull << i))
5513 incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5514 }
77241056
MM		 5515}
5516
5517static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5518{
5519 char buf[96];
2c5b521a 5520 int i = 0;
77241056
MM		 5521
5522 dd_dev_info(dd, "SDMA Error: %s\n",
17fb4f29 5523 sdma_err_status_string(buf, sizeof(buf), reg));
77241056
MM		 5524
5525 if (reg & ALL_SDMA_FREEZE_ERR)
5526 start_freeze_handling(dd->pport, 0);
2c5b521a
JR		 5527
5528 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5529 if (reg & (1ull << i))
5530 incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5531 }
77241056
MM		 5532}
5533
69a00b8e 5534static inline void __count_port_discards(struct hfi1_pportdata *ppd)
77241056 5535{
69a00b8e
MM		 5536 incr_cntr64(&ppd->port_xmit_discards);
5537}
77241056 5538
69a00b8e
MM		 5539static void count_port_inactive(struct hfi1_devdata *dd)
5540{
5541 __count_port_discards(dd->pport);
77241056
MM		 5542}
5543
5544/*
5545 * We have had a "disallowed packet" error during egress. Determine the
5546 * integrity check which failed, and update relevant error counter, etc.
5547 *
5548 * Note that the SEND_EGRESS_ERR_INFO register has only a single
5549 * bit of state per integrity check, and so we can miss the reason for an
5550 * egress error if more than one packet fails the same integrity check
5551 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5552 */
69a00b8e
MM		 5553static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5554 int vl)
77241056
MM		 5555{
5556 struct hfi1_pportdata *ppd = dd->pport;
5557 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5558 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5559 char buf[96];
5560
5561 /* clear down all observed info as quickly as possible after read */
5562 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5563
5564 dd_dev_info(dd,
17fb4f29
JJ		 5565 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5566 info, egress_err_info_string(buf, sizeof(buf), info), src);
77241056
MM		 5567
5568 /* Eventually add other counters for each bit */
69a00b8e
MM		 5569 if (info & PORT_DISCARD_EGRESS_ERRS) {
5570 int weight, i;
77241056 5571
69a00b8e 5572 /*
4c9e7aac
DL		 5573 * Count all applicable bits as individual errors and
5574 * attribute them to the packet that triggered this handler.
5575 * This may not be completely accurate due to limitations
5576 * on the available hardware error information. There is
5577 * a single information register and any number of error
5578 * packets may have occurred and contributed to it before
5579 * this routine is called. This means that:
5580 * a) If multiple packets with the same error occur before
5581 * this routine is called, earlier packets are missed.
5582 * There is only a single bit for each error type.
5583 * b) Errors may not be attributed to the correct VL.
5584 * The driver is attributing all bits in the info register
5585 * to the packet that triggered this call, but bits
5586 * could be an accumulation of different packets with
5587 * different VLs.
5588 * c) A single error packet may have multiple counts attached
5589 * to it. There is no way for the driver to know if
5590 * multiple bits set in the info register are due to a
5591 * single packet or multiple packets. The driver assumes
5592 * multiple packets.
69a00b8e 5593 */
4c9e7aac 5594 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
69a00b8e
MM		 5595 for (i = 0; i < weight; i++) {
5596 __count_port_discards(ppd);
5597 if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5598 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5599 else if (vl == 15)
5600 incr_cntr64(&ppd->port_xmit_discards_vl
5601 [C_VL_15]);
5602 }
77241056
MM		 5603 }
5604}
5605
5606/*
5607 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5608 * register. Does it represent a 'port inactive' error?
5609 */
5610static inline int port_inactive_err(u64 posn)
5611{
5612 return (posn >= SEES(TX_LINKDOWN) &&
5613 posn <= SEES(TX_INCORRECT_LINK_STATE));
5614}
5615
5616/*
5617 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5618 * register. Does it represent a 'disallowed packet' error?
5619 */
69a00b8e 5620static inline int disallowed_pkt_err(int posn)
77241056
MM		 5621{
5622 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5623 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5624}
5625
69a00b8e
MM		 5626/*
5627 * Input value is a bit position of one of the SDMA engine disallowed
5628 * packet errors. Return which engine. Use of this must be guarded by
5629 * disallowed_pkt_err().
5630 */
5631static inline int disallowed_pkt_engine(int posn)
5632{
5633 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5634}
5635
5636/*
5637 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot
5638 * be done.
5639 */
5640static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5641{
5642 struct sdma_vl_map *m;
5643 int vl;
5644
5645 /* range check */
5646 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5647 return -1;
5648
5649 rcu_read_lock();
5650 m = rcu_dereference(dd->sdma_map);
5651 vl = m->engine_to_vl[engine];
5652 rcu_read_unlock();
5653
5654 return vl;
5655}
5656
5657/*
5658 * Translate the send context (sofware index) into a VL. Return -1 if the
5659 * translation cannot be done.
5660 */
5661static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5662{
5663 struct send_context_info *sci;
5664 struct send_context *sc;
5665 int i;
5666
5667 sci = &dd->send_contexts[sw_index];
5668
5669 /* there is no information for user (PSM) and ack contexts */
44306f15 5670 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
69a00b8e
MM		 5671 return -1;
5672
5673 sc = sci->sc;
5674 if (!sc)
5675 return -1;
5676 if (dd->vld[15].sc == sc)
5677 return 15;
5678 for (i = 0; i < num_vls; i++)
5679 if (dd->vld[i].sc == sc)
5680 return i;
5681
5682 return -1;
5683}
5684
77241056
MM		 5685static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5686{
5687 u64 reg_copy = reg, handled = 0;
5688 char buf[96];
2c5b521a 5689 int i = 0;
77241056
MM		 5690
5691 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5692 start_freeze_handling(dd->pport, 0);
69a00b8e
MM		 5693 else if (is_ax(dd) &&
5694 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5695 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
77241056
MM		 5696 start_freeze_handling(dd->pport, 0);
5697
5698 while (reg_copy) {
5699 int posn = fls64(reg_copy);
69a00b8e 5700 /* fls64() returns a 1-based offset, we want it zero based */
77241056 5701 int shift = posn - 1;
69a00b8e 5702 u64 mask = 1ULL << shift;
77241056
MM		 5703
5704 if (port_inactive_err(shift)) {
5705 count_port_inactive(dd);
69a00b8e 5706 handled |= mask;
77241056 5707 } else if (disallowed_pkt_err(shift)) {
69a00b8e
MM		 5708 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5709
5710 handle_send_egress_err_info(dd, vl);
5711 handled |= mask;
77241056 5712 }
69a00b8e 5713 reg_copy &= ~mask;
77241056
MM		 5714 }
5715
5716 reg &= ~handled;
5717
5718 if (reg)
5719 dd_dev_info(dd, "Egress Error: %s\n",
17fb4f29 5720 egress_err_status_string(buf, sizeof(buf), reg));
2c5b521a
JR		 5721
5722 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5723 if (reg & (1ull << i))
5724 incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5725 }
77241056
MM		 5726}
5727
5728static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5729{
5730 char buf[96];
2c5b521a 5731 int i = 0;
77241056
MM		 5732
5733 dd_dev_info(dd, "Send Error: %s\n",
17fb4f29 5734 send_err_status_string(buf, sizeof(buf), reg));
77241056 5735
2c5b521a
JR		 5736 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5737 if (reg & (1ull << i))
5738 incr_cntr64(&dd->send_err_status_cnt[i]);
5739 }
77241056
MM		 5740}
5741
5742/*
5743 * The maximum number of times the error clear down will loop before
5744 * blocking a repeating error. This value is arbitrary.
5745 */
5746#define MAX_CLEAR_COUNT 20
5747
5748/*
5749 * Clear and handle an error register. All error interrupts are funneled
5750 * through here to have a central location to correctly handle single-
5751 * or multi-shot errors.
5752 *
5753 * For non per-context registers, call this routine with a context value
5754 * of 0 so the per-context offset is zero.
5755 *
5756 * If the handler loops too many times, assume that something is wrong
5757 * and can't be fixed, so mask the error bits.
5758 */
5759static void interrupt_clear_down(struct hfi1_devdata *dd,
5760 u32 context,
5761 const struct err_reg_info *eri)
5762{
5763 u64 reg;
5764 u32 count;
5765
5766 /* read in a loop until no more errors are seen */
5767 count = 0;
5768 while (1) {
5769 reg = read_kctxt_csr(dd, context, eri->status);
5770 if (reg == 0)
5771 break;
5772 write_kctxt_csr(dd, context, eri->clear, reg);
5773 if (likely(eri->handler))
5774 eri->handler(dd, context, reg);
5775 count++;
5776 if (count > MAX_CLEAR_COUNT) {
5777 u64 mask;
5778
5779 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
17fb4f29 5780 eri->desc, reg);
77241056
MM		 5781 /*
5782 * Read-modify-write so any other masked bits
5783 * remain masked.
5784 */
5785 mask = read_kctxt_csr(dd, context, eri->mask);
5786 mask &= ~reg;
5787 write_kctxt_csr(dd, context, eri->mask, mask);
5788 break;
5789 }
5790 }
5791}
5792
5793/*
5794 * CCE block "misc" interrupt. Source is < 16.
5795 */
5796static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5797{
5798 const struct err_reg_info *eri = &misc_errs[source];
5799
5800 if (eri->handler) {
5801 interrupt_clear_down(dd, 0, eri);
5802 } else {
5803 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
17fb4f29 5804 source);
77241056
MM		 5805 }
5806}
5807
5808static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5809{
5810 return flag_string(buf, buf_len, flags,
17fb4f29
JJ		 5811 sc_err_status_flags,
5812 ARRAY_SIZE(sc_err_status_flags));
77241056
MM		 5813}
5814
5815/*
5816 * Send context error interrupt. Source (hw_context) is < 160.
5817 *
5818 * All send context errors cause the send context to halt. The normal
5819 * clear-down mechanism cannot be used because we cannot clear the
5820 * error bits until several other long-running items are done first.
5821 * This is OK because with the context halted, nothing else is going
5822 * to happen on it anyway.
5823 */
5824static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5825 unsigned int hw_context)
5826{
5827 struct send_context_info *sci;
5828 struct send_context *sc;
5829 char flags[96];
5830 u64 status;
5831 u32 sw_index;
2c5b521a 5832 int i = 0;
77241056
MM		 5833
5834 sw_index = dd->hw_to_sw[hw_context];
5835 if (sw_index >= dd->num_send_contexts) {
5836 dd_dev_err(dd,
17fb4f29
JJ		 5837 "out of range sw index %u for send context %u\n",
5838 sw_index, hw_context);
77241056
MM		 5839 return;
5840 }
5841 sci = &dd->send_contexts[sw_index];
5842 sc = sci->sc;
5843 if (!sc) {
5844 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
17fb4f29 5845 sw_index, hw_context);
77241056
MM		 5846 return;
5847 }
5848
5849 /* tell the software that a halt has begun */
5850 sc_stop(sc, SCF_HALTED);
5851
5852 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5853
5854 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
17fb4f29
JJ		 5855 send_context_err_status_string(flags, sizeof(flags),
5856 status));
77241056
MM		 5857
5858 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
69a00b8e 5859 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
77241056
MM		 5860
5861 /*
5862 * Automatically restart halted kernel contexts out of interrupt
5863 * context. User contexts must ask the driver to restart the context.
5864 */
5865 if (sc->type != SC_USER)
5866 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
2c5b521a
JR		 5867
5868 /*
5869 * Update the counters for the corresponding status bits.
5870 * Note that these particular counters are aggregated over all
5871 * 160 contexts.
5872 */
5873 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5874 if (status & (1ull << i))
5875 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
5876 }
77241056
MM		 5877}
5878
5879static void handle_sdma_eng_err(struct hfi1_devdata *dd,
5880 unsigned int source, u64 status)
5881{
5882 struct sdma_engine *sde;
2c5b521a 5883 int i = 0;
77241056
MM		 5884
5885 sde = &dd->per_sdma[source];
5886#ifdef CONFIG_SDMA_VERBOSITY
5887 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5888 slashstrip(__FILE__), __LINE__, __func__);
5889 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
5890 sde->this_idx, source, (unsigned long long)status);
5891#endif
a699c6c2 5892 sde->err_cnt++;
77241056 5893 sdma_engine_error(sde, status);
2c5b521a
JR		 5894
5895 /*
5896 * Update the counters for the corresponding status bits.
5897 * Note that these particular counters are aggregated over
5898 * all 16 DMA engines.
5899 */
5900 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
5901 if (status & (1ull << i))
5902 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
5903 }
77241056
MM		 5904}
5905
5906/*
5907 * CCE block SDMA error interrupt. Source is < 16.
5908 */
5909static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
5910{
5911#ifdef CONFIG_SDMA_VERBOSITY
5912 struct sdma_engine *sde = &dd->per_sdma[source];
5913
5914 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5915 slashstrip(__FILE__), __LINE__, __func__);
5916 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
5917 source);
5918 sdma_dumpstate(sde);
5919#endif
5920 interrupt_clear_down(dd, source, &sdma_eng_err);
5921}
5922
5923/*
5924 * CCE block "various" interrupt. Source is < 8.
5925 */
5926static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
5927{
5928 const struct err_reg_info *eri = &various_err[source];
5929
5930 /*
5931 * TCritInt cannot go through interrupt_clear_down()
5932 * because it is not a second tier interrupt. The handler
5933 * should be called directly.
5934 */
5935 if (source == TCRIT_INT_SOURCE)
5936 handle_temp_err(dd);
5937 else if (eri->handler)
5938 interrupt_clear_down(dd, 0, eri);
5939 else
5940 dd_dev_info(dd,
17fb4f29
JJ		 5941 "%s: Unimplemented/reserved interrupt %d\n",
5942 __func__, source);
77241056
MM		 5943}
5944
5945static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
5946{
8ebd4cf1 5947 /* src_ctx is always zero */
77241056
MM		 5948 struct hfi1_pportdata *ppd = dd->pport;
5949 unsigned long flags;
5950 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
5951
5952 if (reg & QSFP_HFI0_MODPRST_N) {
77241056 5953 if (!qsfp_mod_present(ppd)) {
e8aa284b
EH		 5954 dd_dev_info(dd, "%s: QSFP module removed\n",
5955 __func__);
5956
77241056
MM		 5957 ppd->driver_link_ready = 0;
5958 /*
5959 * Cable removed, reset all our information about the
5960 * cache and cable capabilities
5961 */
5962
5963 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5964 /*
5965 * We don't set cache_refresh_required here as we expect
5966 * an interrupt when a cable is inserted
5967 */
5968 ppd->qsfp_info.cache_valid = 0;
8ebd4cf1
EH		 5969 ppd->qsfp_info.reset_needed = 0;
5970 ppd->qsfp_info.limiting_active = 0;
77241056 5971 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
17fb4f29 5972 flags);
8ebd4cf1
EH		 5973 /* Invert the ModPresent pin now to detect plug-in */
5974 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
5975 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
a9c05e35
BM		 5976
5977 if ((ppd->offline_disabled_reason >
5978 HFI1_ODR_MASK(
e1bf0d5e 5979 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
a9c05e35
BM		 5980 (ppd->offline_disabled_reason ==
5981 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
5982 ppd->offline_disabled_reason =
5983 HFI1_ODR_MASK(
e1bf0d5e 5984 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
a9c05e35 5985
77241056
MM		 5986 if (ppd->host_link_state == HLS_DN_POLL) {
5987 /*
5988 * The link is still in POLL. This means
5989 * that the normal link down processing
5990 * will not happen. We have to do it here
5991 * before turning the DC off.
5992 */
5993 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
5994 }
5995 } else {
e8aa284b
EH		 5996 dd_dev_info(dd, "%s: QSFP module inserted\n",
5997 __func__);
5998
77241056
MM		 5999 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6000 ppd->qsfp_info.cache_valid = 0;
6001 ppd->qsfp_info.cache_refresh_required = 1;
6002 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
17fb4f29 6003 flags);
77241056 6004
8ebd4cf1
EH		 6005 /*
6006 * Stop inversion of ModPresent pin to detect
6007 * removal of the cable
6008 */
77241056 6009 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
8ebd4cf1
EH		 6010 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6011 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6012
6013 ppd->offline_disabled_reason =
6014 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
77241056
MM		 6015 }
6016 }
6017
6018 if (reg & QSFP_HFI0_INT_N) {
e8aa284b 6019 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
17fb4f29 6020 __func__);
77241056
MM		 6021 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6022 ppd->qsfp_info.check_interrupt_flags = 1;
77241056
MM		 6023 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6024 }
6025
6026 /* Schedule the QSFP work only if there is a cable attached. */
6027 if (qsfp_mod_present(ppd))
6028 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
6029}
6030
6031static int request_host_lcb_access(struct hfi1_devdata *dd)
6032{
6033 int ret;
6034
6035 ret = do_8051_command(dd, HCMD_MISC,
17fb4f29
JJ		 6036 (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6037 LOAD_DATA_FIELD_ID_SHIFT, NULL);
77241056
MM		 6038 if (ret != HCMD_SUCCESS) {
6039 dd_dev_err(dd, "%s: command failed with error %d\n",
17fb4f29 6040 __func__, ret);
77241056
MM		 6041 }
6042 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6043}
6044
6045static int request_8051_lcb_access(struct hfi1_devdata *dd)
6046{
6047 int ret;
6048
6049 ret = do_8051_command(dd, HCMD_MISC,
17fb4f29
JJ		 6050 (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6051 LOAD_DATA_FIELD_ID_SHIFT, NULL);
77241056
MM		 6052 if (ret != HCMD_SUCCESS) {
6053 dd_dev_err(dd, "%s: command failed with error %d\n",
17fb4f29 6054 __func__, ret);
77241056
MM		 6055 }
6056 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6057}
6058
6059/*
6060 * Set the LCB selector - allow host access. The DCC selector always
6061 * points to the host.
6062 */
6063static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6064{
6065 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
17fb4f29
JJ		 6066 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6067 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
77241056
MM		 6068}
6069
6070/*
6071 * Clear the LCB selector - allow 8051 access. The DCC selector always
6072 * points to the host.
6073 */
6074static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6075{
6076 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
17fb4f29 6077 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
77241056
MM		 6078}
6079
6080/*
6081 * Acquire LCB access from the 8051. If the host already has access,
6082 * just increment a counter. Otherwise, inform the 8051 that the
6083 * host is taking access.
6084 *
6085 * Returns:
6086 * 0 on success
6087 * -EBUSY if the 8051 has control and cannot be disturbed
6088 * -errno if unable to acquire access from the 8051
6089 */
6090int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6091{
6092 struct hfi1_pportdata *ppd = dd->pport;
6093 int ret = 0;
6094
6095 /*
6096 * Use the host link state lock so the operation of this routine
6097 * { link state check, selector change, count increment } can occur
6098 * as a unit against a link state change. Otherwise there is a
6099 * race between the state change and the count increment.
6100 */
6101 if (sleep_ok) {
6102 mutex_lock(&ppd->hls_lock);
6103 } else {
951842b0 6104 while (!mutex_trylock(&ppd->hls_lock))
77241056
MM		 6105 udelay(1);
6106 }
6107
6108 /* this access is valid only when the link is up */
0c7f77af 6109 if (ppd->host_link_state & HLS_DOWN) {
77241056 6110 dd_dev_info(dd, "%s: link state %s not up\n",
17fb4f29 6111 __func__, link_state_name(ppd->host_link_state));
77241056
MM		 6112 ret = -EBUSY;
6113 goto done;
6114 }
6115
6116 if (dd->lcb_access_count == 0) {
6117 ret = request_host_lcb_access(dd);
6118 if (ret) {
6119 dd_dev_err(dd,
17fb4f29
JJ		 6120 "%s: unable to acquire LCB access, err %d\n",
6121 __func__, ret);
77241056
MM		 6122 goto done;
6123 }
6124 set_host_lcb_access(dd);
6125 }
6126 dd->lcb_access_count++;
6127done:
6128 mutex_unlock(&ppd->hls_lock);
6129 return ret;
6130}
6131
6132/*
6133 * Release LCB access by decrementing the use count. If the count is moving
6134 * from 1 to 0, inform 8051 that it has control back.
6135 *
6136 * Returns:
6137 * 0 on success
6138 * -errno if unable to release access to the 8051
6139 */
6140int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6141{
6142 int ret = 0;
6143
6144 /*
6145 * Use the host link state lock because the acquire needed it.
6146 * Here, we only need to keep { selector change, count decrement }
6147 * as a unit.
6148 */
6149 if (sleep_ok) {
6150 mutex_lock(&dd->pport->hls_lock);
6151 } else {
951842b0 6152 while (!mutex_trylock(&dd->pport->hls_lock))
77241056
MM		 6153 udelay(1);
6154 }
6155
6156 if (dd->lcb_access_count == 0) {
6157 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
17fb4f29 6158 __func__);
77241056
MM		 6159 goto done;
6160 }
6161
6162 if (dd->lcb_access_count == 1) {
6163 set_8051_lcb_access(dd);
6164 ret = request_8051_lcb_access(dd);
6165 if (ret) {
6166 dd_dev_err(dd,
17fb4f29
JJ		 6167 "%s: unable to release LCB access, err %d\n",
6168 __func__, ret);
77241056
MM		 6169 /* restore host access if the grant didn't work */
6170 set_host_lcb_access(dd);
6171 goto done;
6172 }
6173 }
6174 dd->lcb_access_count--;
6175done:
6176 mutex_unlock(&dd->pport->hls_lock);
6177 return ret;
6178}
6179
6180/*
6181 * Initialize LCB access variables and state. Called during driver load,
6182 * after most of the initialization is finished.
6183 *
6184 * The DC default is LCB access on for the host. The driver defaults to
6185 * leaving access to the 8051. Assign access now - this constrains the call
6186 * to this routine to be after all LCB set-up is done. In particular, after
6187 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6188 */
6189static void init_lcb_access(struct hfi1_devdata *dd)
6190{
6191 dd->lcb_access_count = 0;
6192}
6193
6194/*
6195 * Write a response back to a 8051 request.
6196 */
6197static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6198{
6199 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
17fb4f29
JJ		 6200 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6201 (u64)return_code <<
6202 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6203 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
77241056
MM		 6204}
6205
6206/*
cbac386a 6207 * Handle host requests from the 8051.
77241056 6208 */
145dd2b3 6209static void handle_8051_request(struct hfi1_pportdata *ppd)
77241056 6210{
cbac386a 6211 struct hfi1_devdata *dd = ppd->dd;
77241056 6212 u64 reg;
cbac386a 6213 u16 data = 0;
145dd2b3 6214 u8 type;
77241056
MM		 6215
6216 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6217 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6218 return; /* no request */
6219
6220 /* zero out COMPLETED so the response is seen */
6221 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6222
6223 /* extract request details */
6224 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6225 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6226 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6227 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6228
6229 switch (type) {
6230 case HREQ_LOAD_CONFIG:
6231 case HREQ_SAVE_CONFIG:
6232 case HREQ_READ_CONFIG:
6233 case HREQ_SET_TX_EQ_ABS:
6234 case HREQ_SET_TX_EQ_REL:
145dd2b3 6235 case HREQ_ENABLE:
77241056 6236 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
17fb4f29 6237 type);
77241056
MM		 6238 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6239 break;
77241056
MM		 6240 case HREQ_CONFIG_DONE:
6241 hreq_response(dd, HREQ_SUCCESS, 0);
6242 break;
6243
6244 case HREQ_INTERFACE_TEST:
6245 hreq_response(dd, HREQ_SUCCESS, data);
6246 break;
77241056
MM		 6247 default:
6248 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6249 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6250 break;
6251 }
6252}
6253
6254static void write_global_credit(struct hfi1_devdata *dd,
6255 u8 vau, u16 total, u16 shared)
6256{
6257 write_csr(dd, SEND_CM_GLOBAL_CREDIT,
17fb4f29
JJ		 6258 ((u64)total <<
6259 SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) |
6260 ((u64)shared <<
6261 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) |
6262 ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT));
77241056
MM		 6263}
6264
6265/*
6266 * Set up initial VL15 credits of the remote. Assumes the rest of
6267 * the CM credit registers are zero from a previous global or credit reset .
6268 */
6269void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf)
6270{
6271 /* leave shared count at zero for both global and VL15 */
6272 write_global_credit(dd, vau, vl15buf, 0);
6273
6274 /* We may need some credits for another VL when sending packets
6275 * with the snoop interface. Dividing it down the middle for VL15
6276 * and VL0 should suffice.
6277 */
6278 if (unlikely(dd->hfi1_snoop.mode_flag == HFI1_PORT_SNOOP_MODE)) {
6279 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)(vl15buf >> 1)
6280 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6281 write_csr(dd, SEND_CM_CREDIT_VL, (u64)(vl15buf >> 1)
6282 << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT);
6283 } else {
6284 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6285 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6286 }
6287}
6288
6289/*
6290 * Zero all credit details from the previous connection and
6291 * reset the CM manager's internal counters.
6292 */
6293void reset_link_credits(struct hfi1_devdata *dd)
6294{
6295 int i;
6296
6297 /* remove all previous VL credit limits */
6298 for (i = 0; i < TXE_NUM_DATA_VL; i++)
8638b77f 6299 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
77241056
MM		 6300 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6301 write_global_credit(dd, 0, 0, 0);
6302 /* reset the CM block */
6303 pio_send_control(dd, PSC_CM_RESET);
6304}
6305
6306/* convert a vCU to a CU */
6307static u32 vcu_to_cu(u8 vcu)
6308{
6309 return 1 << vcu;
6310}
6311
6312/* convert a CU to a vCU */
6313static u8 cu_to_vcu(u32 cu)
6314{
6315 return ilog2(cu);
6316}
6317
6318/* convert a vAU to an AU */
6319static u32 vau_to_au(u8 vau)
6320{
6321 return 8 * (1 << vau);
6322}
6323
6324static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6325{
6326 ppd->sm_trap_qp = 0x0;
6327 ppd->sa_qp = 0x1;
6328}
6329
6330/*
6331 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset.
6332 */
6333static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6334{
6335 u64 reg;
6336
6337 /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6338 write_csr(dd, DC_LCB_CFG_RUN, 0);
6339 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6340 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
17fb4f29 6341 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
77241056
MM		 6342 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6343 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6344 reg = read_csr(dd, DCC_CFG_RESET);
17fb4f29
JJ		 6345 write_csr(dd, DCC_CFG_RESET, reg |
6346 (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) |
6347 (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
50e5dcbe 6348 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
77241056
MM		 6349 if (!abort) {
6350 udelay(1); /* must hold for the longer of 16cclks or 20ns */
6351 write_csr(dd, DCC_CFG_RESET, reg);
6352 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6353 }
6354}
6355
6356/*
6357 * This routine should be called after the link has been transitioned to
6358 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6359 * reset).
6360 *
6361 * The expectation is that the caller of this routine would have taken
6362 * care of properly transitioning the link into the correct state.
6363 */
6364static void dc_shutdown(struct hfi1_devdata *dd)
6365{
6366 unsigned long flags;
6367
6368 spin_lock_irqsave(&dd->dc8051_lock, flags);
6369 if (dd->dc_shutdown) {
6370 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6371 return;
6372 }
6373 dd->dc_shutdown = 1;
6374 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6375 /* Shutdown the LCB */
6376 lcb_shutdown(dd, 1);
4d114fdd
JJ		 6377 /*
6378 * Going to OFFLINE would have causes the 8051 to put the
77241056 6379 * SerDes into reset already. Just need to shut down the 8051,
4d114fdd
JJ		 6380 * itself.
6381 */
77241056
MM		 6382 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6383}
6384
4d114fdd
JJ		 6385/*
6386 * Calling this after the DC has been brought out of reset should not
6387 * do any damage.
6388 */
77241056
MM		 6389static void dc_start(struct hfi1_devdata *dd)
6390{
6391 unsigned long flags;
6392 int ret;
6393
6394 spin_lock_irqsave(&dd->dc8051_lock, flags);
6395 if (!dd->dc_shutdown)
6396 goto done;
6397 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6398 /* Take the 8051 out of reset */
6399 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6400 /* Wait until 8051 is ready */
6401 ret = wait_fm_ready(dd, TIMEOUT_8051_START);
6402 if (ret) {
6403 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
17fb4f29 6404 __func__);
77241056
MM		 6405 }
6406 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6407 write_csr(dd, DCC_CFG_RESET, 0x10);
6408 /* lcb_shutdown() with abort=1 does not restore these */
6409 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6410 spin_lock_irqsave(&dd->dc8051_lock, flags);
6411 dd->dc_shutdown = 0;
6412done:
6413 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6414}
6415
6416/*
6417 * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6418 */
6419static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6420{
6421 u64 rx_radr, tx_radr;
6422 u32 version;
6423
6424 if (dd->icode != ICODE_FPGA_EMULATION)
6425 return;
6426
6427 /*
6428 * These LCB defaults on emulator _s are good, nothing to do here:
6429 * LCB_CFG_TX_FIFOS_RADR
6430 * LCB_CFG_RX_FIFOS_RADR
6431 * LCB_CFG_LN_DCLK
6432 * LCB_CFG_IGNORE_LOST_RCLK
6433 */
6434 if (is_emulator_s(dd))
6435 return;
6436 /* else this is _p */
6437
6438 version = emulator_rev(dd);
995deafa 6439 if (!is_ax(dd))
77241056
MM		 6440 version = 0x2d; /* all B0 use 0x2d or higher settings */
6441
6442 if (version <= 0x12) {
6443 /* release 0x12 and below */
6444
6445 /*
6446 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6447 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6448 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6449 */
6450 rx_radr =
6451 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6452 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6453 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6454 /*
6455 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6456 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6457 */
6458 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6459 } else if (version <= 0x18) {
6460 /* release 0x13 up to 0x18 */
6461 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6462 rx_radr =
6463 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6464 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6465 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6466 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6467 } else if (version == 0x19) {
6468 /* release 0x19 */
6469 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6470 rx_radr =
6471 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6472 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6473 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6474 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6475 } else if (version == 0x1a) {
6476 /* release 0x1a */
6477 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6478 rx_radr =
6479 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6480 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6481 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6482 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6483 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6484 } else {
6485 /* release 0x1b and higher */
6486 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6487 rx_radr =
6488 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6489 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6490 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6491 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6492 }
6493
6494 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6495 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6496 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
17fb4f29 6497 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
77241056
MM		 6498 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6499}
6500
6501/*
6502 * Handle a SMA idle message
6503 *
6504 * This is a work-queue function outside of the interrupt.
6505 */
6506void handle_sma_message(struct work_struct *work)
6507{
6508 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6509 sma_message_work);
6510 struct hfi1_devdata *dd = ppd->dd;
6511 u64 msg;
6512 int ret;
6513
4d114fdd
JJ		 6514 /*
6515 * msg is bytes 1-4 of the 40-bit idle message - the command code
6516 * is stripped off
6517 */
77241056
MM		 6518 ret = read_idle_sma(dd, &msg);
6519 if (ret)
6520 return;
6521 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6522 /*
6523 * React to the SMA message. Byte[1] (0 for us) is the command.
6524 */
6525 switch (msg & 0xff) {
6526 case SMA_IDLE_ARM:
6527 /*
6528 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6529 * State Transitions
6530 *
6531 * Only expected in INIT or ARMED, discard otherwise.
6532 */
6533 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6534 ppd->neighbor_normal = 1;
6535 break;
6536 case SMA_IDLE_ACTIVE:
6537 /*
6538 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6539 * State Transitions
6540 *
6541 * Can activate the node. Discard otherwise.
6542 */
d0d236ea
JJ		 6543 if (ppd->host_link_state == HLS_UP_ARMED &&
6544 ppd->is_active_optimize_enabled) {
77241056
MM		 6545 ppd->neighbor_normal = 1;
6546 ret = set_link_state(ppd, HLS_UP_ACTIVE);
6547 if (ret)
6548 dd_dev_err(
6549 dd,
6550 "%s: received Active SMA idle message, couldn't set link to Active\n",
6551 __func__);
6552 }
6553 break;
6554 default:
6555 dd_dev_err(dd,
17fb4f29
JJ		 6556 "%s: received unexpected SMA idle message 0x%llx\n",
6557 __func__, msg);
77241056
MM		 6558 break;
6559 }
6560}
6561
6562static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6563{
6564 u64 rcvctrl;
6565 unsigned long flags;
6566
6567 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6568 rcvctrl = read_csr(dd, RCV_CTRL);
6569 rcvctrl |= add;
6570 rcvctrl &= ~clear;
6571 write_csr(dd, RCV_CTRL, rcvctrl);
6572 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6573}
6574
6575static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6576{
6577 adjust_rcvctrl(dd, add, 0);
6578}
6579
6580static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6581{
6582 adjust_rcvctrl(dd, 0, clear);
6583}
6584
6585/*
6586 * Called from all interrupt handlers to start handling an SPC freeze.
6587 */
6588void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6589{
6590 struct hfi1_devdata *dd = ppd->dd;
6591 struct send_context *sc;
6592 int i;
6593
6594 if (flags & FREEZE_SELF)
6595 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6596
6597 /* enter frozen mode */
6598 dd->flags |= HFI1_FROZEN;
6599
6600 /* notify all SDMA engines that they are going into a freeze */
6601 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6602
6603 /* do halt pre-handling on all enabled send contexts */
6604 for (i = 0; i < dd->num_send_contexts; i++) {
6605 sc = dd->send_contexts[i].sc;
6606 if (sc && (sc->flags & SCF_ENABLED))
6607 sc_stop(sc, SCF_FROZEN | SCF_HALTED);
6608 }
6609
6610 /* Send context are frozen. Notify user space */
6611 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6612
6613 if (flags & FREEZE_ABORT) {
6614 dd_dev_err(dd,
6615 "Aborted freeze recovery. Please REBOOT system\n");
6616 return;
6617 }
6618 /* queue non-interrupt handler */
6619 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6620}
6621
6622/*
6623 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6624 * depending on the "freeze" parameter.
6625 *
6626 * No need to return an error if it times out, our only option
6627 * is to proceed anyway.
6628 */
6629static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6630{
6631 unsigned long timeout;
6632 u64 reg;
6633
6634 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6635 while (1) {
6636 reg = read_csr(dd, CCE_STATUS);
6637 if (freeze) {
6638 /* waiting until all indicators are set */
6639 if ((reg & ALL_FROZE) == ALL_FROZE)
6640 return; /* all done */
6641 } else {
6642 /* waiting until all indicators are clear */
6643 if ((reg & ALL_FROZE) == 0)
6644 return; /* all done */
6645 }
6646
6647 if (time_after(jiffies, timeout)) {
6648 dd_dev_err(dd,
17fb4f29
JJ		 6649 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6650 freeze ? "" : "un", reg & ALL_FROZE,
6651 freeze ? ALL_FROZE : 0ull);
77241056
MM		 6652 return;
6653 }
6654 usleep_range(80, 120);
6655 }
6656}
6657
6658/*
6659 * Do all freeze handling for the RXE block.
6660 */
6661static void rxe_freeze(struct hfi1_devdata *dd)
6662{
6663 int i;
6664
6665 /* disable port */
6666 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6667
6668 /* disable all receive contexts */
6669 for (i = 0; i < dd->num_rcv_contexts; i++)
6670 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
6671}
6672
6673/*
6674 * Unfreeze handling for the RXE block - kernel contexts only.
6675 * This will also enable the port. User contexts will do unfreeze
6676 * handling on a per-context basis as they call into the driver.
6677 *
6678 */
6679static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6680{
566c157c 6681 u32 rcvmask;
77241056
MM		 6682 int i;
6683
6684 /* enable all kernel contexts */
566c157c
MH		 6685 for (i = 0; i < dd->n_krcv_queues; i++) {
6686 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6687 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6688 rcvmask |= HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, DMA_RTAIL) ?
6689 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6690 hfi1_rcvctrl(dd, rcvmask, i);
6691 }
77241056
MM		 6692
6693 /* enable port */
6694 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6695}
6696
6697/*
6698 * Non-interrupt SPC freeze handling.
6699 *
6700 * This is a work-queue function outside of the triggering interrupt.
6701 */
6702void handle_freeze(struct work_struct *work)
6703{
6704 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6705 freeze_work);
6706 struct hfi1_devdata *dd = ppd->dd;
6707
6708 /* wait for freeze indicators on all affected blocks */
77241056
MM		 6709 wait_for_freeze_status(dd, 1);
6710
6711 /* SPC is now frozen */
6712
6713 /* do send PIO freeze steps */
6714 pio_freeze(dd);
6715
6716 /* do send DMA freeze steps */
6717 sdma_freeze(dd);
6718
6719 /* do send egress freeze steps - nothing to do */
6720
6721 /* do receive freeze steps */
6722 rxe_freeze(dd);
6723
6724 /*
6725 * Unfreeze the hardware - clear the freeze, wait for each
6726 * block's frozen bit to clear, then clear the frozen flag.
6727 */
6728 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6729 wait_for_freeze_status(dd, 0);
6730
995deafa 6731 if (is_ax(dd)) {
77241056
MM		 6732 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6733 wait_for_freeze_status(dd, 1);
6734 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6735 wait_for_freeze_status(dd, 0);
6736 }
6737
6738 /* do send PIO unfreeze steps for kernel contexts */
6739 pio_kernel_unfreeze(dd);
6740
6741 /* do send DMA unfreeze steps */
6742 sdma_unfreeze(dd);
6743
6744 /* do send egress unfreeze steps - nothing to do */
6745
6746 /* do receive unfreeze steps for kernel contexts */
6747 rxe_kernel_unfreeze(dd);
6748
6749 /*
6750 * The unfreeze procedure touches global device registers when
6751 * it disables and re-enables RXE. Mark the device unfrozen
6752 * after all that is done so other parts of the driver waiting
6753 * for the device to unfreeze don't do things out of order.
6754 *
6755 * The above implies that the meaning of HFI1_FROZEN flag is
6756 * "Device has gone into freeze mode and freeze mode handling
6757 * is still in progress."
6758 *
6759 * The flag will be removed when freeze mode processing has
6760 * completed.
6761 */
6762 dd->flags &= ~HFI1_FROZEN;
6763 wake_up(&dd->event_queue);
6764
6765 /* no longer frozen */
77241056
MM		 6766}
6767
6768/*
6769 * Handle a link up interrupt from the 8051.
6770 *
6771 * This is a work-queue function outside of the interrupt.
6772 */
6773void handle_link_up(struct work_struct *work)
6774{
6775 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
17fb4f29 6776 link_up_work);
77241056
MM		 6777 set_link_state(ppd, HLS_UP_INIT);
6778
6779 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6780 read_ltp_rtt(ppd->dd);
6781 /*
6782 * OPA specifies that certain counters are cleared on a transition
6783 * to link up, so do that.
6784 */
6785 clear_linkup_counters(ppd->dd);
6786 /*
6787 * And (re)set link up default values.
6788 */
6789 set_linkup_defaults(ppd);
6790
6791 /* enforce link speed enabled */
6792 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6793 /* oops - current speed is not enabled, bounce */
6794 dd_dev_err(ppd->dd,
17fb4f29
JJ		 6795 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6796 ppd->link_speed_active, ppd->link_speed_enabled);
77241056 6797 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
17fb4f29 6798 OPA_LINKDOWN_REASON_SPEED_POLICY);
77241056 6799 set_link_state(ppd, HLS_DN_OFFLINE);
8ebd4cf1 6800 tune_serdes(ppd);
77241056
MM		 6801 start_link(ppd);
6802 }
6803}
6804
4d114fdd
JJ		 6805/*
6806 * Several pieces of LNI information were cached for SMA in ppd.
6807 * Reset these on link down
6808 */
77241056
MM		 6809static void reset_neighbor_info(struct hfi1_pportdata *ppd)
6810{
6811 ppd->neighbor_guid = 0;
6812 ppd->neighbor_port_number = 0;
6813 ppd->neighbor_type = 0;
6814 ppd->neighbor_fm_security = 0;
6815}
6816
feb831dd
DL		 6817static const char * const link_down_reason_strs[] = {
6818 [OPA_LINKDOWN_REASON_NONE] = "None",
6819 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Recive error 0",
6820 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
6821 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
6822 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
6823 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
6824 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
6825 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
6826 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
6827 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
6828 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
6829 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
6830 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
6831 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
6832 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
6833 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
6834 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
6835 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
6836 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
6837 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
6838 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
6839 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
6840 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
6841 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
6842 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
6843 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
6844 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
6845 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
6846 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
6847 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
6848 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
6849 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
6850 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
6851 "Excessive buffer overrun",
6852 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
6853 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
6854 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
6855 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
6856 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
6857 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
6858 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
6859 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
6860 "Local media not installed",
6861 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
6862 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
6863 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
6864 "End to end not installed",
6865 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
6866 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
6867 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
6868 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
6869 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
6870 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
6871};
6872
6873/* return the neighbor link down reason string */
6874static const char *link_down_reason_str(u8 reason)
6875{
6876 const char *str = NULL;
6877
6878 if (reason < ARRAY_SIZE(link_down_reason_strs))
6879 str = link_down_reason_strs[reason];
6880 if (!str)
6881 str = "(invalid)";
6882
6883 return str;
6884}
6885
77241056
MM		 6886/*
6887 * Handle a link down interrupt from the 8051.
6888 *
6889 * This is a work-queue function outside of the interrupt.
6890 */
6891void handle_link_down(struct work_struct *work)
6892{
6893 u8 lcl_reason, neigh_reason = 0;
feb831dd 6894 u8 link_down_reason;
77241056 6895 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
feb831dd
DL		 6896 link_down_work);
6897 int was_up;
6898 static const char ldr_str[] = "Link down reason: ";
77241056 6899
8ebd4cf1
EH		 6900 if ((ppd->host_link_state &
6901 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
6902 ppd->port_type == PORT_TYPE_FIXED)
6903 ppd->offline_disabled_reason =
6904 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
6905
6906 /* Go offline first, then deal with reading/writing through 8051 */
feb831dd 6907 was_up = !!(ppd->host_link_state & HLS_UP);
77241056
MM		 6908 set_link_state(ppd, HLS_DN_OFFLINE);
6909
feb831dd
DL		 6910 if (was_up) {
6911 lcl_reason = 0;
6912 /* link down reason is only valid if the link was up */
6913 read_link_down_reason(ppd->dd, &link_down_reason);
6914 switch (link_down_reason) {
6915 case LDR_LINK_TRANSFER_ACTIVE_LOW:
6916 /* the link went down, no idle message reason */
6917 dd_dev_info(ppd->dd, "%sUnexpected link down\n",
6918 ldr_str);
6919 break;
6920 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
6921 /*
6922 * The neighbor reason is only valid if an idle message
6923 * was received for it.
6924 */
6925 read_planned_down_reason_code(ppd->dd, &neigh_reason);
6926 dd_dev_info(ppd->dd,
6927 "%sNeighbor link down message %d, %s\n",
6928 ldr_str, neigh_reason,
6929 link_down_reason_str(neigh_reason));
6930 break;
6931 case LDR_RECEIVED_HOST_OFFLINE_REQ:
6932 dd_dev_info(ppd->dd,
6933 "%sHost requested link to go offline\n",
6934 ldr_str);
6935 break;
6936 default:
6937 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
6938 ldr_str, link_down_reason);
6939 break;
6940 }
77241056 6941
feb831dd
DL		 6942 /*
6943 * If no reason, assume peer-initiated but missed
6944 * LinkGoingDown idle flits.
6945 */
6946 if (neigh_reason == 0)
6947 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
6948 } else {
6949 /* went down while polling or going up */
6950 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
6951 }
77241056
MM		 6952
6953 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
6954
015e91fb
DL		 6955 /* inform the SMA when the link transitions from up to down */
6956 if (was_up && ppd->local_link_down_reason.sma == 0 &&
6957 ppd->neigh_link_down_reason.sma == 0) {
6958 ppd->local_link_down_reason.sma =
6959 ppd->local_link_down_reason.latest;
6960 ppd->neigh_link_down_reason.sma =
6961 ppd->neigh_link_down_reason.latest;
6962 }
6963
77241056
MM		 6964 reset_neighbor_info(ppd);
6965
6966 /* disable the port */
6967 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6968
4d114fdd
JJ		 6969 /*
6970 * If there is no cable attached, turn the DC off. Otherwise,
6971 * start the link bring up.
6972 */
623bba2d 6973 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) {
77241056 6974 dc_shutdown(ppd->dd);
8ebd4cf1
EH		 6975 } else {
6976 tune_serdes(ppd);
77241056 6977 start_link(ppd);
8ebd4cf1 6978 }
77241056
MM		 6979}
6980
6981void handle_link_bounce(struct work_struct *work)
6982{
6983 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6984 link_bounce_work);
6985
6986 /*
6987 * Only do something if the link is currently up.
6988 */
6989 if (ppd->host_link_state & HLS_UP) {
6990 set_link_state(ppd, HLS_DN_OFFLINE);
8ebd4cf1 6991 tune_serdes(ppd);
77241056
MM		 6992 start_link(ppd);
6993 } else {
6994 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
17fb4f29 6995 __func__, link_state_name(ppd->host_link_state));
77241056
MM		 6996 }
6997}
6998
6999/*
7000 * Mask conversion: Capability exchange to Port LTP. The capability
7001 * exchange has an implicit 16b CRC that is mandatory.
7002 */
7003static int cap_to_port_ltp(int cap)
7004{
7005 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
7006
7007 if (cap & CAP_CRC_14B)
7008 port_ltp |= PORT_LTP_CRC_MODE_14;
7009 if (cap & CAP_CRC_48B)
7010 port_ltp |= PORT_LTP_CRC_MODE_48;
7011 if (cap & CAP_CRC_12B_16B_PER_LANE)
7012 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7013
7014 return port_ltp;
7015}
7016
7017/*
7018 * Convert an OPA Port LTP mask to capability mask
7019 */
7020int port_ltp_to_cap(int port_ltp)
7021{
7022 int cap_mask = 0;
7023
7024 if (port_ltp & PORT_LTP_CRC_MODE_14)
7025 cap_mask |= CAP_CRC_14B;
7026 if (port_ltp & PORT_LTP_CRC_MODE_48)
7027 cap_mask |= CAP_CRC_48B;
7028 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7029 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7030
7031 return cap_mask;
7032}
7033
7034/*
7035 * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
7036 */
7037static int lcb_to_port_ltp(int lcb_crc)
7038{
7039 int port_ltp = 0;
7040
7041 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7042 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7043 else if (lcb_crc == LCB_CRC_48B)
7044 port_ltp = PORT_LTP_CRC_MODE_48;
7045 else if (lcb_crc == LCB_CRC_14B)
7046 port_ltp = PORT_LTP_CRC_MODE_14;
7047 else
7048 port_ltp = PORT_LTP_CRC_MODE_16;
7049
7050 return port_ltp;
7051}
7052
7053/*
7054 * Our neighbor has indicated that we are allowed to act as a fabric
7055 * manager, so place the full management partition key in the second
7056 * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note
7057 * that we should already have the limited management partition key in
7058 * array element 1, and also that the port is not yet up when
7059 * add_full_mgmt_pkey() is invoked.
7060 */
7061static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7062{
7063 struct hfi1_devdata *dd = ppd->dd;
7064
8764522e
DL		 7065 /* Sanity check - ppd->pkeys[2] should be 0, or already initalized */
7066 if (!((ppd->pkeys[2] == 0) || (ppd->pkeys[2] == FULL_MGMT_P_KEY)))
7067 dd_dev_warn(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n",
7068 __func__, ppd->pkeys[2], FULL_MGMT_P_KEY);
77241056
MM		 7069 ppd->pkeys[2] = FULL_MGMT_P_KEY;
7070 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
34d351f8 7071 hfi1_event_pkey_change(ppd->dd, ppd->port);
77241056
MM		 7072}
7073
3ec5fa28 7074static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd)
ce8b2fd0 7075{
3ec5fa28
SS		 7076 if (ppd->pkeys[2] != 0) {
7077 ppd->pkeys[2] = 0;
7078 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
34d351f8 7079 hfi1_event_pkey_change(ppd->dd, ppd->port);
3ec5fa28 7080 }
ce8b2fd0
SS		 7081}
7082
77241056
MM		 7083/*
7084 * Convert the given link width to the OPA link width bitmask.
7085 */
7086static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7087{
7088 switch (width) {
7089 case 0:
7090 /*
7091 * Simulator and quick linkup do not set the width.
7092 * Just set it to 4x without complaint.
7093 */
7094 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7095 return OPA_LINK_WIDTH_4X;
7096 return 0; /* no lanes up */
7097 case 1: return OPA_LINK_WIDTH_1X;
7098 case 2: return OPA_LINK_WIDTH_2X;
7099 case 3: return OPA_LINK_WIDTH_3X;
7100 default:
7101 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
17fb4f29 7102 __func__, width);
77241056
MM		 7103 /* fall through */
7104 case 4: return OPA_LINK_WIDTH_4X;
7105 }
7106}
7107
7108/*
7109 * Do a population count on the bottom nibble.
7110 */
7111static const u8 bit_counts[16] = {
7112 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7113};
f4d507cd 7114
77241056
MM		 7115static inline u8 nibble_to_count(u8 nibble)
7116{
7117 return bit_counts[nibble & 0xf];
7118}
7119
7120/*
7121 * Read the active lane information from the 8051 registers and return
7122 * their widths.
7123 *
7124 * Active lane information is found in these 8051 registers:
7125 * enable_lane_tx
7126 * enable_lane_rx
7127 */
7128static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7129 u16 *rx_width)
7130{
7131 u16 tx, rx;
7132 u8 enable_lane_rx;
7133 u8 enable_lane_tx;
7134 u8 tx_polarity_inversion;
7135 u8 rx_polarity_inversion;
7136 u8 max_rate;
7137
7138 /* read the active lanes */
7139 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
17fb4f29 7140 &rx_polarity_inversion, &max_rate);
77241056
MM		 7141 read_local_lni(dd, &enable_lane_rx);
7142
7143 /* convert to counts */
7144 tx = nibble_to_count(enable_lane_tx);
7145 rx = nibble_to_count(enable_lane_rx);
7146
7147 /*
7148 * Set link_speed_active here, overriding what was set in
7149 * handle_verify_cap(). The ASIC 8051 firmware does not correctly
7150 * set the max_rate field in handle_verify_cap until v0.19.
7151 */
d0d236ea
JJ		 7152 if ((dd->icode == ICODE_RTL_SILICON) &&
7153 (dd->dc8051_ver < dc8051_ver(0, 19))) {
77241056
MM		 7154 /* max_rate: 0 = 12.5G, 1 = 25G */
7155 switch (max_rate) {
7156 case 0:
7157 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7158 break;
7159 default:
7160 dd_dev_err(dd,
17fb4f29
JJ		 7161 "%s: unexpected max rate %d, using 25Gb\n",
7162 __func__, (int)max_rate);
77241056
MM		 7163 /* fall through */
7164 case 1:
7165 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7166 break;
7167 }
7168 }
7169
7170 dd_dev_info(dd,
17fb4f29
JJ		 7171 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7172 enable_lane_tx, tx, enable_lane_rx, rx);
77241056
MM		 7173 *tx_width = link_width_to_bits(dd, tx);
7174 *rx_width = link_width_to_bits(dd, rx);
7175}
7176
7177/*
7178 * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7179 * Valid after the end of VerifyCap and during LinkUp. Does not change
7180 * after link up. I.e. look elsewhere for downgrade information.
7181 *
7182 * Bits are:
7183 * + bits [7:4] contain the number of active transmitters
7184 * + bits [3:0] contain the number of active receivers
7185 * These are numbers 1 through 4 and can be different values if the
7186 * link is asymmetric.
7187 *
7188 * verify_cap_local_fm_link_width[0] retains its original value.
7189 */
7190static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7191 u16 *rx_width)
7192{
7193 u16 widths, tx, rx;
7194 u8 misc_bits, local_flags;
7195 u16 active_tx, active_rx;
7196
7197 read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
7198 tx = widths >> 12;
7199 rx = (widths >> 8) & 0xf;
7200
7201 *tx_width = link_width_to_bits(dd, tx);
7202 *rx_width = link_width_to_bits(dd, rx);
7203
7204 /* print the active widths */
7205 get_link_widths(dd, &active_tx, &active_rx);
7206}
7207
7208/*
7209 * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7210 * hardware information when the link first comes up.
7211 *
7212 * The link width is not available until after VerifyCap.AllFramesReceived
7213 * (the trigger for handle_verify_cap), so this is outside that routine
7214 * and should be called when the 8051 signals linkup.
7215 */
7216void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7217{
7218 u16 tx_width, rx_width;
7219
7220 /* get end-of-LNI link widths */
7221 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7222
7223 /* use tx_width as the link is supposed to be symmetric on link up */
7224 ppd->link_width_active = tx_width;
7225 /* link width downgrade active (LWD.A) starts out matching LW.A */
7226 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7227 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7228 /* per OPA spec, on link up LWD.E resets to LWD.S */
7229 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7230 /* cache the active egress rate (units {10^6 bits/sec]) */
7231 ppd->current_egress_rate = active_egress_rate(ppd);
7232}
7233
7234/*
7235 * Handle a verify capabilities interrupt from the 8051.
7236 *
7237 * This is a work-queue function outside of the interrupt.
7238 */
7239void handle_verify_cap(struct work_struct *work)
7240{
7241 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7242 link_vc_work);
7243 struct hfi1_devdata *dd = ppd->dd;
7244 u64 reg;
7245 u8 power_management;
7246 u8 continious;
7247 u8 vcu;
7248 u8 vau;
7249 u8 z;
7250 u16 vl15buf;
7251 u16 link_widths;
7252 u16 crc_mask;
7253 u16 crc_val;
7254 u16 device_id;
7255 u16 active_tx, active_rx;
7256 u8 partner_supported_crc;
7257 u8 remote_tx_rate;
7258 u8 device_rev;
7259
7260 set_link_state(ppd, HLS_VERIFY_CAP);
7261
7262 lcb_shutdown(dd, 0);
7263 adjust_lcb_for_fpga_serdes(dd);
7264
7265 /*
7266 * These are now valid:
7267 * remote VerifyCap fields in the general LNI config
7268 * CSR DC8051_STS_REMOTE_GUID
7269 * CSR DC8051_STS_REMOTE_NODE_TYPE
7270 * CSR DC8051_STS_REMOTE_FM_SECURITY
7271 * CSR DC8051_STS_REMOTE_PORT_NO
7272 */
7273
7274 read_vc_remote_phy(dd, &power_management, &continious);
17fb4f29
JJ		 7275 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7276 &partner_supported_crc);
77241056
MM		 7277 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7278 read_remote_device_id(dd, &device_id, &device_rev);
7279 /*
7280 * And the 'MgmtAllowed' information, which is exchanged during
7281 * LNI, is also be available at this point.
7282 */
7283 read_mgmt_allowed(dd, &ppd->mgmt_allowed);
7284 /* print the active widths */
7285 get_link_widths(dd, &active_tx, &active_rx);
7286 dd_dev_info(dd,
17fb4f29
JJ		 7287 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7288 (int)power_management, (int)continious);
77241056 7289 dd_dev_info(dd,
17fb4f29
JJ		 7290 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7291 (int)vau, (int)z, (int)vcu, (int)vl15buf,
7292 (int)partner_supported_crc);
77241056 7293 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
17fb4f29 7294 (u32)remote_tx_rate, (u32)link_widths);
77241056 7295 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
17fb4f29 7296 (u32)device_id, (u32)device_rev);
77241056
MM		 7297 /*
7298 * The peer vAU value just read is the peer receiver value. HFI does
7299 * not support a transmit vAU of 0 (AU == 8). We advertised that
7300 * with Z=1 in the fabric capabilities sent to the peer. The peer
7301 * will see our Z=1, and, if it advertised a vAU of 0, will move its
7302 * receive to vAU of 1 (AU == 16). Do the same here. We do not care
7303 * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7304 * subject to the Z value exception.
7305 */
7306 if (vau == 0)
7307 vau = 1;
7308 set_up_vl15(dd, vau, vl15buf);
7309
7310 /* set up the LCB CRC mode */
7311 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7312
7313 /* order is important: use the lowest bit in common */
7314 if (crc_mask & CAP_CRC_14B)
7315 crc_val = LCB_CRC_14B;
7316 else if (crc_mask & CAP_CRC_48B)
7317 crc_val = LCB_CRC_48B;
7318 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7319 crc_val = LCB_CRC_12B_16B_PER_LANE;
7320 else
7321 crc_val = LCB_CRC_16B;
7322
7323 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7324 write_csr(dd, DC_LCB_CFG_CRC_MODE,
7325 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7326
7327 /* set (14b only) or clear sideband credit */
7328 reg = read_csr(dd, SEND_CM_CTRL);
7329 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7330 write_csr(dd, SEND_CM_CTRL,
17fb4f29 7331 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
77241056
MM		 7332 } else {
7333 write_csr(dd, SEND_CM_CTRL,
17fb4f29 7334 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
77241056
MM		 7335 }
7336
7337 ppd->link_speed_active = 0; /* invalid value */
7338 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
7339 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7340 switch (remote_tx_rate) {
7341 case 0:
7342 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7343 break;
7344 case 1:
7345 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7346 break;
7347 }
7348 } else {
7349 /* actual rate is highest bit of the ANDed rates */
7350 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7351
7352 if (rate & 2)
7353 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7354 else if (rate & 1)
7355 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7356 }
7357 if (ppd->link_speed_active == 0) {
7358 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
17fb4f29 7359 __func__, (int)remote_tx_rate);
77241056
MM		 7360 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7361 }
7362
7363 /*
7364 * Cache the values of the supported, enabled, and active
7365 * LTP CRC modes to return in 'portinfo' queries. But the bit
7366 * flags that are returned in the portinfo query differ from
7367 * what's in the link_crc_mask, crc_sizes, and crc_val
7368 * variables. Convert these here.
7369 */
7370 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7371 /* supported crc modes */
7372 ppd->port_ltp_crc_mode |=
7373 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7374 /* enabled crc modes */
7375 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7376 /* active crc mode */
7377
7378 /* set up the remote credit return table */
7379 assign_remote_cm_au_table(dd, vcu);
7380
7381 /*
7382 * The LCB is reset on entry to handle_verify_cap(), so this must
7383 * be applied on every link up.
7384 *
7385 * Adjust LCB error kill enable to kill the link if
7386 * these RBUF errors are seen:
7387 * REPLAY_BUF_MBE_SMASK
7388 * FLIT_INPUT_BUF_MBE_SMASK
7389 */
995deafa 7390 if (is_ax(dd)) { /* fixed in B0 */
77241056
MM		 7391 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7392 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7393 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7394 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7395 }
7396
7397 /* pull LCB fifos out of reset - all fifo clocks must be stable */
7398 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7399
7400 /* give 8051 access to the LCB CSRs */
7401 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7402 set_8051_lcb_access(dd);
7403
7404 ppd->neighbor_guid =
7405 read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
7406 ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
7407 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
7408 ppd->neighbor_type =
7409 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
7410 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
7411 ppd->neighbor_fm_security =
7412 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
7413 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
7414 dd_dev_info(dd,
17fb4f29
JJ		 7415 "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
7416 ppd->neighbor_guid, ppd->neighbor_type,
7417 ppd->mgmt_allowed, ppd->neighbor_fm_security);
77241056
MM		 7418 if (ppd->mgmt_allowed)
7419 add_full_mgmt_pkey(ppd);
7420
7421 /* tell the 8051 to go to LinkUp */
7422 set_link_state(ppd, HLS_GOING_UP);
7423}
7424
7425/*
7426 * Apply the link width downgrade enabled policy against the current active
7427 * link widths.
7428 *
7429 * Called when the enabled policy changes or the active link widths change.
7430 */
7431void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
7432{
77241056 7433 int do_bounce = 0;
323fd785
DL		 7434 int tries;
7435 u16 lwde;
77241056
MM		 7436 u16 tx, rx;
7437
323fd785
DL		 7438 /* use the hls lock to avoid a race with actual link up */
7439 tries = 0;
7440retry:
77241056
MM		 7441 mutex_lock(&ppd->hls_lock);
7442 /* only apply if the link is up */
0c7f77af 7443 if (ppd->host_link_state & HLS_DOWN) {
323fd785
DL		 7444 /* still going up..wait and retry */
7445 if (ppd->host_link_state & HLS_GOING_UP) {
7446 if (++tries < 1000) {
7447 mutex_unlock(&ppd->hls_lock);
7448 usleep_range(100, 120); /* arbitrary */
7449 goto retry;
7450 }
7451 dd_dev_err(ppd->dd,
7452 "%s: giving up waiting for link state change\n",
7453 __func__);
7454 }
7455 goto done;
7456 }
7457
7458 lwde = ppd->link_width_downgrade_enabled;
77241056
MM		 7459
7460 if (refresh_widths) {
7461 get_link_widths(ppd->dd, &tx, &rx);
7462 ppd->link_width_downgrade_tx_active = tx;
7463 ppd->link_width_downgrade_rx_active = rx;
7464 }
7465
f9b5635c
DL		 7466 if (ppd->link_width_downgrade_tx_active == 0 ||
7467 ppd->link_width_downgrade_rx_active == 0) {
7468 /* the 8051 reported a dead link as a downgrade */
7469 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7470 } else if (lwde == 0) {
77241056
MM		 7471 /* downgrade is disabled */
7472
7473 /* bounce if not at starting active width */
7474 if ((ppd->link_width_active !=
17fb4f29
JJ		 7475 ppd->link_width_downgrade_tx_active) ||
7476 (ppd->link_width_active !=
7477 ppd->link_width_downgrade_rx_active)) {
77241056 7478 dd_dev_err(ppd->dd,
17fb4f29 7479 "Link downgrade is disabled and link has downgraded, downing link\n");
77241056 7480 dd_dev_err(ppd->dd,
17fb4f29
JJ		 7481 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
7482 ppd->link_width_active,
7483 ppd->link_width_downgrade_tx_active,
7484 ppd->link_width_downgrade_rx_active);
77241056
MM		 7485 do_bounce = 1;
7486 }
d0d236ea
JJ		 7487 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7488 (lwde & ppd->link_width_downgrade_rx_active) == 0) {
77241056
MM		 7489 /* Tx or Rx is outside the enabled policy */
7490 dd_dev_err(ppd->dd,
17fb4f29 7491 "Link is outside of downgrade allowed, downing link\n");
77241056 7492 dd_dev_err(ppd->dd,
17fb4f29
JJ		 7493 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7494 lwde, ppd->link_width_downgrade_tx_active,
7495 ppd->link_width_downgrade_rx_active);
77241056
MM		 7496 do_bounce = 1;
7497 }
7498
323fd785
DL		 7499done:
7500 mutex_unlock(&ppd->hls_lock);
7501
77241056
MM		 7502 if (do_bounce) {
7503 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
17fb4f29 7504 OPA_LINKDOWN_REASON_WIDTH_POLICY);
77241056 7505 set_link_state(ppd, HLS_DN_OFFLINE);
8ebd4cf1 7506 tune_serdes(ppd);
77241056
MM		 7507 start_link(ppd);
7508 }
7509}
7510
7511/*
7512 * Handle a link downgrade interrupt from the 8051.
7513 *
7514 * This is a work-queue function outside of the interrupt.
7515 */
7516void handle_link_downgrade(struct work_struct *work)
7517{
7518 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7519 link_downgrade_work);
7520
7521 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7522 apply_link_downgrade_policy(ppd, 1);
7523}
7524
7525static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7526{
7527 return flag_string(buf, buf_len, flags, dcc_err_flags,
7528 ARRAY_SIZE(dcc_err_flags));
7529}
7530
7531static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7532{
7533 return flag_string(buf, buf_len, flags, lcb_err_flags,
7534 ARRAY_SIZE(lcb_err_flags));
7535}
7536
7537static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7538{
7539 return flag_string(buf, buf_len, flags, dc8051_err_flags,
7540 ARRAY_SIZE(dc8051_err_flags));
7541}
7542
7543static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7544{
7545 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7546 ARRAY_SIZE(dc8051_info_err_flags));
7547}
7548
7549static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7550{
7551 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7552 ARRAY_SIZE(dc8051_info_host_msg_flags));
7553}
7554
7555static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7556{
7557 struct hfi1_pportdata *ppd = dd->pport;
7558 u64 info, err, host_msg;
7559 int queue_link_down = 0;
7560 char buf[96];
7561
7562 /* look at the flags */
7563 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7564 /* 8051 information set by firmware */
7565 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7566 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7567 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7568 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7569 host_msg = (info >>
7570 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7571 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7572
7573 /*
7574 * Handle error flags.
7575 */
7576 if (err & FAILED_LNI) {
7577 /*
7578 * LNI error indications are cleared by the 8051
7579 * only when starting polling. Only pay attention
7580 * to them when in the states that occur during
7581 * LNI.
7582 */
7583 if (ppd->host_link_state
7584 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7585 queue_link_down = 1;
7586 dd_dev_info(dd, "Link error: %s\n",
17fb4f29
JJ		 7587 dc8051_info_err_string(buf,
7588 sizeof(buf),
7589 err &
7590 FAILED_LNI));
77241056
MM		 7591 }
7592 err &= ~(u64)FAILED_LNI;
7593 }
6d014530
DL		 7594 /* unknown frames can happen durning LNI, just count */
7595 if (err & UNKNOWN_FRAME) {
7596 ppd->unknown_frame_count++;
7597 err &= ~(u64)UNKNOWN_FRAME;
7598 }
77241056
MM		 7599 if (err) {
7600 /* report remaining errors, but do not do anything */
7601 dd_dev_err(dd, "8051 info error: %s\n",
17fb4f29
JJ		 7602 dc8051_info_err_string(buf, sizeof(buf),
7603 err));
77241056
MM		 7604 }
7605
7606 /*
7607 * Handle host message flags.
7608 */
7609 if (host_msg & HOST_REQ_DONE) {
7610 /*
7611 * Presently, the driver does a busy wait for
7612 * host requests to complete. This is only an
7613 * informational message.
7614 * NOTE: The 8051 clears the host message
7615 * information *on the next 8051 command*.
7616 * Therefore, when linkup is achieved,
7617 * this flag will still be set.
7618 */
7619 host_msg &= ~(u64)HOST_REQ_DONE;
7620 }
7621 if (host_msg & BC_SMA_MSG) {
7622 queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
7623 host_msg &= ~(u64)BC_SMA_MSG;
7624 }
7625 if (host_msg & LINKUP_ACHIEVED) {
7626 dd_dev_info(dd, "8051: Link up\n");
7627 queue_work(ppd->hfi1_wq, &ppd->link_up_work);
7628 host_msg &= ~(u64)LINKUP_ACHIEVED;
7629 }
7630 if (host_msg & EXT_DEVICE_CFG_REQ) {
145dd2b3 7631 handle_8051_request(ppd);
77241056
MM		 7632 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7633 }
7634 if (host_msg & VERIFY_CAP_FRAME) {
7635 queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
7636 host_msg &= ~(u64)VERIFY_CAP_FRAME;
7637 }
7638 if (host_msg & LINK_GOING_DOWN) {
7639 const char *extra = "";
7640 /* no downgrade action needed if going down */
7641 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7642 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7643 extra = " (ignoring downgrade)";
7644 }
7645 dd_dev_info(dd, "8051: Link down%s\n", extra);
7646 queue_link_down = 1;
7647 host_msg &= ~(u64)LINK_GOING_DOWN;
7648 }
7649 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7650 queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
7651 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7652 }
7653 if (host_msg) {
7654 /* report remaining messages, but do not do anything */
7655 dd_dev_info(dd, "8051 info host message: %s\n",
17fb4f29
JJ		 7656 dc8051_info_host_msg_string(buf,
7657 sizeof(buf),
7658 host_msg));
77241056
MM		 7659 }
7660
7661 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7662 }
7663 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7664 /*
7665 * Lost the 8051 heartbeat. If this happens, we
7666 * receive constant interrupts about it. Disable
7667 * the interrupt after the first.
7668 */
7669 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7670 write_csr(dd, DC_DC8051_ERR_EN,
17fb4f29
JJ		 7671 read_csr(dd, DC_DC8051_ERR_EN) &
7672 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
77241056
MM		 7673
7674 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7675 }
7676 if (reg) {
7677 /* report the error, but do not do anything */
7678 dd_dev_err(dd, "8051 error: %s\n",
17fb4f29 7679 dc8051_err_string(buf, sizeof(buf), reg));
77241056
MM		 7680 }
7681
7682 if (queue_link_down) {
4d114fdd
JJ		 7683 /*
7684 * if the link is already going down or disabled, do not
7685 * queue another
7686 */
d0d236ea
JJ		 7687 if ((ppd->host_link_state &
7688 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7689 ppd->link_enabled == 0) {
77241056 7690 dd_dev_info(dd, "%s: not queuing link down\n",
17fb4f29 7691 __func__);
77241056
MM		 7692 } else {
7693 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
7694 }
7695 }
7696}
7697
7698static const char * const fm_config_txt[] = {
7699[0] =
7700 "BadHeadDist: Distance violation between two head flits",
7701[1] =
7702 "BadTailDist: Distance violation between two tail flits",
7703[2] =
7704 "BadCtrlDist: Distance violation between two credit control flits",
7705[3] =
7706 "BadCrdAck: Credits return for unsupported VL",
7707[4] =
7708 "UnsupportedVLMarker: Received VL Marker",
7709[5] =
7710 "BadPreempt: Exceeded the preemption nesting level",
7711[6] =
7712 "BadControlFlit: Received unsupported control flit",
7713/* no 7 */
7714[8] =
7715 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7716};
7717
7718static const char * const port_rcv_txt[] = {
7719[1] =
7720 "BadPktLen: Illegal PktLen",
7721[2] =
7722 "PktLenTooLong: Packet longer than PktLen",
7723[3] =
7724 "PktLenTooShort: Packet shorter than PktLen",
7725[4] =
7726 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7727[5] =
7728 "BadDLID: Illegal DLID (0, doesn't match HFI)",
7729[6] =
7730 "BadL2: Illegal L2 opcode",
7731[7] =
7732 "BadSC: Unsupported SC",
7733[9] =
7734 "BadRC: Illegal RC",
7735[11] =
7736 "PreemptError: Preempting with same VL",
7737[12] =
7738 "PreemptVL15: Preempting a VL15 packet",
7739};
7740
7741#define OPA_LDR_FMCONFIG_OFFSET 16
7742#define OPA_LDR_PORTRCV_OFFSET 0
7743static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7744{
7745 u64 info, hdr0, hdr1;
7746 const char *extra;
7747 char buf[96];
7748 struct hfi1_pportdata *ppd = dd->pport;
7749 u8 lcl_reason = 0;
7750 int do_bounce = 0;
7751
7752 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7753 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7754 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7755 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7756 /* set status bit */
7757 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7758 }
7759 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7760 }
7761
7762 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7763 struct hfi1_pportdata *ppd = dd->pport;
7764 /* this counter saturates at (2^32) - 1 */
7765 if (ppd->link_downed < (u32)UINT_MAX)
7766 ppd->link_downed++;
7767 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7768 }
7769
7770 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7771 u8 reason_valid = 1;
7772
7773 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7774 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7775 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7776 /* set status bit */
7777 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7778 }
7779 switch (info) {
7780 case 0:
7781 case 1:
7782 case 2:
7783 case 3:
7784 case 4:
7785 case 5:
7786 case 6:
7787 extra = fm_config_txt[info];
7788 break;
7789 case 8:
7790 extra = fm_config_txt[info];
7791 if (ppd->port_error_action &
7792 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7793 do_bounce = 1;
7794 /*
7795 * lcl_reason cannot be derived from info
7796 * for this error
7797 */
7798 lcl_reason =
7799 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7800 }
7801 break;
7802 default:
7803 reason_valid = 0;
7804 snprintf(buf, sizeof(buf), "reserved%lld", info);
7805 extra = buf;
7806 break;
7807 }
7808
7809 if (reason_valid && !do_bounce) {
7810 do_bounce = ppd->port_error_action &
7811 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7812 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7813 }
7814
7815 /* just report this */
7816 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra);
7817 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7818 }
7819
7820 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
7821 u8 reason_valid = 1;
7822
7823 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
7824 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
7825 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
7826 if (!(dd->err_info_rcvport.status_and_code &
7827 OPA_EI_STATUS_SMASK)) {
7828 dd->err_info_rcvport.status_and_code =
7829 info & OPA_EI_CODE_SMASK;
7830 /* set status bit */
7831 dd->err_info_rcvport.status_and_code |=
7832 OPA_EI_STATUS_SMASK;
4d114fdd
JJ		 7833 /*
7834 * save first 2 flits in the packet that caused
7835 * the error
7836 */
48a0cc13
BVA		 7837 dd->err_info_rcvport.packet_flit1 = hdr0;
7838 dd->err_info_rcvport.packet_flit2 = hdr1;
77241056
MM		 7839 }
7840 switch (info) {
7841 case 1:
7842 case 2:
7843 case 3:
7844 case 4:
7845 case 5:
7846 case 6:
7847 case 7:
7848 case 9:
7849 case 11:
7850 case 12:
7851 extra = port_rcv_txt[info];
7852 break;
7853 default:
7854 reason_valid = 0;
7855 snprintf(buf, sizeof(buf), "reserved%lld", info);
7856 extra = buf;
7857 break;
7858 }
7859
7860 if (reason_valid && !do_bounce) {
7861 do_bounce = ppd->port_error_action &
7862 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
7863 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
7864 }
7865
7866 /* just report this */
7867 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra);
7868 dd_dev_info(dd, " hdr0 0x%llx, hdr1 0x%llx\n",
17fb4f29 7869 hdr0, hdr1);
77241056
MM		 7870
7871 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
7872 }
7873
7874 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
7875 /* informative only */
7876 dd_dev_info(dd, "8051 access to LCB blocked\n");
7877 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
7878 }
7879 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
7880 /* informative only */
7881 dd_dev_info(dd, "host access to LCB blocked\n");
7882 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
7883 }
7884
7885 /* report any remaining errors */
7886 if (reg)
7887 dd_dev_info(dd, "DCC Error: %s\n",
17fb4f29 7888 dcc_err_string(buf, sizeof(buf), reg));
77241056
MM		 7889
7890 if (lcl_reason == 0)
7891 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
7892
7893 if (do_bounce) {
7894 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
7895 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
7896 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
7897 }
7898}
7899
7900static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7901{
7902 char buf[96];
7903
7904 dd_dev_info(dd, "LCB Error: %s\n",
17fb4f29 7905 lcb_err_string(buf, sizeof(buf), reg));
77241056
MM		 7906}
7907
7908/*
7909 * CCE block DC interrupt. Source is < 8.
7910 */
7911static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
7912{
7913 const struct err_reg_info *eri = &dc_errs[source];
7914
7915 if (eri->handler) {
7916 interrupt_clear_down(dd, 0, eri);
7917 } else if (source == 3 /* dc_lbm_int */) {
7918 /*
7919 * This indicates that a parity error has occurred on the
7920 * address/control lines presented to the LBM. The error
7921 * is a single pulse, there is no associated error flag,
7922 * and it is non-maskable. This is because if a parity
7923 * error occurs on the request the request is dropped.
7924 * This should never occur, but it is nice to know if it
7925 * ever does.
7926 */
7927 dd_dev_err(dd, "Parity error in DC LBM block\n");
7928 } else {
7929 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
7930 }
7931}
7932
7933/*
7934 * TX block send credit interrupt. Source is < 160.
7935 */
7936static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
7937{
7938 sc_group_release_update(dd, source);
7939}
7940
7941/*
7942 * TX block SDMA interrupt. Source is < 48.
7943 *
7944 * SDMA interrupts are grouped by type:
7945 *
7946 * 0 - N-1 = SDma
7947 * N - 2N-1 = SDmaProgress
7948 * 2N - 3N-1 = SDmaIdle
7949 */
7950static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
7951{
7952 /* what interrupt */
7953 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
7954 /* which engine */
7955 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
7956
7957#ifdef CONFIG_SDMA_VERBOSITY
7958 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
7959 slashstrip(__FILE__), __LINE__, __func__);
7960 sdma_dumpstate(&dd->per_sdma[which]);
7961#endif
7962
7963 if (likely(what < 3 && which < dd->num_sdma)) {
7964 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
7965 } else {
7966 /* should not happen */
7967 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
7968 }
7969}
7970
7971/*
7972 * RX block receive available interrupt. Source is < 160.
7973 */
7974static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
7975{
7976 struct hfi1_ctxtdata *rcd;
7977 char *err_detail;
7978
7979 if (likely(source < dd->num_rcv_contexts)) {
7980 rcd = dd->rcd[source];
7981 if (rcd) {
7982 if (source < dd->first_user_ctxt)
f4f30031 7983 rcd->do_interrupt(rcd, 0);
77241056
MM		 7984 else
7985 handle_user_interrupt(rcd);
7986 return; /* OK */
7987 }
7988 /* received an interrupt, but no rcd */
7989 err_detail = "dataless";
7990 } else {
7991 /* received an interrupt, but are not using that context */
7992 err_detail = "out of range";
7993 }
7994 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
17fb4f29 7995 err_detail, source);
77241056
MM		 7996}
7997
7998/*
7999 * RX block receive urgent interrupt. Source is < 160.
8000 */
8001static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8002{
8003 struct hfi1_ctxtdata *rcd;
8004 char *err_detail;
8005
8006 if (likely(source < dd->num_rcv_contexts)) {
8007 rcd = dd->rcd[source];
8008 if (rcd) {
8009 /* only pay attention to user urgent interrupts */
8010 if (source >= dd->first_user_ctxt)
8011 handle_user_interrupt(rcd);
8012 return; /* OK */
8013 }
8014 /* received an interrupt, but no rcd */
8015 err_detail = "dataless";
8016 } else {
8017 /* received an interrupt, but are not using that context */
8018 err_detail = "out of range";
8019 }
8020 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
17fb4f29 8021 err_detail, source);
77241056
MM		 8022}
8023
8024/*
8025 * Reserved range interrupt. Should not be called in normal operation.
8026 */
8027static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8028{
8029 char name[64];
8030
8031 dd_dev_err(dd, "unexpected %s interrupt\n",
17fb4f29 8032 is_reserved_name(name, sizeof(name), source));
77241056
MM		 8033}
8034
8035static const struct is_table is_table[] = {
4d114fdd
JJ		 8036/*
8037 * start end
8038 * name func interrupt func
8039 */
77241056
MM		 8040{ IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
8041 is_misc_err_name, is_misc_err_int },
8042{ IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
8043 is_sdma_eng_err_name, is_sdma_eng_err_int },
8044{ IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8045 is_sendctxt_err_name, is_sendctxt_err_int },
8046{ IS_SDMA_START, IS_SDMA_END,
8047 is_sdma_eng_name, is_sdma_eng_int },
8048{ IS_VARIOUS_START, IS_VARIOUS_END,
8049 is_various_name, is_various_int },
8050{ IS_DC_START, IS_DC_END,
8051 is_dc_name, is_dc_int },
8052{ IS_RCVAVAIL_START, IS_RCVAVAIL_END,
8053 is_rcv_avail_name, is_rcv_avail_int },
8054{ IS_RCVURGENT_START, IS_RCVURGENT_END,
8055 is_rcv_urgent_name, is_rcv_urgent_int },
8056{ IS_SENDCREDIT_START, IS_SENDCREDIT_END,
8057 is_send_credit_name, is_send_credit_int},
8058{ IS_RESERVED_START, IS_RESERVED_END,
8059 is_reserved_name, is_reserved_int},
8060};
8061
8062/*
8063 * Interrupt source interrupt - called when the given source has an interrupt.
8064 * Source is a bit index into an array of 64-bit integers.
8065 */
8066static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8067{
8068 const struct is_table *entry;
8069
8070 /* avoids a double compare by walking the table in-order */
8071 for (entry = &is_table[0]; entry->is_name; entry++) {
8072 if (source < entry->end) {
8073 trace_hfi1_interrupt(dd, entry, source);
8074 entry->is_int(dd, source - entry->start);
8075 return;
8076 }
8077 }
8078 /* fell off the end */
8079 dd_dev_err(dd, "invalid interrupt source %u\n", source);
8080}
8081
8082/*
8083 * General interrupt handler. This is able to correctly handle
8084 * all interrupts in case INTx is used.
8085 */
8086static irqreturn_t general_interrupt(int irq, void *data)
8087{
8088 struct hfi1_devdata *dd = data;
8089 u64 regs[CCE_NUM_INT_CSRS];
8090 u32 bit;
8091 int i;
8092
8093 this_cpu_inc(*dd->int_counter);
8094
8095 /* phase 1: scan and clear all handled interrupts */
8096 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8097 if (dd->gi_mask[i] == 0) {
8098 regs[i] = 0; /* used later */
8099 continue;
8100 }
8101 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8102 dd->gi_mask[i];
8103 /* only clear if anything is set */
8104 if (regs[i])
8105 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8106 }
8107
8108 /* phase 2: call the appropriate handler */
8109 for_each_set_bit(bit, (unsigned long *)&regs[0],
17fb4f29 8110 CCE_NUM_INT_CSRS * 64) {
77241056
MM		 8111 is_interrupt(dd, bit);
8112 }
8113
8114 return IRQ_HANDLED;
8115}
8116
8117static irqreturn_t sdma_interrupt(int irq, void *data)
8118{
8119 struct sdma_engine *sde = data;
8120 struct hfi1_devdata *dd = sde->dd;
8121 u64 status;
8122
8123#ifdef CONFIG_SDMA_VERBOSITY
8124 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8125 slashstrip(__FILE__), __LINE__, __func__);
8126 sdma_dumpstate(sde);
8127#endif
8128
8129 this_cpu_inc(*dd->int_counter);
8130
8131 /* This read_csr is really bad in the hot path */
8132 status = read_csr(dd,
17fb4f29
JJ		 8133 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8134 & sde->imask;
77241056
MM		 8135 if (likely(status)) {
8136 /* clear the interrupt(s) */
8137 write_csr(dd,
17fb4f29
JJ		 8138 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8139 status);
77241056
MM		 8140
8141 /* handle the interrupt(s) */
8142 sdma_engine_interrupt(sde, status);
8143 } else
8144 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
17fb4f29 8145 sde->this_idx);
77241056
MM		 8146
8147 return IRQ_HANDLED;
8148}
8149
8150/*
ecd42f8d
DL		 8151 * Clear the receive interrupt. Use a read of the interrupt clear CSR
8152 * to insure that the write completed. This does NOT guarantee that
8153 * queued DMA writes to memory from the chip are pushed.
f4f30031
DL		 8154 */
8155static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8156{
8157 struct hfi1_devdata *dd = rcd->dd;
8158 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8159
8160 mmiowb(); /* make sure everything before is written */
8161 write_csr(dd, addr, rcd->imask);
8162 /* force the above write on the chip and get a value back */
8163 (void)read_csr(dd, addr);
8164}
8165
8166/* force the receive interrupt */
fb9036dd 8167void force_recv_intr(struct hfi1_ctxtdata *rcd)
f4f30031
DL		 8168{
8169 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8170}
8171
ecd42f8d
DL		 8172/*
8173 * Return non-zero if a packet is present.
8174 *
8175 * This routine is called when rechecking for packets after the RcvAvail
8176 * interrupt has been cleared down. First, do a quick check of memory for
8177 * a packet present. If not found, use an expensive CSR read of the context
8178 * tail to determine the actual tail. The CSR read is necessary because there
8179 * is no method to push pending DMAs to memory other than an interrupt and we
8180 * are trying to determine if we need to force an interrupt.
8181 */
f4f30031
DL		 8182static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8183{
ecd42f8d
DL		 8184 u32 tail;
8185 int present;
8186
f4f30031 8187 if (!HFI1_CAP_IS_KSET(DMA_RTAIL))
ecd42f8d 8188 present = (rcd->seq_cnt ==
f4f30031 8189 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
ecd42f8d
DL		 8190 else /* is RDMA rtail */
8191 present = (rcd->head != get_rcvhdrtail(rcd));
8192
8193 if (present)
8194 return 1;
f4f30031 8195
ecd42f8d
DL		 8196 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8197 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8198 return rcd->head != tail;
f4f30031
DL		 8199}
8200
8201/*
8202 * Receive packet IRQ handler. This routine expects to be on its own IRQ.
8203 * This routine will try to handle packets immediately (latency), but if
8204 * it finds too many, it will invoke the thread handler (bandwitdh). The
16733b88 8205 * chip receive interrupt is *not* cleared down until this or the thread (if
f4f30031
DL		 8206 * invoked) is finished. The intent is to avoid extra interrupts while we
8207 * are processing packets anyway.
77241056
MM		 8208 */
8209static irqreturn_t receive_context_interrupt(int irq, void *data)
8210{
8211 struct hfi1_ctxtdata *rcd = data;
8212 struct hfi1_devdata *dd = rcd->dd;
f4f30031
DL		 8213 int disposition;
8214 int present;
77241056
MM		 8215
8216 trace_hfi1_receive_interrupt(dd, rcd->ctxt);
8217 this_cpu_inc(*dd->int_counter);
affa48de 8218 aspm_ctx_disable(rcd);
77241056 8219
f4f30031
DL		 8220 /* receive interrupt remains blocked while processing packets */
8221 disposition = rcd->do_interrupt(rcd, 0);
77241056 8222
f4f30031
DL		 8223 /*
8224 * Too many packets were seen while processing packets in this
8225 * IRQ handler. Invoke the handler thread. The receive interrupt
8226 * remains blocked.
8227 */
8228 if (disposition == RCV_PKT_LIMIT)
8229 return IRQ_WAKE_THREAD;
8230
8231 /*
8232 * The packet processor detected no more packets. Clear the receive
8233 * interrupt and recheck for a packet packet that may have arrived
8234 * after the previous check and interrupt clear. If a packet arrived,
8235 * force another interrupt.
8236 */
8237 clear_recv_intr(rcd);
8238 present = check_packet_present(rcd);
8239 if (present)
8240 force_recv_intr(rcd);
8241
8242 return IRQ_HANDLED;
8243}
8244
8245/*
8246 * Receive packet thread handler. This expects to be invoked with the
8247 * receive interrupt still blocked.
8248 */
8249static irqreturn_t receive_context_thread(int irq, void *data)
8250{
8251 struct hfi1_ctxtdata *rcd = data;
8252 int present;
8253
8254 /* receive interrupt is still blocked from the IRQ handler */
8255 (void)rcd->do_interrupt(rcd, 1);
8256
8257 /*
8258 * The packet processor will only return if it detected no more
8259 * packets. Hold IRQs here so we can safely clear the interrupt and
8260 * recheck for a packet that may have arrived after the previous
8261 * check and the interrupt clear. If a packet arrived, force another
8262 * interrupt.
8263 */
8264 local_irq_disable();
8265 clear_recv_intr(rcd);
8266 present = check_packet_present(rcd);
8267 if (present)
8268 force_recv_intr(rcd);
8269 local_irq_enable();
77241056
MM		 8270
8271 return IRQ_HANDLED;
8272}
8273
8274/* ========================================================================= */
8275
8276u32 read_physical_state(struct hfi1_devdata *dd)
8277{
8278 u64 reg;
8279
8280 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8281 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8282 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8283}
8284
fb9036dd 8285u32 read_logical_state(struct hfi1_devdata *dd)
77241056
MM		 8286{
8287 u64 reg;
8288
8289 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8290 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8291 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8292}
8293
8294static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8295{
8296 u64 reg;
8297
8298 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8299 /* clear current state, set new state */
8300 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8301 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8302 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8303}
8304
8305/*
8306 * Use the 8051 to read a LCB CSR.
8307 */
8308static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8309{
8310 u32 regno;
8311 int ret;
8312
8313 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8314 if (acquire_lcb_access(dd, 0) == 0) {
8315 *data = read_csr(dd, addr);
8316 release_lcb_access(dd, 0);
8317 return 0;
8318 }
8319 return -EBUSY;
8320 }
8321
8322 /* register is an index of LCB registers: (offset - base) / 8 */
8323 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8324 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8325 if (ret != HCMD_SUCCESS)
8326 return -EBUSY;
8327 return 0;
8328}
8329
8330/*
8331 * Read an LCB CSR. Access may not be in host control, so check.
8332 * Return 0 on success, -EBUSY on failure.
8333 */
8334int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8335{
8336 struct hfi1_pportdata *ppd = dd->pport;
8337
8338 /* if up, go through the 8051 for the value */
8339 if (ppd->host_link_state & HLS_UP)
8340 return read_lcb_via_8051(dd, addr, data);
8341 /* if going up or down, no access */
8342 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8343 return -EBUSY;
8344 /* otherwise, host has access */
8345 *data = read_csr(dd, addr);
8346 return 0;
8347}
8348
8349/*
8350 * Use the 8051 to write a LCB CSR.
8351 */
8352static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8353{
3bf40d65
DL		 8354 u32 regno;
8355 int ret;
77241056 8356
3bf40d65
DL		 8357 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8358 (dd->dc8051_ver < dc8051_ver(0, 20))) {
8359 if (acquire_lcb_access(dd, 0) == 0) {
8360 write_csr(dd, addr, data);
8361 release_lcb_access(dd, 0);
8362 return 0;
8363 }
8364 return -EBUSY;
77241056 8365 }
3bf40d65
DL		 8366
8367 /* register is an index of LCB registers: (offset - base) / 8 */
8368 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8369 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8370 if (ret != HCMD_SUCCESS)
8371 return -EBUSY;
8372 return 0;
77241056
MM		 8373}
8374
8375/*
8376 * Write an LCB CSR. Access may not be in host control, so check.
8377 * Return 0 on success, -EBUSY on failure.
8378 */
8379int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8380{
8381 struct hfi1_pportdata *ppd = dd->pport;
8382
8383 /* if up, go through the 8051 for the value */
8384 if (ppd->host_link_state & HLS_UP)
8385 return write_lcb_via_8051(dd, addr, data);
8386 /* if going up or down, no access */
8387 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8388 return -EBUSY;
8389 /* otherwise, host has access */
8390 write_csr(dd, addr, data);
8391 return 0;
8392}
8393
8394/*
8395 * Returns:
8396 * < 0 = Linux error, not able to get access
8397 * > 0 = 8051 command RETURN_CODE
8398 */
8399static int do_8051_command(
8400 struct hfi1_devdata *dd,
8401 u32 type,
8402 u64 in_data,
8403 u64 *out_data)
8404{
8405 u64 reg, completed;
8406 int return_code;
8407 unsigned long flags;
8408 unsigned long timeout;
8409
8410 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8411
8412 /*
8413 * Alternative to holding the lock for a long time:
8414 * - keep busy wait - have other users bounce off
8415 */
8416 spin_lock_irqsave(&dd->dc8051_lock, flags);
8417
8418 /* We can't send any commands to the 8051 if it's in reset */
8419 if (dd->dc_shutdown) {
8420 return_code = -ENODEV;
8421 goto fail;
8422 }
8423
8424 /*
8425 * If an 8051 host command timed out previously, then the 8051 is
8426 * stuck.
8427 *
8428 * On first timeout, attempt to reset and restart the entire DC
8429 * block (including 8051). (Is this too big of a hammer?)
8430 *
8431 * If the 8051 times out a second time, the reset did not bring it
8432 * back to healthy life. In that case, fail any subsequent commands.
8433 */
8434 if (dd->dc8051_timed_out) {
8435 if (dd->dc8051_timed_out > 1) {
8436 dd_dev_err(dd,
8437 "Previous 8051 host command timed out, skipping command %u\n",
8438 type);
8439 return_code = -ENXIO;
8440 goto fail;
8441 }
8442 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
8443 dc_shutdown(dd);
8444 dc_start(dd);
8445 spin_lock_irqsave(&dd->dc8051_lock, flags);
8446 }
8447
8448 /*
8449 * If there is no timeout, then the 8051 command interface is
8450 * waiting for a command.
8451 */
8452
3bf40d65
DL		 8453 /*
8454 * When writing a LCB CSR, out_data contains the full value to
8455 * to be written, while in_data contains the relative LCB
8456 * address in 7:0. Do the work here, rather than the caller,
8457 * of distrubting the write data to where it needs to go:
8458 *
8459 * Write data
8460 * 39:00 -> in_data[47:8]
8461 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8462 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8463 */
8464 if (type == HCMD_WRITE_LCB_CSR) {
8465 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8466 reg = ((((*out_data) >> 40) & 0xff) <<
8467 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8468 | ((((*out_data) >> 48) & 0xffff) <<
8469 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8470 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8471 }
8472
77241056
MM		 8473 /*
8474 * Do two writes: the first to stabilize the type and req_data, the
8475 * second to activate.
8476 */
8477 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8478 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8479 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8480 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8481 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8482 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8483 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8484
8485 /* wait for completion, alternate: interrupt */
8486 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8487 while (1) {
8488 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8489 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8490 if (completed)
8491 break;
8492 if (time_after(jiffies, timeout)) {
8493 dd->dc8051_timed_out++;
8494 dd_dev_err(dd, "8051 host command %u timeout\n", type);
8495 if (out_data)
8496 *out_data = 0;
8497 return_code = -ETIMEDOUT;
8498 goto fail;
8499 }
8500 udelay(2);
8501 }
8502
8503 if (out_data) {
8504 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8505 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8506 if (type == HCMD_READ_LCB_CSR) {
8507 /* top 16 bits are in a different register */
8508 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8509 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8510 << (48
8511 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8512 }
8513 }
8514 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8515 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8516 dd->dc8051_timed_out = 0;
8517 /*
8518 * Clear command for next user.
8519 */
8520 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8521
8522fail:
8523 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
8524
8525 return return_code;
8526}
8527
8528static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8529{
8530 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8531}
8532
8ebd4cf1
EH		 8533int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8534 u8 lane_id, u32 config_data)
77241056
MM		 8535{
8536 u64 data;
8537 int ret;
8538
8539 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8540 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8541 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8542 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8543 if (ret != HCMD_SUCCESS) {
8544 dd_dev_err(dd,
17fb4f29
JJ		 8545 "load 8051 config: field id %d, lane %d, err %d\n",
8546 (int)field_id, (int)lane_id, ret);
77241056
MM		 8547 }
8548 return ret;
8549}
8550
8551/*
8552 * Read the 8051 firmware "registers". Use the RAM directly. Always
8553 * set the result, even on error.
8554 * Return 0 on success, -errno on failure
8555 */
8ebd4cf1
EH		 8556int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8557 u32 *result)
77241056
MM		 8558{
8559 u64 big_data;
8560 u32 addr;
8561 int ret;
8562
8563 /* address start depends on the lane_id */
8564 if (lane_id < 4)
8565 addr = (4 * NUM_GENERAL_FIELDS)
8566 + (lane_id * 4 * NUM_LANE_FIELDS);
8567 else
8568 addr = 0;
8569 addr += field_id * 4;
8570
8571 /* read is in 8-byte chunks, hardware will truncate the address down */
8572 ret = read_8051_data(dd, addr, 8, &big_data);
8573
8574 if (ret == 0) {
8575 /* extract the 4 bytes we want */
8576 if (addr & 0x4)
8577 *result = (u32)(big_data >> 32);
8578 else
8579 *result = (u32)big_data;
8580 } else {
8581 *result = 0;
8582 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
17fb4f29 8583 __func__, lane_id, field_id);
77241056
MM		 8584 }
8585
8586 return ret;
8587}
8588
8589static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8590 u8 continuous)
8591{
8592 u32 frame;
8593
8594 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8595 | power_management << POWER_MANAGEMENT_SHIFT;
8596 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8597 GENERAL_CONFIG, frame);
8598}
8599
8600static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8601 u16 vl15buf, u8 crc_sizes)
8602{
8603 u32 frame;
8604
8605 frame = (u32)vau << VAU_SHIFT
8606 | (u32)z << Z_SHIFT
8607 | (u32)vcu << VCU_SHIFT
8608 | (u32)vl15buf << VL15BUF_SHIFT
8609 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8610 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8611 GENERAL_CONFIG, frame);
8612}
8613
8614static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
8615 u8 *flag_bits, u16 *link_widths)
8616{
8617 u32 frame;
8618
8619 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
17fb4f29 8620 &frame);
77241056
MM		 8621 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8622 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8623 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8624}
8625
8626static int write_vc_local_link_width(struct hfi1_devdata *dd,
8627 u8 misc_bits,
8628 u8 flag_bits,
8629 u16 link_widths)
8630{
8631 u32 frame;
8632
8633 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8634 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8635 | (u32)link_widths << LINK_WIDTH_SHIFT;
8636 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
8637 frame);
8638}
8639
8640static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8641 u8 device_rev)
8642{
8643 u32 frame;
8644
8645 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8646 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8647 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8648}
8649
8650static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8651 u8 *device_rev)
8652{
8653 u32 frame;
8654
8655 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8656 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8657 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8658 & REMOTE_DEVICE_REV_MASK;
8659}
8660
8661void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b)
8662{
8663 u32 frame;
8664
8665 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8666 *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK;
8667 *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK;
8668}
8669
8670static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8671 u8 *continuous)
8672{
8673 u32 frame;
8674
8675 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8676 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8677 & POWER_MANAGEMENT_MASK;
8678 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
8679 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
8680}
8681
8682static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
8683 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
8684{
8685 u32 frame;
8686
8687 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
8688 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
8689 *z = (frame >> Z_SHIFT) & Z_MASK;
8690 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
8691 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
8692 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
8693}
8694
8695static void read_vc_remote_link_width(struct hfi1_devdata *dd,
8696 u8 *remote_tx_rate,
8697 u16 *link_widths)
8698{
8699 u32 frame;
8700
8701 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
17fb4f29 8702 &frame);
77241056
MM		 8703 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
8704 & REMOTE_TX_RATE_MASK;
8705 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8706}
8707
8708static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
8709{
8710 u32 frame;
8711
8712 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
8713 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
8714}
8715
8716static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed)
8717{
8718 u32 frame;
8719
8720 read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame);
8721 *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK;
8722}
8723
8724static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
8725{
8726 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
8727}
8728
8729static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
8730{
8731 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
8732}
8733
8734void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
8735{
8736 u32 frame;
8737 int ret;
8738
8739 *link_quality = 0;
8740 if (dd->pport->host_link_state & HLS_UP) {
8741 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
17fb4f29 8742 &frame);
77241056
MM		 8743 if (ret == 0)
8744 *link_quality = (frame >> LINK_QUALITY_SHIFT)
8745 & LINK_QUALITY_MASK;
8746 }
8747}
8748
8749static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
8750{
8751 u32 frame;
8752
8753 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
8754 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
8755}
8756
feb831dd
DL		 8757static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
8758{
8759 u32 frame;
8760
8761 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
8762 *ldr = (frame & 0xff);
8763}
8764
77241056
MM		 8765static int read_tx_settings(struct hfi1_devdata *dd,
8766 u8 *enable_lane_tx,
8767 u8 *tx_polarity_inversion,
8768 u8 *rx_polarity_inversion,
8769 u8 *max_rate)
8770{
8771 u32 frame;
8772 int ret;
8773
8774 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
8775 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
8776 & ENABLE_LANE_TX_MASK;
8777 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
8778 & TX_POLARITY_INVERSION_MASK;
8779 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
8780 & RX_POLARITY_INVERSION_MASK;
8781 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
8782 return ret;
8783}
8784
8785static int write_tx_settings(struct hfi1_devdata *dd,
8786 u8 enable_lane_tx,
8787 u8 tx_polarity_inversion,
8788 u8 rx_polarity_inversion,
8789 u8 max_rate)
8790{
8791 u32 frame;
8792
8793 /* no need to mask, all variable sizes match field widths */
8794 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
8795 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
8796 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
8797 | max_rate << MAX_RATE_SHIFT;
8798 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
8799}
8800
8801static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
8802{
8803 u32 frame, version, prod_id;
8804 int ret, lane;
8805
8806 /* 4 lanes */
8807 for (lane = 0; lane < 4; lane++) {
8808 ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
8809 if (ret) {
17fb4f29
JJ		 8810 dd_dev_err(dd,
8811 "Unable to read lane %d firmware details\n",
8812 lane);
77241056
MM		 8813 continue;
8814 }
8815 version = (frame >> SPICO_ROM_VERSION_SHIFT)
8816 & SPICO_ROM_VERSION_MASK;
8817 prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
8818 & SPICO_ROM_PROD_ID_MASK;
8819 dd_dev_info(dd,
17fb4f29
JJ		 8820 "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
8821 lane, version, prod_id);
77241056
MM		 8822 }
8823}
8824
8825/*
8826 * Read an idle LCB message.
8827 *
8828 * Returns 0 on success, -EINVAL on error
8829 */
8830static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
8831{
8832 int ret;
8833
17fb4f29 8834 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
77241056
MM		 8835 if (ret != HCMD_SUCCESS) {
8836 dd_dev_err(dd, "read idle message: type %d, err %d\n",
17fb4f29 8837 (u32)type, ret);
77241056
MM		 8838 return -EINVAL;
8839 }
8840 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
8841 /* return only the payload as we already know the type */
8842 *data_out >>= IDLE_PAYLOAD_SHIFT;
8843 return 0;
8844}
8845
8846/*
8847 * Read an idle SMA message. To be done in response to a notification from
8848 * the 8051.
8849 *
8850 * Returns 0 on success, -EINVAL on error
8851 */
8852static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
8853{
17fb4f29
JJ		 8854 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
8855 data);
77241056
MM		 8856}
8857
8858/*
8859 * Send an idle LCB message.
8860 *
8861 * Returns 0 on success, -EINVAL on error
8862 */
8863static int send_idle_message(struct hfi1_devdata *dd, u64 data)
8864{
8865 int ret;
8866
8867 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
8868 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
8869 if (ret != HCMD_SUCCESS) {
8870 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
17fb4f29 8871 data, ret);
77241056
MM		 8872 return -EINVAL;
8873 }
8874 return 0;
8875}
8876
8877/*
8878 * Send an idle SMA message.
8879 *
8880 * Returns 0 on success, -EINVAL on error
8881 */
8882int send_idle_sma(struct hfi1_devdata *dd, u64 message)
8883{
8884 u64 data;
8885
17fb4f29
JJ		 8886 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
8887 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
77241056
MM		 8888 return send_idle_message(dd, data);
8889}
8890
8891/*
8892 * Initialize the LCB then do a quick link up. This may or may not be
8893 * in loopback.
8894 *
8895 * return 0 on success, -errno on error
8896 */
8897static int do_quick_linkup(struct hfi1_devdata *dd)
8898{
8899 u64 reg;
8900 unsigned long timeout;
8901 int ret;
8902
8903 lcb_shutdown(dd, 0);
8904
8905 if (loopback) {
8906 /* LCB_CFG_LOOPBACK.VAL = 2 */
8907 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
8908 write_csr(dd, DC_LCB_CFG_LOOPBACK,
17fb4f29 8909 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
77241056
MM		 8910 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
8911 }
8912
8913 /* start the LCBs */
8914 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
8915 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
8916
8917 /* simulator only loopback steps */
8918 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8919 /* LCB_CFG_RUN.EN = 1 */
8920 write_csr(dd, DC_LCB_CFG_RUN,
17fb4f29 8921 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
77241056
MM		 8922
8923 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
8924 timeout = jiffies + msecs_to_jiffies(10);
8925 while (1) {
17fb4f29 8926 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
77241056
MM		 8927 if (reg)
8928 break;
8929 if (time_after(jiffies, timeout)) {
8930 dd_dev_err(dd,
17fb4f29 8931 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
77241056
MM		 8932 return -ETIMEDOUT;
8933 }
8934 udelay(2);
8935 }
8936
8937 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
17fb4f29 8938 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
77241056
MM		 8939 }
8940
8941 if (!loopback) {
8942 /*
8943 * When doing quick linkup and not in loopback, both
8944 * sides must be done with LCB set-up before either
8945 * starts the quick linkup. Put a delay here so that
8946 * both sides can be started and have a chance to be
8947 * done with LCB set up before resuming.
8948 */
8949 dd_dev_err(dd,
17fb4f29 8950 "Pausing for peer to be finished with LCB set up\n");
77241056 8951 msleep(5000);
17fb4f29 8952 dd_dev_err(dd, "Continuing with quick linkup\n");
77241056
MM		 8953 }
8954
8955 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
8956 set_8051_lcb_access(dd);
8957
8958 /*
8959 * State "quick" LinkUp request sets the physical link state to
8960 * LinkUp without a verify capability sequence.
8961 * This state is in simulator v37 and later.
8962 */
8963 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
8964 if (ret != HCMD_SUCCESS) {
8965 dd_dev_err(dd,
17fb4f29
JJ		 8966 "%s: set physical link state to quick LinkUp failed with return %d\n",
8967 __func__, ret);
77241056
MM		 8968
8969 set_host_lcb_access(dd);
8970 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
8971
8972 if (ret >= 0)
8973 ret = -EINVAL;
8974 return ret;
8975 }
8976
8977 return 0; /* success */
8978}
8979
8980/*
8981 * Set the SerDes to internal loopback mode.
8982 * Returns 0 on success, -errno on error.
8983 */
8984static int set_serdes_loopback_mode(struct hfi1_devdata *dd)
8985{
8986 int ret;
8987
8988 ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK);
8989 if (ret == HCMD_SUCCESS)
8990 return 0;
8991 dd_dev_err(dd,
17fb4f29
JJ		 8992 "Set physical link state to SerDes Loopback failed with return %d\n",
8993 ret);
77241056
MM		 8994 if (ret >= 0)
8995 ret = -EINVAL;
8996 return ret;
8997}
8998
8999/*
9000 * Do all special steps to set up loopback.
9001 */
9002static int init_loopback(struct hfi1_devdata *dd)
9003{
9004 dd_dev_info(dd, "Entering loopback mode\n");
9005
9006 /* all loopbacks should disable self GUID check */
9007 write_csr(dd, DC_DC8051_CFG_MODE,
17fb4f29 9008 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
77241056
MM		 9009
9010 /*
9011 * The simulator has only one loopback option - LCB. Switch
9012 * to that option, which includes quick link up.
9013 *
9014 * Accept all valid loopback values.
9015 */
d0d236ea
JJ		 9016 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9017 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9018 loopback == LOOPBACK_CABLE)) {
77241056
MM		 9019 loopback = LOOPBACK_LCB;
9020 quick_linkup = 1;
9021 return 0;
9022 }
9023
9024 /* handle serdes loopback */
9025 if (loopback == LOOPBACK_SERDES) {
9026 /* internal serdes loopack needs quick linkup on RTL */
9027 if (dd->icode == ICODE_RTL_SILICON)
9028 quick_linkup = 1;
9029 return set_serdes_loopback_mode(dd);
9030 }
9031
9032 /* LCB loopback - handled at poll time */
9033 if (loopback == LOOPBACK_LCB) {
9034 quick_linkup = 1; /* LCB is always quick linkup */
9035
9036 /* not supported in emulation due to emulation RTL changes */
9037 if (dd->icode == ICODE_FPGA_EMULATION) {
9038 dd_dev_err(dd,
17fb4f29 9039 "LCB loopback not supported in emulation\n");
77241056
MM		 9040 return -EINVAL;
9041 }
9042 return 0;
9043 }
9044
9045 /* external cable loopback requires no extra steps */
9046 if (loopback == LOOPBACK_CABLE)
9047 return 0;
9048
9049 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9050 return -EINVAL;
9051}
9052
9053/*
9054 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
9055 * used in the Verify Capability link width attribute.
9056 */
9057static u16 opa_to_vc_link_widths(u16 opa_widths)
9058{
9059 int i;
9060 u16 result = 0;
9061
9062 static const struct link_bits {
9063 u16 from;
9064 u16 to;
9065 } opa_link_xlate[] = {
8638b77f
JJ		 9066 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) },
9067 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) },
9068 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) },
9069 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) },
77241056
MM		 9070 };
9071
9072 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9073 if (opa_widths & opa_link_xlate[i].from)
9074 result |= opa_link_xlate[i].to;
9075 }
9076 return result;
9077}
9078
9079/*
9080 * Set link attributes before moving to polling.
9081 */
9082static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9083{
9084 struct hfi1_devdata *dd = ppd->dd;
9085 u8 enable_lane_tx;
9086 u8 tx_polarity_inversion;
9087 u8 rx_polarity_inversion;
9088 int ret;
9089
9090 /* reset our fabric serdes to clear any lingering problems */
9091 fabric_serdes_reset(dd);
9092
9093 /* set the local tx rate - need to read-modify-write */
9094 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
17fb4f29 9095 &rx_polarity_inversion, &ppd->local_tx_rate);
77241056
MM		 9096 if (ret)
9097 goto set_local_link_attributes_fail;
9098
9099 if (dd->dc8051_ver < dc8051_ver(0, 20)) {
9100 /* set the tx rate to the fastest enabled */
9101 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9102 ppd->local_tx_rate = 1;
9103 else
9104 ppd->local_tx_rate = 0;
9105 } else {
9106 /* set the tx rate to all enabled */
9107 ppd->local_tx_rate = 0;
9108 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9109 ppd->local_tx_rate |= 2;
9110 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9111 ppd->local_tx_rate |= 1;
9112 }
febffe2c
EH		 9113
9114 enable_lane_tx = 0xF; /* enable all four lanes */
77241056 9115 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
17fb4f29 9116 rx_polarity_inversion, ppd->local_tx_rate);
77241056
MM		 9117 if (ret != HCMD_SUCCESS)
9118 goto set_local_link_attributes_fail;
9119
9120 /*
9121 * DC supports continuous updates.
9122 */
17fb4f29
JJ		 9123 ret = write_vc_local_phy(dd,
9124 0 /* no power management */,
9125 1 /* continuous updates */);
77241056
MM		 9126 if (ret != HCMD_SUCCESS)
9127 goto set_local_link_attributes_fail;
9128
9129 /* z=1 in the next call: AU of 0 is not supported by the hardware */
9130 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9131 ppd->port_crc_mode_enabled);
9132 if (ret != HCMD_SUCCESS)
9133 goto set_local_link_attributes_fail;
9134
9135 ret = write_vc_local_link_width(dd, 0, 0,
17fb4f29
JJ		 9136 opa_to_vc_link_widths(
9137 ppd->link_width_enabled));
77241056
MM		 9138 if (ret != HCMD_SUCCESS)
9139 goto set_local_link_attributes_fail;
9140
9141 /* let peer know who we are */
9142 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9143 if (ret == HCMD_SUCCESS)
9144 return 0;
9145
9146set_local_link_attributes_fail:
9147 dd_dev_err(dd,
17fb4f29
JJ		 9148 "Failed to set local link attributes, return 0x%x\n",
9149 ret);
77241056
MM		 9150 return ret;
9151}
9152
9153/*
623bba2d
EH		 9154 * Call this to start the link.
9155 * Do not do anything if the link is disabled.
9156 * Returns 0 if link is disabled, moved to polling, or the driver is not ready.
77241056
MM		 9157 */
9158int start_link(struct hfi1_pportdata *ppd)
9159{
9160 if (!ppd->link_enabled) {
9161 dd_dev_info(ppd->dd,
17fb4f29
JJ		 9162 "%s: stopping link start because link is disabled\n",
9163 __func__);
77241056
MM		 9164 return 0;
9165 }
9166 if (!ppd->driver_link_ready) {
9167 dd_dev_info(ppd->dd,
17fb4f29
JJ		 9168 "%s: stopping link start because driver is not ready\n",
9169 __func__);
77241056
MM		 9170 return 0;
9171 }
9172
3ec5fa28
SS		 9173 /*
9174 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the
9175 * pkey table can be configured properly if the HFI unit is connected
9176 * to switch port with MgmtAllowed=NO
9177 */
9178 clear_full_mgmt_pkey(ppd);
9179
623bba2d 9180 return set_link_state(ppd, HLS_DN_POLL);
77241056
MM		 9181}
9182
8ebd4cf1
EH		 9183static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9184{
9185 struct hfi1_devdata *dd = ppd->dd;
9186 u64 mask;
9187 unsigned long timeout;
9188
9189 /*
9190 * Check for QSFP interrupt for t_init (SFF 8679)
9191 */
9192 timeout = jiffies + msecs_to_jiffies(2000);
9193 while (1) {
9194 mask = read_csr(dd, dd->hfi1_id ?
9195 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9196 if (!(mask & QSFP_HFI0_INT_N)) {
9197 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR :
9198 ASIC_QSFP1_CLEAR, QSFP_HFI0_INT_N);
9199 break;
9200 }
9201 if (time_after(jiffies, timeout)) {
9202 dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9203 __func__);
9204 break;
9205 }
9206 udelay(2);
9207 }
9208}
9209
9210static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9211{
9212 struct hfi1_devdata *dd = ppd->dd;
9213 u64 mask;
9214
9215 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9216 if (enable)
9217 mask |= (u64)QSFP_HFI0_INT_N;
9218 else
9219 mask &= ~(u64)QSFP_HFI0_INT_N;
9220 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9221}
9222
9223void reset_qsfp(struct hfi1_pportdata *ppd)
77241056
MM		 9224{
9225 struct hfi1_devdata *dd = ppd->dd;
9226 u64 mask, qsfp_mask;
9227
8ebd4cf1
EH		 9228 /* Disable INT_N from triggering QSFP interrupts */
9229 set_qsfp_int_n(ppd, 0);
9230
9231 /* Reset the QSFP */
77241056 9232 mask = (u64)QSFP_HFI0_RESET_N;
77241056
MM		 9233
9234 qsfp_mask = read_csr(dd,
17fb4f29 9235 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
77241056
MM		 9236 qsfp_mask &= ~mask;
9237 write_csr(dd,
17fb4f29 9238 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
77241056
MM		 9239
9240 udelay(10);
9241
9242 qsfp_mask |= mask;
9243 write_csr(dd,
17fb4f29 9244 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
8ebd4cf1
EH		 9245
9246 wait_for_qsfp_init(ppd);
9247
9248 /*
9249 * Allow INT_N to trigger the QSFP interrupt to watch
9250 * for alarms and warnings
9251 */
9252 set_qsfp_int_n(ppd, 1);
77241056
MM		 9253}
9254
9255static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9256 u8 *qsfp_interrupt_status)
9257{
9258 struct hfi1_devdata *dd = ppd->dd;
9259
9260 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
17fb4f29
JJ		 9261 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9262 dd_dev_info(dd, "%s: QSFP cable on fire\n",
9263 __func__);
77241056
MM		 9264
9265 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
17fb4f29
JJ		 9266 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9267 dd_dev_info(dd, "%s: QSFP cable temperature too low\n",
9268 __func__);
77241056 9269
0c7f77af
EH		 9270 /*
9271 * The remaining alarms/warnings don't matter if the link is down.
9272 */
9273 if (ppd->host_link_state & HLS_DOWN)
9274 return 0;
9275
77241056 9276 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
17fb4f29
JJ		 9277 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9278 dd_dev_info(dd, "%s: QSFP supply voltage too high\n",
9279 __func__);
77241056
MM		 9280
9281 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
17fb4f29
JJ		 9282 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9283 dd_dev_info(dd, "%s: QSFP supply voltage too low\n",
9284 __func__);
77241056
MM		 9285
9286 /* Byte 2 is vendor specific */
9287
9288 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9289 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9290 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too high\n",
9291 __func__);
77241056
MM		 9292
9293 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9294 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9295 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too low\n",
9296 __func__);
77241056
MM		 9297
9298 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9299 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9300 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too high\n",
9301 __func__);
77241056
MM		 9302
9303 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9304 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9305 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too low\n",
9306 __func__);
77241056
MM		 9307
9308 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
17fb4f29
JJ		 9309 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9310 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too high\n",
9311 __func__);
77241056
MM		 9312
9313 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
17fb4f29
JJ		 9314 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9315 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too low\n",
9316 __func__);
77241056
MM		 9317
9318 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
17fb4f29
JJ		 9319 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9320 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too high\n",
9321 __func__);
77241056
MM		 9322
9323 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
17fb4f29
JJ		 9324 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9325 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too low\n",
9326 __func__);
77241056
MM		 9327
9328 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9329 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9330 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too high\n",
9331 __func__);
77241056
MM		 9332
9333 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9334 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9335 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too low\n",
9336 __func__);
77241056
MM		 9337
9338 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
17fb4f29
JJ		 9339 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9340 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too high\n",
9341 __func__);
77241056
MM		 9342
9343 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
17fb4f29
JJ		 9344 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9345 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too low\n",
9346 __func__);
77241056
MM		 9347
9348 /* Bytes 9-10 and 11-12 are reserved */
9349 /* Bytes 13-15 are vendor specific */
9350
9351 return 0;
9352}
9353
623bba2d 9354/* This routine will only be scheduled if the QSFP module present is asserted */
8ebd4cf1 9355void qsfp_event(struct work_struct *work)
77241056
MM		 9356{
9357 struct qsfp_data *qd;
9358 struct hfi1_pportdata *ppd;
9359 struct hfi1_devdata *dd;
9360
9361 qd = container_of(work, struct qsfp_data, qsfp_work);
9362 ppd = qd->ppd;
9363 dd = ppd->dd;
9364
9365 /* Sanity check */
9366 if (!qsfp_mod_present(ppd))
9367 return;
9368
9369 /*
0c7f77af
EH		 9370 * Turn DC back on after cable has been re-inserted. Up until
9371 * now, the DC has been in reset to save power.
77241056
MM		 9372 */
9373 dc_start(dd);
9374
9375 if (qd->cache_refresh_required) {
8ebd4cf1 9376 set_qsfp_int_n(ppd, 0);
77241056 9377
8ebd4cf1
EH		 9378 wait_for_qsfp_init(ppd);
9379
9380 /*
9381 * Allow INT_N to trigger the QSFP interrupt to watch
9382 * for alarms and warnings
77241056 9383 */
8ebd4cf1
EH		 9384 set_qsfp_int_n(ppd, 1);
9385
9386 tune_serdes(ppd);
9387
9388 start_link(ppd);
77241056
MM		 9389 }
9390
9391 if (qd->check_interrupt_flags) {
9392 u8 qsfp_interrupt_status[16] = {0,};
9393
765a6fac
DL		 9394 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9395 &qsfp_interrupt_status[0], 16) != 16) {
77241056 9396 dd_dev_info(dd,
17fb4f29
JJ		 9397 "%s: Failed to read status of QSFP module\n",
9398 __func__);
77241056
MM		 9399 } else {
9400 unsigned long flags;
77241056 9401
8ebd4cf1
EH		 9402 handle_qsfp_error_conditions(
9403 ppd, qsfp_interrupt_status);
77241056
MM		 9404 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9405 ppd->qsfp_info.check_interrupt_flags = 0;
9406 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
17fb4f29 9407 flags);
77241056
MM		 9408 }
9409 }
9410}
9411
8ebd4cf1 9412static void init_qsfp_int(struct hfi1_devdata *dd)
77241056 9413{
8ebd4cf1
EH		 9414 struct hfi1_pportdata *ppd = dd->pport;
9415 u64 qsfp_mask, cce_int_mask;
9416 const int qsfp1_int_smask = QSFP1_INT % 64;
9417 const int qsfp2_int_smask = QSFP2_INT % 64;
77241056 9418
8ebd4cf1
EH		 9419 /*
9420 * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0
9421 * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR,
9422 * therefore just one of QSFP1_INT/QSFP2_INT can be used to find
9423 * the index of the appropriate CSR in the CCEIntMask CSR array
9424 */
9425 cce_int_mask = read_csr(dd, CCE_INT_MASK +
9426 (8 * (QSFP1_INT / 64)));
9427 if (dd->hfi1_id) {
9428 cce_int_mask &= ~((u64)1 << qsfp1_int_smask);
9429 write_csr(dd, CCE_INT_MASK + (8 * (QSFP1_INT / 64)),
9430 cce_int_mask);
9431 } else {
9432 cce_int_mask &= ~((u64)1 << qsfp2_int_smask);
9433 write_csr(dd, CCE_INT_MASK + (8 * (QSFP2_INT / 64)),
9434 cce_int_mask);
77241056
MM		 9435 }
9436
77241056
MM		 9437 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9438 /* Clear current status to avoid spurious interrupts */
8ebd4cf1
EH		 9439 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9440 qsfp_mask);
9441 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9442 qsfp_mask);
9443
9444 set_qsfp_int_n(ppd, 0);
77241056
MM		 9445
9446 /* Handle active low nature of INT_N and MODPRST_N pins */
9447 if (qsfp_mod_present(ppd))
9448 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9449 write_csr(dd,
9450 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9451 qsfp_mask);
77241056
MM		 9452}
9453
bbdeb33d
DL		 9454/*
9455 * Do a one-time initialize of the LCB block.
9456 */
9457static void init_lcb(struct hfi1_devdata *dd)
9458{
a59329d5
DL		 9459 /* simulator does not correctly handle LCB cclk loopback, skip */
9460 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9461 return;
9462
bbdeb33d
DL		 9463 /* the DC has been reset earlier in the driver load */
9464
9465 /* set LCB for cclk loopback on the port */
9466 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9467 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9468 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9469 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9470 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9471 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9472 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9473}
9474
77241056
MM		 9475int bringup_serdes(struct hfi1_pportdata *ppd)
9476{
9477 struct hfi1_devdata *dd = ppd->dd;
9478 u64 guid;
9479 int ret;
9480
9481 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9482 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9483
9484 guid = ppd->guid;
9485 if (!guid) {
9486 if (dd->base_guid)
9487 guid = dd->base_guid + ppd->port - 1;
9488 ppd->guid = guid;
9489 }
9490
77241056
MM		 9491 /* Set linkinit_reason on power up per OPA spec */
9492 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9493
bbdeb33d
DL		 9494 /* one-time init of the LCB */
9495 init_lcb(dd);
9496
77241056
MM		 9497 if (loopback) {
9498 ret = init_loopback(dd);
9499 if (ret < 0)
9500 return ret;
9501 }
9502
9775a991
EH		 9503 get_port_type(ppd);
9504 if (ppd->port_type == PORT_TYPE_QSFP) {
9505 set_qsfp_int_n(ppd, 0);
9506 wait_for_qsfp_init(ppd);
9507 set_qsfp_int_n(ppd, 1);
9508 }
9509
9510 /*
9511 * Tune the SerDes to a ballpark setting for
8ebd4cf1
EH		 9512 * optimal signal and bit error rate
9513 * Needs to be done before starting the link
9514 */
9515 tune_serdes(ppd);
9516
77241056
MM		 9517 return start_link(ppd);
9518}
9519
9520void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9521{
9522 struct hfi1_devdata *dd = ppd->dd;
9523
9524 /*
9525 * Shut down the link and keep it down. First turn off that the
9526 * driver wants to allow the link to be up (driver_link_ready).
9527 * Then make sure the link is not automatically restarted
9528 * (link_enabled). Cancel any pending restart. And finally
9529 * go offline.
9530 */
9531 ppd->driver_link_ready = 0;
9532 ppd->link_enabled = 0;
9533
8ebd4cf1
EH		 9534 ppd->offline_disabled_reason =
9535 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_SMA_DISABLED);
77241056 9536 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
17fb4f29 9537 OPA_LINKDOWN_REASON_SMA_DISABLED);
77241056
MM		 9538 set_link_state(ppd, HLS_DN_OFFLINE);
9539
9540 /* disable the port */
9541 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9542}
9543
9544static inline int init_cpu_counters(struct hfi1_devdata *dd)
9545{
9546 struct hfi1_pportdata *ppd;
9547 int i;
9548
9549 ppd = (struct hfi1_pportdata *)(dd + 1);
9550 for (i = 0; i < dd->num_pports; i++, ppd++) {
4eb06882
DD		 9551 ppd->ibport_data.rvp.rc_acks = NULL;
9552 ppd->ibport_data.rvp.rc_qacks = NULL;
9553 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9554 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9555 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9556 if (!ppd->ibport_data.rvp.rc_acks ||
9557 !ppd->ibport_data.rvp.rc_delayed_comp ||
9558 !ppd->ibport_data.rvp.rc_qacks)
77241056
MM		 9559 return -ENOMEM;
9560 }
9561
9562 return 0;
9563}
9564
9565static const char * const pt_names[] = {
9566 "expected",
9567 "eager",
9568 "invalid"
9569};
9570
9571static const char *pt_name(u32 type)
9572{
9573 return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
9574}
9575
9576/*
9577 * index is the index into the receive array
9578 */
9579void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9580 u32 type, unsigned long pa, u16 order)
9581{
9582 u64 reg;
9583 void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
9584 (dd->kregbase + RCV_ARRAY));
9585
9586 if (!(dd->flags & HFI1_PRESENT))
9587 goto done;
9588
9589 if (type == PT_INVALID) {
9590 pa = 0;
9591 } else if (type > PT_INVALID) {
9592 dd_dev_err(dd,
17fb4f29
JJ		 9593 "unexpected receive array type %u for index %u, not handled\n",
9594 type, index);
77241056
MM		 9595 goto done;
9596 }
9597
9598 hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
9599 pt_name(type), index, pa, (unsigned long)order);
9600
9601#define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */
9602 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9603 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9604 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9605 << RCV_ARRAY_RT_ADDR_SHIFT;
9606 writeq(reg, base + (index * 8));
9607
9608 if (type == PT_EAGER)
9609 /*
9610 * Eager entries are written one-by-one so we have to push them
9611 * after we write the entry.
9612 */
9613 flush_wc();
9614done:
9615 return;
9616}
9617
9618void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9619{
9620 struct hfi1_devdata *dd = rcd->dd;
9621 u32 i;
9622
9623 /* this could be optimized */
9624 for (i = rcd->eager_base; i < rcd->eager_base +
9625 rcd->egrbufs.alloced; i++)
9626 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9627
9628 for (i = rcd->expected_base;
9629 i < rcd->expected_base + rcd->expected_count; i++)
9630 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9631}
9632
9633int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
9634 struct hfi1_ctxt_info *kinfo)
9635{
9636 kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) |
9637 HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U);
9638 return 0;
9639}
9640
9641struct hfi1_message_header *hfi1_get_msgheader(
9642 struct hfi1_devdata *dd, __le32 *rhf_addr)
9643{
9644 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
9645
9646 return (struct hfi1_message_header *)
9647 (rhf_addr - dd->rhf_offset + offset);
9648}
9649
9650static const char * const ib_cfg_name_strings[] = {
9651 "HFI1_IB_CFG_LIDLMC",
9652 "HFI1_IB_CFG_LWID_DG_ENB",
9653 "HFI1_IB_CFG_LWID_ENB",
9654 "HFI1_IB_CFG_LWID",
9655 "HFI1_IB_CFG_SPD_ENB",
9656 "HFI1_IB_CFG_SPD",
9657 "HFI1_IB_CFG_RXPOL_ENB",
9658 "HFI1_IB_CFG_LREV_ENB",
9659 "HFI1_IB_CFG_LINKLATENCY",
9660 "HFI1_IB_CFG_HRTBT",
9661 "HFI1_IB_CFG_OP_VLS",
9662 "HFI1_IB_CFG_VL_HIGH_CAP",
9663 "HFI1_IB_CFG_VL_LOW_CAP",
9664 "HFI1_IB_CFG_OVERRUN_THRESH",
9665 "HFI1_IB_CFG_PHYERR_THRESH",
9666 "HFI1_IB_CFG_LINKDEFAULT",
9667 "HFI1_IB_CFG_PKEYS",
9668 "HFI1_IB_CFG_MTU",
9669 "HFI1_IB_CFG_LSTATE",
9670 "HFI1_IB_CFG_VL_HIGH_LIMIT",
9671 "HFI1_IB_CFG_PMA_TICKS",
9672 "HFI1_IB_CFG_PORT"
9673};
9674
9675static const char *ib_cfg_name(int which)
9676{
9677 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
9678 return "invalid";
9679 return ib_cfg_name_strings[which];
9680}
9681
9682int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
9683{
9684 struct hfi1_devdata *dd = ppd->dd;
9685 int val = 0;
9686
9687 switch (which) {
9688 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
9689 val = ppd->link_width_enabled;
9690 break;
9691 case HFI1_IB_CFG_LWID: /* currently active Link-width */
9692 val = ppd->link_width_active;
9693 break;
9694 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
9695 val = ppd->link_speed_enabled;
9696 break;
9697 case HFI1_IB_CFG_SPD: /* current Link speed */
9698 val = ppd->link_speed_active;
9699 break;
9700
9701 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
9702 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
9703 case HFI1_IB_CFG_LINKLATENCY:
9704 goto unimplemented;
9705
9706 case HFI1_IB_CFG_OP_VLS:
9707 val = ppd->vls_operational;
9708 break;
9709 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
9710 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
9711 break;
9712 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
9713 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
9714 break;
9715 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
9716 val = ppd->overrun_threshold;
9717 break;
9718 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
9719 val = ppd->phy_error_threshold;
9720 break;
9721 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
9722 val = dd->link_default;
9723 break;
9724
9725 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
9726 case HFI1_IB_CFG_PMA_TICKS:
9727 default:
9728unimplemented:
9729 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
9730 dd_dev_info(
9731 dd,
9732 "%s: which %s: not implemented\n",
9733 __func__,
9734 ib_cfg_name(which));
9735 break;
9736 }
9737
9738 return val;
9739}
9740
9741/*
9742 * The largest MAD packet size.
9743 */
9744#define MAX_MAD_PACKET 2048
9745
9746/*
9747 * Return the maximum header bytes that can go on the _wire_
9748 * for this device. This count includes the ICRC which is
9749 * not part of the packet held in memory but it is appended
9750 * by the HW.
9751 * This is dependent on the device's receive header entry size.
9752 * HFI allows this to be set per-receive context, but the
9753 * driver presently enforces a global value.
9754 */
9755u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
9756{
9757 /*
9758 * The maximum non-payload (MTU) bytes in LRH.PktLen are
9759 * the Receive Header Entry Size minus the PBC (or RHF) size
9760 * plus one DW for the ICRC appended by HW.
9761 *
9762 * dd->rcd[0].rcvhdrqentsize is in DW.
9763 * We use rcd[0] as all context will have the same value. Also,
9764 * the first kernel context would have been allocated by now so
9765 * we are guaranteed a valid value.
9766 */
9767 return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
9768}
9769
9770/*
9771 * Set Send Length
9772 * @ppd - per port data
9773 *
9774 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck*
9775 * registers compare against LRH.PktLen, so use the max bytes included
9776 * in the LRH.
9777 *
9778 * This routine changes all VL values except VL15, which it maintains at
9779 * the same value.
9780 */
9781static void set_send_length(struct hfi1_pportdata *ppd)
9782{
9783 struct hfi1_devdata *dd = ppd->dd;
6cc6ad2e
HC		 9784 u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
9785 u32 maxvlmtu = dd->vld[15].mtu;
77241056
MM		 9786 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
9787 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
9788 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
b4ba6633 9789 int i, j;
44306f15 9790 u32 thres;
77241056
MM		 9791
9792 for (i = 0; i < ppd->vls_supported; i++) {
9793 if (dd->vld[i].mtu > maxvlmtu)
9794 maxvlmtu = dd->vld[i].mtu;
9795 if (i <= 3)
9796 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
9797 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
9798 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
9799 else
9800 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
9801 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
9802 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
9803 }
9804 write_csr(dd, SEND_LEN_CHECK0, len1);
9805 write_csr(dd, SEND_LEN_CHECK1, len2);
9806 /* adjust kernel credit return thresholds based on new MTUs */
9807 /* all kernel receive contexts have the same hdrqentsize */
9808 for (i = 0; i < ppd->vls_supported; i++) {
44306f15
JX		 9809 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
9810 sc_mtu_to_threshold(dd->vld[i].sc,
9811 dd->vld[i].mtu,
17fb4f29 9812 dd->rcd[0]->rcvhdrqentsize));
b4ba6633
JJ		 9813 for (j = 0; j < INIT_SC_PER_VL; j++)
9814 sc_set_cr_threshold(
9815 pio_select_send_context_vl(dd, j, i),
9816 thres);
44306f15
JX		 9817 }
9818 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
9819 sc_mtu_to_threshold(dd->vld[15].sc,
9820 dd->vld[15].mtu,
9821 dd->rcd[0]->rcvhdrqentsize));
9822 sc_set_cr_threshold(dd->vld[15].sc, thres);
77241056
MM		 9823
9824 /* Adjust maximum MTU for the port in DC */
9825 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
9826 (ilog2(maxvlmtu >> 8) + 1);
9827 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
9828 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
9829 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
9830 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
9831 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
9832}
9833
9834static void set_lidlmc(struct hfi1_pportdata *ppd)
9835{
9836 int i;
9837 u64 sreg = 0;
9838 struct hfi1_devdata *dd = ppd->dd;
9839 u32 mask = ~((1U << ppd->lmc) - 1);
9840 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
9841
9842 if (dd->hfi1_snoop.mode_flag)
9843 dd_dev_info(dd, "Set lid/lmc while snooping");
9844
9845 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
9846 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
9847 c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
8638b77f 9848 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
77241056
MM		 9849 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
9850 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
9851 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
9852
9853 /*
9854 * Iterate over all the send contexts and set their SLID check
9855 */
9856 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
9857 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
9858 (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
9859 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
9860
9861 for (i = 0; i < dd->chip_send_contexts; i++) {
9862 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
9863 i, (u32)sreg);
9864 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
9865 }
9866
9867 /* Now we have to do the same thing for the sdma engines */
9868 sdma_update_lmc(dd, mask, ppd->lid);
9869}
9870
9871static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
9872{
9873 unsigned long timeout;
9874 u32 curr_state;
9875
9876 timeout = jiffies + msecs_to_jiffies(msecs);
9877 while (1) {
9878 curr_state = read_physical_state(dd);
9879 if (curr_state == state)
9880 break;
9881 if (time_after(jiffies, timeout)) {
9882 dd_dev_err(dd,
17fb4f29
JJ		 9883 "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
9884 state, curr_state);
77241056
MM		 9885 return -ETIMEDOUT;
9886 }
9887 usleep_range(1950, 2050); /* sleep 2ms-ish */
9888 }
9889
9890 return 0;
9891}
9892
9893/*
9894 * Helper for set_link_state(). Do not call except from that routine.
9895 * Expects ppd->hls_mutex to be held.
9896 *
9897 * @rem_reason value to be sent to the neighbor
9898 *
9899 * LinkDownReasons only set if transition succeeds.
9900 */
9901static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
9902{
9903 struct hfi1_devdata *dd = ppd->dd;
9904 u32 pstate, previous_state;
9905 u32 last_local_state;
9906 u32 last_remote_state;
9907 int ret;
9908 int do_transition;
9909 int do_wait;
9910
9911 previous_state = ppd->host_link_state;
9912 ppd->host_link_state = HLS_GOING_OFFLINE;
9913 pstate = read_physical_state(dd);
9914 if (pstate == PLS_OFFLINE) {
9915 do_transition = 0; /* in right state */
9916 do_wait = 0; /* ...no need to wait */
9917 } else if ((pstate & 0xff) == PLS_OFFLINE) {
9918 do_transition = 0; /* in an offline transient state */
9919 do_wait = 1; /* ...wait for it to settle */
9920 } else {
9921 do_transition = 1; /* need to move to offline */
9922 do_wait = 1; /* ...will need to wait */
9923 }
9924
9925 if (do_transition) {
9926 ret = set_physical_link_state(dd,
bf640096 9927 (rem_reason << 8) | PLS_OFFLINE);
77241056
MM		 9928
9929 if (ret != HCMD_SUCCESS) {
9930 dd_dev_err(dd,
17fb4f29
JJ		 9931 "Failed to transition to Offline link state, return %d\n",
9932 ret);
77241056
MM		 9933 return -EINVAL;
9934 }
a9c05e35
BM		 9935 if (ppd->offline_disabled_reason ==
9936 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
77241056 9937 ppd->offline_disabled_reason =
a9c05e35 9938 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
77241056
MM		 9939 }
9940
9941 if (do_wait) {
9942 /* it can take a while for the link to go down */
dc060245 9943 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
77241056
MM		 9944 if (ret < 0)
9945 return ret;
9946 }
9947
9948 /* make sure the logical state is also down */
9949 wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
9950
9951 /*
9952 * Now in charge of LCB - must be after the physical state is
9953 * offline.quiet and before host_link_state is changed.
9954 */
9955 set_host_lcb_access(dd);
9956 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
9957 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
9958
8ebd4cf1
EH		 9959 if (ppd->port_type == PORT_TYPE_QSFP &&
9960 ppd->qsfp_info.limiting_active &&
9961 qsfp_mod_present(ppd)) {
765a6fac
DL		 9962 int ret;
9963
9964 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
9965 if (ret == 0) {
9966 set_qsfp_tx(ppd, 0);
9967 release_chip_resource(dd, qsfp_resource(dd));
9968 } else {
9969 /* not fatal, but should warn */
9970 dd_dev_err(dd,
9971 "Unable to acquire lock to turn off QSFP TX\n");
9972 }
8ebd4cf1
EH		 9973 }
9974
77241056
MM		 9975 /*
9976 * The LNI has a mandatory wait time after the physical state
9977 * moves to Offline.Quiet. The wait time may be different
9978 * depending on how the link went down. The 8051 firmware
9979 * will observe the needed wait time and only move to ready
9980 * when that is completed. The largest of the quiet timeouts
05087f3b
DL		 9981 * is 6s, so wait that long and then at least 0.5s more for
9982 * other transitions, and another 0.5s for a buffer.
77241056 9983 */
05087f3b 9984 ret = wait_fm_ready(dd, 7000);
77241056
MM		 9985 if (ret) {
9986 dd_dev_err(dd,
17fb4f29 9987 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
77241056
MM		 9988 /* state is really offline, so make it so */
9989 ppd->host_link_state = HLS_DN_OFFLINE;
9990 return ret;
9991 }
9992
9993 /*
9994 * The state is now offline and the 8051 is ready to accept host
9995 * requests.
9996 * - change our state
9997 * - notify others if we were previously in a linkup state
9998 */
9999 ppd->host_link_state = HLS_DN_OFFLINE;
10000 if (previous_state & HLS_UP) {
10001 /* went down while link was up */
10002 handle_linkup_change(dd, 0);
10003 } else if (previous_state
10004 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
10005 /* went down while attempting link up */
10006 /* byte 1 of last_*_state is the failure reason */
10007 read_last_local_state(dd, &last_local_state);
10008 read_last_remote_state(dd, &last_remote_state);
10009 dd_dev_err(dd,
17fb4f29
JJ		 10010 "LNI failure last states: local 0x%08x, remote 0x%08x\n",
10011 last_local_state, last_remote_state);
77241056
MM		 10012 }
10013
10014 /* the active link width (downgrade) is 0 on link down */
10015 ppd->link_width_active = 0;
10016 ppd->link_width_downgrade_tx_active = 0;
10017 ppd->link_width_downgrade_rx_active = 0;
10018 ppd->current_egress_rate = 0;
10019 return 0;
10020}
10021
10022/* return the link state name */
10023static const char *link_state_name(u32 state)
10024{
10025 const char *name;
10026 int n = ilog2(state);
10027 static const char * const names[] = {
10028 [__HLS_UP_INIT_BP] = "INIT",
10029 [__HLS_UP_ARMED_BP] = "ARMED",
10030 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
10031 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
10032 [__HLS_DN_POLL_BP] = "POLL",
10033 [__HLS_DN_DISABLE_BP] = "DISABLE",
10034 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
10035 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
10036 [__HLS_GOING_UP_BP] = "GOING_UP",
10037 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10038 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10039 };
10040
10041 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10042 return name ? name : "unknown";
10043}
10044
10045/* return the link state reason name */
10046static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10047{
10048 if (state == HLS_UP_INIT) {
10049 switch (ppd->linkinit_reason) {
10050 case OPA_LINKINIT_REASON_LINKUP:
10051 return "(LINKUP)";
10052 case OPA_LINKINIT_REASON_FLAPPING:
10053 return "(FLAPPING)";
10054 case OPA_LINKINIT_OUTSIDE_POLICY:
10055 return "(OUTSIDE_POLICY)";
10056 case OPA_LINKINIT_QUARANTINED:
10057 return "(QUARANTINED)";
10058 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10059 return "(INSUFIC_CAPABILITY)";
10060 default:
10061 break;
10062 }
10063 }
10064 return "";
10065}
10066
10067/*
10068 * driver_physical_state - convert the driver's notion of a port's
10069 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
10070 * Return -1 (converted to a u32) to indicate error.
10071 */
10072u32 driver_physical_state(struct hfi1_pportdata *ppd)
10073{
10074 switch (ppd->host_link_state) {
10075 case HLS_UP_INIT:
10076 case HLS_UP_ARMED:
10077 case HLS_UP_ACTIVE:
10078 return IB_PORTPHYSSTATE_LINKUP;
10079 case HLS_DN_POLL:
10080 return IB_PORTPHYSSTATE_POLLING;
10081 case HLS_DN_DISABLE:
10082 return IB_PORTPHYSSTATE_DISABLED;
10083 case HLS_DN_OFFLINE:
10084 return OPA_PORTPHYSSTATE_OFFLINE;
10085 case HLS_VERIFY_CAP:
10086 return IB_PORTPHYSSTATE_POLLING;
10087 case HLS_GOING_UP:
10088 return IB_PORTPHYSSTATE_POLLING;
10089 case HLS_GOING_OFFLINE:
10090 return OPA_PORTPHYSSTATE_OFFLINE;
10091 case HLS_LINK_COOLDOWN:
10092 return OPA_PORTPHYSSTATE_OFFLINE;
10093 case HLS_DN_DOWNDEF:
10094 default:
10095 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10096 ppd->host_link_state);
10097 return -1;
10098 }
10099}
10100
10101/*
10102 * driver_logical_state - convert the driver's notion of a port's
10103 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
10104 * (converted to a u32) to indicate error.
10105 */
10106u32 driver_logical_state(struct hfi1_pportdata *ppd)
10107{
0c7f77af 10108 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
77241056
MM		 10109 return IB_PORT_DOWN;
10110
10111 switch (ppd->host_link_state & HLS_UP) {
10112 case HLS_UP_INIT:
10113 return IB_PORT_INIT;
10114 case HLS_UP_ARMED:
10115 return IB_PORT_ARMED;
10116 case HLS_UP_ACTIVE:
10117 return IB_PORT_ACTIVE;
10118 default:
10119 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10120 ppd->host_link_state);
10121 return -1;
10122 }
10123}
10124
10125void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10126 u8 neigh_reason, u8 rem_reason)
10127{
10128 if (ppd->local_link_down_reason.latest == 0 &&
10129 ppd->neigh_link_down_reason.latest == 0) {
10130 ppd->local_link_down_reason.latest = lcl_reason;
10131 ppd->neigh_link_down_reason.latest = neigh_reason;
10132 ppd->remote_link_down_reason = rem_reason;
10133 }
10134}
10135
10136/*
10137 * Change the physical and/or logical link state.
10138 *
10139 * Do not call this routine while inside an interrupt. It contains
10140 * calls to routines that can take multiple seconds to finish.
10141 *
10142 * Returns 0 on success, -errno on failure.
10143 */
10144int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10145{
10146 struct hfi1_devdata *dd = ppd->dd;
10147 struct ib_event event = {.device = NULL};
10148 int ret1, ret = 0;
77241056
MM		 10149 int orig_new_state, poll_bounce;
10150
10151 mutex_lock(&ppd->hls_lock);
10152
10153 orig_new_state = state;
10154 if (state == HLS_DN_DOWNDEF)
10155 state = dd->link_default;
10156
10157 /* interpret poll -> poll as a link bounce */
d0d236ea
JJ		 10158 poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10159 state == HLS_DN_POLL;
77241056
MM		 10160
10161 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
17fb4f29
JJ		 10162 link_state_name(ppd->host_link_state),
10163 link_state_name(orig_new_state),
10164 poll_bounce ? "(bounce) " : "",
10165 link_state_reason_name(ppd, state));
77241056 10166
77241056
MM		 10167 /*
10168 * If we're going to a (HLS_*) link state that implies the logical
10169 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10170 * reset is_sm_config_started to 0.
10171 */
10172 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10173 ppd->is_sm_config_started = 0;
10174
10175 /*
10176 * Do nothing if the states match. Let a poll to poll link bounce
10177 * go through.
10178 */
10179 if (ppd->host_link_state == state && !poll_bounce)
10180 goto done;
10181
10182 switch (state) {
10183 case HLS_UP_INIT:
d0d236ea
JJ		 10184 if (ppd->host_link_state == HLS_DN_POLL &&
10185 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
77241056
MM		 10186 /*
10187 * Quick link up jumps from polling to here.
10188 *
10189 * Whether in normal or loopback mode, the
10190 * simulator jumps from polling to link up.
10191 * Accept that here.
10192 */
17fb4f29 10193 /* OK */
77241056
MM		 10194 } else if (ppd->host_link_state != HLS_GOING_UP) {
10195 goto unexpected;
10196 }
10197
10198 ppd->host_link_state = HLS_UP_INIT;
10199 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10200 if (ret) {
10201 /* logical state didn't change, stay at going_up */
10202 ppd->host_link_state = HLS_GOING_UP;
10203 dd_dev_err(dd,
17fb4f29
JJ		 10204 "%s: logical state did not change to INIT\n",
10205 __func__);
77241056
MM		 10206 } else {
10207 /* clear old transient LINKINIT_REASON code */
10208 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10209 ppd->linkinit_reason =
10210 OPA_LINKINIT_REASON_LINKUP;
10211
10212 /* enable the port */
10213 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10214
10215 handle_linkup_change(dd, 1);
10216 }
10217 break;
10218 case HLS_UP_ARMED:
10219 if (ppd->host_link_state != HLS_UP_INIT)
10220 goto unexpected;
10221
10222 ppd->host_link_state = HLS_UP_ARMED;
10223 set_logical_state(dd, LSTATE_ARMED);
10224 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10225 if (ret) {
10226 /* logical state didn't change, stay at init */
10227 ppd->host_link_state = HLS_UP_INIT;
10228 dd_dev_err(dd,
17fb4f29
JJ		 10229 "%s: logical state did not change to ARMED\n",
10230 __func__);
77241056
MM		 10231 }
10232 /*
10233 * The simulator does not currently implement SMA messages,
10234 * so neighbor_normal is not set. Set it here when we first
10235 * move to Armed.
10236 */
10237 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10238 ppd->neighbor_normal = 1;
10239 break;
10240 case HLS_UP_ACTIVE:
10241 if (ppd->host_link_state != HLS_UP_ARMED)
10242 goto unexpected;
10243
10244 ppd->host_link_state = HLS_UP_ACTIVE;
10245 set_logical_state(dd, LSTATE_ACTIVE);
10246 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10247 if (ret) {
10248 /* logical state didn't change, stay at armed */
10249 ppd->host_link_state = HLS_UP_ARMED;
10250 dd_dev_err(dd,
17fb4f29
JJ		 10251 "%s: logical state did not change to ACTIVE\n",
10252 __func__);
77241056 10253 } else {
77241056
MM		 10254 /* tell all engines to go running */
10255 sdma_all_running(dd);
10256
10257 /* Signal the IB layer that the port has went active */
ec3f2c12 10258 event.device = &dd->verbs_dev.rdi.ibdev;
77241056
MM		 10259 event.element.port_num = ppd->port;
10260 event.event = IB_EVENT_PORT_ACTIVE;
10261 }
10262 break;
10263 case HLS_DN_POLL:
10264 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10265 ppd->host_link_state == HLS_DN_OFFLINE) &&
10266 dd->dc_shutdown)
10267 dc_start(dd);
10268 /* Hand LED control to the DC */
10269 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10270
10271 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10272 u8 tmp = ppd->link_enabled;
10273
10274 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10275 if (ret) {
10276 ppd->link_enabled = tmp;
10277 break;
10278 }
10279 ppd->remote_link_down_reason = 0;
10280
10281 if (ppd->driver_link_ready)
10282 ppd->link_enabled = 1;
10283 }
10284
fb9036dd 10285 set_all_slowpath(ppd->dd);
77241056
MM		 10286 ret = set_local_link_attributes(ppd);
10287 if (ret)
10288 break;
10289
10290 ppd->port_error_action = 0;
10291 ppd->host_link_state = HLS_DN_POLL;
10292
10293 if (quick_linkup) {
10294 /* quick linkup does not go into polling */
10295 ret = do_quick_linkup(dd);
10296 } else {
10297 ret1 = set_physical_link_state(dd, PLS_POLLING);
10298 if (ret1 != HCMD_SUCCESS) {
10299 dd_dev_err(dd,
17fb4f29
JJ		 10300 "Failed to transition to Polling link state, return 0x%x\n",
10301 ret1);
77241056
MM		 10302 ret = -EINVAL;
10303 }
10304 }
a9c05e35
BM		 10305 ppd->offline_disabled_reason =
10306 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
77241056
MM		 10307 /*
10308 * If an error occurred above, go back to offline. The
10309 * caller may reschedule another attempt.
10310 */
10311 if (ret)
10312 goto_offline(ppd, 0);
10313 break;
10314 case HLS_DN_DISABLE:
10315 /* link is disabled */
10316 ppd->link_enabled = 0;
10317
10318 /* allow any state to transition to disabled */
10319
10320 /* must transition to offline first */
10321 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10322 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10323 if (ret)
10324 break;
10325 ppd->remote_link_down_reason = 0;
10326 }
10327
10328 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10329 if (ret1 != HCMD_SUCCESS) {
10330 dd_dev_err(dd,
17fb4f29
JJ		 10331 "Failed to transition to Disabled link state, return 0x%x\n",
10332 ret1);
77241056
MM		 10333 ret = -EINVAL;
10334 break;
10335 }
10336 ppd->host_link_state = HLS_DN_DISABLE;
10337 dc_shutdown(dd);
10338 break;
10339 case HLS_DN_OFFLINE:
10340 if (ppd->host_link_state == HLS_DN_DISABLE)
10341 dc_start(dd);
10342
10343 /* allow any state to transition to offline */
10344 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10345 if (!ret)
10346 ppd->remote_link_down_reason = 0;
10347 break;
10348 case HLS_VERIFY_CAP:
10349 if (ppd->host_link_state != HLS_DN_POLL)
10350 goto unexpected;
10351 ppd->host_link_state = HLS_VERIFY_CAP;
10352 break;
10353 case HLS_GOING_UP:
10354 if (ppd->host_link_state != HLS_VERIFY_CAP)
10355 goto unexpected;
10356
10357 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10358 if (ret1 != HCMD_SUCCESS) {
10359 dd_dev_err(dd,
17fb4f29
JJ		 10360 "Failed to transition to link up state, return 0x%x\n",
10361 ret1);
77241056
MM		 10362 ret = -EINVAL;
10363 break;
10364 }
10365 ppd->host_link_state = HLS_GOING_UP;
10366 break;
10367
10368 case HLS_GOING_OFFLINE: /* transient within goto_offline() */
10369 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */
10370 default:
10371 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
17fb4f29 10372 __func__, state);
77241056
MM		 10373 ret = -EINVAL;
10374 break;
10375 }
10376
77241056
MM		 10377 goto done;
10378
10379unexpected:
10380 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
17fb4f29
JJ		 10381 __func__, link_state_name(ppd->host_link_state),
10382 link_state_name(state));
77241056
MM		 10383 ret = -EINVAL;
10384
10385done:
10386 mutex_unlock(&ppd->hls_lock);
10387
10388 if (event.device)
10389 ib_dispatch_event(&event);
10390
10391 return ret;
10392}
10393
10394int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10395{
10396 u64 reg;
10397 int ret = 0;
10398
10399 switch (which) {
10400 case HFI1_IB_CFG_LIDLMC:
10401 set_lidlmc(ppd);
10402 break;
10403 case HFI1_IB_CFG_VL_HIGH_LIMIT:
10404 /*
10405 * The VL Arbitrator high limit is sent in units of 4k
10406 * bytes, while HFI stores it in units of 64 bytes.
10407 */
8638b77f 10408 val *= 4096 / 64;
77241056
MM		 10409 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10410 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10411 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10412 break;
10413 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10414 /* HFI only supports POLL as the default link down state */
10415 if (val != HLS_DN_POLL)
10416 ret = -EINVAL;
10417 break;
10418 case HFI1_IB_CFG_OP_VLS:
10419 if (ppd->vls_operational != val) {
10420 ppd->vls_operational = val;
10421 if (!ppd->port)
10422 ret = -EINVAL;
77241056
MM		 10423 }
10424 break;
10425 /*
10426 * For link width, link width downgrade, and speed enable, always AND
10427 * the setting with what is actually supported. This has two benefits.
10428 * First, enabled can't have unsupported values, no matter what the
10429 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean
10430 * "fill in with your supported value" have all the bits in the
10431 * field set, so simply ANDing with supported has the desired result.
10432 */
10433 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
10434 ppd->link_width_enabled = val & ppd->link_width_supported;
10435 break;
10436 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
10437 ppd->link_width_downgrade_enabled =
10438 val & ppd->link_width_downgrade_supported;
10439 break;
10440 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10441 ppd->link_speed_enabled = val & ppd->link_speed_supported;
10442 break;
10443 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10444 /*
10445 * HFI does not follow IB specs, save this value
10446 * so we can report it, if asked.
10447 */
10448 ppd->overrun_threshold = val;
10449 break;
10450 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10451 /*
10452 * HFI does not follow IB specs, save this value
10453 * so we can report it, if asked.
10454 */
10455 ppd->phy_error_threshold = val;
10456 break;
10457
10458 case HFI1_IB_CFG_MTU:
10459 set_send_length(ppd);
10460 break;
10461
10462 case HFI1_IB_CFG_PKEYS:
10463 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10464 set_partition_keys(ppd);
10465 break;
10466
10467 default:
10468 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10469 dd_dev_info(ppd->dd,
17fb4f29
JJ		 10470 "%s: which %s, val 0x%x: not implemented\n",
10471 __func__, ib_cfg_name(which), val);
77241056
MM		 10472 break;
10473 }
10474 return ret;
10475}
10476
10477/* begin functions related to vl arbitration table caching */
10478static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
10479{
10480 int i;
10481
10482 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10483 VL_ARB_LOW_PRIO_TABLE_SIZE);
10484 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10485 VL_ARB_HIGH_PRIO_TABLE_SIZE);
10486
10487 /*
10488 * Note that we always return values directly from the
10489 * 'vl_arb_cache' (and do no CSR reads) in response to a
10490 * 'Get(VLArbTable)'. This is obviously correct after a
10491 * 'Set(VLArbTable)', since the cache will then be up to
10492 * date. But it's also correct prior to any 'Set(VLArbTable)'
10493 * since then both the cache, and the relevant h/w registers
10494 * will be zeroed.
10495 */
10496
10497 for (i = 0; i < MAX_PRIO_TABLE; i++)
10498 spin_lock_init(&ppd->vl_arb_cache[i].lock);
10499}
10500
10501/*
10502 * vl_arb_lock_cache
10503 *
10504 * All other vl_arb_* functions should be called only after locking
10505 * the cache.
10506 */
10507static inline struct vl_arb_cache *
10508vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
10509{
10510 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
10511 return NULL;
10512 spin_lock(&ppd->vl_arb_cache[idx].lock);
10513 return &ppd->vl_arb_cache[idx];
10514}
10515
10516static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
10517{
10518 spin_unlock(&ppd->vl_arb_cache[idx].lock);
10519}
10520
10521static void vl_arb_get_cache(struct vl_arb_cache *cache,
10522 struct ib_vl_weight_elem *vl)
10523{
10524 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
10525}
10526
10527static void vl_arb_set_cache(struct vl_arb_cache *cache,
10528 struct ib_vl_weight_elem *vl)
10529{
10530 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10531}
10532
10533static int vl_arb_match_cache(struct vl_arb_cache *cache,
10534 struct ib_vl_weight_elem *vl)
10535{
10536 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10537}
f4d507cd 10538
77241056
MM		 10539/* end functions related to vl arbitration table caching */
10540
10541static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
10542 u32 size, struct ib_vl_weight_elem *vl)
10543{
10544 struct hfi1_devdata *dd = ppd->dd;
10545 u64 reg;
10546 unsigned int i, is_up = 0;
10547 int drain, ret = 0;
10548
10549 mutex_lock(&ppd->hls_lock);
10550
10551 if (ppd->host_link_state & HLS_UP)
10552 is_up = 1;
10553
10554 drain = !is_ax(dd) && is_up;
10555
10556 if (drain)
10557 /*
10558 * Before adjusting VL arbitration weights, empty per-VL
10559 * FIFOs, otherwise a packet whose VL weight is being
10560 * set to 0 could get stuck in a FIFO with no chance to
10561 * egress.
10562 */
10563 ret = stop_drain_data_vls(dd);
10564
10565 if (ret) {
10566 dd_dev_err(
10567 dd,
10568 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
10569 __func__);
10570 goto err;
10571 }
10572
10573 for (i = 0; i < size; i++, vl++) {
10574 /*
10575 * NOTE: The low priority shift and mask are used here, but
10576 * they are the same for both the low and high registers.
10577 */
10578 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
10579 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
10580 | (((u64)vl->weight
10581 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
10582 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
10583 write_csr(dd, target + (i * 8), reg);
10584 }
10585 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
10586
10587 if (drain)
10588 open_fill_data_vls(dd); /* reopen all VLs */
10589
10590err:
10591 mutex_unlock(&ppd->hls_lock);
10592
10593 return ret;
10594}
10595
10596/*
10597 * Read one credit merge VL register.
10598 */
10599static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
10600 struct vl_limit *vll)
10601{
10602 u64 reg = read_csr(dd, csr);
10603
10604 vll->dedicated = cpu_to_be16(
10605 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
10606 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
10607 vll->shared = cpu_to_be16(
10608 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
10609 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
10610}
10611
10612/*
10613 * Read the current credit merge limits.
10614 */
10615static int get_buffer_control(struct hfi1_devdata *dd,
10616 struct buffer_control *bc, u16 *overall_limit)
10617{
10618 u64 reg;
10619 int i;
10620
10621 /* not all entries are filled in */
10622 memset(bc, 0, sizeof(*bc));
10623
10624 /* OPA and HFI have a 1-1 mapping */
10625 for (i = 0; i < TXE_NUM_DATA_VL; i++)
8638b77f 10626 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
77241056
MM		 10627
10628 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
10629 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
10630
10631 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10632 bc->overall_shared_limit = cpu_to_be16(
10633 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
10634 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
10635 if (overall_limit)
10636 *overall_limit = (reg
10637 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
10638 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
10639 return sizeof(struct buffer_control);
10640}
10641
10642static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10643{
10644 u64 reg;
10645 int i;
10646
10647 /* each register contains 16 SC->VLnt mappings, 4 bits each */
10648 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
10649 for (i = 0; i < sizeof(u64); i++) {
10650 u8 byte = *(((u8 *)&reg) + i);
10651
10652 dp->vlnt[2 * i] = byte & 0xf;
10653 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
10654 }
10655
10656 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
10657 for (i = 0; i < sizeof(u64); i++) {
10658 u8 byte = *(((u8 *)&reg) + i);
10659
10660 dp->vlnt[16 + (2 * i)] = byte & 0xf;
10661 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
10662 }
10663 return sizeof(struct sc2vlnt);
10664}
10665
10666static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
10667 struct ib_vl_weight_elem *vl)
10668{
10669 unsigned int i;
10670
10671 for (i = 0; i < nelems; i++, vl++) {
10672 vl->vl = 0xf;
10673 vl->weight = 0;
10674 }
10675}
10676
10677static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10678{
10679 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
17fb4f29
JJ		 10680 DC_SC_VL_VAL(15_0,
10681 0, dp->vlnt[0] & 0xf,
10682 1, dp->vlnt[1] & 0xf,
10683 2, dp->vlnt[2] & 0xf,
10684 3, dp->vlnt[3] & 0xf,
10685 4, dp->vlnt[4] & 0xf,
10686 5, dp->vlnt[5] & 0xf,
10687 6, dp->vlnt[6] & 0xf,
10688 7, dp->vlnt[7] & 0xf,
10689 8, dp->vlnt[8] & 0xf,
10690 9, dp->vlnt[9] & 0xf,
10691 10, dp->vlnt[10] & 0xf,
10692 11, dp->vlnt[11] & 0xf,
10693 12, dp->vlnt[12] & 0xf,
10694 13, dp->vlnt[13] & 0xf,
10695 14, dp->vlnt[14] & 0xf,
10696 15, dp->vlnt[15] & 0xf));
77241056 10697 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
17fb4f29
JJ		 10698 DC_SC_VL_VAL(31_16,
10699 16, dp->vlnt[16] & 0xf,
10700 17, dp->vlnt[17] & 0xf,
10701 18, dp->vlnt[18] & 0xf,
10702 19, dp->vlnt[19] & 0xf,
10703 20, dp->vlnt[20] & 0xf,
10704 21, dp->vlnt[21] & 0xf,
10705 22, dp->vlnt[22] & 0xf,
10706 23, dp->vlnt[23] & 0xf,
10707 24, dp->vlnt[24] & 0xf,
10708 25, dp->vlnt[25] & 0xf,
10709 26, dp->vlnt[26] & 0xf,
10710 27, dp->vlnt[27] & 0xf,
10711 28, dp->vlnt[28] & 0xf,
10712 29, dp->vlnt[29] & 0xf,
10713 30, dp->vlnt[30] & 0xf,
10714 31, dp->vlnt[31] & 0xf));
77241056
MM		 10715}
10716
10717static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
10718 u16 limit)
10719{
10720 if (limit != 0)
10721 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
17fb4f29 10722 what, (int)limit, idx);
77241056
MM		 10723}
10724
10725/* change only the shared limit portion of SendCmGLobalCredit */
10726static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
10727{
10728 u64 reg;
10729
10730 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10731 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
10732 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
10733 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10734}
10735
10736/* change only the total credit limit portion of SendCmGLobalCredit */
10737static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
10738{
10739 u64 reg;
10740
10741 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10742 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
10743 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
10744 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10745}
10746
10747/* set the given per-VL shared limit */
10748static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
10749{
10750 u64 reg;
10751 u32 addr;
10752
10753 if (vl < TXE_NUM_DATA_VL)
10754 addr = SEND_CM_CREDIT_VL + (8 * vl);
10755 else
10756 addr = SEND_CM_CREDIT_VL15;
10757
10758 reg = read_csr(dd, addr);
10759 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
10760 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
10761 write_csr(dd, addr, reg);
10762}
10763
10764/* set the given per-VL dedicated limit */
10765static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
10766{
10767 u64 reg;
10768 u32 addr;
10769
10770 if (vl < TXE_NUM_DATA_VL)
10771 addr = SEND_CM_CREDIT_VL + (8 * vl);
10772 else
10773 addr = SEND_CM_CREDIT_VL15;
10774
10775 reg = read_csr(dd, addr);
10776 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
10777 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
10778 write_csr(dd, addr, reg);
10779}
10780
10781/* spin until the given per-VL status mask bits clear */
10782static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
10783 const char *which)
10784{
10785 unsigned long timeout;
10786 u64 reg;
10787
10788 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
10789 while (1) {
10790 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
10791
10792 if (reg == 0)
10793 return; /* success */
10794 if (time_after(jiffies, timeout))
10795 break; /* timed out */
10796 udelay(1);
10797 }
10798
10799 dd_dev_err(dd,
17fb4f29
JJ		 10800 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
10801 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
77241056
MM		 10802 /*
10803 * If this occurs, it is likely there was a credit loss on the link.
10804 * The only recovery from that is a link bounce.
10805 */
10806 dd_dev_err(dd,
17fb4f29 10807 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
77241056
MM		 10808}
10809
10810/*
10811 * The number of credits on the VLs may be changed while everything
10812 * is "live", but the following algorithm must be followed due to
10813 * how the hardware is actually implemented. In particular,
10814 * Return_Credit_Status[] is the only correct status check.
10815 *
10816 * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
10817 * set Global_Shared_Credit_Limit = 0
10818 * use_all_vl = 1
10819 * mask0 = all VLs that are changing either dedicated or shared limits
10820 * set Shared_Limit[mask0] = 0
10821 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
10822 * if (changing any dedicated limit)
10823 * mask1 = all VLs that are lowering dedicated limits
10824 * lower Dedicated_Limit[mask1]
10825 * spin until Return_Credit_Status[mask1] == 0
10826 * raise Dedicated_Limits
10827 * raise Shared_Limits
10828 * raise Global_Shared_Credit_Limit
10829 *
10830 * lower = if the new limit is lower, set the limit to the new value
10831 * raise = if the new limit is higher than the current value (may be changed
10832 * earlier in the algorithm), set the new limit to the new value
10833 */
8a4d3444
MM		 10834int set_buffer_control(struct hfi1_pportdata *ppd,
10835 struct buffer_control *new_bc)
77241056 10836{
8a4d3444 10837 struct hfi1_devdata *dd = ppd->dd;
77241056
MM		 10838 u64 changing_mask, ld_mask, stat_mask;
10839 int change_count;
10840 int i, use_all_mask;
10841 int this_shared_changing;
8a4d3444 10842 int vl_count = 0, ret;
77241056
MM		 10843 /*
10844 * A0: add the variable any_shared_limit_changing below and in the
10845 * algorithm above. If removing A0 support, it can be removed.
10846 */
10847 int any_shared_limit_changing;
10848 struct buffer_control cur_bc;
10849 u8 changing[OPA_MAX_VLS];
10850 u8 lowering_dedicated[OPA_MAX_VLS];
10851 u16 cur_total;
10852 u32 new_total = 0;
10853 const u64 all_mask =
10854 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
10855 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
10856 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
10857 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
10858 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
10859 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
10860 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
10861 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
10862 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
10863
10864#define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
10865#define NUM_USABLE_VLS 16 /* look at VL15 and less */
10866
77241056
MM		 10867 /* find the new total credits, do sanity check on unused VLs */
10868 for (i = 0; i < OPA_MAX_VLS; i++) {
10869 if (valid_vl(i)) {
10870 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
10871 continue;
10872 }
10873 nonzero_msg(dd, i, "dedicated",
17fb4f29 10874 be16_to_cpu(new_bc->vl[i].dedicated));
77241056 10875 nonzero_msg(dd, i, "shared",
17fb4f29 10876 be16_to_cpu(new_bc->vl[i].shared));
77241056
MM		 10877 new_bc->vl[i].dedicated = 0;
10878 new_bc->vl[i].shared = 0;
10879 }
10880 new_total += be16_to_cpu(new_bc->overall_shared_limit);
bff14bb6 10881
77241056
MM		 10882 /* fetch the current values */
10883 get_buffer_control(dd, &cur_bc, &cur_total);
10884
10885 /*
10886 * Create the masks we will use.
10887 */
10888 memset(changing, 0, sizeof(changing));
10889 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
4d114fdd
JJ		 10890 /*
10891 * NOTE: Assumes that the individual VL bits are adjacent and in
10892 * increasing order
10893 */
77241056
MM		 10894 stat_mask =
10895 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
10896 changing_mask = 0;
10897 ld_mask = 0;
10898 change_count = 0;
10899 any_shared_limit_changing = 0;
10900 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
10901 if (!valid_vl(i))
10902 continue;
10903 this_shared_changing = new_bc->vl[i].shared
10904 != cur_bc.vl[i].shared;
10905 if (this_shared_changing)
10906 any_shared_limit_changing = 1;
d0d236ea
JJ		 10907 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
10908 this_shared_changing) {
77241056
MM		 10909 changing[i] = 1;
10910 changing_mask |= stat_mask;
10911 change_count++;
10912 }
10913 if (be16_to_cpu(new_bc->vl[i].dedicated) <
10914 be16_to_cpu(cur_bc.vl[i].dedicated)) {
10915 lowering_dedicated[i] = 1;
10916 ld_mask |= stat_mask;
10917 }
10918 }
10919
10920 /* bracket the credit change with a total adjustment */
10921 if (new_total > cur_total)
10922 set_global_limit(dd, new_total);
10923
10924 /*
10925 * Start the credit change algorithm.
10926 */
10927 use_all_mask = 0;
10928 if ((be16_to_cpu(new_bc->overall_shared_limit) <
995deafa
MM		 10929 be16_to_cpu(cur_bc.overall_shared_limit)) ||
10930 (is_ax(dd) && any_shared_limit_changing)) {
77241056
MM		 10931 set_global_shared(dd, 0);
10932 cur_bc.overall_shared_limit = 0;
10933 use_all_mask = 1;
10934 }
10935
10936 for (i = 0; i < NUM_USABLE_VLS; i++) {
10937 if (!valid_vl(i))
10938 continue;
10939
10940 if (changing[i]) {
10941 set_vl_shared(dd, i, 0);
10942 cur_bc.vl[i].shared = 0;
10943 }
10944 }
10945
10946 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
17fb4f29 10947 "shared");
77241056
MM		 10948
10949 if (change_count > 0) {
10950 for (i = 0; i < NUM_USABLE_VLS; i++) {
10951 if (!valid_vl(i))
10952 continue;
10953
10954 if (lowering_dedicated[i]) {
10955 set_vl_dedicated(dd, i,
17fb4f29
JJ		 10956 be16_to_cpu(new_bc->
10957 vl[i].dedicated));
77241056
MM		 10958 cur_bc.vl[i].dedicated =
10959 new_bc->vl[i].dedicated;
10960 }
10961 }
10962
10963 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
10964
10965 /* now raise all dedicated that are going up */
10966 for (i = 0; i < NUM_USABLE_VLS; i++) {
10967 if (!valid_vl(i))
10968 continue;
10969
10970 if (be16_to_cpu(new_bc->vl[i].dedicated) >
10971 be16_to_cpu(cur_bc.vl[i].dedicated))
10972 set_vl_dedicated(dd, i,
17fb4f29
JJ		 10973 be16_to_cpu(new_bc->
10974 vl[i].dedicated));
77241056
MM		 10975 }
10976 }
10977
10978 /* next raise all shared that are going up */
10979 for (i = 0; i < NUM_USABLE_VLS; i++) {
10980 if (!valid_vl(i))
10981 continue;
10982
10983 if (be16_to_cpu(new_bc->vl[i].shared) >
10984 be16_to_cpu(cur_bc.vl[i].shared))
10985 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
10986 }
10987
10988 /* finally raise the global shared */
10989 if (be16_to_cpu(new_bc->overall_shared_limit) >
17fb4f29 10990 be16_to_cpu(cur_bc.overall_shared_limit))
77241056 10991 set_global_shared(dd,
17fb4f29 10992 be16_to_cpu(new_bc->overall_shared_limit));
77241056
MM		 10993
10994 /* bracket the credit change with a total adjustment */
10995 if (new_total < cur_total)
10996 set_global_limit(dd, new_total);
8a4d3444
MM		 10997
10998 /*
10999 * Determine the actual number of operational VLS using the number of
11000 * dedicated and shared credits for each VL.
11001 */
11002 if (change_count > 0) {
11003 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11004 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
11005 be16_to_cpu(new_bc->vl[i].shared) > 0)
11006 vl_count++;
11007 ppd->actual_vls_operational = vl_count;
11008 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
11009 ppd->actual_vls_operational :
11010 ppd->vls_operational,
11011 NULL);
11012 if (ret == 0)
11013 ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11014 ppd->actual_vls_operational :
11015 ppd->vls_operational, NULL);
11016 if (ret)
11017 return ret;
11018 }
77241056
MM		 11019 return 0;
11020}
11021
11022/*
11023 * Read the given fabric manager table. Return the size of the
11024 * table (in bytes) on success, and a negative error code on
11025 * failure.
11026 */
11027int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11028
11029{
11030 int size;
11031 struct vl_arb_cache *vlc;
11032
11033 switch (which) {
11034 case FM_TBL_VL_HIGH_ARB:
11035 size = 256;
11036 /*
11037 * OPA specifies 128 elements (of 2 bytes each), though
11038 * HFI supports only 16 elements in h/w.
11039 */
11040 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11041 vl_arb_get_cache(vlc, t);
11042 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11043 break;
11044 case FM_TBL_VL_LOW_ARB:
11045 size = 256;
11046 /*
11047 * OPA specifies 128 elements (of 2 bytes each), though
11048 * HFI supports only 16 elements in h/w.
11049 */
11050 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11051 vl_arb_get_cache(vlc, t);
11052 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11053 break;
11054 case FM_TBL_BUFFER_CONTROL:
11055 size = get_buffer_control(ppd->dd, t, NULL);
11056 break;
11057 case FM_TBL_SC2VLNT:
11058 size = get_sc2vlnt(ppd->dd, t);
11059 break;
11060 case FM_TBL_VL_PREEMPT_ELEMS:
11061 size = 256;
11062 /* OPA specifies 128 elements, of 2 bytes each */
11063 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11064 break;
11065 case FM_TBL_VL_PREEMPT_MATRIX:
11066 size = 256;
11067 /*
11068 * OPA specifies that this is the same size as the VL
11069 * arbitration tables (i.e., 256 bytes).
11070 */
11071 break;
11072 default:
11073 return -EINVAL;
11074 }
11075 return size;
11076}
11077
11078/*
11079 * Write the given fabric manager table.
11080 */
11081int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11082{
11083 int ret = 0;
11084 struct vl_arb_cache *vlc;
11085
11086 switch (which) {
11087 case FM_TBL_VL_HIGH_ARB:
11088 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11089 if (vl_arb_match_cache(vlc, t)) {
11090 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11091 break;
11092 }
11093 vl_arb_set_cache(vlc, t);
11094 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11095 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11096 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11097 break;
11098 case FM_TBL_VL_LOW_ARB:
11099 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11100 if (vl_arb_match_cache(vlc, t)) {
11101 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11102 break;
11103 }
11104 vl_arb_set_cache(vlc, t);
11105 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11106 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11107 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11108 break;
11109 case FM_TBL_BUFFER_CONTROL:
8a4d3444 11110 ret = set_buffer_control(ppd, t);
77241056
MM		 11111 break;
11112 case FM_TBL_SC2VLNT:
11113 set_sc2vlnt(ppd->dd, t);
11114 break;
11115 default:
11116 ret = -EINVAL;
11117 }
11118 return ret;
11119}
11120
11121/*
11122 * Disable all data VLs.
11123 *
11124 * Return 0 if disabled, non-zero if the VLs cannot be disabled.
11125 */
11126static int disable_data_vls(struct hfi1_devdata *dd)
11127{
995deafa 11128 if (is_ax(dd))
77241056
MM		 11129 return 1;
11130
11131 pio_send_control(dd, PSC_DATA_VL_DISABLE);
11132
11133 return 0;
11134}
11135
11136/*
11137 * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11138 * Just re-enables all data VLs (the "fill" part happens
11139 * automatically - the name was chosen for symmetry with
11140 * stop_drain_data_vls()).
11141 *
11142 * Return 0 if successful, non-zero if the VLs cannot be enabled.
11143 */
11144int open_fill_data_vls(struct hfi1_devdata *dd)
11145{
995deafa 11146 if (is_ax(dd))
77241056
MM		 11147 return 1;
11148
11149 pio_send_control(dd, PSC_DATA_VL_ENABLE);
11150
11151 return 0;
11152}
11153
11154/*
11155 * drain_data_vls() - assumes that disable_data_vls() has been called,
11156 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11157 * engines to drop to 0.
11158 */
11159static void drain_data_vls(struct hfi1_devdata *dd)
11160{
11161 sc_wait(dd);
11162 sdma_wait(dd);
11163 pause_for_credit_return(dd);
11164}
11165
11166/*
11167 * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11168 *
11169 * Use open_fill_data_vls() to resume using data VLs. This pair is
11170 * meant to be used like this:
11171 *
11172 * stop_drain_data_vls(dd);
11173 * // do things with per-VL resources
11174 * open_fill_data_vls(dd);
11175 */
11176int stop_drain_data_vls(struct hfi1_devdata *dd)
11177{
11178 int ret;
11179
11180 ret = disable_data_vls(dd);
11181 if (ret == 0)
11182 drain_data_vls(dd);
11183
11184 return ret;
11185}
11186
11187/*
11188 * Convert a nanosecond time to a cclock count. No matter how slow
11189 * the cclock, a non-zero ns will always have a non-zero result.
11190 */
11191u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11192{
11193 u32 cclocks;
11194
11195 if (dd->icode == ICODE_FPGA_EMULATION)
11196 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11197 else /* simulation pretends to be ASIC */
11198 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11199 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */
11200 cclocks = 1;
11201 return cclocks;
11202}
11203
11204/*
11205 * Convert a cclock count to nanoseconds. Not matter how slow
11206 * the cclock, a non-zero cclocks will always have a non-zero result.
11207 */
11208u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11209{
11210 u32 ns;
11211
11212 if (dd->icode == ICODE_FPGA_EMULATION)
11213 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11214 else /* simulation pretends to be ASIC */
11215 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11216 if (cclocks && !ns)
11217 ns = 1;
11218 return ns;
11219}
11220
11221/*
11222 * Dynamically adjust the receive interrupt timeout for a context based on
11223 * incoming packet rate.
11224 *
11225 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11226 */
11227static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11228{
11229 struct hfi1_devdata *dd = rcd->dd;
11230 u32 timeout = rcd->rcvavail_timeout;
11231
11232 /*
11233 * This algorithm doubles or halves the timeout depending on whether
11234 * the number of packets received in this interrupt were less than or
11235 * greater equal the interrupt count.
11236 *
11237 * The calculations below do not allow a steady state to be achieved.
11238 * Only at the endpoints it is possible to have an unchanging
11239 * timeout.
11240 */
11241 if (npkts < rcv_intr_count) {
11242 /*
11243 * Not enough packets arrived before the timeout, adjust
11244 * timeout downward.
11245 */
11246 if (timeout < 2) /* already at minimum? */
11247 return;
11248 timeout >>= 1;
11249 } else {
11250 /*
11251 * More than enough packets arrived before the timeout, adjust
11252 * timeout upward.
11253 */
11254 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11255 return;
11256 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11257 }
11258
11259 rcd->rcvavail_timeout = timeout;
4d114fdd
JJ		 11260 /*
11261 * timeout cannot be larger than rcv_intr_timeout_csr which has already
11262 * been verified to be in range
11263 */
77241056 11264 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
17fb4f29
JJ		 11265 (u64)timeout <<
11266 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
77241056
MM		 11267}
11268
11269void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11270 u32 intr_adjust, u32 npkts)
11271{
11272 struct hfi1_devdata *dd = rcd->dd;
11273 u64 reg;
11274 u32 ctxt = rcd->ctxt;
11275
11276 /*
11277 * Need to write timeout register before updating RcvHdrHead to ensure
11278 * that a new value is used when the HW decides to restart counting.
11279 */
11280 if (intr_adjust)
11281 adjust_rcv_timeout(rcd, npkts);
11282 if (updegr) {
11283 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11284 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11285 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11286 }
11287 mmiowb();
11288 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11289 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11290 << RCV_HDR_HEAD_HEAD_SHIFT);
11291 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11292 mmiowb();
11293}
11294
11295u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11296{
11297 u32 head, tail;
11298
11299 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11300 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11301
11302 if (rcd->rcvhdrtail_kvaddr)
11303 tail = get_rcvhdrtail(rcd);
11304 else
11305 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11306
11307 return head == tail;
11308}
11309
11310/*
11311 * Context Control and Receive Array encoding for buffer size:
11312 * 0x0 invalid
11313 * 0x1 4 KB
11314 * 0x2 8 KB
11315 * 0x3 16 KB
11316 * 0x4 32 KB
11317 * 0x5 64 KB
11318 * 0x6 128 KB
11319 * 0x7 256 KB
11320 * 0x8 512 KB (Receive Array only)
11321 * 0x9 1 MB (Receive Array only)
11322 * 0xa 2 MB (Receive Array only)
11323 *
11324 * 0xB-0xF - reserved (Receive Array only)
11325 *
11326 *
11327 * This routine assumes that the value has already been sanity checked.
11328 */
11329static u32 encoded_size(u32 size)
11330{
11331 switch (size) {
8638b77f
JJ		 11332 case 4 * 1024: return 0x1;
11333 case 8 * 1024: return 0x2;
11334 case 16 * 1024: return 0x3;
11335 case 32 * 1024: return 0x4;
11336 case 64 * 1024: return 0x5;
11337 case 128 * 1024: return 0x6;
11338 case 256 * 1024: return 0x7;
11339 case 512 * 1024: return 0x8;
11340 case 1 * 1024 * 1024: return 0x9;
11341 case 2 * 1024 * 1024: return 0xa;
77241056
MM		 11342 }
11343 return 0x1; /* if invalid, go with the minimum size */
11344}
11345
11346void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
11347{
11348 struct hfi1_ctxtdata *rcd;
11349 u64 rcvctrl, reg;
11350 int did_enable = 0;
11351
11352 rcd = dd->rcd[ctxt];
11353 if (!rcd)
11354 return;
11355
11356 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
11357
11358 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
11359 /* if the context already enabled, don't do the extra steps */
d0d236ea
JJ		 11360 if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
11361 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
77241056
MM		 11362 /* reset the tail and hdr addresses, and sequence count */
11363 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
11364 rcd->rcvhdrq_phys);
11365 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
11366 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11367 rcd->rcvhdrqtailaddr_phys);
11368 rcd->seq_cnt = 1;
11369
11370 /* reset the cached receive header queue head value */
11371 rcd->head = 0;
11372
11373 /*
11374 * Zero the receive header queue so we don't get false
11375 * positives when checking the sequence number. The
11376 * sequence numbers could land exactly on the same spot.
11377 * E.g. a rcd restart before the receive header wrapped.
11378 */
11379 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
11380
11381 /* starting timeout */
11382 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
11383
11384 /* enable the context */
11385 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
11386
11387 /* clean the egr buffer size first */
11388 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11389 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
11390 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
11391 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
11392
11393 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
11394 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
11395 did_enable = 1;
11396
11397 /* zero RcvEgrIndexHead */
11398 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
11399
11400 /* set eager count and base index */
11401 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
11402 & RCV_EGR_CTRL_EGR_CNT_MASK)
11403 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
11404 (((rcd->eager_base >> RCV_SHIFT)
11405 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
11406 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
11407 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
11408
11409 /*
11410 * Set TID (expected) count and base index.
11411 * rcd->expected_count is set to individual RcvArray entries,
11412 * not pairs, and the CSR takes a pair-count in groups of
11413 * four, so divide by 8.
11414 */
11415 reg = (((rcd->expected_count >> RCV_SHIFT)
11416 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
11417 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
11418 (((rcd->expected_base >> RCV_SHIFT)
11419 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
11420 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
11421 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
82c2611d
NV		 11422 if (ctxt == HFI1_CTRL_CTXT)
11423 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
77241056
MM		 11424 }
11425 if (op & HFI1_RCVCTRL_CTXT_DIS) {
11426 write_csr(dd, RCV_VL15, 0);
46b010d3
MB		 11427 /*
11428 * When receive context is being disabled turn on tail
11429 * update with a dummy tail address and then disable
11430 * receive context.
11431 */
11432 if (dd->rcvhdrtail_dummy_physaddr) {
11433 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11434 dd->rcvhdrtail_dummy_physaddr);
566c157c 11435 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
46b010d3
MB		 11436 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11437 }
11438
77241056
MM		 11439 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
11440 }
11441 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB)
11442 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11443 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
11444 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11445 if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys)
11446 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
566c157c
MH		 11447 if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
11448 /* See comment on RcvCtxtCtrl.TailUpd above */
11449 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
11450 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11451 }
77241056
MM		 11452 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
11453 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11454 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
11455 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11456 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
4d114fdd
JJ		 11457 /*
11458 * In one-packet-per-eager mode, the size comes from
11459 * the RcvArray entry.
11460 */
77241056
MM		 11461 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11462 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11463 }
11464 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
11465 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11466 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
11467 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11468 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
11469 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11470 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
11471 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11472 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
11473 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11474 rcd->rcvctrl = rcvctrl;
11475 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
11476 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl);
11477
11478 /* work around sticky RcvCtxtStatus.BlockedRHQFull */
d0d236ea
JJ		 11479 if (did_enable &&
11480 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
77241056
MM		 11481 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11482 if (reg != 0) {
11483 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
17fb4f29 11484 ctxt, reg);
77241056
MM		 11485 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11486 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
11487 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
11488 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11489 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11490 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
17fb4f29 11491 ctxt, reg, reg == 0 ? "not" : "still");
77241056
MM		 11492 }
11493 }
11494
11495 if (did_enable) {
11496 /*
11497 * The interrupt timeout and count must be set after
11498 * the context is enabled to take effect.
11499 */
11500 /* set interrupt timeout */
11501 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
17fb4f29 11502 (u64)rcd->rcvavail_timeout <<
77241056
MM		 11503 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11504
11505 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
11506 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
11507 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11508 }
11509
11510 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
11511 /*
11512 * If the context has been disabled and the Tail Update has
46b010d3
MB		 11513 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
11514 * so it doesn't contain an address that is invalid.
77241056 11515 */
46b010d3
MB		 11516 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11517 dd->rcvhdrtail_dummy_physaddr);
77241056
MM		 11518}
11519
582e05c3 11520u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
77241056
MM		 11521{
11522 int ret;
11523 u64 val = 0;
11524
11525 if (namep) {
11526 ret = dd->cntrnameslen;
77241056
MM		 11527 *namep = dd->cntrnames;
11528 } else {
11529 const struct cntr_entry *entry;
11530 int i, j;
11531
11532 ret = (dd->ndevcntrs) * sizeof(u64);
77241056
MM		 11533
11534 /* Get the start of the block of counters */
11535 *cntrp = dd->cntrs;
11536
11537 /*
11538 * Now go and fill in each counter in the block.
11539 */
11540 for (i = 0; i < DEV_CNTR_LAST; i++) {
11541 entry = &dev_cntrs[i];
11542 hfi1_cdbg(CNTR, "reading %s", entry->name);
11543 if (entry->flags & CNTR_DISABLED) {
11544 /* Nothing */
11545 hfi1_cdbg(CNTR, "\tDisabled\n");
11546 } else {
11547 if (entry->flags & CNTR_VL) {
11548 hfi1_cdbg(CNTR, "\tPer VL\n");
11549 for (j = 0; j < C_VL_COUNT; j++) {
11550 val = entry->rw_cntr(entry,
11551 dd, j,
11552 CNTR_MODE_R,
11553 0);
11554 hfi1_cdbg(
11555 CNTR,
11556 "\t\tRead 0x%llx for %d\n",
11557 val, j);
11558 dd->cntrs[entry->offset + j] =
11559 val;
11560 }
a699c6c2
VM		 11561 } else if (entry->flags & CNTR_SDMA) {
11562 hfi1_cdbg(CNTR,
11563 "\t Per SDMA Engine\n");
11564 for (j = 0; j < dd->chip_sdma_engines;
11565 j++) {
11566 val =
11567 entry->rw_cntr(entry, dd, j,
11568 CNTR_MODE_R, 0);
11569 hfi1_cdbg(CNTR,
11570 "\t\tRead 0x%llx for %d\n",
11571 val, j);
11572 dd->cntrs[entry->offset + j] =
11573 val;
11574 }
77241056
MM		 11575 } else {
11576 val = entry->rw_cntr(entry, dd,
11577 CNTR_INVALID_VL,
11578 CNTR_MODE_R, 0);
11579 dd->cntrs[entry->offset] = val;
11580 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11581 }
11582 }
11583 }
11584 }
11585 return ret;
11586}
11587
11588/*
11589 * Used by sysfs to create files for hfi stats to read
11590 */
582e05c3 11591u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
77241056
MM		 11592{
11593 int ret;
11594 u64 val = 0;
11595
11596 if (namep) {
582e05c3
DL		 11597 ret = ppd->dd->portcntrnameslen;
11598 *namep = ppd->dd->portcntrnames;
77241056
MM		 11599 } else {
11600 const struct cntr_entry *entry;
77241056
MM		 11601 int i, j;
11602
582e05c3 11603 ret = ppd->dd->nportcntrs * sizeof(u64);
77241056
MM		 11604 *cntrp = ppd->cntrs;
11605
11606 for (i = 0; i < PORT_CNTR_LAST; i++) {
11607 entry = &port_cntrs[i];
11608 hfi1_cdbg(CNTR, "reading %s", entry->name);
11609 if (entry->flags & CNTR_DISABLED) {
11610 /* Nothing */
11611 hfi1_cdbg(CNTR, "\tDisabled\n");
11612 continue;
11613 }
11614
11615 if (entry->flags & CNTR_VL) {
11616 hfi1_cdbg(CNTR, "\tPer VL");
11617 for (j = 0; j < C_VL_COUNT; j++) {
11618 val = entry->rw_cntr(entry, ppd, j,
11619 CNTR_MODE_R,
11620 0);
11621 hfi1_cdbg(
11622 CNTR,
11623 "\t\tRead 0x%llx for %d",
11624 val, j);
11625 ppd->cntrs[entry->offset + j] = val;
11626 }
11627 } else {
11628 val = entry->rw_cntr(entry, ppd,
11629 CNTR_INVALID_VL,
11630 CNTR_MODE_R,
11631 0);
11632 ppd->cntrs[entry->offset] = val;
11633 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11634 }
11635 }
11636 }
11637 return ret;
11638}
11639
11640static void free_cntrs(struct hfi1_devdata *dd)
11641{
11642 struct hfi1_pportdata *ppd;
11643 int i;
11644
11645 if (dd->synth_stats_timer.data)
11646 del_timer_sync(&dd->synth_stats_timer);
11647 dd->synth_stats_timer.data = 0;
11648 ppd = (struct hfi1_pportdata *)(dd + 1);
11649 for (i = 0; i < dd->num_pports; i++, ppd++) {
11650 kfree(ppd->cntrs);
11651 kfree(ppd->scntrs);
4eb06882
DD		 11652 free_percpu(ppd->ibport_data.rvp.rc_acks);
11653 free_percpu(ppd->ibport_data.rvp.rc_qacks);
11654 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
77241056
MM		 11655 ppd->cntrs = NULL;
11656 ppd->scntrs = NULL;
4eb06882
DD		 11657 ppd->ibport_data.rvp.rc_acks = NULL;
11658 ppd->ibport_data.rvp.rc_qacks = NULL;
11659 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
77241056
MM		 11660 }
11661 kfree(dd->portcntrnames);
11662 dd->portcntrnames = NULL;
11663 kfree(dd->cntrs);
11664 dd->cntrs = NULL;
11665 kfree(dd->scntrs);
11666 dd->scntrs = NULL;
11667 kfree(dd->cntrnames);
11668 dd->cntrnames = NULL;
11669}
11670
11671#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
11672#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
11673
11674static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
11675 u64 *psval, void *context, int vl)
11676{
11677 u64 val;
11678 u64 sval = *psval;
11679
11680 if (entry->flags & CNTR_DISABLED) {
11681 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11682 return 0;
11683 }
11684
11685 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11686
11687 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
11688
11689 /* If its a synthetic counter there is more work we need to do */
11690 if (entry->flags & CNTR_SYNTH) {
11691 if (sval == CNTR_MAX) {
11692 /* No need to read already saturated */
11693 return CNTR_MAX;
11694 }
11695
11696 if (entry->flags & CNTR_32BIT) {
11697 /* 32bit counters can wrap multiple times */
11698 u64 upper = sval >> 32;
11699 u64 lower = (sval << 32) >> 32;
11700
11701 if (lower > val) { /* hw wrapped */
11702 if (upper == CNTR_32BIT_MAX)
11703 val = CNTR_MAX;
11704 else
11705 upper++;
11706 }
11707
11708 if (val != CNTR_MAX)
11709 val = (upper << 32) | val;
11710
11711 } else {
11712 /* If we rolled we are saturated */
11713 if ((val < sval) || (val > CNTR_MAX))
11714 val = CNTR_MAX;
11715 }
11716 }
11717
11718 *psval = val;
11719
11720 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11721
11722 return val;
11723}
11724
11725static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
11726 struct cntr_entry *entry,
11727 u64 *psval, void *context, int vl, u64 data)
11728{
11729 u64 val;
11730
11731 if (entry->flags & CNTR_DISABLED) {
11732 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11733 return 0;
11734 }
11735
11736 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11737
11738 if (entry->flags & CNTR_SYNTH) {
11739 *psval = data;
11740 if (entry->flags & CNTR_32BIT) {
11741 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11742 (data << 32) >> 32);
11743 val = data; /* return the full 64bit value */
11744 } else {
11745 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11746 data);
11747 }
11748 } else {
11749 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
11750 }
11751
11752 *psval = val;
11753
11754 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11755
11756 return val;
11757}
11758
11759u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
11760{
11761 struct cntr_entry *entry;
11762 u64 *sval;
11763
11764 entry = &dev_cntrs[index];
11765 sval = dd->scntrs + entry->offset;
11766
11767 if (vl != CNTR_INVALID_VL)
11768 sval += vl;
11769
11770 return read_dev_port_cntr(dd, entry, sval, dd, vl);
11771}
11772
11773u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
11774{
11775 struct cntr_entry *entry;
11776 u64 *sval;
11777
11778 entry = &dev_cntrs[index];
11779 sval = dd->scntrs + entry->offset;
11780
11781 if (vl != CNTR_INVALID_VL)
11782 sval += vl;
11783
11784 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
11785}
11786
11787u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
11788{
11789 struct cntr_entry *entry;
11790 u64 *sval;
11791
11792 entry = &port_cntrs[index];
11793 sval = ppd->scntrs + entry->offset;
11794
11795 if (vl != CNTR_INVALID_VL)
11796 sval += vl;
11797
11798 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11799 (index <= C_RCV_HDR_OVF_LAST)) {
11800 /* We do not want to bother for disabled contexts */
11801 return 0;
11802 }
11803
11804 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
11805}
11806
11807u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
11808{
11809 struct cntr_entry *entry;
11810 u64 *sval;
11811
11812 entry = &port_cntrs[index];
11813 sval = ppd->scntrs + entry->offset;
11814
11815 if (vl != CNTR_INVALID_VL)
11816 sval += vl;
11817
11818 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11819 (index <= C_RCV_HDR_OVF_LAST)) {
11820 /* We do not want to bother for disabled contexts */
11821 return 0;
11822 }
11823
11824 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
11825}
11826
11827static void update_synth_timer(unsigned long opaque)
11828{
11829 u64 cur_tx;
11830 u64 cur_rx;
11831 u64 total_flits;
11832 u8 update = 0;
11833 int i, j, vl;
11834 struct hfi1_pportdata *ppd;
11835 struct cntr_entry *entry;
11836
11837 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
11838
11839 /*
11840 * Rather than keep beating on the CSRs pick a minimal set that we can
11841 * check to watch for potential roll over. We can do this by looking at
11842 * the number of flits sent/recv. If the total flits exceeds 32bits then
11843 * we have to iterate all the counters and update.
11844 */
11845 entry = &dev_cntrs[C_DC_RCV_FLITS];
11846 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
11847
11848 entry = &dev_cntrs[C_DC_XMIT_FLITS];
11849 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
11850
11851 hfi1_cdbg(
11852 CNTR,
11853 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
11854 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
11855
11856 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
11857 /*
11858 * May not be strictly necessary to update but it won't hurt and
11859 * simplifies the logic here.
11860 */
11861 update = 1;
11862 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
11863 dd->unit);
11864 } else {
11865 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
11866 hfi1_cdbg(CNTR,
11867 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
11868 total_flits, (u64)CNTR_32BIT_MAX);
11869 if (total_flits >= CNTR_32BIT_MAX) {
11870 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
11871 dd->unit);
11872 update = 1;
11873 }
11874 }
11875
11876 if (update) {
11877 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
11878 for (i = 0; i < DEV_CNTR_LAST; i++) {
11879 entry = &dev_cntrs[i];
11880 if (entry->flags & CNTR_VL) {
11881 for (vl = 0; vl < C_VL_COUNT; vl++)
11882 read_dev_cntr(dd, i, vl);
11883 } else {
11884 read_dev_cntr(dd, i, CNTR_INVALID_VL);
11885 }
11886 }
11887 ppd = (struct hfi1_pportdata *)(dd + 1);
11888 for (i = 0; i < dd->num_pports; i++, ppd++) {
11889 for (j = 0; j < PORT_CNTR_LAST; j++) {
11890 entry = &port_cntrs[j];
11891 if (entry->flags & CNTR_VL) {
11892 for (vl = 0; vl < C_VL_COUNT; vl++)
11893 read_port_cntr(ppd, j, vl);
11894 } else {
11895 read_port_cntr(ppd, j, CNTR_INVALID_VL);
11896 }
11897 }
11898 }
11899
11900 /*
11901 * We want the value in the register. The goal is to keep track
11902 * of the number of "ticks" not the counter value. In other
11903 * words if the register rolls we want to notice it and go ahead
11904 * and force an update.
11905 */
11906 entry = &dev_cntrs[C_DC_XMIT_FLITS];
11907 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
11908 CNTR_MODE_R, 0);
11909
11910 entry = &dev_cntrs[C_DC_RCV_FLITS];
11911 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
11912 CNTR_MODE_R, 0);
11913
11914 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
11915 dd->unit, dd->last_tx, dd->last_rx);
11916
11917 } else {
11918 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
11919 }
11920
48a0cc13 11921 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
77241056
MM		 11922}
11923
11924#define C_MAX_NAME 13 /* 12 chars + one for /0 */
11925static int init_cntrs(struct hfi1_devdata *dd)
11926{
c024c554 11927 int i, rcv_ctxts, j;
77241056
MM		 11928 size_t sz;
11929 char *p;
11930 char name[C_MAX_NAME];
11931 struct hfi1_pportdata *ppd;
11d2b114
SS		 11932 const char *bit_type_32 = ",32";
11933 const int bit_type_32_sz = strlen(bit_type_32);
77241056
MM		 11934
11935 /* set up the stats timer; the add_timer is done at the end */
24523a94
MFW		 11936 setup_timer(&dd->synth_stats_timer, update_synth_timer,
11937 (unsigned long)dd);
77241056
MM		 11938
11939 /***********************/
11940 /* per device counters */
11941 /***********************/
11942
11943 /* size names and determine how many we have*/
11944 dd->ndevcntrs = 0;
11945 sz = 0;
77241056
MM		 11946
11947 for (i = 0; i < DEV_CNTR_LAST; i++) {
77241056
MM		 11948 if (dev_cntrs[i].flags & CNTR_DISABLED) {
11949 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
11950 continue;
11951 }
11952
11953 if (dev_cntrs[i].flags & CNTR_VL) {
c024c554 11954 dev_cntrs[i].offset = dd->ndevcntrs;
77241056 11955 for (j = 0; j < C_VL_COUNT; j++) {
77241056 11956 snprintf(name, C_MAX_NAME, "%s%d",
17fb4f29 11957 dev_cntrs[i].name, vl_from_idx(j));
77241056 11958 sz += strlen(name);
11d2b114
SS		 11959 /* Add ",32" for 32-bit counters */
11960 if (dev_cntrs[i].flags & CNTR_32BIT)
11961 sz += bit_type_32_sz;
77241056 11962 sz++;
77241056 11963 dd->ndevcntrs++;
77241056 11964 }
a699c6c2 11965 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
c024c554 11966 dev_cntrs[i].offset = dd->ndevcntrs;
a699c6c2 11967 for (j = 0; j < dd->chip_sdma_engines; j++) {
a699c6c2
VM		 11968 snprintf(name, C_MAX_NAME, "%s%d",
11969 dev_cntrs[i].name, j);
77241056 11970 sz += strlen(name);
11d2b114
SS		 11971 /* Add ",32" for 32-bit counters */
11972 if (dev_cntrs[i].flags & CNTR_32BIT)
11973 sz += bit_type_32_sz;
77241056 11974 sz++;
77241056 11975 dd->ndevcntrs++;
77241056
MM		 11976 }
11977 } else {
11d2b114 11978 /* +1 for newline. */
77241056 11979 sz += strlen(dev_cntrs[i].name) + 1;
11d2b114
SS		 11980 /* Add ",32" for 32-bit counters */
11981 if (dev_cntrs[i].flags & CNTR_32BIT)
11982 sz += bit_type_32_sz;
c024c554 11983 dev_cntrs[i].offset = dd->ndevcntrs;
77241056 11984 dd->ndevcntrs++;
77241056
MM		 11985 }
11986 }
11987
11988 /* allocate space for the counter values */
c024c554 11989 dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
77241056
MM		 11990 if (!dd->cntrs)
11991 goto bail;
11992
c024c554 11993 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
77241056
MM		 11994 if (!dd->scntrs)
11995 goto bail;
11996
77241056
MM		 11997 /* allocate space for the counter names */
11998 dd->cntrnameslen = sz;
11999 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
12000 if (!dd->cntrnames)
12001 goto bail;
12002
12003 /* fill in the names */
c024c554 12004 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
77241056
MM		 12005 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12006 /* Nothing */
11d2b114
SS		 12007 } else if (dev_cntrs[i].flags & CNTR_VL) {
12008 for (j = 0; j < C_VL_COUNT; j++) {
11d2b114
SS		 12009 snprintf(name, C_MAX_NAME, "%s%d",
12010 dev_cntrs[i].name,
12011 vl_from_idx(j));
12012 memcpy(p, name, strlen(name));
12013 p += strlen(name);
12014
12015 /* Counter is 32 bits */
12016 if (dev_cntrs[i].flags & CNTR_32BIT) {
12017 memcpy(p, bit_type_32, bit_type_32_sz);
12018 p += bit_type_32_sz;
77241056 12019 }
11d2b114
SS		 12020
12021 *p++ = '\n';
12022 }
12023 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12024 for (j = 0; j < dd->chip_sdma_engines; j++) {
11d2b114
SS		 12025 snprintf(name, C_MAX_NAME, "%s%d",
12026 dev_cntrs[i].name, j);
12027 memcpy(p, name, strlen(name));
12028 p += strlen(name);
12029
12030 /* Counter is 32 bits */
12031 if (dev_cntrs[i].flags & CNTR_32BIT) {
12032 memcpy(p, bit_type_32, bit_type_32_sz);
12033 p += bit_type_32_sz;
a699c6c2 12034 }
11d2b114 12035
77241056
MM		 12036 *p++ = '\n';
12037 }
11d2b114
SS		 12038 } else {
12039 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12040 p += strlen(dev_cntrs[i].name);
12041
12042 /* Counter is 32 bits */
12043 if (dev_cntrs[i].flags & CNTR_32BIT) {
12044 memcpy(p, bit_type_32, bit_type_32_sz);
12045 p += bit_type_32_sz;
12046 }
12047
12048 *p++ = '\n';
77241056
MM		 12049 }
12050 }
12051
12052 /*********************/
12053 /* per port counters */
12054 /*********************/
12055
12056 /*
12057 * Go through the counters for the overflows and disable the ones we
12058 * don't need. This varies based on platform so we need to do it
12059 * dynamically here.
12060 */
12061 rcv_ctxts = dd->num_rcv_contexts;
12062 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12063 i <= C_RCV_HDR_OVF_LAST; i++) {
12064 port_cntrs[i].flags |= CNTR_DISABLED;
12065 }
12066
12067 /* size port counter names and determine how many we have*/
12068 sz = 0;
12069 dd->nportcntrs = 0;
12070 for (i = 0; i < PORT_CNTR_LAST; i++) {
77241056
MM		 12071 if (port_cntrs[i].flags & CNTR_DISABLED) {
12072 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12073 continue;
12074 }
12075
12076 if (port_cntrs[i].flags & CNTR_VL) {
77241056
MM		 12077 port_cntrs[i].offset = dd->nportcntrs;
12078 for (j = 0; j < C_VL_COUNT; j++) {
77241056 12079 snprintf(name, C_MAX_NAME, "%s%d",
17fb4f29 12080 port_cntrs[i].name, vl_from_idx(j));
77241056 12081 sz += strlen(name);
11d2b114
SS		 12082 /* Add ",32" for 32-bit counters */
12083 if (port_cntrs[i].flags & CNTR_32BIT)
12084 sz += bit_type_32_sz;
77241056 12085 sz++;
77241056
MM		 12086 dd->nportcntrs++;
12087 }
12088 } else {
11d2b114 12089 /* +1 for newline */
77241056 12090 sz += strlen(port_cntrs[i].name) + 1;
11d2b114
SS		 12091 /* Add ",32" for 32-bit counters */
12092 if (port_cntrs[i].flags & CNTR_32BIT)
12093 sz += bit_type_32_sz;
77241056
MM		 12094 port_cntrs[i].offset = dd->nportcntrs;
12095 dd->nportcntrs++;
77241056
MM		 12096 }
12097 }
12098
12099 /* allocate space for the counter names */
12100 dd->portcntrnameslen = sz;
12101 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12102 if (!dd->portcntrnames)
12103 goto bail;
12104
12105 /* fill in port cntr names */
12106 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12107 if (port_cntrs[i].flags & CNTR_DISABLED)
12108 continue;
12109
12110 if (port_cntrs[i].flags & CNTR_VL) {
12111 for (j = 0; j < C_VL_COUNT; j++) {
77241056 12112 snprintf(name, C_MAX_NAME, "%s%d",
17fb4f29 12113 port_cntrs[i].name, vl_from_idx(j));
77241056
MM		 12114 memcpy(p, name, strlen(name));
12115 p += strlen(name);
11d2b114
SS		 12116
12117 /* Counter is 32 bits */
12118 if (port_cntrs[i].flags & CNTR_32BIT) {
12119 memcpy(p, bit_type_32, bit_type_32_sz);
12120 p += bit_type_32_sz;
12121 }
12122
77241056
MM		 12123 *p++ = '\n';
12124 }
12125 } else {
12126 memcpy(p, port_cntrs[i].name,
12127 strlen(port_cntrs[i].name));
12128 p += strlen(port_cntrs[i].name);
11d2b114
SS		 12129
12130 /* Counter is 32 bits */
12131 if (port_cntrs[i].flags & CNTR_32BIT) {
12132 memcpy(p, bit_type_32, bit_type_32_sz);
12133 p += bit_type_32_sz;
12134 }
12135
77241056
MM		 12136 *p++ = '\n';
12137 }
12138 }
12139
12140 /* allocate per port storage for counter values */
12141 ppd = (struct hfi1_pportdata *)(dd + 1);
12142 for (i = 0; i < dd->num_pports; i++, ppd++) {
12143 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12144 if (!ppd->cntrs)
12145 goto bail;
12146
12147 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12148 if (!ppd->scntrs)
12149 goto bail;
12150 }
12151
12152 /* CPU counters need to be allocated and zeroed */
12153 if (init_cpu_counters(dd))
12154 goto bail;
12155
12156 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12157 return 0;
12158bail:
12159 free_cntrs(dd);
12160 return -ENOMEM;
12161}
12162
77241056
MM		 12163static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12164{
12165 switch (chip_lstate) {
12166 default:
12167 dd_dev_err(dd,
17fb4f29
JJ		 12168 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12169 chip_lstate);
77241056
MM		 12170 /* fall through */
12171 case LSTATE_DOWN:
12172 return IB_PORT_DOWN;
12173 case LSTATE_INIT:
12174 return IB_PORT_INIT;
12175 case LSTATE_ARMED:
12176 return IB_PORT_ARMED;
12177 case LSTATE_ACTIVE:
12178 return IB_PORT_ACTIVE;
12179 }
12180}
12181
12182u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12183{
12184 /* look at the HFI meta-states only */
12185 switch (chip_pstate & 0xf0) {
12186 default:
12187 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
17fb4f29 12188 chip_pstate);
77241056
MM		 12189 /* fall through */
12190 case PLS_DISABLED:
12191 return IB_PORTPHYSSTATE_DISABLED;
12192 case PLS_OFFLINE:
12193 return OPA_PORTPHYSSTATE_OFFLINE;
12194 case PLS_POLLING:
12195 return IB_PORTPHYSSTATE_POLLING;
12196 case PLS_CONFIGPHY:
12197 return IB_PORTPHYSSTATE_TRAINING;
12198 case PLS_LINKUP:
12199 return IB_PORTPHYSSTATE_LINKUP;
12200 case PLS_PHYTEST:
12201 return IB_PORTPHYSSTATE_PHY_TEST;
12202 }
12203}
12204
12205/* return the OPA port logical state name */
12206const char *opa_lstate_name(u32 lstate)
12207{
12208 static const char * const port_logical_names[] = {
12209 "PORT_NOP",
12210 "PORT_DOWN",
12211 "PORT_INIT",
12212 "PORT_ARMED",
12213 "PORT_ACTIVE",
12214 "PORT_ACTIVE_DEFER",
12215 };
12216 if (lstate < ARRAY_SIZE(port_logical_names))
12217 return port_logical_names[lstate];
12218 return "unknown";
12219}
12220
12221/* return the OPA port physical state name */
12222const char *opa_pstate_name(u32 pstate)
12223{
12224 static const char * const port_physical_names[] = {
12225 "PHYS_NOP",
12226 "reserved1",
12227 "PHYS_POLL",
12228 "PHYS_DISABLED",
12229 "PHYS_TRAINING",
12230 "PHYS_LINKUP",
12231 "PHYS_LINK_ERR_RECOVER",
12232 "PHYS_PHY_TEST",
12233 "reserved8",
12234 "PHYS_OFFLINE",
12235 "PHYS_GANGED",
12236 "PHYS_TEST",
12237 };
12238 if (pstate < ARRAY_SIZE(port_physical_names))
12239 return port_physical_names[pstate];
12240 return "unknown";
12241}
12242
12243/*
12244 * Read the hardware link state and set the driver's cached value of it.
12245 * Return the (new) current value.
12246 */
12247u32 get_logical_state(struct hfi1_pportdata *ppd)
12248{
12249 u32 new_state;
12250
12251 new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
12252 if (new_state != ppd->lstate) {
12253 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
17fb4f29 12254 opa_lstate_name(new_state), new_state);
77241056
MM		 12255 ppd->lstate = new_state;
12256 }
12257 /*
12258 * Set port status flags in the page mapped into userspace
12259 * memory. Do it here to ensure a reliable state - this is
12260 * the only function called by all state handling code.
12261 * Always set the flags due to the fact that the cache value
12262 * might have been changed explicitly outside of this
12263 * function.
12264 */
12265 if (ppd->statusp) {
12266 switch (ppd->lstate) {
12267 case IB_PORT_DOWN:
12268 case IB_PORT_INIT:
12269 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12270 HFI1_STATUS_IB_READY);
12271 break;
12272 case IB_PORT_ARMED:
12273 *ppd->statusp |= HFI1_STATUS_IB_CONF;
12274 break;
12275 case IB_PORT_ACTIVE:
12276 *ppd->statusp |= HFI1_STATUS_IB_READY;
12277 break;
12278 }
12279 }
12280 return ppd->lstate;
12281}
12282
12283/**
12284 * wait_logical_linkstate - wait for an IB link state change to occur
12285 * @ppd: port device
12286 * @state: the state to wait for
12287 * @msecs: the number of milliseconds to wait
12288 *
12289 * Wait up to msecs milliseconds for IB link state change to occur.
12290 * For now, take the easy polling route.
12291 * Returns 0 if state reached, otherwise -ETIMEDOUT.
12292 */
12293static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12294 int msecs)
12295{
12296 unsigned long timeout;
12297
12298 timeout = jiffies + msecs_to_jiffies(msecs);
12299 while (1) {
12300 if (get_logical_state(ppd) == state)
12301 return 0;
12302 if (time_after(jiffies, timeout))
12303 break;
12304 msleep(20);
12305 }
12306 dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
12307
12308 return -ETIMEDOUT;
12309}
12310
12311u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
12312{
77241056
MM		 12313 u32 pstate;
12314 u32 ib_pstate;
12315
12316 pstate = read_physical_state(ppd->dd);
12317 ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
f45c8dc8 12318 if (ppd->last_pstate != ib_pstate) {
77241056 12319 dd_dev_info(ppd->dd,
17fb4f29
JJ		 12320 "%s: physical state changed to %s (0x%x), phy 0x%x\n",
12321 __func__, opa_pstate_name(ib_pstate), ib_pstate,
12322 pstate);
f45c8dc8 12323 ppd->last_pstate = ib_pstate;
77241056
MM		 12324 }
12325 return ib_pstate;
12326}
12327
12328/*
12329 * Read/modify/write ASIC_QSFP register bits as selected by mask
12330 * data: 0 or 1 in the positions depending on what needs to be written
12331 * dir: 0 for read, 1 for write
12332 * mask: select by setting
12333 * I2CCLK (bit 0)
12334 * I2CDATA (bit 1)
12335 */
12336u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
12337 u32 mask)
12338{
12339 u64 qsfp_oe, target_oe;
12340
12341 target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
12342 if (mask) {
12343 /* We are writing register bits, so lock access */
12344 dir &= mask;
12345 data &= mask;
12346
12347 qsfp_oe = read_csr(dd, target_oe);
12348 qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir;
12349 write_csr(dd, target_oe, qsfp_oe);
12350 }
12351 /* We are exclusively reading bits here, but it is unlikely
12352 * we'll get valid data when we set the direction of the pin
12353 * in the same call, so read should call this function again
12354 * to get valid data
12355 */
12356 return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
12357}
12358
12359#define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12360(r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12361
12362#define SET_STATIC_RATE_CONTROL_SMASK(r) \
12363(r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12364
12365int hfi1_init_ctxt(struct send_context *sc)
12366{
d125a6c6 12367 if (sc) {
77241056
MM		 12368 struct hfi1_devdata *dd = sc->dd;
12369 u64 reg;
12370 u8 set = (sc->type == SC_USER ?
12371 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
12372 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
12373 reg = read_kctxt_csr(dd, sc->hw_context,
12374 SEND_CTXT_CHECK_ENABLE);
12375 if (set)
12376 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
12377 else
12378 SET_STATIC_RATE_CONTROL_SMASK(reg);
12379 write_kctxt_csr(dd, sc->hw_context,
12380 SEND_CTXT_CHECK_ENABLE, reg);
12381 }
12382 return 0;
12383}
12384
12385int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
12386{
12387 int ret = 0;
12388 u64 reg;
12389
12390 if (dd->icode != ICODE_RTL_SILICON) {
12391 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
12392 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
12393 __func__);
12394 return -EINVAL;
12395 }
12396 reg = read_csr(dd, ASIC_STS_THERM);
12397 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
12398 ASIC_STS_THERM_CURR_TEMP_MASK);
12399 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
12400 ASIC_STS_THERM_LO_TEMP_MASK);
12401 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
12402 ASIC_STS_THERM_HI_TEMP_MASK);
12403 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
12404 ASIC_STS_THERM_CRIT_TEMP_MASK);
12405 /* triggers is a 3-bit value - 1 bit per trigger. */
12406 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
12407
12408 return ret;
12409}
12410
12411/* ========================================================================= */
12412
12413/*
12414 * Enable/disable chip from delivering interrupts.
12415 */
12416void set_intr_state(struct hfi1_devdata *dd, u32 enable)
12417{
12418 int i;
12419
12420 /*
12421 * In HFI, the mask needs to be 1 to allow interrupts.
12422 */
12423 if (enable) {
77241056
MM		 12424 /* enable all interrupts */
12425 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 12426 write_csr(dd, CCE_INT_MASK + (8 * i), ~(u64)0);
77241056 12427
8ebd4cf1 12428 init_qsfp_int(dd);
77241056
MM		 12429 } else {
12430 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 12431 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
77241056
MM		 12432 }
12433}
12434
12435/*
12436 * Clear all interrupt sources on the chip.
12437 */
12438static void clear_all_interrupts(struct hfi1_devdata *dd)
12439{
12440 int i;
12441
12442 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 12443 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
77241056
MM		 12444
12445 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
12446 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
12447 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
12448 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
12449 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
12450 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
12451 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
12452 for (i = 0; i < dd->chip_send_contexts; i++)
12453 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
12454 for (i = 0; i < dd->chip_sdma_engines; i++)
12455 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
12456
12457 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
12458 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
12459 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
12460}
12461
12462/* Move to pcie.c? */
12463static void disable_intx(struct pci_dev *pdev)
12464{
12465 pci_intx(pdev, 0);
12466}
12467
12468static void clean_up_interrupts(struct hfi1_devdata *dd)
12469{
12470 int i;
12471
12472 /* remove irqs - must happen before disabling/turning off */
12473 if (dd->num_msix_entries) {
12474 /* MSI-X */
12475 struct hfi1_msix_entry *me = dd->msix_entries;
12476
12477 for (i = 0; i < dd->num_msix_entries; i++, me++) {
d125a6c6 12478 if (!me->arg) /* => no irq, no affinity */
957558c9
MH		 12479 continue;
12480 hfi1_put_irq_affinity(dd, &dd->msix_entries[i]);
77241056
MM		 12481 free_irq(me->msix.vector, me->arg);
12482 }
12483 } else {
12484 /* INTx */
12485 if (dd->requested_intx_irq) {
12486 free_irq(dd->pcidev->irq, dd);
12487 dd->requested_intx_irq = 0;
12488 }
12489 }
12490
12491 /* turn off interrupts */
12492 if (dd->num_msix_entries) {
12493 /* MSI-X */
6e5b6131 12494 pci_disable_msix(dd->pcidev);
77241056
MM		 12495 } else {
12496 /* INTx */
12497 disable_intx(dd->pcidev);
12498 }
12499
12500 /* clean structures */
77241056
MM		 12501 kfree(dd->msix_entries);
12502 dd->msix_entries = NULL;
12503 dd->num_msix_entries = 0;
12504}
12505
12506/*
12507 * Remap the interrupt source from the general handler to the given MSI-X
12508 * interrupt.
12509 */
12510static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
12511{
12512 u64 reg;
12513 int m, n;
12514
12515 /* clear from the handled mask of the general interrupt */
12516 m = isrc / 64;
12517 n = isrc % 64;
12518 dd->gi_mask[m] &= ~((u64)1 << n);
12519
12520 /* direct the chip source to the given MSI-X interrupt */
12521 m = isrc / 8;
12522 n = isrc % 8;
8638b77f
JJ		 12523 reg = read_csr(dd, CCE_INT_MAP + (8 * m));
12524 reg &= ~((u64)0xff << (8 * n));
12525 reg |= ((u64)msix_intr & 0xff) << (8 * n);
12526 write_csr(dd, CCE_INT_MAP + (8 * m), reg);
77241056
MM		 12527}
12528
12529static void remap_sdma_interrupts(struct hfi1_devdata *dd,
12530 int engine, int msix_intr)
12531{
12532 /*
12533 * SDMA engine interrupt sources grouped by type, rather than
12534 * engine. Per-engine interrupts are as follows:
12535 * SDMA
12536 * SDMAProgress
12537 * SDMAIdle
12538 */
8638b77f 12539 remap_intr(dd, IS_SDMA_START + 0 * TXE_NUM_SDMA_ENGINES + engine,
17fb4f29 12540 msix_intr);
8638b77f 12541 remap_intr(dd, IS_SDMA_START + 1 * TXE_NUM_SDMA_ENGINES + engine,
17fb4f29 12542 msix_intr);
8638b77f 12543 remap_intr(dd, IS_SDMA_START + 2 * TXE_NUM_SDMA_ENGINES + engine,
17fb4f29 12544 msix_intr);
77241056
MM		 12545}
12546
77241056
MM		 12547static int request_intx_irq(struct hfi1_devdata *dd)
12548{
12549 int ret;
12550
9805071e
JJ		 12551 snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d",
12552 dd->unit);
77241056 12553 ret = request_irq(dd->pcidev->irq, general_interrupt,
17fb4f29 12554 IRQF_SHARED, dd->intx_name, dd);
77241056
MM		 12555 if (ret)
12556 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n",
17fb4f29 12557 ret);
77241056
MM		 12558 else
12559 dd->requested_intx_irq = 1;
12560 return ret;
12561}
12562
12563static int request_msix_irqs(struct hfi1_devdata *dd)
12564{
77241056
MM		 12565 int first_general, last_general;
12566 int first_sdma, last_sdma;
12567 int first_rx, last_rx;
957558c9 12568 int i, ret = 0;
77241056
MM		 12569
12570 /* calculate the ranges we are going to use */
12571 first_general = 0;
f3ff8189
JJ		 12572 last_general = first_general + 1;
12573 first_sdma = last_general;
12574 last_sdma = first_sdma + dd->num_sdma;
12575 first_rx = last_sdma;
77241056
MM		 12576 last_rx = first_rx + dd->n_krcv_queues;
12577
77241056
MM		 12578 /*
12579 * Sanity check - the code expects all SDMA chip source
12580 * interrupts to be in the same CSR, starting at bit 0. Verify
12581 * that this is true by checking the bit location of the start.
12582 */
12583 BUILD_BUG_ON(IS_SDMA_START % 64);
12584
12585 for (i = 0; i < dd->num_msix_entries; i++) {
12586 struct hfi1_msix_entry *me = &dd->msix_entries[i];
12587 const char *err_info;
12588 irq_handler_t handler;
f4f30031 12589 irq_handler_t thread = NULL;
77241056
MM		 12590 void *arg;
12591 int idx;
12592 struct hfi1_ctxtdata *rcd = NULL;
12593 struct sdma_engine *sde = NULL;
12594
12595 /* obtain the arguments to request_irq */
12596 if (first_general <= i && i < last_general) {
12597 idx = i - first_general;
12598 handler = general_interrupt;
12599 arg = dd;
12600 snprintf(me->name, sizeof(me->name),
9805071e 12601 DRIVER_NAME "_%d", dd->unit);
77241056 12602 err_info = "general";
957558c9 12603 me->type = IRQ_GENERAL;
77241056
MM		 12604 } else if (first_sdma <= i && i < last_sdma) {
12605 idx = i - first_sdma;
12606 sde = &dd->per_sdma[idx];
12607 handler = sdma_interrupt;
12608 arg = sde;
12609 snprintf(me->name, sizeof(me->name),
9805071e 12610 DRIVER_NAME "_%d sdma%d", dd->unit, idx);
77241056
MM		 12611 err_info = "sdma";
12612 remap_sdma_interrupts(dd, idx, i);
957558c9 12613 me->type = IRQ_SDMA;
77241056
MM		 12614 } else if (first_rx <= i && i < last_rx) {
12615 idx = i - first_rx;
12616 rcd = dd->rcd[idx];
12617 /* no interrupt if no rcd */
12618 if (!rcd)
12619 continue;
12620 /*
12621 * Set the interrupt register and mask for this
12622 * context's interrupt.
12623 */
8638b77f 12624 rcd->ireg = (IS_RCVAVAIL_START + idx) / 64;
77241056 12625 rcd->imask = ((u64)1) <<
8638b77f 12626 ((IS_RCVAVAIL_START + idx) % 64);
77241056 12627 handler = receive_context_interrupt;
f4f30031 12628 thread = receive_context_thread;
77241056
MM		 12629 arg = rcd;
12630 snprintf(me->name, sizeof(me->name),
9805071e 12631 DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
77241056 12632 err_info = "receive context";
66c0933b 12633 remap_intr(dd, IS_RCVAVAIL_START + idx, i);
957558c9 12634 me->type = IRQ_RCVCTXT;
77241056
MM		 12635 } else {
12636 /* not in our expected range - complain, then
4d114fdd
JJ		 12637 * ignore it
12638 */
77241056 12639 dd_dev_err(dd,
17fb4f29 12640 "Unexpected extra MSI-X interrupt %d\n", i);
77241056
MM		 12641 continue;
12642 }
12643 /* no argument, no interrupt */
d125a6c6 12644 if (!arg)
77241056
MM		 12645 continue;
12646 /* make sure the name is terminated */
8638b77f 12647 me->name[sizeof(me->name) - 1] = 0;
77241056 12648
f4f30031 12649 ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
17fb4f29 12650 me->name, arg);
77241056
MM		 12651 if (ret) {
12652 dd_dev_err(dd,
17fb4f29
JJ		 12653 "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
12654 err_info, me->msix.vector, idx, ret);
77241056
MM		 12655 return ret;
12656 }
12657 /*
12658 * assign arg after request_irq call, so it will be
12659 * cleaned up
12660 */
12661 me->arg = arg;
12662
957558c9
MH		 12663 ret = hfi1_get_irq_affinity(dd, me);
12664 if (ret)
12665 dd_dev_err(dd,
12666 "unable to pin IRQ %d\n", ret);
77241056
MM		 12667 }
12668
77241056 12669 return ret;
77241056
MM		 12670}
12671
12672/*
12673 * Set the general handler to accept all interrupts, remap all
12674 * chip interrupts back to MSI-X 0.
12675 */
12676static void reset_interrupts(struct hfi1_devdata *dd)
12677{
12678 int i;
12679
12680 /* all interrupts handled by the general handler */
12681 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12682 dd->gi_mask[i] = ~(u64)0;
12683
12684 /* all chip interrupts map to MSI-X 0 */
12685 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
8638b77f 12686 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
77241056
MM		 12687}
12688
12689static int set_up_interrupts(struct hfi1_devdata *dd)
12690{
12691 struct hfi1_msix_entry *entries;
12692 u32 total, request;
12693 int i, ret;
12694 int single_interrupt = 0; /* we expect to have all the interrupts */
12695
12696 /*
12697 * Interrupt count:
12698 * 1 general, "slow path" interrupt (includes the SDMA engines
12699 * slow source, SDMACleanupDone)
12700 * N interrupts - one per used SDMA engine
12701 * M interrupt - one per kernel receive context
12702 */
12703 total = 1 + dd->num_sdma + dd->n_krcv_queues;
12704
12705 entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
12706 if (!entries) {
77241056
MM		 12707 ret = -ENOMEM;
12708 goto fail;
12709 }
12710 /* 1-1 MSI-X entry assignment */
12711 for (i = 0; i < total; i++)
12712 entries[i].msix.entry = i;
12713
12714 /* ask for MSI-X interrupts */
12715 request = total;
12716 request_msix(dd, &request, entries);
12717
12718 if (request == 0) {
12719 /* using INTx */
12720 /* dd->num_msix_entries already zero */
12721 kfree(entries);
12722 single_interrupt = 1;
12723 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
12724 } else {
12725 /* using MSI-X */
12726 dd->num_msix_entries = request;
12727 dd->msix_entries = entries;
12728
12729 if (request != total) {
12730 /* using MSI-X, with reduced interrupts */
12731 dd_dev_err(
12732 dd,
12733 "cannot handle reduced interrupt case, want %u, got %u\n",
12734 total, request);
12735 ret = -EINVAL;
12736 goto fail;
12737 }
12738 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
12739 }
12740
12741 /* mask all interrupts */
12742 set_intr_state(dd, 0);
12743 /* clear all pending interrupts */
12744 clear_all_interrupts(dd);
12745
12746 /* reset general handler mask, chip MSI-X mappings */
12747 reset_interrupts(dd);
12748
12749 if (single_interrupt)
12750 ret = request_intx_irq(dd);
12751 else
12752 ret = request_msix_irqs(dd);
12753 if (ret)
12754 goto fail;
12755
12756 return 0;
12757
12758fail:
12759 clean_up_interrupts(dd);
12760 return ret;
12761}
12762
12763/*
12764 * Set up context values in dd. Sets:
12765 *
12766 * num_rcv_contexts - number of contexts being used
12767 * n_krcv_queues - number of kernel contexts
12768 * first_user_ctxt - first non-kernel context in array of contexts
12769 * freectxts - number of free user contexts
12770 * num_send_contexts - number of PIO send contexts being used
12771 */
12772static int set_up_context_variables(struct hfi1_devdata *dd)
12773{
12774 int num_kernel_contexts;
77241056
MM		 12775 int total_contexts;
12776 int ret;
12777 unsigned ngroups;
8f000f7f
DL		 12778 int qos_rmt_count;
12779 int user_rmt_reduced;
77241056
MM		 12780
12781 /*
33a9eb52
DL		 12782 * Kernel receive contexts:
12783 * - min of 2 or 1 context/numa (excluding control context)
82c2611d 12784 * - Context 0 - control context (VL15/multicast/error)
33a9eb52
DL		 12785 * - Context 1 - first kernel context
12786 * - Context 2 - second kernel context
12787 * ...
77241056
MM		 12788 */
12789 if (n_krcvqs)
82c2611d 12790 /*
33a9eb52
DL		 12791 * n_krcvqs is the sum of module parameter kernel receive
12792 * contexts, krcvqs[]. It does not include the control
12793 * context, so add that.
82c2611d 12794 */
33a9eb52 12795 num_kernel_contexts = n_krcvqs + 1;
77241056 12796 else
0edf80ea 12797 num_kernel_contexts = num_online_nodes() + 1;
77241056
MM		 12798 num_kernel_contexts =
12799 max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
12800 /*
12801 * Every kernel receive context needs an ACK send context.
12802 * one send context is allocated for each VL{0-7} and VL15
12803 */
12804 if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
12805 dd_dev_err(dd,
12806 "Reducing # kernel rcv contexts to: %d, from %d\n",
12807 (int)(dd->chip_send_contexts - num_vls - 1),
12808 (int)num_kernel_contexts);
12809 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
12810 }
12811 /*
0852d241
JJ		 12812 * User contexts:
12813 * - default to 1 user context per real (non-HT) CPU core if
12814 * num_user_contexts is negative
77241056 12815 */
2ce6bf22 12816 if (num_user_contexts < 0)
0852d241
JJ		 12817 num_user_contexts =
12818 cpumask_weight(&dd->affinity->real_cpu_mask);
77241056
MM		 12819
12820 total_contexts = num_kernel_contexts + num_user_contexts;
12821
12822 /*
12823 * Adjust the counts given a global max.
12824 */
12825 if (total_contexts > dd->chip_rcv_contexts) {
12826 dd_dev_err(dd,
12827 "Reducing # user receive contexts to: %d, from %d\n",
12828 (int)(dd->chip_rcv_contexts - num_kernel_contexts),
12829 (int)num_user_contexts);
12830 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts;
12831 /* recalculate */
12832 total_contexts = num_kernel_contexts + num_user_contexts;
12833 }
12834
8f000f7f
DL		 12835 /* each user context requires an entry in the RMT */
12836 qos_rmt_count = qos_rmt_entries(dd, NULL, NULL);
12837 if (qos_rmt_count + num_user_contexts > NUM_MAP_ENTRIES) {
12838 user_rmt_reduced = NUM_MAP_ENTRIES - qos_rmt_count;
12839 dd_dev_err(dd,
12840 "RMT size is reducing the number of user receive contexts from %d to %d\n",
12841 (int)num_user_contexts,
12842 user_rmt_reduced);
12843 /* recalculate */
12844 num_user_contexts = user_rmt_reduced;
12845 total_contexts = num_kernel_contexts + num_user_contexts;
12846 }
12847
77241056
MM		 12848 /* the first N are kernel contexts, the rest are user contexts */
12849 dd->num_rcv_contexts = total_contexts;
12850 dd->n_krcv_queues = num_kernel_contexts;
12851 dd->first_user_ctxt = num_kernel_contexts;
affa48de 12852 dd->num_user_contexts = num_user_contexts;
77241056
MM		 12853 dd->freectxts = num_user_contexts;
12854 dd_dev_info(dd,
17fb4f29
JJ		 12855 "rcv contexts: chip %d, used %d (kernel %d, user %d)\n",
12856 (int)dd->chip_rcv_contexts,
12857 (int)dd->num_rcv_contexts,
12858 (int)dd->n_krcv_queues,
12859 (int)dd->num_rcv_contexts - dd->n_krcv_queues);
77241056
MM		 12860
12861 /*
12862 * Receive array allocation:
12863 * All RcvArray entries are divided into groups of 8. This
12864 * is required by the hardware and will speed up writes to
12865 * consecutive entries by using write-combining of the entire
12866 * cacheline.
12867 *
12868 * The number of groups are evenly divided among all contexts.
12869 * any left over groups will be given to the first N user
12870 * contexts.
12871 */
12872 dd->rcv_entries.group_size = RCV_INCREMENT;
12873 ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size;
12874 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
12875 dd->rcv_entries.nctxt_extra = ngroups -
12876 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
12877 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
12878 dd->rcv_entries.ngroups,
12879 dd->rcv_entries.nctxt_extra);
12880 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
12881 MAX_EAGER_ENTRIES * 2) {
12882 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
12883 dd->rcv_entries.group_size;
12884 dd_dev_info(dd,
17fb4f29
JJ		 12885 "RcvArray group count too high, change to %u\n",
12886 dd->rcv_entries.ngroups);
77241056
MM		 12887 dd->rcv_entries.nctxt_extra = 0;
12888 }
12889 /*
12890 * PIO send contexts
12891 */
12892 ret = init_sc_pools_and_sizes(dd);
12893 if (ret >= 0) { /* success */
12894 dd->num_send_contexts = ret;
12895 dd_dev_info(
12896 dd,
44306f15 12897 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
77241056
MM		 12898 dd->chip_send_contexts,
12899 dd->num_send_contexts,
12900 dd->sc_sizes[SC_KERNEL].count,
12901 dd->sc_sizes[SC_ACK].count,
44306f15
JX		 12902 dd->sc_sizes[SC_USER].count,
12903 dd->sc_sizes[SC_VL15].count);
77241056
MM		 12904 ret = 0; /* success */
12905 }
12906
12907 return ret;
12908}
12909
12910/*
12911 * Set the device/port partition key table. The MAD code
12912 * will ensure that, at least, the partial management
12913 * partition key is present in the table.
12914 */
12915static void set_partition_keys(struct hfi1_pportdata *ppd)
12916{
12917 struct hfi1_devdata *dd = ppd->dd;
12918 u64 reg = 0;
12919 int i;
12920
12921 dd_dev_info(dd, "Setting partition keys\n");
12922 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
12923 reg |= (ppd->pkeys[i] &
12924 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
12925 ((i % 4) *
12926 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
12927 /* Each register holds 4 PKey values. */
12928 if ((i % 4) == 3) {
12929 write_csr(dd, RCV_PARTITION_KEY +
12930 ((i - 3) * 2), reg);
12931 reg = 0;
12932 }
12933 }
12934
12935 /* Always enable HW pkeys check when pkeys table is set */
12936 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
12937}
12938
12939/*
12940 * These CSRs and memories are uninitialized on reset and must be
12941 * written before reading to set the ECC/parity bits.
12942 *
12943 * NOTE: All user context CSRs that are not mmaped write-only
12944 * (e.g. the TID flows) must be initialized even if the driver never
12945 * reads them.
12946 */
12947static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
12948{
12949 int i, j;
12950
12951 /* CceIntMap */
12952 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
8638b77f 12953 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
77241056
MM		 12954
12955 /* SendCtxtCreditReturnAddr */
12956 for (i = 0; i < dd->chip_send_contexts; i++)
12957 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
12958
12959 /* PIO Send buffers */
12960 /* SDMA Send buffers */
4d114fdd
JJ		 12961 /*
12962 * These are not normally read, and (presently) have no method
12963 * to be read, so are not pre-initialized
12964 */
77241056
MM		 12965
12966 /* RcvHdrAddr */
12967 /* RcvHdrTailAddr */
12968 /* RcvTidFlowTable */
12969 for (i = 0; i < dd->chip_rcv_contexts; i++) {
12970 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
12971 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
12972 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
8638b77f 12973 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
77241056
MM		 12974 }
12975
12976 /* RcvArray */
12977 for (i = 0; i < dd->chip_rcv_array_count; i++)
8638b77f 12978 write_csr(dd, RCV_ARRAY + (8 * i),
17fb4f29 12979 RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
77241056
MM		 12980
12981 /* RcvQPMapTable */
12982 for (i = 0; i < 32; i++)
12983 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
12984}
12985
12986/*
12987 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
12988 */
12989static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
12990 u64 ctrl_bits)
12991{
12992 unsigned long timeout;
12993 u64 reg;
12994
12995 /* is the condition present? */
12996 reg = read_csr(dd, CCE_STATUS);
12997 if ((reg & status_bits) == 0)
12998 return;
12999
13000 /* clear the condition */
13001 write_csr(dd, CCE_CTRL, ctrl_bits);
13002
13003 /* wait for the condition to clear */
13004 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
13005 while (1) {
13006 reg = read_csr(dd, CCE_STATUS);
13007 if ((reg & status_bits) == 0)
13008 return;
13009 if (time_after(jiffies, timeout)) {
13010 dd_dev_err(dd,
17fb4f29
JJ		 13011 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13012 status_bits, reg & status_bits);
77241056
MM		 13013 return;
13014 }
13015 udelay(1);
13016 }
13017}
13018
13019/* set CCE CSRs to chip reset defaults */
13020static void reset_cce_csrs(struct hfi1_devdata *dd)
13021{
13022 int i;
13023
13024 /* CCE_REVISION read-only */
13025 /* CCE_REVISION2 read-only */
13026 /* CCE_CTRL - bits clear automatically */
13027 /* CCE_STATUS read-only, use CceCtrl to clear */
13028 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13029 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13030 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13031 for (i = 0; i < CCE_NUM_SCRATCH; i++)
13032 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13033 /* CCE_ERR_STATUS read-only */
13034 write_csr(dd, CCE_ERR_MASK, 0);
13035 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13036 /* CCE_ERR_FORCE leave alone */
13037 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13038 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13039 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13040 /* CCE_PCIE_CTRL leave alone */
13041 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13042 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13043 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
17fb4f29 13044 CCE_MSIX_TABLE_UPPER_RESETCSR);
77241056
MM		 13045 }
13046 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13047 /* CCE_MSIX_PBA read-only */
13048 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13049 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13050 }
13051 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13052 write_csr(dd, CCE_INT_MAP, 0);
13053 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13054 /* CCE_INT_STATUS read-only */
13055 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13056 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13057 /* CCE_INT_FORCE leave alone */
13058 /* CCE_INT_BLOCKED read-only */
13059 }
13060 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13061 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13062}
13063
77241056
MM		 13064/* set MISC CSRs to chip reset defaults */
13065static void reset_misc_csrs(struct hfi1_devdata *dd)
13066{
13067 int i;
13068
13069 for (i = 0; i < 32; i++) {
13070 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13071 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13072 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13073 }
4d114fdd
JJ		 13074 /*
13075 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13076 * only be written 128-byte chunks
13077 */
77241056
MM		 13078 /* init RSA engine to clear lingering errors */
13079 write_csr(dd, MISC_CFG_RSA_CMD, 1);
13080 write_csr(dd, MISC_CFG_RSA_MU, 0);
13081 write_csr(dd, MISC_CFG_FW_CTRL, 0);
13082 /* MISC_STS_8051_DIGEST read-only */
13083 /* MISC_STS_SBM_DIGEST read-only */
13084 /* MISC_STS_PCIE_DIGEST read-only */
13085 /* MISC_STS_FAB_DIGEST read-only */
13086 /* MISC_ERR_STATUS read-only */
13087 write_csr(dd, MISC_ERR_MASK, 0);
13088 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13089 /* MISC_ERR_FORCE leave alone */
13090}
13091
13092/* set TXE CSRs to chip reset defaults */
13093static void reset_txe_csrs(struct hfi1_devdata *dd)
13094{
13095 int i;
13096
13097 /*
13098 * TXE Kernel CSRs
13099 */
13100 write_csr(dd, SEND_CTRL, 0);
13101 __cm_reset(dd, 0); /* reset CM internal state */
13102 /* SEND_CONTEXTS read-only */
13103 /* SEND_DMA_ENGINES read-only */
13104 /* SEND_PIO_MEM_SIZE read-only */
13105 /* SEND_DMA_MEM_SIZE read-only */
13106 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13107 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */
13108 /* SEND_PIO_ERR_STATUS read-only */
13109 write_csr(dd, SEND_PIO_ERR_MASK, 0);
13110 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13111 /* SEND_PIO_ERR_FORCE leave alone */
13112 /* SEND_DMA_ERR_STATUS read-only */
13113 write_csr(dd, SEND_DMA_ERR_MASK, 0);
13114 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13115 /* SEND_DMA_ERR_FORCE leave alone */
13116 /* SEND_EGRESS_ERR_STATUS read-only */
13117 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13118 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13119 /* SEND_EGRESS_ERR_FORCE leave alone */
13120 write_csr(dd, SEND_BTH_QP, 0);
13121 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13122 write_csr(dd, SEND_SC2VLT0, 0);
13123 write_csr(dd, SEND_SC2VLT1, 0);
13124 write_csr(dd, SEND_SC2VLT2, 0);
13125 write_csr(dd, SEND_SC2VLT3, 0);
13126 write_csr(dd, SEND_LEN_CHECK0, 0);
13127 write_csr(dd, SEND_LEN_CHECK1, 0);
13128 /* SEND_ERR_STATUS read-only */
13129 write_csr(dd, SEND_ERR_MASK, 0);
13130 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13131 /* SEND_ERR_FORCE read-only */
13132 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
8638b77f 13133 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
77241056 13134 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
8638b77f
JJ		 13135 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13136 for (i = 0; i < dd->chip_send_contexts / NUM_CONTEXTS_PER_SET; i++)
13137 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
77241056 13138 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
8638b77f 13139 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
77241056 13140 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
8638b77f 13141 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
77241056 13142 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
17fb4f29 13143 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
77241056
MM		 13144 /* SEND_CM_CREDIT_USED_STATUS read-only */
13145 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13146 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13147 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13148 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13149 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13150 for (i = 0; i < TXE_NUM_DATA_VL; i++)
8638b77f 13151 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
77241056
MM		 13152 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13153 /* SEND_CM_CREDIT_USED_VL read-only */
13154 /* SEND_CM_CREDIT_USED_VL15 read-only */
13155 /* SEND_EGRESS_CTXT_STATUS read-only */
13156 /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13157 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13158 /* SEND_EGRESS_ERR_INFO read-only */
13159 /* SEND_EGRESS_ERR_SOURCE read-only */
13160
13161 /*
13162 * TXE Per-Context CSRs
13163 */
13164 for (i = 0; i < dd->chip_send_contexts; i++) {
13165 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13166 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13167 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13168 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13169 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13170 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13171 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13172 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13173 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13174 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13175 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13176 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13177 }
13178
13179 /*
13180 * TXE Per-SDMA CSRs
13181 */
13182 for (i = 0; i < dd->chip_sdma_engines; i++) {
13183 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13184 /* SEND_DMA_STATUS read-only */
13185 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13186 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13187 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13188 /* SEND_DMA_HEAD read-only */
13189 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13190 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13191 /* SEND_DMA_IDLE_CNT read-only */
13192 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13193 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13194 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13195 /* SEND_DMA_ENG_ERR_STATUS read-only */
13196 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13197 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13198 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13199 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13200 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13201 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13202 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13203 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13204 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13205 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13206 }
13207}
13208
13209/*
13210 * Expect on entry:
13211 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13212 */
13213static void init_rbufs(struct hfi1_devdata *dd)
13214{
13215 u64 reg;
13216 int count;
13217
13218 /*
13219 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13220 * clear.
13221 */
13222 count = 0;
13223 while (1) {
13224 reg = read_csr(dd, RCV_STATUS);
13225 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13226 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13227 break;
13228 /*
13229 * Give up after 1ms - maximum wait time.
13230 *
13231 * RBuf size is 148KiB. Slowest possible is PCIe Gen1 x1 at
13232 * 250MB/s bandwidth. Lower rate to 66% for overhead to get:
13233 * 148 KB / (66% * 250MB/s) = 920us
13234 */
13235 if (count++ > 500) {
13236 dd_dev_err(dd,
17fb4f29
JJ		 13237 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13238 __func__, reg);
77241056
MM		 13239 break;
13240 }
13241 udelay(2); /* do not busy-wait the CSR */
13242 }
13243
13244 /* start the init - expect RcvCtrl to be 0 */
13245 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13246
13247 /*
13248 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief
13249 * period after the write before RcvStatus.RxRbufInitDone is valid.
13250 * The delay in the first run through the loop below is sufficient and
13251 * required before the first read of RcvStatus.RxRbufInintDone.
13252 */
13253 read_csr(dd, RCV_CTRL);
13254
13255 /* wait for the init to finish */
13256 count = 0;
13257 while (1) {
13258 /* delay is required first time through - see above */
13259 udelay(2); /* do not busy-wait the CSR */
13260 reg = read_csr(dd, RCV_STATUS);
13261 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13262 break;
13263
13264 /* give up after 100us - slowest possible at 33MHz is 73us */
13265 if (count++ > 50) {
13266 dd_dev_err(dd,
17fb4f29
JJ		 13267 "%s: RcvStatus.RxRbufInit not set, continuing\n",
13268 __func__);
77241056
MM		 13269 break;
13270 }
13271 }
13272}
13273
13274/* set RXE CSRs to chip reset defaults */
13275static void reset_rxe_csrs(struct hfi1_devdata *dd)
13276{
13277 int i, j;
13278
13279 /*
13280 * RXE Kernel CSRs
13281 */
13282 write_csr(dd, RCV_CTRL, 0);
13283 init_rbufs(dd);
13284 /* RCV_STATUS read-only */
13285 /* RCV_CONTEXTS read-only */
13286 /* RCV_ARRAY_CNT read-only */
13287 /* RCV_BUF_SIZE read-only */
13288 write_csr(dd, RCV_BTH_QP, 0);
13289 write_csr(dd, RCV_MULTICAST, 0);
13290 write_csr(dd, RCV_BYPASS, 0);
13291 write_csr(dd, RCV_VL15, 0);
13292 /* this is a clear-down */
13293 write_csr(dd, RCV_ERR_INFO,
17fb4f29 13294 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
77241056
MM		 13295 /* RCV_ERR_STATUS read-only */
13296 write_csr(dd, RCV_ERR_MASK, 0);
13297 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13298 /* RCV_ERR_FORCE leave alone */
13299 for (i = 0; i < 32; i++)
13300 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13301 for (i = 0; i < 4; i++)
13302 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13303 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13304 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13305 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13306 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13307 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) {
13308 write_csr(dd, RCV_RSM_CFG + (8 * i), 0);
13309 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0);
13310 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0);
13311 }
13312 for (i = 0; i < 32; i++)
13313 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13314
13315 /*
13316 * RXE Kernel and User Per-Context CSRs
13317 */
13318 for (i = 0; i < dd->chip_rcv_contexts; i++) {
13319 /* kernel */
13320 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13321 /* RCV_CTXT_STATUS read-only */
13322 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13323 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13324 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13325 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13326 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13327 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13328 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13329 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13330 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13331 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13332
13333 /* user */
13334 /* RCV_HDR_TAIL read-only */
13335 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13336 /* RCV_EGR_INDEX_TAIL read-only */
13337 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13338 /* RCV_EGR_OFFSET_TAIL read-only */
13339 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
17fb4f29
JJ		 13340 write_uctxt_csr(dd, i,
13341 RCV_TID_FLOW_TABLE + (8 * j), 0);
77241056
MM		 13342 }
13343 }
13344}
13345
13346/*
13347 * Set sc2vl tables.
13348 *
13349 * They power on to zeros, so to avoid send context errors
13350 * they need to be set:
13351 *
13352 * SC 0-7 -> VL 0-7 (respectively)
13353 * SC 15 -> VL 15
13354 * otherwise
13355 * -> VL 0
13356 */
13357static void init_sc2vl_tables(struct hfi1_devdata *dd)
13358{
13359 int i;
13360 /* init per architecture spec, constrained by hardware capability */
13361
13362 /* HFI maps sent packets */
13363 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13364 0,
13365 0, 0, 1, 1,
13366 2, 2, 3, 3,
13367 4, 4, 5, 5,
13368 6, 6, 7, 7));
13369 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13370 1,
13371 8, 0, 9, 0,
13372 10, 0, 11, 0,
13373 12, 0, 13, 0,
13374 14, 0, 15, 15));
13375 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13376 2,
13377 16, 0, 17, 0,
13378 18, 0, 19, 0,
13379 20, 0, 21, 0,
13380 22, 0, 23, 0));
13381 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13382 3,
13383 24, 0, 25, 0,
13384 26, 0, 27, 0,
13385 28, 0, 29, 0,
13386 30, 0, 31, 0));
13387
13388 /* DC maps received packets */
13389 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13390 15_0,
13391 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
13392 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13393 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13394 31_16,
13395 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13396 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13397
13398 /* initialize the cached sc2vl values consistently with h/w */
13399 for (i = 0; i < 32; i++) {
13400 if (i < 8 || i == 15)
13401 *((u8 *)(dd->sc2vl) + i) = (u8)i;
13402 else
13403 *((u8 *)(dd->sc2vl) + i) = 0;
13404 }
13405}
13406
13407/*
13408 * Read chip sizes and then reset parts to sane, disabled, values. We cannot
13409 * depend on the chip going through a power-on reset - a driver may be loaded
13410 * and unloaded many times.
13411 *
13412 * Do not write any CSR values to the chip in this routine - there may be
13413 * a reset following the (possible) FLR in this routine.
13414 *
13415 */
13416static void init_chip(struct hfi1_devdata *dd)
13417{
13418 int i;
13419
13420 /*
13421 * Put the HFI CSRs in a known state.
13422 * Combine this with a DC reset.
13423 *
13424 * Stop the device from doing anything while we do a
13425 * reset. We know there are no other active users of
13426 * the device since we are now in charge. Turn off
13427 * off all outbound and inbound traffic and make sure
13428 * the device does not generate any interrupts.
13429 */
13430
13431 /* disable send contexts and SDMA engines */
13432 write_csr(dd, SEND_CTRL, 0);
13433 for (i = 0; i < dd->chip_send_contexts; i++)
13434 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13435 for (i = 0; i < dd->chip_sdma_engines; i++)
13436 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13437 /* disable port (turn off RXE inbound traffic) and contexts */
13438 write_csr(dd, RCV_CTRL, 0);
13439 for (i = 0; i < dd->chip_rcv_contexts; i++)
13440 write_csr(dd, RCV_CTXT_CTRL, 0);
13441 /* mask all interrupt sources */
13442 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
8638b77f 13443 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
77241056
MM		 13444
13445 /*
13446 * DC Reset: do a full DC reset before the register clear.
13447 * A recommended length of time to hold is one CSR read,
13448 * so reread the CceDcCtrl. Then, hold the DC in reset
13449 * across the clear.
13450 */
13451 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
50e5dcbe 13452 (void)read_csr(dd, CCE_DC_CTRL);
77241056
MM		 13453
13454 if (use_flr) {
13455 /*
13456 * A FLR will reset the SPC core and part of the PCIe.
13457 * The parts that need to be restored have already been
13458 * saved.
13459 */
13460 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13461
13462 /* do the FLR, the DC reset will remain */
13463 hfi1_pcie_flr(dd);
13464
13465 /* restore command and BARs */
13466 restore_pci_variables(dd);
13467
995deafa 13468 if (is_ax(dd)) {
77241056
MM		 13469 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13470 hfi1_pcie_flr(dd);
13471 restore_pci_variables(dd);
13472 }
77241056
MM		 13473 } else {
13474 dd_dev_info(dd, "Resetting CSRs with writes\n");
13475 reset_cce_csrs(dd);
13476 reset_txe_csrs(dd);
13477 reset_rxe_csrs(dd);
77241056
MM		 13478 reset_misc_csrs(dd);
13479 }
13480 /* clear the DC reset */
13481 write_csr(dd, CCE_DC_CTRL, 0);
7c03ed85 13482
77241056 13483 /* Set the LED off */
773d0451
SS		 13484 setextled(dd, 0);
13485
77241056
MM		 13486 /*
13487 * Clear the QSFP reset.
72a67ba2 13488 * An FLR enforces a 0 on all out pins. The driver does not touch
77241056 13489 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and
72a67ba2 13490 * anything plugged constantly in reset, if it pays attention
77241056 13491 * to RESET_N.
72a67ba2 13492 * Prime examples of this are optical cables. Set all pins high.
77241056
MM		 13493 * I2CCLK and I2CDAT will change per direction, and INT_N and
13494 * MODPRS_N are input only and their value is ignored.
13495 */
72a67ba2
EH		 13496 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
13497 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
a2ee27a4 13498 init_chip_resources(dd);
77241056
MM		 13499}
13500
13501static void init_early_variables(struct hfi1_devdata *dd)
13502{
13503 int i;
13504
13505 /* assign link credit variables */
13506 dd->vau = CM_VAU;
13507 dd->link_credits = CM_GLOBAL_CREDITS;
995deafa 13508 if (is_ax(dd))
77241056
MM		 13509 dd->link_credits--;
13510 dd->vcu = cu_to_vcu(hfi1_cu);
13511 /* enough room for 8 MAD packets plus header - 17K */
13512 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
13513 if (dd->vl15_init > dd->link_credits)
13514 dd->vl15_init = dd->link_credits;
13515
13516 write_uninitialized_csrs_and_memories(dd);
13517
13518 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
13519 for (i = 0; i < dd->num_pports; i++) {
13520 struct hfi1_pportdata *ppd = &dd->pport[i];
13521
13522 set_partition_keys(ppd);
13523 }
13524 init_sc2vl_tables(dd);
13525}
13526
13527static void init_kdeth_qp(struct hfi1_devdata *dd)
13528{
13529 /* user changed the KDETH_QP */
13530 if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
13531 /* out of range or illegal value */
13532 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
13533 kdeth_qp = 0;
13534 }
13535 if (kdeth_qp == 0) /* not set, or failed range check */
13536 kdeth_qp = DEFAULT_KDETH_QP;
13537
13538 write_csr(dd, SEND_BTH_QP,
17fb4f29
JJ		 13539 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
13540 SEND_BTH_QP_KDETH_QP_SHIFT);
77241056
MM		 13541
13542 write_csr(dd, RCV_BTH_QP,
17fb4f29
JJ		 13543 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
13544 RCV_BTH_QP_KDETH_QP_SHIFT);
77241056
MM		 13545}
13546
13547/**
13548 * init_qpmap_table
13549 * @dd - device data
13550 * @first_ctxt - first context
13551 * @last_ctxt - first context
13552 *
13553 * This return sets the qpn mapping table that
13554 * is indexed by qpn[8:1].
13555 *
13556 * The routine will round robin the 256 settings
13557 * from first_ctxt to last_ctxt.
13558 *
13559 * The first/last looks ahead to having specialized
13560 * receive contexts for mgmt and bypass. Normal
13561 * verbs traffic will assumed to be on a range
13562 * of receive contexts.
13563 */
13564static void init_qpmap_table(struct hfi1_devdata *dd,
13565 u32 first_ctxt,
13566 u32 last_ctxt)
13567{
13568 u64 reg = 0;
13569 u64 regno = RCV_QP_MAP_TABLE;
13570 int i;
13571 u64 ctxt = first_ctxt;
13572
60d585ad 13573 for (i = 0; i < 256; i++) {
77241056 13574 reg |= ctxt << (8 * (i % 8));
77241056
MM		 13575 ctxt++;
13576 if (ctxt > last_ctxt)
13577 ctxt = first_ctxt;
60d585ad 13578 if (i % 8 == 7) {
77241056
MM		 13579 write_csr(dd, regno, reg);
13580 reg = 0;
13581 regno += 8;
13582 }
13583 }
77241056
MM		 13584
13585 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
13586 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
13587}
13588
372cc85a
DL		 13589struct rsm_map_table {
13590 u64 map[NUM_MAP_REGS];
13591 unsigned int used;
13592};
13593
b12349ae
DL		 13594struct rsm_rule_data {
13595 u8 offset;
13596 u8 pkt_type;
13597 u32 field1_off;
13598 u32 field2_off;
13599 u32 index1_off;
13600 u32 index1_width;
13601 u32 index2_off;
13602 u32 index2_width;
13603 u32 mask1;
13604 u32 value1;
13605 u32 mask2;
13606 u32 value2;
13607};
13608
372cc85a
DL		 13609/*
13610 * Return an initialized RMT map table for users to fill in. OK if it
13611 * returns NULL, indicating no table.
13612 */
13613static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
13614{
13615 struct rsm_map_table *rmt;
13616 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */
13617
13618 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
13619 if (rmt) {
13620 memset(rmt->map, rxcontext, sizeof(rmt->map));
13621 rmt->used = 0;
13622 }
13623
13624 return rmt;
13625}
13626
13627/*
13628 * Write the final RMT map table to the chip and free the table. OK if
13629 * table is NULL.
13630 */
13631static void complete_rsm_map_table(struct hfi1_devdata *dd,
13632 struct rsm_map_table *rmt)
13633{
13634 int i;
13635
13636 if (rmt) {
13637 /* write table to chip */
13638 for (i = 0; i < NUM_MAP_REGS; i++)
13639 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
13640
13641 /* enable RSM */
13642 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
13643 }
13644}
13645
b12349ae
DL		 13646/*
13647 * Add a receive side mapping rule.
13648 */
13649static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
13650 struct rsm_rule_data *rrd)
13651{
13652 write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
13653 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
13654 1ull << rule_index | /* enable bit */
13655 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
13656 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
13657 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
13658 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
13659 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
13660 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
13661 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
13662 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
13663 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
13664 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
13665 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
13666 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
13667 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
13668}
13669
4a818bed
DL		 13670/* return the number of RSM map table entries that will be used for QOS */
13671static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
13672 unsigned int *np)
13673{
13674 int i;
13675 unsigned int m, n;
13676 u8 max_by_vl = 0;
13677
13678 /* is QOS active at all? */
13679 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
13680 num_vls == 1 ||
13681 krcvqsset <= 1)
13682 goto no_qos;
13683
13684 /* determine bits for qpn */
13685 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
13686 if (krcvqs[i] > max_by_vl)
13687 max_by_vl = krcvqs[i];
13688 if (max_by_vl > 32)
13689 goto no_qos;
13690 m = ilog2(__roundup_pow_of_two(max_by_vl));
13691
13692 /* determine bits for vl */
13693 n = ilog2(__roundup_pow_of_two(num_vls));
13694
13695 /* reject if too much is used */
13696 if ((m + n) > 7)
13697 goto no_qos;
13698
13699 if (mp)
13700 *mp = m;
13701 if (np)
13702 *np = n;
13703
13704 return 1 << (m + n);
13705
13706no_qos:
13707 if (mp)
13708 *mp = 0;
13709 if (np)
13710 *np = 0;
13711 return 0;
13712}
13713
77241056
MM		 13714/**
13715 * init_qos - init RX qos
13716 * @dd - device data
372cc85a 13717 * @rmt - RSM map table
77241056 13718 *
33a9eb52
DL		 13719 * This routine initializes Rule 0 and the RSM map table to implement
13720 * quality of service (qos).
77241056 13721 *
33a9eb52
DL		 13722 * If all of the limit tests succeed, qos is applied based on the array
13723 * interpretation of krcvqs where entry 0 is VL0.
77241056 13724 *
33a9eb52
DL		 13725 * The number of vl bits (n) and the number of qpn bits (m) are computed to
13726 * feed both the RSM map table and the single rule.
77241056 13727 */
372cc85a 13728static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
77241056 13729{
b12349ae 13730 struct rsm_rule_data rrd;
77241056 13731 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
372cc85a 13732 unsigned int rmt_entries;
77241056 13733 u64 reg;
77241056 13734
4a818bed 13735 if (!rmt)
77241056 13736 goto bail;
4a818bed
DL		 13737 rmt_entries = qos_rmt_entries(dd, &m, &n);
13738 if (rmt_entries == 0)
77241056 13739 goto bail;
4a818bed
DL		 13740 qpns_per_vl = 1 << m;
13741
372cc85a
DL		 13742 /* enough room in the map table? */
13743 rmt_entries = 1 << (m + n);
13744 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
859bcad9 13745 goto bail;
4a818bed 13746
372cc85a 13747 /* add qos entries to the the RSM map table */
33a9eb52 13748 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
77241056
MM		 13749 unsigned tctxt;
13750
13751 for (qpn = 0, tctxt = ctxt;
13752 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
13753 unsigned idx, regoff, regidx;
13754
372cc85a
DL		 13755 /* generate the index the hardware will produce */
13756 idx = rmt->used + ((qpn << n) ^ i);
77241056
MM		 13757 regoff = (idx % 8) * 8;
13758 regidx = idx / 8;
372cc85a
DL		 13759 /* replace default with context number */
13760 reg = rmt->map[regidx];
77241056
MM		 13761 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
13762 << regoff);
13763 reg |= (u64)(tctxt++) << regoff;
372cc85a 13764 rmt->map[regidx] = reg;
77241056
MM		 13765 if (tctxt == ctxt + krcvqs[i])
13766 tctxt = ctxt;
13767 }
13768 ctxt += krcvqs[i];
13769 }
b12349ae
DL		 13770
13771 rrd.offset = rmt->used;
13772 rrd.pkt_type = 2;
13773 rrd.field1_off = LRH_BTH_MATCH_OFFSET;
13774 rrd.field2_off = LRH_SC_MATCH_OFFSET;
13775 rrd.index1_off = LRH_SC_SELECT_OFFSET;
13776 rrd.index1_width = n;
13777 rrd.index2_off = QPN_SELECT_OFFSET;
13778 rrd.index2_width = m + n;
13779 rrd.mask1 = LRH_BTH_MASK;
13780 rrd.value1 = LRH_BTH_VALUE;
13781 rrd.mask2 = LRH_SC_MASK;
13782 rrd.value2 = LRH_SC_VALUE;
13783
13784 /* add rule 0 */
13785 add_rsm_rule(dd, 0, &rrd);
13786
372cc85a
DL		 13787 /* mark RSM map entries as used */
13788 rmt->used += rmt_entries;
33a9eb52
DL		 13789 /* map everything else to the mcast/err/vl15 context */
13790 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
77241056
MM		 13791 dd->qos_shift = n + 1;
13792 return;
13793bail:
13794 dd->qos_shift = 1;
82c2611d 13795 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
77241056
MM		 13796}
13797
8f000f7f
DL		 13798static void init_user_fecn_handling(struct hfi1_devdata *dd,
13799 struct rsm_map_table *rmt)
13800{
13801 struct rsm_rule_data rrd;
13802 u64 reg;
13803 int i, idx, regoff, regidx;
13804 u8 offset;
13805
13806 /* there needs to be enough room in the map table */
13807 if (rmt->used + dd->num_user_contexts >= NUM_MAP_ENTRIES) {
13808 dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n");
13809 return;
13810 }
13811
13812 /*
13813 * RSM will extract the destination context as an index into the
13814 * map table. The destination contexts are a sequential block
13815 * in the range first_user_ctxt...num_rcv_contexts-1 (inclusive).
13816 * Map entries are accessed as offset + extracted value. Adjust
13817 * the added offset so this sequence can be placed anywhere in
13818 * the table - as long as the entries themselves do not wrap.
13819 * There are only enough bits in offset for the table size, so
13820 * start with that to allow for a "negative" offset.
13821 */
13822 offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used -
13823 (int)dd->first_user_ctxt);
13824
13825 for (i = dd->first_user_ctxt, idx = rmt->used;
13826 i < dd->num_rcv_contexts; i++, idx++) {
13827 /* replace with identity mapping */
13828 regoff = (idx % 8) * 8;
13829 regidx = idx / 8;
13830 reg = rmt->map[regidx];
13831 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
13832 reg |= (u64)i << regoff;
13833 rmt->map[regidx] = reg;
13834 }
13835
13836 /*
13837 * For RSM intercept of Expected FECN packets:
13838 * o packet type 0 - expected
13839 * o match on F (bit 95), using select/match 1, and
13840 * o match on SH (bit 133), using select/match 2.
13841 *
13842 * Use index 1 to extract the 8-bit receive context from DestQP
13843 * (start at bit 64). Use that as the RSM map table index.
13844 */
13845 rrd.offset = offset;
13846 rrd.pkt_type = 0;
13847 rrd.field1_off = 95;
13848 rrd.field2_off = 133;
13849 rrd.index1_off = 64;
13850 rrd.index1_width = 8;
13851 rrd.index2_off = 0;
13852 rrd.index2_width = 0;
13853 rrd.mask1 = 1;
13854 rrd.value1 = 1;
13855 rrd.mask2 = 1;
13856 rrd.value2 = 1;
13857
13858 /* add rule 1 */
13859 add_rsm_rule(dd, 1, &rrd);
13860
13861 rmt->used += dd->num_user_contexts;
13862}
13863
77241056
MM		 13864static void init_rxe(struct hfi1_devdata *dd)
13865{
372cc85a
DL		 13866 struct rsm_map_table *rmt;
13867
77241056
MM		 13868 /* enable all receive errors */
13869 write_csr(dd, RCV_ERR_MASK, ~0ull);
372cc85a
DL		 13870
13871 rmt = alloc_rsm_map_table(dd);
13872 /* set up QOS, including the QPN map table */
13873 init_qos(dd, rmt);
8f000f7f 13874 init_user_fecn_handling(dd, rmt);
372cc85a
DL		 13875 complete_rsm_map_table(dd, rmt);
13876 kfree(rmt);
13877
77241056
MM		 13878 /*
13879 * make sure RcvCtrl.RcvWcb <= PCIe Device Control
13880 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
13881 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one
13882 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
13883 * Max_PayLoad_Size set to its minimum of 128.
13884 *
13885 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
13886 * (64 bytes). Max_Payload_Size is possibly modified upward in
13887 * tune_pcie_caps() which is called after this routine.
13888 */
13889}
13890
13891static void init_other(struct hfi1_devdata *dd)
13892{
13893 /* enable all CCE errors */
13894 write_csr(dd, CCE_ERR_MASK, ~0ull);
13895 /* enable *some* Misc errors */
13896 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
13897 /* enable all DC errors, except LCB */
13898 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
13899 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
13900}
13901
13902/*
13903 * Fill out the given AU table using the given CU. A CU is defined in terms
13904 * AUs. The table is a an encoding: given the index, how many AUs does that
13905 * represent?
13906 *
13907 * NOTE: Assumes that the register layout is the same for the
13908 * local and remote tables.
13909 */
13910static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
13911 u32 csr0to3, u32 csr4to7)
13912{
13913 write_csr(dd, csr0to3,
17fb4f29
JJ		 13914 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
13915 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
13916 2ull * cu <<
13917 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
13918 4ull * cu <<
13919 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
77241056 13920 write_csr(dd, csr4to7,
17fb4f29
JJ		 13921 8ull * cu <<
13922 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
13923 16ull * cu <<
13924 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
13925 32ull * cu <<
13926 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
13927 64ull * cu <<
13928 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
77241056
MM		 13929}
13930
13931static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
13932{
13933 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
17fb4f29 13934 SEND_CM_LOCAL_AU_TABLE4_TO7);
77241056
MM		 13935}
13936
13937void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
13938{
13939 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
17fb4f29 13940 SEND_CM_REMOTE_AU_TABLE4_TO7);
77241056
MM		 13941}
13942
13943static void init_txe(struct hfi1_devdata *dd)
13944{
13945 int i;
13946
13947 /* enable all PIO, SDMA, general, and Egress errors */
13948 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
13949 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
13950 write_csr(dd, SEND_ERR_MASK, ~0ull);
13951 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
13952
13953 /* enable all per-context and per-SDMA engine errors */
13954 for (i = 0; i < dd->chip_send_contexts; i++)
13955 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
13956 for (i = 0; i < dd->chip_sdma_engines; i++)
13957 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
13958
13959 /* set the local CU to AU mapping */
13960 assign_local_cm_au_table(dd, dd->vcu);
13961
13962 /*
13963 * Set reasonable default for Credit Return Timer
13964 * Don't set on Simulator - causes it to choke.
13965 */
13966 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
13967 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
13968}
13969
13970int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
13971{
13972 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
13973 unsigned sctxt;
13974 int ret = 0;
13975 u64 reg;
13976
13977 if (!rcd || !rcd->sc) {
13978 ret = -EINVAL;
13979 goto done;
13980 }
13981 sctxt = rcd->sc->hw_context;
13982 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
13983 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
13984 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
13985 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
13986 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
13987 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
13988 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
13989 /*
13990 * Enable send-side J_KEY integrity check, unless this is A0 h/w
77241056 13991 */
995deafa 13992 if (!is_ax(dd)) {
77241056
MM		 13993 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
13994 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
13995 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
13996 }
13997
13998 /* Enable J_KEY check on receive context. */
13999 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
14000 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
14001 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
14002 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
14003done:
14004 return ret;
14005}
14006
14007int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
14008{
14009 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
14010 unsigned sctxt;
14011 int ret = 0;
14012 u64 reg;
14013
14014 if (!rcd || !rcd->sc) {
14015 ret = -EINVAL;
14016 goto done;
14017 }
14018 sctxt = rcd->sc->hw_context;
14019 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14020 /*
14021 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
14022 * This check would not have been enabled for A0 h/w, see
14023 * set_ctxt_jkey().
14024 */
995deafa 14025 if (!is_ax(dd)) {
77241056
MM		 14026 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14027 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14028 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14029 }
14030 /* Turn off the J_KEY on the receive side */
14031 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
14032done:
14033 return ret;
14034}
14035
14036int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
14037{
14038 struct hfi1_ctxtdata *rcd;
14039 unsigned sctxt;
14040 int ret = 0;
14041 u64 reg;
14042
e490974e 14043 if (ctxt < dd->num_rcv_contexts) {
77241056 14044 rcd = dd->rcd[ctxt];
e490974e 14045 } else {
77241056
MM		 14046 ret = -EINVAL;
14047 goto done;
14048 }
14049 if (!rcd || !rcd->sc) {
14050 ret = -EINVAL;
14051 goto done;
14052 }
14053 sctxt = rcd->sc->hw_context;
14054 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14055 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14056 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14057 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14058 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
e38d1e4f 14059 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
77241056
MM		 14060 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14061done:
14062 return ret;
14063}
14064
14065int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt)
14066{
14067 struct hfi1_ctxtdata *rcd;
14068 unsigned sctxt;
14069 int ret = 0;
14070 u64 reg;
14071
e490974e 14072 if (ctxt < dd->num_rcv_contexts) {
77241056 14073 rcd = dd->rcd[ctxt];
e490974e 14074 } else {
77241056
MM		 14075 ret = -EINVAL;
14076 goto done;
14077 }
14078 if (!rcd || !rcd->sc) {
14079 ret = -EINVAL;
14080 goto done;
14081 }
14082 sctxt = rcd->sc->hw_context;
14083 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14084 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14085 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14086 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14087done:
14088 return ret;
14089}
14090
14091/*
14092 * Start doing the clean up the the chip. Our clean up happens in multiple
14093 * stages and this is just the first.
14094 */
14095void hfi1_start_cleanup(struct hfi1_devdata *dd)
14096{
affa48de 14097 aspm_exit(dd);
77241056
MM		 14098 free_cntrs(dd);
14099 free_rcverr(dd);
14100 clean_up_interrupts(dd);
a2ee27a4 14101 finish_chip_resources(dd);
77241056
MM		 14102}
14103
14104#define HFI_BASE_GUID(dev) \
14105 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14106
14107/*
78eb129d
DL		 14108 * Information can be shared between the two HFIs on the same ASIC
14109 * in the same OS. This function finds the peer device and sets
14110 * up a shared structure.
77241056 14111 */
78eb129d 14112static int init_asic_data(struct hfi1_devdata *dd)
77241056
MM		 14113{
14114 unsigned long flags;
14115 struct hfi1_devdata *tmp, *peer = NULL;
98f179a5 14116 struct hfi1_asic_data *asic_data;
78eb129d 14117 int ret = 0;
77241056 14118
98f179a5
TS		 14119 /* pre-allocate the asic structure in case we are the first device */
14120 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14121 if (!asic_data)
14122 return -ENOMEM;
14123
77241056
MM		 14124 spin_lock_irqsave(&hfi1_devs_lock, flags);
14125 /* Find our peer device */
14126 list_for_each_entry(tmp, &hfi1_dev_list, list) {
14127 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
14128 dd->unit != tmp->unit) {
14129 peer = tmp;
14130 break;
14131 }
14132 }
14133
78eb129d 14134 if (peer) {
98f179a5 14135 /* use already allocated structure */
78eb129d 14136 dd->asic_data = peer->asic_data;
98f179a5 14137 kfree(asic_data);
78eb129d 14138 } else {
98f179a5 14139 dd->asic_data = asic_data;
78eb129d
DL		 14140 mutex_init(&dd->asic_data->asic_resource_mutex);
14141 }
14142 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
77241056 14143 spin_unlock_irqrestore(&hfi1_devs_lock, flags);
78eb129d 14144 return ret;
77241056
MM		 14145}
14146
5d9157aa
DL		 14147/*
14148 * Set dd->boardname. Use a generic name if a name is not returned from
14149 * EFI variable space.
14150 *
14151 * Return 0 on success, -ENOMEM if space could not be allocated.
14152 */
14153static int obtain_boardname(struct hfi1_devdata *dd)
14154{
14155 /* generic board description */
14156 const char generic[] =
14157 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14158 unsigned long size;
14159 int ret;
14160
14161 ret = read_hfi1_efi_var(dd, "description", &size,
14162 (void **)&dd->boardname);
14163 if (ret) {
845f876d 14164 dd_dev_info(dd, "Board description not found\n");
5d9157aa
DL		 14165 /* use generic description */
14166 dd->boardname = kstrdup(generic, GFP_KERNEL);
14167 if (!dd->boardname)
14168 return -ENOMEM;
14169 }
14170 return 0;
14171}
14172
24487dd3
KW		 14173/*
14174 * Check the interrupt registers to make sure that they are mapped correctly.
14175 * It is intended to help user identify any mismapping by VMM when the driver
14176 * is running in a VM. This function should only be called before interrupt
14177 * is set up properly.
14178 *
14179 * Return 0 on success, -EINVAL on failure.
14180 */
14181static int check_int_registers(struct hfi1_devdata *dd)
14182{
14183 u64 reg;
14184 u64 all_bits = ~(u64)0;
14185 u64 mask;
14186
14187 /* Clear CceIntMask[0] to avoid raising any interrupts */
14188 mask = read_csr(dd, CCE_INT_MASK);
14189 write_csr(dd, CCE_INT_MASK, 0ull);
14190 reg = read_csr(dd, CCE_INT_MASK);
14191 if (reg)
14192 goto err_exit;
14193
14194 /* Clear all interrupt status bits */
14195 write_csr(dd, CCE_INT_CLEAR, all_bits);
14196 reg = read_csr(dd, CCE_INT_STATUS);
14197 if (reg)
14198 goto err_exit;
14199
14200 /* Set all interrupt status bits */
14201 write_csr(dd, CCE_INT_FORCE, all_bits);
14202 reg = read_csr(dd, CCE_INT_STATUS);
14203 if (reg != all_bits)
14204 goto err_exit;
14205
14206 /* Restore the interrupt mask */
14207 write_csr(dd, CCE_INT_CLEAR, all_bits);
14208 write_csr(dd, CCE_INT_MASK, mask);
14209
14210 return 0;
14211err_exit:
14212 write_csr(dd, CCE_INT_MASK, mask);
14213 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14214 return -EINVAL;
14215}
14216
77241056 14217/**
7c03ed85 14218 * Allocate and initialize the device structure for the hfi.
77241056
MM		 14219 * @dev: the pci_dev for hfi1_ib device
14220 * @ent: pci_device_id struct for this dev
14221 *
14222 * Also allocates, initializes, and returns the devdata struct for this
14223 * device instance
14224 *
14225 * This is global, and is called directly at init to set up the
14226 * chip-specific function pointers for later use.
14227 */
14228struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
14229 const struct pci_device_id *ent)
14230{
14231 struct hfi1_devdata *dd;
14232 struct hfi1_pportdata *ppd;
14233 u64 reg;
14234 int i, ret;
14235 static const char * const inames[] = { /* implementation names */
14236 "RTL silicon",
14237 "RTL VCS simulation",
14238 "RTL FPGA emulation",
14239 "Functional simulator"
14240 };
24487dd3 14241 struct pci_dev *parent = pdev->bus->self;
77241056 14242
17fb4f29
JJ		 14243 dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
14244 sizeof(struct hfi1_pportdata));
77241056
MM		 14245 if (IS_ERR(dd))
14246 goto bail;
14247 ppd = dd->pport;
14248 for (i = 0; i < dd->num_pports; i++, ppd++) {
14249 int vl;
14250 /* init common fields */
14251 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
14252 /* DC supports 4 link widths */
14253 ppd->link_width_supported =
14254 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
14255 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
14256 ppd->link_width_downgrade_supported =
14257 ppd->link_width_supported;
14258 /* start out enabling only 4X */
14259 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
14260 ppd->link_width_downgrade_enabled =
14261 ppd->link_width_downgrade_supported;
14262 /* link width active is 0 when link is down */
14263 /* link width downgrade active is 0 when link is down */
14264
d0d236ea
JJ		 14265 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
14266 num_vls > HFI1_MAX_VLS_SUPPORTED) {
77241056
MM		 14267 hfi1_early_err(&pdev->dev,
14268 "Invalid num_vls %u, using %u VLs\n",
14269 num_vls, HFI1_MAX_VLS_SUPPORTED);
14270 num_vls = HFI1_MAX_VLS_SUPPORTED;
14271 }
14272 ppd->vls_supported = num_vls;
14273 ppd->vls_operational = ppd->vls_supported;
8a4d3444 14274 ppd->actual_vls_operational = ppd->vls_supported;
77241056
MM		 14275 /* Set the default MTU. */
14276 for (vl = 0; vl < num_vls; vl++)
14277 dd->vld[vl].mtu = hfi1_max_mtu;
14278 dd->vld[15].mtu = MAX_MAD_PACKET;
14279 /*
14280 * Set the initial values to reasonable default, will be set
14281 * for real when link is up.
14282 */
14283 ppd->lstate = IB_PORT_DOWN;
14284 ppd->overrun_threshold = 0x4;
14285 ppd->phy_error_threshold = 0xf;
14286 ppd->port_crc_mode_enabled = link_crc_mask;
14287 /* initialize supported LTP CRC mode */
14288 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
14289 /* initialize enabled LTP CRC mode */
14290 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
14291 /* start in offline */
14292 ppd->host_link_state = HLS_DN_OFFLINE;
14293 init_vl_arb_caches(ppd);
f45c8dc8 14294 ppd->last_pstate = 0xff; /* invalid value */
77241056
MM		 14295 }
14296
14297 dd->link_default = HLS_DN_POLL;
14298
14299 /*
14300 * Do remaining PCIe setup and save PCIe values in dd.
14301 * Any error printing is already done by the init code.
14302 * On return, we have the chip mapped.
14303 */
14304 ret = hfi1_pcie_ddinit(dd, pdev, ent);
14305 if (ret < 0)
14306 goto bail_free;
14307
14308 /* verify that reads actually work, save revision for reset check */
14309 dd->revision = read_csr(dd, CCE_REVISION);
14310 if (dd->revision == ~(u64)0) {
14311 dd_dev_err(dd, "cannot read chip CSRs\n");
14312 ret = -EINVAL;
14313 goto bail_cleanup;
14314 }
14315 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
14316 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
14317 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
14318 & CCE_REVISION_CHIP_REV_MINOR_MASK;
14319
24487dd3
KW		 14320 /*
14321 * Check interrupt registers mapping if the driver has no access to
14322 * the upstream component. In this case, it is likely that the driver
14323 * is running in a VM.
14324 */
14325 if (!parent) {
14326 ret = check_int_registers(dd);
14327 if (ret)
14328 goto bail_cleanup;
14329 }
14330
4d114fdd
JJ		 14331 /*
14332 * obtain the hardware ID - NOT related to unit, which is a
14333 * software enumeration
14334 */
77241056
MM		 14335 reg = read_csr(dd, CCE_REVISION2);
14336 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
14337 & CCE_REVISION2_HFI_ID_MASK;
14338 /* the variable size will remove unwanted bits */
14339 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
14340 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
14341 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
17fb4f29
JJ		 14342 dd->icode < ARRAY_SIZE(inames) ?
14343 inames[dd->icode] : "unknown", (int)dd->irev);
77241056
MM		 14344
14345 /* speeds the hardware can support */
14346 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
14347 /* speeds allowed to run at */
14348 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
14349 /* give a reasonable active value, will be set on link up */
14350 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
14351
14352 dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS);
14353 dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS);
14354 dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES);
14355 dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE);
14356 dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE);
14357 /* fix up link widths for emulation _p */
14358 ppd = dd->pport;
14359 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
14360 ppd->link_width_supported =
14361 ppd->link_width_enabled =
14362 ppd->link_width_downgrade_supported =
14363 ppd->link_width_downgrade_enabled =
14364 OPA_LINK_WIDTH_1X;
14365 }
14366 /* insure num_vls isn't larger than number of sdma engines */
14367 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) {
14368 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
11a5909b
DL		 14369 num_vls, dd->chip_sdma_engines);
14370 num_vls = dd->chip_sdma_engines;
14371 ppd->vls_supported = dd->chip_sdma_engines;
8a4d3444 14372 ppd->vls_operational = ppd->vls_supported;
77241056
MM		 14373 }
14374
14375 /*
14376 * Convert the ns parameter to the 64 * cclocks used in the CSR.
14377 * Limit the max if larger than the field holds. If timeout is
14378 * non-zero, then the calculated field will be at least 1.
14379 *
14380 * Must be after icode is set up - the cclock rate depends
14381 * on knowing the hardware being used.
14382 */
14383 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
14384 if (dd->rcv_intr_timeout_csr >
14385 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
14386 dd->rcv_intr_timeout_csr =
14387 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
14388 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
14389 dd->rcv_intr_timeout_csr = 1;
14390
7c03ed85
EH		 14391 /* needs to be done before we look for the peer device */
14392 read_guid(dd);
14393
78eb129d
DL		 14394 /* set up shared ASIC data with peer device */
14395 ret = init_asic_data(dd);
14396 if (ret)
14397 goto bail_cleanup;
7c03ed85 14398
77241056
MM		 14399 /* obtain chip sizes, reset chip CSRs */
14400 init_chip(dd);
14401
14402 /* read in the PCIe link speed information */
14403 ret = pcie_speeds(dd);
14404 if (ret)
14405 goto bail_cleanup;
14406
c3838b39
EH		 14407 /* Needs to be called before hfi1_firmware_init */
14408 get_platform_config(dd);
14409
77241056
MM		 14410 /* read in firmware */
14411 ret = hfi1_firmware_init(dd);
14412 if (ret)
14413 goto bail_cleanup;
14414
14415 /*
14416 * In general, the PCIe Gen3 transition must occur after the
14417 * chip has been idled (so it won't initiate any PCIe transactions
14418 * e.g. an interrupt) and before the driver changes any registers
14419 * (the transition will reset the registers).
14420 *
14421 * In particular, place this call after:
14422 * - init_chip() - the chip will not initiate any PCIe transactions
14423 * - pcie_speeds() - reads the current link speed
14424 * - hfi1_firmware_init() - the needed firmware is ready to be
14425 * downloaded
14426 */
14427 ret = do_pcie_gen3_transition(dd);
14428 if (ret)
14429 goto bail_cleanup;
14430
14431 /* start setting dd values and adjusting CSRs */
14432 init_early_variables(dd);
14433
14434 parse_platform_config(dd);
14435
5d9157aa
DL		 14436 ret = obtain_boardname(dd);
14437 if (ret)
77241056 14438 goto bail_cleanup;
77241056
MM		 14439
14440 snprintf(dd->boardversion, BOARD_VERS_MAX,
5d9157aa 14441 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
77241056 14442 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
77241056
MM		 14443 (u32)dd->majrev,
14444 (u32)dd->minrev,
14445 (dd->revision >> CCE_REVISION_SW_SHIFT)
14446 & CCE_REVISION_SW_MASK);
14447
0852d241
JJ		 14448 /*
14449 * The real cpu mask is part of the affinity struct but has to be
14450 * initialized earlier than the rest of the affinity struct because it
14451 * is needed to calculate the number of user contexts in
14452 * set_up_context_variables(). However, hfi1_dev_affinity_init(),
14453 * which initializes the rest of the affinity struct members,
14454 * depends on set_up_context_variables() for the number of kernel
14455 * contexts, so it cannot be called before set_up_context_variables().
14456 */
14457 ret = init_real_cpu_mask(dd);
14458 if (ret)
14459 goto bail_cleanup;
14460
77241056
MM		 14461 ret = set_up_context_variables(dd);
14462 if (ret)
14463 goto bail_cleanup;
14464
14465 /* set initial RXE CSRs */
14466 init_rxe(dd);
14467 /* set initial TXE CSRs */
14468 init_txe(dd);
14469 /* set initial non-RXE, non-TXE CSRs */
14470 init_other(dd);
14471 /* set up KDETH QP prefix in both RX and TX CSRs */
14472 init_kdeth_qp(dd);
14473
0852d241 14474 hfi1_dev_affinity_init(dd);
957558c9 14475
77241056
MM		 14476 /* send contexts must be set up before receive contexts */
14477 ret = init_send_contexts(dd);
14478 if (ret)
14479 goto bail_cleanup;
14480
14481 ret = hfi1_create_ctxts(dd);
14482 if (ret)
14483 goto bail_cleanup;
14484
14485 dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
14486 /*
14487 * rcd[0] is guaranteed to be valid by this point. Also, all
14488 * context are using the same value, as per the module parameter.
14489 */
14490 dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
14491
14492 ret = init_pervl_scs(dd);
14493 if (ret)
14494 goto bail_cleanup;
14495
14496 /* sdma init */
14497 for (i = 0; i < dd->num_pports; ++i) {
14498 ret = sdma_init(dd, i);
14499 if (ret)
14500 goto bail_cleanup;
14501 }
14502
14503 /* use contexts created by hfi1_create_ctxts */
14504 ret = set_up_interrupts(dd);
14505 if (ret)
14506 goto bail_cleanup;
14507
14508 /* set up LCB access - must be after set_up_interrupts() */
14509 init_lcb_access(dd);
14510
14511 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
14512 dd->base_guid & 0xFFFFFF);
14513
14514 dd->oui1 = dd->base_guid >> 56 & 0xFF;
14515 dd->oui2 = dd->base_guid >> 48 & 0xFF;
14516 dd->oui3 = dd->base_guid >> 40 & 0xFF;
14517
14518 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
14519 if (ret)
14520 goto bail_clear_intr;
14521 check_fabric_firmware_versions(dd);
14522
14523 thermal_init(dd);
14524
14525 ret = init_cntrs(dd);
14526 if (ret)
14527 goto bail_clear_intr;
14528
14529 ret = init_rcverr(dd);
14530 if (ret)
14531 goto bail_free_cntrs;
14532
14533 ret = eprom_init(dd);
14534 if (ret)
14535 goto bail_free_rcverr;
14536
14537 goto bail;
14538
14539bail_free_rcverr:
14540 free_rcverr(dd);
14541bail_free_cntrs:
14542 free_cntrs(dd);
14543bail_clear_intr:
14544 clean_up_interrupts(dd);
14545bail_cleanup:
14546 hfi1_pcie_ddcleanup(dd);
14547bail_free:
14548 hfi1_free_devdata(dd);
14549 dd = ERR_PTR(ret);
14550bail:
14551 return dd;
14552}
14553
14554static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
14555 u32 dw_len)
14556{
14557 u32 delta_cycles;
14558 u32 current_egress_rate = ppd->current_egress_rate;
14559 /* rates here are in units of 10^6 bits/sec */
14560
14561 if (desired_egress_rate == -1)
14562 return 0; /* shouldn't happen */
14563
14564 if (desired_egress_rate >= current_egress_rate)
14565 return 0; /* we can't help go faster, only slower */
14566
14567 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
14568 egress_cycles(dw_len * 4, current_egress_rate);
14569
14570 return (u16)delta_cycles;
14571}
14572
77241056
MM		 14573/**
14574 * create_pbc - build a pbc for transmission
14575 * @flags: special case flags or-ed in built pbc
14576 * @srate: static rate
14577 * @vl: vl
14578 * @dwlen: dword length (header words + data words + pbc words)
14579 *
14580 * Create a PBC with the given flags, rate, VL, and length.
14581 *
14582 * NOTE: The PBC created will not insert any HCRC - all callers but one are
14583 * for verbs, which does not use this PSM feature. The lone other caller
14584 * is for the diagnostic interface which calls this if the user does not
14585 * supply their own PBC.
14586 */
14587u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
14588 u32 dw_len)
14589{
14590 u64 pbc, delay = 0;
14591
14592 if (unlikely(srate_mbs))
14593 delay = delay_cycles(ppd, srate_mbs, dw_len);
14594
14595 pbc = flags
14596 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
14597 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
14598 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
14599 | (dw_len & PBC_LENGTH_DWS_MASK)
14600 << PBC_LENGTH_DWS_SHIFT;
14601
14602 return pbc;
14603}
14604
14605#define SBUS_THERMAL 0x4f
14606#define SBUS_THERM_MONITOR_MODE 0x1
14607
14608#define THERM_FAILURE(dev, ret, reason) \
14609 dd_dev_err((dd), \
14610 "Thermal sensor initialization failed: %s (%d)\n", \
14611 (reason), (ret))
14612
14613/*
cde10afa 14614 * Initialize the thermal sensor.
77241056
MM		 14615 *
14616 * After initialization, enable polling of thermal sensor through
14617 * SBus interface. In order for this to work, the SBus Master
14618 * firmware has to be loaded due to the fact that the HW polling
14619 * logic uses SBus interrupts, which are not supported with
14620 * default firmware. Otherwise, no data will be returned through
14621 * the ASIC_STS_THERM CSR.
14622 */
14623static int thermal_init(struct hfi1_devdata *dd)
14624{
14625 int ret = 0;
14626
14627 if (dd->icode != ICODE_RTL_SILICON ||
a453698b 14628 check_chip_resource(dd, CR_THERM_INIT, NULL))
77241056
MM		 14629 return ret;
14630
576531fd
DL		 14631 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
14632 if (ret) {
14633 THERM_FAILURE(dd, ret, "Acquire SBus");
14634 return ret;
14635 }
14636
77241056 14637 dd_dev_info(dd, "Initializing thermal sensor\n");
4ef98989
JAQ		 14638 /* Disable polling of thermal readings */
14639 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
14640 msleep(100);
77241056
MM		 14641 /* Thermal Sensor Initialization */
14642 /* Step 1: Reset the Thermal SBus Receiver */
14643 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14644 RESET_SBUS_RECEIVER, 0);
14645 if (ret) {
14646 THERM_FAILURE(dd, ret, "Bus Reset");
14647 goto done;
14648 }
14649 /* Step 2: Set Reset bit in Thermal block */
14650 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14651 WRITE_SBUS_RECEIVER, 0x1);
14652 if (ret) {
14653 THERM_FAILURE(dd, ret, "Therm Block Reset");
14654 goto done;
14655 }
14656 /* Step 3: Write clock divider value (100MHz -> 2MHz) */
14657 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
14658 WRITE_SBUS_RECEIVER, 0x32);
14659 if (ret) {
14660 THERM_FAILURE(dd, ret, "Write Clock Div");
14661 goto done;
14662 }
14663 /* Step 4: Select temperature mode */
14664 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
14665 WRITE_SBUS_RECEIVER,
14666 SBUS_THERM_MONITOR_MODE);
14667 if (ret) {
14668 THERM_FAILURE(dd, ret, "Write Mode Sel");
14669 goto done;
14670 }
14671 /* Step 5: De-assert block reset and start conversion */
14672 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14673 WRITE_SBUS_RECEIVER, 0x2);
14674 if (ret) {
14675 THERM_FAILURE(dd, ret, "Write Reset Deassert");
14676 goto done;
14677 }
14678 /* Step 5.1: Wait for first conversion (21.5ms per spec) */
14679 msleep(22);
14680
14681 /* Enable polling of thermal readings */
14682 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
a453698b
DL		 14683
14684 /* Set initialized flag */
14685 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
14686 if (ret)
14687 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
14688
77241056 14689done:
576531fd 14690 release_chip_resource(dd, CR_SBUS);
77241056
MM		 14691 return ret;
14692}
14693
14694static void handle_temp_err(struct hfi1_devdata *dd)
14695{
14696 struct hfi1_pportdata *ppd = &dd->pport[0];
14697 /*
14698 * Thermal Critical Interrupt
14699 * Put the device into forced freeze mode, take link down to
14700 * offline, and put DC into reset.
14701 */
14702 dd_dev_emerg(dd,
14703 "Critical temperature reached! Forcing device into freeze mode!\n");
14704 dd->flags |= HFI1_FORCED_FREEZE;
8638b77f 14705 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
77241056
MM		 14706 /*
14707 * Shut DC down as much and as quickly as possible.
14708 *
14709 * Step 1: Take the link down to OFFLINE. This will cause the
14710 * 8051 to put the Serdes in reset. However, we don't want to
14711 * go through the entire link state machine since we want to
14712 * shutdown ASAP. Furthermore, this is not a graceful shutdown
14713 * but rather an attempt to save the chip.
14714 * Code below is almost the same as quiet_serdes() but avoids
14715 * all the extra work and the sleeps.
14716 */
14717 ppd->driver_link_ready = 0;
14718 ppd->link_enabled = 0;
bf640096
HC		 14719 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
14720 PLS_OFFLINE);
77241056
MM		 14721 /*
14722 * Step 2: Shutdown LCB and 8051
14723 * After shutdown, do not restore DC_CFG_RESET value.
14724 */
14725 dc_shutdown(dd);
14726}
Ubuntu Artful kernel mirror