]>
Commit | Line | Data |
---|---|---|
4863dea3 SG |
1 | /* |
2 | * Copyright (C) 2015 Cavium, Inc. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of version 2 of the GNU General Public License | |
6 | * as published by the Free Software Foundation. | |
7 | */ | |
8 | ||
9 | #include <linux/module.h> | |
10 | #include <linux/interrupt.h> | |
11 | #include <linux/pci.h> | |
12 | #include <linux/etherdevice.h> | |
13 | #include <linux/of.h> | |
14 | ||
15 | #include "nic_reg.h" | |
16 | #include "nic.h" | |
17 | #include "q_struct.h" | |
18 | #include "thunder_bgx.h" | |
19 | ||
20 | #define DRV_NAME "thunder-nic" | |
21 | #define DRV_VERSION "1.0" | |
22 | ||
23 | struct nicpf { | |
24 | struct pci_dev *pdev; | |
25 | u8 rev_id; | |
26 | #define NIC_NODE_ID_MASK 0x300000000000 | |
27 | #define NIC_NODE_ID(x) ((x & NODE_ID_MASK) >> 44) | |
28 | u8 node; | |
29 | unsigned int flags; | |
30 | u8 num_vf_en; /* No of VF enabled */ | |
31 | bool vf_enabled[MAX_NUM_VFS_SUPPORTED]; | |
32 | void __iomem *reg_base; /* Register start address */ | |
33 | struct pkind_cfg pkind; | |
34 | #define NIC_SET_VF_LMAC_MAP(bgx, lmac) (((bgx & 0xF) << 4) | (lmac & 0xF)) | |
35 | #define NIC_GET_BGX_FROM_VF_LMAC_MAP(map) ((map >> 4) & 0xF) | |
36 | #define NIC_GET_LMAC_FROM_VF_LMAC_MAP(map) (map & 0xF) | |
37 | u8 vf_lmac_map[MAX_LMAC]; | |
38 | struct delayed_work dwork; | |
39 | struct workqueue_struct *check_link; | |
40 | u8 link[MAX_LMAC]; | |
41 | u8 duplex[MAX_LMAC]; | |
42 | u32 speed[MAX_LMAC]; | |
43 | u16 cpi_base[MAX_NUM_VFS_SUPPORTED]; | |
44 | u16 rss_ind_tbl_size; | |
45 | bool mbx_lock[MAX_NUM_VFS_SUPPORTED]; | |
46 | ||
47 | /* MSI-X */ | |
48 | bool msix_enabled; | |
49 | u8 num_vec; | |
50 | struct msix_entry msix_entries[NIC_PF_MSIX_VECTORS]; | |
51 | bool irq_allocated[NIC_PF_MSIX_VECTORS]; | |
52 | }; | |
53 | ||
54 | /* Supported devices */ | |
55 | static const struct pci_device_id nic_id_table[] = { | |
56 | { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_NIC_PF) }, | |
57 | { 0, } /* end of table */ | |
58 | }; | |
59 | ||
60 | MODULE_AUTHOR("Sunil Goutham"); | |
61 | MODULE_DESCRIPTION("Cavium Thunder NIC Physical Function Driver"); | |
62 | MODULE_LICENSE("GPL v2"); | |
63 | MODULE_VERSION(DRV_VERSION); | |
64 | MODULE_DEVICE_TABLE(pci, nic_id_table); | |
65 | ||
66 | /* The Cavium ThunderX network controller can *only* be found in SoCs | |
67 | * containing the ThunderX ARM64 CPU implementation. All accesses to the device | |
68 | * registers on this platform are implicitly strongly ordered with respect | |
69 | * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use | |
70 | * with no memory barriers in this driver. The readq()/writeq() functions add | |
71 | * explicit ordering operation which in this case are redundant, and only | |
72 | * add overhead. | |
73 | */ | |
74 | ||
75 | /* Register read/write APIs */ | |
76 | static void nic_reg_write(struct nicpf *nic, u64 offset, u64 val) | |
77 | { | |
78 | writeq_relaxed(val, nic->reg_base + offset); | |
79 | } | |
80 | ||
81 | static u64 nic_reg_read(struct nicpf *nic, u64 offset) | |
82 | { | |
83 | return readq_relaxed(nic->reg_base + offset); | |
84 | } | |
85 | ||
86 | /* PF -> VF mailbox communication APIs */ | |
87 | static void nic_enable_mbx_intr(struct nicpf *nic) | |
88 | { | |
89 | /* Enable mailbox interrupt for all 128 VFs */ | |
90 | nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, ~0ull); | |
91 | nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(u64), ~0ull); | |
92 | } | |
93 | ||
94 | static void nic_clear_mbx_intr(struct nicpf *nic, int vf, int mbx_reg) | |
95 | { | |
96 | nic_reg_write(nic, NIC_PF_MAILBOX_INT + (mbx_reg << 3), BIT_ULL(vf)); | |
97 | } | |
98 | ||
99 | static u64 nic_get_mbx_addr(int vf) | |
100 | { | |
101 | return NIC_PF_VF_0_127_MAILBOX_0_1 + (vf << NIC_VF_NUM_SHIFT); | |
102 | } | |
103 | ||
104 | /* Send a mailbox message to VF | |
105 | * @vf: vf to which this message to be sent | |
106 | * @mbx: Message to be sent | |
107 | */ | |
108 | static void nic_send_msg_to_vf(struct nicpf *nic, int vf, union nic_mbx *mbx) | |
109 | { | |
110 | void __iomem *mbx_addr = nic->reg_base + nic_get_mbx_addr(vf); | |
111 | u64 *msg = (u64 *)mbx; | |
112 | ||
113 | /* In first revision HW, mbox interrupt is triggerred | |
114 | * when PF writes to MBOX(1), in next revisions when | |
115 | * PF writes to MBOX(0) | |
116 | */ | |
117 | if (nic->rev_id == 0) { | |
118 | /* see the comment for nic_reg_write()/nic_reg_read() | |
119 | * functions above | |
120 | */ | |
121 | writeq_relaxed(msg[0], mbx_addr); | |
122 | writeq_relaxed(msg[1], mbx_addr + 8); | |
123 | } else { | |
124 | writeq_relaxed(msg[1], mbx_addr + 8); | |
125 | writeq_relaxed(msg[0], mbx_addr); | |
126 | } | |
127 | } | |
128 | ||
129 | /* Responds to VF's READY message with VF's | |
130 | * ID, node, MAC address e.t.c | |
131 | * @vf: VF which sent READY message | |
132 | */ | |
133 | static void nic_mbx_send_ready(struct nicpf *nic, int vf) | |
134 | { | |
135 | union nic_mbx mbx = {}; | |
136 | int bgx_idx, lmac; | |
137 | const char *mac; | |
138 | ||
139 | mbx.nic_cfg.msg = NIC_MBOX_MSG_READY; | |
140 | mbx.nic_cfg.vf_id = vf; | |
141 | ||
142 | mbx.nic_cfg.tns_mode = NIC_TNS_BYPASS_MODE; | |
143 | ||
144 | bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); | |
145 | lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); | |
146 | ||
147 | mac = bgx_get_lmac_mac(nic->node, bgx_idx, lmac); | |
148 | if (mac) | |
149 | ether_addr_copy((u8 *)&mbx.nic_cfg.mac_addr, mac); | |
150 | ||
151 | mbx.nic_cfg.node_id = nic->node; | |
152 | nic_send_msg_to_vf(nic, vf, &mbx); | |
153 | } | |
154 | ||
155 | /* ACKs VF's mailbox message | |
156 | * @vf: VF to which ACK to be sent | |
157 | */ | |
158 | static void nic_mbx_send_ack(struct nicpf *nic, int vf) | |
159 | { | |
160 | union nic_mbx mbx = {}; | |
161 | ||
162 | mbx.msg.msg = NIC_MBOX_MSG_ACK; | |
163 | nic_send_msg_to_vf(nic, vf, &mbx); | |
164 | } | |
165 | ||
166 | /* NACKs VF's mailbox message that PF is not able to | |
167 | * complete the action | |
168 | * @vf: VF to which ACK to be sent | |
169 | */ | |
170 | static void nic_mbx_send_nack(struct nicpf *nic, int vf) | |
171 | { | |
172 | union nic_mbx mbx = {}; | |
173 | ||
174 | mbx.msg.msg = NIC_MBOX_MSG_NACK; | |
175 | nic_send_msg_to_vf(nic, vf, &mbx); | |
176 | } | |
177 | ||
178 | /* Flush all in flight receive packets to memory and | |
179 | * bring down an active RQ | |
180 | */ | |
181 | static int nic_rcv_queue_sw_sync(struct nicpf *nic) | |
182 | { | |
183 | u16 timeout = ~0x00; | |
184 | ||
185 | nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x01); | |
186 | /* Wait till sync cycle is finished */ | |
187 | while (timeout) { | |
188 | if (nic_reg_read(nic, NIC_PF_SW_SYNC_RX_DONE) & 0x1) | |
189 | break; | |
190 | timeout--; | |
191 | } | |
192 | nic_reg_write(nic, NIC_PF_SW_SYNC_RX, 0x00); | |
193 | if (!timeout) { | |
194 | dev_err(&nic->pdev->dev, "Receive queue software sync failed"); | |
195 | return 1; | |
196 | } | |
197 | return 0; | |
198 | } | |
199 | ||
200 | /* Get BGX Rx/Tx stats and respond to VF's request */ | |
201 | static void nic_get_bgx_stats(struct nicpf *nic, struct bgx_stats_msg *bgx) | |
202 | { | |
203 | int bgx_idx, lmac; | |
204 | union nic_mbx mbx = {}; | |
205 | ||
206 | bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[bgx->vf_id]); | |
207 | lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[bgx->vf_id]); | |
208 | ||
209 | mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS; | |
210 | mbx.bgx_stats.vf_id = bgx->vf_id; | |
211 | mbx.bgx_stats.rx = bgx->rx; | |
212 | mbx.bgx_stats.idx = bgx->idx; | |
213 | if (bgx->rx) | |
214 | mbx.bgx_stats.stats = bgx_get_rx_stats(nic->node, bgx_idx, | |
215 | lmac, bgx->idx); | |
216 | else | |
217 | mbx.bgx_stats.stats = bgx_get_tx_stats(nic->node, bgx_idx, | |
218 | lmac, bgx->idx); | |
219 | nic_send_msg_to_vf(nic, bgx->vf_id, &mbx); | |
220 | } | |
221 | ||
222 | /* Update hardware min/max frame size */ | |
223 | static int nic_update_hw_frs(struct nicpf *nic, int new_frs, int vf) | |
224 | { | |
225 | if ((new_frs > NIC_HW_MAX_FRS) || (new_frs < NIC_HW_MIN_FRS)) { | |
226 | dev_err(&nic->pdev->dev, | |
227 | "Invalid MTU setting from VF%d rejected, should be between %d and %d\n", | |
228 | vf, NIC_HW_MIN_FRS, NIC_HW_MAX_FRS); | |
229 | return 1; | |
230 | } | |
231 | new_frs += ETH_HLEN; | |
232 | if (new_frs <= nic->pkind.maxlen) | |
233 | return 0; | |
234 | ||
235 | nic->pkind.maxlen = new_frs; | |
236 | nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG, *(u64 *)&nic->pkind); | |
237 | return 0; | |
238 | } | |
239 | ||
240 | /* Set minimum transmit packet size */ | |
241 | static void nic_set_tx_pkt_pad(struct nicpf *nic, int size) | |
242 | { | |
243 | int lmac; | |
244 | u64 lmac_cfg; | |
245 | ||
246 | /* Max value that can be set is 60 */ | |
247 | if (size > 60) | |
248 | size = 60; | |
249 | ||
250 | for (lmac = 0; lmac < (MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX); lmac++) { | |
251 | lmac_cfg = nic_reg_read(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3)); | |
252 | lmac_cfg &= ~(0xF << 2); | |
253 | lmac_cfg |= ((size / 4) << 2); | |
254 | nic_reg_write(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3), lmac_cfg); | |
255 | } | |
256 | } | |
257 | ||
258 | /* Function to check number of LMACs present and set VF::LMAC mapping. | |
259 | * Mapping will be used while initializing channels. | |
260 | */ | |
261 | static void nic_set_lmac_vf_mapping(struct nicpf *nic) | |
262 | { | |
263 | unsigned bgx_map = bgx_get_map(nic->node); | |
264 | int bgx, next_bgx_lmac = 0; | |
265 | int lmac, lmac_cnt = 0; | |
266 | u64 lmac_credit; | |
267 | ||
268 | nic->num_vf_en = 0; | |
269 | ||
270 | for (bgx = 0; bgx < NIC_MAX_BGX; bgx++) { | |
271 | if (!(bgx_map & (1 << bgx))) | |
272 | continue; | |
273 | lmac_cnt = bgx_get_lmac_count(nic->node, bgx); | |
274 | for (lmac = 0; lmac < lmac_cnt; lmac++) | |
275 | nic->vf_lmac_map[next_bgx_lmac++] = | |
276 | NIC_SET_VF_LMAC_MAP(bgx, lmac); | |
277 | nic->num_vf_en += lmac_cnt; | |
278 | ||
279 | /* Program LMAC credits */ | |
280 | lmac_credit = (1ull << 1); /* channel credit enable */ | |
281 | lmac_credit |= (0x1ff << 2); /* Max outstanding pkt count */ | |
282 | /* 48KB BGX Tx buffer size, each unit is of size 16bytes */ | |
283 | lmac_credit |= (((((48 * 1024) / lmac_cnt) - | |
284 | NIC_HW_MAX_FRS) / 16) << 12); | |
285 | lmac = bgx * MAX_LMAC_PER_BGX; | |
286 | for (; lmac < lmac_cnt + (bgx * MAX_LMAC_PER_BGX); lmac++) | |
287 | nic_reg_write(nic, | |
288 | NIC_PF_LMAC_0_7_CREDIT + (lmac * 8), | |
289 | lmac_credit); | |
290 | } | |
291 | } | |
292 | ||
293 | #define BGX0_BLOCK 8 | |
294 | #define BGX1_BLOCK 9 | |
295 | ||
296 | static void nic_init_hw(struct nicpf *nic) | |
297 | { | |
298 | int i; | |
299 | ||
300 | /* Reset NIC, in case the driver is repeatedly inserted and removed */ | |
301 | nic_reg_write(nic, NIC_PF_SOFT_RESET, 1); | |
302 | ||
303 | /* Enable NIC HW block */ | |
304 | nic_reg_write(nic, NIC_PF_CFG, 0x3); | |
305 | ||
306 | /* Enable backpressure */ | |
307 | nic_reg_write(nic, NIC_PF_BP_CFG, (1ULL << 6) | 0x03); | |
308 | ||
309 | /* Disable TNS mode on both interfaces */ | |
310 | nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG, | |
311 | (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK); | |
312 | nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8), | |
313 | (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK); | |
314 | nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG, | |
315 | (1ULL << 63) | BGX0_BLOCK); | |
316 | nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8), | |
317 | (1ULL << 63) | BGX1_BLOCK); | |
318 | ||
319 | /* PKIND configuration */ | |
320 | nic->pkind.minlen = 0; | |
321 | nic->pkind.maxlen = NIC_HW_MAX_FRS + ETH_HLEN; | |
322 | nic->pkind.lenerr_en = 1; | |
323 | nic->pkind.rx_hdr = 0; | |
324 | nic->pkind.hdr_sl = 0; | |
325 | ||
326 | for (i = 0; i < NIC_MAX_PKIND; i++) | |
327 | nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG | (i << 3), | |
328 | *(u64 *)&nic->pkind); | |
329 | ||
330 | nic_set_tx_pkt_pad(nic, NIC_HW_MIN_FRS); | |
331 | ||
332 | /* Timer config */ | |
333 | nic_reg_write(nic, NIC_PF_INTR_TIMER_CFG, NICPF_CLK_PER_INT_TICK); | |
334 | } | |
335 | ||
336 | /* Channel parse index configuration */ | |
337 | static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg) | |
338 | { | |
339 | u32 vnic, bgx, lmac, chan; | |
340 | u32 padd, cpi_count = 0; | |
341 | u64 cpi_base, cpi, rssi_base, rssi; | |
342 | u8 qset, rq_idx = 0; | |
343 | ||
344 | vnic = cfg->vf_id; | |
345 | bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); | |
346 | lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); | |
347 | ||
348 | chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF); | |
349 | cpi_base = (lmac * NIC_MAX_CPI_PER_LMAC) + (bgx * NIC_CPI_PER_BGX); | |
350 | rssi_base = (lmac * nic->rss_ind_tbl_size) + (bgx * NIC_RSSI_PER_BGX); | |
351 | ||
352 | /* Rx channel configuration */ | |
353 | nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_BP_CFG | (chan << 3), | |
354 | (1ull << 63) | (vnic << 0)); | |
355 | nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_CFG | (chan << 3), | |
356 | ((u64)cfg->cpi_alg << 62) | (cpi_base << 48)); | |
357 | ||
358 | if (cfg->cpi_alg == CPI_ALG_NONE) | |
359 | cpi_count = 1; | |
360 | else if (cfg->cpi_alg == CPI_ALG_VLAN) /* 3 bits of PCP */ | |
361 | cpi_count = 8; | |
362 | else if (cfg->cpi_alg == CPI_ALG_VLAN16) /* 3 bits PCP + DEI */ | |
363 | cpi_count = 16; | |
364 | else if (cfg->cpi_alg == CPI_ALG_DIFF) /* 6bits DSCP */ | |
365 | cpi_count = NIC_MAX_CPI_PER_LMAC; | |
366 | ||
367 | /* RSS Qset, Qidx mapping */ | |
368 | qset = cfg->vf_id; | |
369 | rssi = rssi_base; | |
370 | for (; rssi < (rssi_base + cfg->rq_cnt); rssi++) { | |
371 | nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3), | |
372 | (qset << 3) | rq_idx); | |
373 | rq_idx++; | |
374 | } | |
375 | ||
376 | rssi = 0; | |
377 | cpi = cpi_base; | |
378 | for (; cpi < (cpi_base + cpi_count); cpi++) { | |
379 | /* Determine port to channel adder */ | |
380 | if (cfg->cpi_alg != CPI_ALG_DIFF) | |
381 | padd = cpi % cpi_count; | |
382 | else | |
383 | padd = cpi % 8; /* 3 bits CS out of 6bits DSCP */ | |
384 | ||
385 | /* Leave RSS_SIZE as '0' to disable RSS */ | |
386 | nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi << 3), | |
387 | (vnic << 24) | (padd << 16) | (rssi_base + rssi)); | |
388 | ||
389 | if ((rssi + 1) >= cfg->rq_cnt) | |
390 | continue; | |
391 | ||
392 | if (cfg->cpi_alg == CPI_ALG_VLAN) | |
393 | rssi++; | |
394 | else if (cfg->cpi_alg == CPI_ALG_VLAN16) | |
395 | rssi = ((cpi - cpi_base) & 0xe) >> 1; | |
396 | else if (cfg->cpi_alg == CPI_ALG_DIFF) | |
397 | rssi = ((cpi - cpi_base) & 0x38) >> 3; | |
398 | } | |
399 | nic->cpi_base[cfg->vf_id] = cpi_base; | |
400 | } | |
401 | ||
402 | /* Responsds to VF with its RSS indirection table size */ | |
403 | static void nic_send_rss_size(struct nicpf *nic, int vf) | |
404 | { | |
405 | union nic_mbx mbx = {}; | |
406 | u64 *msg; | |
407 | ||
408 | msg = (u64 *)&mbx; | |
409 | ||
410 | mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; | |
411 | mbx.rss_size.ind_tbl_size = nic->rss_ind_tbl_size; | |
412 | nic_send_msg_to_vf(nic, vf, &mbx); | |
413 | } | |
414 | ||
415 | /* Receive side scaling configuration | |
416 | * configure: | |
417 | * - RSS index | |
418 | * - indir table i.e hash::RQ mapping | |
419 | * - no of hash bits to consider | |
420 | */ | |
421 | static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg) | |
422 | { | |
423 | u8 qset, idx = 0; | |
424 | u64 cpi_cfg, cpi_base, rssi_base, rssi; | |
425 | ||
426 | cpi_base = nic->cpi_base[cfg->vf_id]; | |
427 | cpi_cfg = nic_reg_read(nic, NIC_PF_CPI_0_2047_CFG | (cpi_base << 3)); | |
428 | rssi_base = (cpi_cfg & 0x0FFF) + cfg->tbl_offset; | |
429 | ||
430 | rssi = rssi_base; | |
431 | qset = cfg->vf_id; | |
432 | ||
433 | for (; rssi < (rssi_base + cfg->tbl_len); rssi++) { | |
434 | nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3), | |
435 | (qset << 3) | (cfg->ind_tbl[idx] & 0x7)); | |
436 | idx++; | |
437 | } | |
438 | ||
439 | cpi_cfg &= ~(0xFULL << 20); | |
440 | cpi_cfg |= (cfg->hash_bits << 20); | |
441 | nic_reg_write(nic, NIC_PF_CPI_0_2047_CFG | (cpi_base << 3), cpi_cfg); | |
442 | } | |
443 | ||
444 | /* 4 level transmit side scheduler configutation | |
445 | * for TNS bypass mode | |
446 | * | |
447 | * Sample configuration for SQ0 | |
448 | * VNIC0-SQ0 -> TL4(0) -> TL3[0] -> TL2[0] -> TL1[0] -> BGX0 | |
449 | * VNIC1-SQ0 -> TL4(8) -> TL3[2] -> TL2[0] -> TL1[0] -> BGX0 | |
450 | * VNIC2-SQ0 -> TL4(16) -> TL3[4] -> TL2[1] -> TL1[0] -> BGX0 | |
451 | * VNIC3-SQ0 -> TL4(24) -> TL3[6] -> TL2[1] -> TL1[0] -> BGX0 | |
452 | * VNIC4-SQ0 -> TL4(512) -> TL3[128] -> TL2[32] -> TL1[1] -> BGX1 | |
453 | * VNIC5-SQ0 -> TL4(520) -> TL3[130] -> TL2[32] -> TL1[1] -> BGX1 | |
454 | * VNIC6-SQ0 -> TL4(528) -> TL3[132] -> TL2[33] -> TL1[1] -> BGX1 | |
455 | * VNIC7-SQ0 -> TL4(536) -> TL3[134] -> TL2[33] -> TL1[1] -> BGX1 | |
456 | */ | |
457 | static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic, u8 sq_idx) | |
458 | { | |
459 | u32 bgx, lmac, chan; | |
460 | u32 tl2, tl3, tl4; | |
461 | u32 rr_quantum; | |
462 | ||
463 | bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); | |
464 | lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); | |
465 | /* 24 bytes for FCS, IPG and preamble */ | |
466 | rr_quantum = ((NIC_HW_MAX_FRS + 24) / 4); | |
467 | ||
468 | tl4 = (lmac * NIC_TL4_PER_LMAC) + (bgx * NIC_TL4_PER_BGX); | |
469 | tl4 += sq_idx; | |
470 | tl3 = tl4 / (NIC_MAX_TL4 / NIC_MAX_TL3); | |
471 | nic_reg_write(nic, NIC_PF_QSET_0_127_SQ_0_7_CFG2 | | |
472 | ((u64)vnic << NIC_QS_ID_SHIFT) | | |
473 | ((u32)sq_idx << NIC_Q_NUM_SHIFT), tl4); | |
474 | nic_reg_write(nic, NIC_PF_TL4_0_1023_CFG | (tl4 << 3), | |
475 | ((u64)vnic << 27) | ((u32)sq_idx << 24) | rr_quantum); | |
476 | ||
477 | nic_reg_write(nic, NIC_PF_TL3_0_255_CFG | (tl3 << 3), rr_quantum); | |
478 | chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF); | |
479 | nic_reg_write(nic, NIC_PF_TL3_0_255_CHAN | (tl3 << 3), chan); | |
480 | /* Enable backpressure on the channel */ | |
481 | nic_reg_write(nic, NIC_PF_CHAN_0_255_TX_CFG | (chan << 3), 1); | |
482 | ||
483 | tl2 = tl3 >> 2; | |
484 | nic_reg_write(nic, NIC_PF_TL3A_0_63_CFG | (tl2 << 3), tl2); | |
485 | nic_reg_write(nic, NIC_PF_TL2_0_63_CFG | (tl2 << 3), rr_quantum); | |
486 | /* No priorities as of now */ | |
487 | nic_reg_write(nic, NIC_PF_TL2_0_63_PRI | (tl2 << 3), 0x00); | |
488 | } | |
489 | ||
490 | /* Interrupt handler to handle mailbox messages from VFs */ | |
491 | static void nic_handle_mbx_intr(struct nicpf *nic, int vf) | |
492 | { | |
493 | union nic_mbx mbx = {}; | |
494 | u64 *mbx_data; | |
495 | u64 mbx_addr; | |
496 | u64 reg_addr; | |
497 | u64 mac_addr; | |
498 | int bgx, lmac; | |
499 | int i; | |
500 | int ret = 0; | |
501 | ||
502 | nic->mbx_lock[vf] = true; | |
503 | ||
504 | mbx_addr = nic_get_mbx_addr(vf); | |
505 | mbx_data = (u64 *)&mbx; | |
506 | ||
507 | for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { | |
508 | *mbx_data = nic_reg_read(nic, mbx_addr); | |
509 | mbx_data++; | |
510 | mbx_addr += sizeof(u64); | |
511 | } | |
512 | ||
513 | dev_dbg(&nic->pdev->dev, "%s: Mailbox msg %d from VF%d\n", | |
514 | __func__, mbx.msg.msg, vf); | |
515 | switch (mbx.msg.msg) { | |
516 | case NIC_MBOX_MSG_READY: | |
517 | nic_mbx_send_ready(nic, vf); | |
518 | nic->link[vf] = 0; | |
519 | nic->duplex[vf] = 0; | |
520 | nic->speed[vf] = 0; | |
521 | ret = 1; | |
522 | break; | |
523 | case NIC_MBOX_MSG_QS_CFG: | |
524 | reg_addr = NIC_PF_QSET_0_127_CFG | | |
525 | (mbx.qs.num << NIC_QS_ID_SHIFT); | |
526 | nic_reg_write(nic, reg_addr, mbx.qs.cfg); | |
527 | break; | |
528 | case NIC_MBOX_MSG_RQ_CFG: | |
529 | reg_addr = NIC_PF_QSET_0_127_RQ_0_7_CFG | | |
530 | (mbx.rq.qs_num << NIC_QS_ID_SHIFT) | | |
531 | (mbx.rq.rq_num << NIC_Q_NUM_SHIFT); | |
532 | nic_reg_write(nic, reg_addr, mbx.rq.cfg); | |
533 | break; | |
534 | case NIC_MBOX_MSG_RQ_BP_CFG: | |
535 | reg_addr = NIC_PF_QSET_0_127_RQ_0_7_BP_CFG | | |
536 | (mbx.rq.qs_num << NIC_QS_ID_SHIFT) | | |
537 | (mbx.rq.rq_num << NIC_Q_NUM_SHIFT); | |
538 | nic_reg_write(nic, reg_addr, mbx.rq.cfg); | |
539 | break; | |
540 | case NIC_MBOX_MSG_RQ_SW_SYNC: | |
541 | ret = nic_rcv_queue_sw_sync(nic); | |
542 | break; | |
543 | case NIC_MBOX_MSG_RQ_DROP_CFG: | |
544 | reg_addr = NIC_PF_QSET_0_127_RQ_0_7_DROP_CFG | | |
545 | (mbx.rq.qs_num << NIC_QS_ID_SHIFT) | | |
546 | (mbx.rq.rq_num << NIC_Q_NUM_SHIFT); | |
547 | nic_reg_write(nic, reg_addr, mbx.rq.cfg); | |
548 | break; | |
549 | case NIC_MBOX_MSG_SQ_CFG: | |
550 | reg_addr = NIC_PF_QSET_0_127_SQ_0_7_CFG | | |
551 | (mbx.sq.qs_num << NIC_QS_ID_SHIFT) | | |
552 | (mbx.sq.sq_num << NIC_Q_NUM_SHIFT); | |
553 | nic_reg_write(nic, reg_addr, mbx.sq.cfg); | |
554 | nic_tx_channel_cfg(nic, mbx.qs.num, mbx.sq.sq_num); | |
555 | break; | |
556 | case NIC_MBOX_MSG_SET_MAC: | |
557 | lmac = mbx.mac.vf_id; | |
558 | bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]); | |
559 | lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]); | |
560 | #ifdef __BIG_ENDIAN | |
561 | mac_addr = cpu_to_be64(mbx.nic_cfg.mac_addr) << 16; | |
562 | #else | |
563 | mac_addr = cpu_to_be64(mbx.nic_cfg.mac_addr) >> 16; | |
564 | #endif | |
565 | bgx_set_lmac_mac(nic->node, bgx, lmac, (u8 *)&mac_addr); | |
566 | break; | |
567 | case NIC_MBOX_MSG_SET_MAX_FRS: | |
568 | ret = nic_update_hw_frs(nic, mbx.frs.max_frs, | |
569 | mbx.frs.vf_id); | |
570 | break; | |
571 | case NIC_MBOX_MSG_CPI_CFG: | |
572 | nic_config_cpi(nic, &mbx.cpi_cfg); | |
573 | break; | |
574 | case NIC_MBOX_MSG_RSS_SIZE: | |
575 | nic_send_rss_size(nic, vf); | |
576 | goto unlock; | |
577 | case NIC_MBOX_MSG_RSS_CFG: | |
578 | case NIC_MBOX_MSG_RSS_CFG_CONT: | |
579 | nic_config_rss(nic, &mbx.rss_cfg); | |
580 | break; | |
581 | case NIC_MBOX_MSG_CFG_DONE: | |
582 | /* Last message of VF config msg sequence */ | |
583 | nic->vf_enabled[vf] = true; | |
584 | goto unlock; | |
585 | case NIC_MBOX_MSG_SHUTDOWN: | |
586 | /* First msg in VF teardown sequence */ | |
587 | nic->vf_enabled[vf] = false; | |
588 | break; | |
589 | case NIC_MBOX_MSG_BGX_STATS: | |
590 | nic_get_bgx_stats(nic, &mbx.bgx_stats); | |
591 | goto unlock; | |
592 | default: | |
593 | dev_err(&nic->pdev->dev, | |
594 | "Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg); | |
595 | break; | |
596 | } | |
597 | ||
598 | if (!ret) | |
599 | nic_mbx_send_ack(nic, vf); | |
600 | else if (mbx.msg.msg != NIC_MBOX_MSG_READY) | |
601 | nic_mbx_send_nack(nic, vf); | |
602 | unlock: | |
603 | nic->mbx_lock[vf] = false; | |
604 | } | |
605 | ||
606 | static void nic_mbx_intr_handler (struct nicpf *nic, int mbx) | |
607 | { | |
608 | u64 intr; | |
609 | u8 vf, vf_per_mbx_reg = 64; | |
610 | ||
611 | intr = nic_reg_read(nic, NIC_PF_MAILBOX_INT + (mbx << 3)); | |
612 | dev_dbg(&nic->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr); | |
613 | for (vf = 0; vf < vf_per_mbx_reg; vf++) { | |
614 | if (intr & (1ULL << vf)) { | |
615 | dev_dbg(&nic->pdev->dev, "Intr from VF %d\n", | |
616 | vf + (mbx * vf_per_mbx_reg)); | |
617 | if ((vf + (mbx * vf_per_mbx_reg)) > nic->num_vf_en) | |
618 | break; | |
619 | nic_handle_mbx_intr(nic, vf + (mbx * vf_per_mbx_reg)); | |
620 | nic_clear_mbx_intr(nic, vf, mbx); | |
621 | } | |
622 | } | |
623 | } | |
624 | ||
625 | static irqreturn_t nic_mbx0_intr_handler (int irq, void *nic_irq) | |
626 | { | |
627 | struct nicpf *nic = (struct nicpf *)nic_irq; | |
628 | ||
629 | nic_mbx_intr_handler(nic, 0); | |
630 | ||
631 | return IRQ_HANDLED; | |
632 | } | |
633 | ||
634 | static irqreturn_t nic_mbx1_intr_handler (int irq, void *nic_irq) | |
635 | { | |
636 | struct nicpf *nic = (struct nicpf *)nic_irq; | |
637 | ||
638 | nic_mbx_intr_handler(nic, 1); | |
639 | ||
640 | return IRQ_HANDLED; | |
641 | } | |
642 | ||
643 | static int nic_enable_msix(struct nicpf *nic) | |
644 | { | |
645 | int i, ret; | |
646 | ||
647 | nic->num_vec = NIC_PF_MSIX_VECTORS; | |
648 | ||
649 | for (i = 0; i < nic->num_vec; i++) | |
650 | nic->msix_entries[i].entry = i; | |
651 | ||
652 | ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec); | |
653 | if (ret) { | |
654 | dev_err(&nic->pdev->dev, | |
655 | "Request for #%d msix vectors failed\n", | |
656 | nic->num_vec); | |
657 | return ret; | |
658 | } | |
659 | ||
660 | nic->msix_enabled = 1; | |
661 | return 0; | |
662 | } | |
663 | ||
664 | static void nic_disable_msix(struct nicpf *nic) | |
665 | { | |
666 | if (nic->msix_enabled) { | |
667 | pci_disable_msix(nic->pdev); | |
668 | nic->msix_enabled = 0; | |
669 | nic->num_vec = 0; | |
670 | } | |
671 | } | |
672 | ||
673 | static void nic_free_all_interrupts(struct nicpf *nic) | |
674 | { | |
675 | int irq; | |
676 | ||
677 | for (irq = 0; irq < nic->num_vec; irq++) { | |
678 | if (nic->irq_allocated[irq]) | |
679 | free_irq(nic->msix_entries[irq].vector, nic); | |
680 | nic->irq_allocated[irq] = false; | |
681 | } | |
682 | } | |
683 | ||
684 | static int nic_register_interrupts(struct nicpf *nic) | |
685 | { | |
686 | int ret; | |
687 | ||
688 | /* Enable MSI-X */ | |
689 | ret = nic_enable_msix(nic); | |
690 | if (ret) | |
691 | return ret; | |
692 | ||
693 | /* Register mailbox interrupt handlers */ | |
694 | ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX0].vector, | |
695 | nic_mbx0_intr_handler, 0, "NIC Mbox0", nic); | |
696 | if (ret) | |
697 | goto fail; | |
698 | ||
699 | nic->irq_allocated[NIC_PF_INTR_ID_MBOX0] = true; | |
700 | ||
701 | ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX1].vector, | |
702 | nic_mbx1_intr_handler, 0, "NIC Mbox1", nic); | |
703 | if (ret) | |
704 | goto fail; | |
705 | ||
706 | nic->irq_allocated[NIC_PF_INTR_ID_MBOX1] = true; | |
707 | ||
708 | /* Enable mailbox interrupt */ | |
709 | nic_enable_mbx_intr(nic); | |
710 | return 0; | |
711 | ||
712 | fail: | |
713 | dev_err(&nic->pdev->dev, "Request irq failed\n"); | |
714 | nic_free_all_interrupts(nic); | |
715 | return ret; | |
716 | } | |
717 | ||
718 | static void nic_unregister_interrupts(struct nicpf *nic) | |
719 | { | |
720 | nic_free_all_interrupts(nic); | |
721 | nic_disable_msix(nic); | |
722 | } | |
723 | ||
724 | static int nic_sriov_init(struct pci_dev *pdev, struct nicpf *nic) | |
725 | { | |
726 | int pos = 0; | |
727 | int err; | |
728 | u16 total_vf_cnt; | |
729 | ||
730 | pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); | |
731 | if (!pos) { | |
732 | dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n"); | |
733 | return -ENODEV; | |
734 | } | |
735 | ||
736 | pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt); | |
737 | if (total_vf_cnt < nic->num_vf_en) | |
738 | nic->num_vf_en = total_vf_cnt; | |
739 | ||
740 | if (!total_vf_cnt) | |
741 | return 0; | |
742 | ||
743 | err = pci_enable_sriov(pdev, nic->num_vf_en); | |
744 | if (err) { | |
745 | dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n", | |
746 | nic->num_vf_en); | |
747 | nic->num_vf_en = 0; | |
748 | return err; | |
749 | } | |
750 | ||
751 | dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n", | |
752 | nic->num_vf_en); | |
753 | ||
754 | nic->flags |= NIC_SRIOV_ENABLED; | |
755 | return 0; | |
756 | } | |
757 | ||
758 | /* Poll for BGX LMAC link status and update corresponding VF | |
759 | * if there is a change, valid only if internal L2 switch | |
760 | * is not present otherwise VF link is always treated as up | |
761 | */ | |
762 | static void nic_poll_for_link(struct work_struct *work) | |
763 | { | |
764 | union nic_mbx mbx = {}; | |
765 | struct nicpf *nic; | |
766 | struct bgx_link_status link; | |
767 | u8 vf, bgx, lmac; | |
768 | ||
769 | nic = container_of(work, struct nicpf, dwork.work); | |
770 | ||
771 | mbx.link_status.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE; | |
772 | ||
773 | for (vf = 0; vf < nic->num_vf_en; vf++) { | |
774 | /* Poll only if VF is UP */ | |
775 | if (!nic->vf_enabled[vf]) | |
776 | continue; | |
777 | ||
778 | /* Get BGX, LMAC indices for the VF */ | |
779 | bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); | |
780 | lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); | |
781 | /* Get interface link status */ | |
782 | bgx_get_lmac_link_state(nic->node, bgx, lmac, &link); | |
783 | ||
784 | /* Inform VF only if link status changed */ | |
785 | if (nic->link[vf] == link.link_up) | |
786 | continue; | |
787 | ||
788 | if (!nic->mbx_lock[vf]) { | |
789 | nic->link[vf] = link.link_up; | |
790 | nic->duplex[vf] = link.duplex; | |
791 | nic->speed[vf] = link.speed; | |
792 | ||
793 | /* Send a mbox message to VF with current link status */ | |
794 | mbx.link_status.link_up = link.link_up; | |
795 | mbx.link_status.duplex = link.duplex; | |
796 | mbx.link_status.speed = link.speed; | |
797 | nic_send_msg_to_vf(nic, vf, &mbx); | |
798 | } | |
799 | } | |
800 | queue_delayed_work(nic->check_link, &nic->dwork, HZ * 2); | |
801 | } | |
802 | ||
803 | static int nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) | |
804 | { | |
805 | struct device *dev = &pdev->dev; | |
806 | struct nicpf *nic; | |
807 | int err; | |
808 | ||
809 | BUILD_BUG_ON(sizeof(union nic_mbx) > 16); | |
810 | ||
811 | nic = devm_kzalloc(dev, sizeof(*nic), GFP_KERNEL); | |
812 | if (!nic) | |
813 | return -ENOMEM; | |
814 | ||
815 | pci_set_drvdata(pdev, nic); | |
816 | ||
817 | nic->pdev = pdev; | |
818 | ||
819 | err = pci_enable_device(pdev); | |
820 | if (err) { | |
821 | dev_err(dev, "Failed to enable PCI device\n"); | |
822 | pci_set_drvdata(pdev, NULL); | |
823 | return err; | |
824 | } | |
825 | ||
826 | err = pci_request_regions(pdev, DRV_NAME); | |
827 | if (err) { | |
828 | dev_err(dev, "PCI request regions failed 0x%x\n", err); | |
829 | goto err_disable_device; | |
830 | } | |
831 | ||
832 | err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); | |
833 | if (err) { | |
834 | dev_err(dev, "Unable to get usable DMA configuration\n"); | |
835 | goto err_release_regions; | |
836 | } | |
837 | ||
838 | err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); | |
839 | if (err) { | |
840 | dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); | |
841 | goto err_release_regions; | |
842 | } | |
843 | ||
844 | /* MAP PF's configuration registers */ | |
845 | nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); | |
846 | if (!nic->reg_base) { | |
847 | dev_err(dev, "Cannot map config register space, aborting\n"); | |
848 | err = -ENOMEM; | |
849 | goto err_release_regions; | |
850 | } | |
851 | ||
852 | pci_read_config_byte(pdev, PCI_REVISION_ID, &nic->rev_id); | |
853 | ||
854 | nic->node = NIC_NODE_ID(pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM)); | |
855 | ||
856 | nic_set_lmac_vf_mapping(nic); | |
857 | ||
858 | /* Initialize hardware */ | |
859 | nic_init_hw(nic); | |
860 | ||
861 | /* Set RSS TBL size for each VF */ | |
862 | nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE; | |
863 | ||
864 | /* Register interrupts */ | |
865 | err = nic_register_interrupts(nic); | |
866 | if (err) | |
867 | goto err_release_regions; | |
868 | ||
869 | /* Configure SRIOV */ | |
870 | err = nic_sriov_init(pdev, nic); | |
871 | if (err) | |
872 | goto err_unregister_interrupts; | |
873 | ||
874 | /* Register a physical link status poll fn() */ | |
875 | nic->check_link = alloc_workqueue("check_link_status", | |
876 | WQ_UNBOUND | WQ_MEM_RECLAIM, 1); | |
877 | if (!nic->check_link) { | |
878 | err = -ENOMEM; | |
879 | goto err_disable_sriov; | |
880 | } | |
881 | ||
882 | INIT_DELAYED_WORK(&nic->dwork, nic_poll_for_link); | |
883 | queue_delayed_work(nic->check_link, &nic->dwork, 0); | |
884 | ||
885 | return 0; | |
886 | ||
887 | err_disable_sriov: | |
888 | if (nic->flags & NIC_SRIOV_ENABLED) | |
889 | pci_disable_sriov(pdev); | |
890 | err_unregister_interrupts: | |
891 | nic_unregister_interrupts(nic); | |
892 | err_release_regions: | |
893 | pci_release_regions(pdev); | |
894 | err_disable_device: | |
895 | pci_disable_device(pdev); | |
896 | pci_set_drvdata(pdev, NULL); | |
897 | return err; | |
898 | } | |
899 | ||
900 | static void nic_remove(struct pci_dev *pdev) | |
901 | { | |
902 | struct nicpf *nic = pci_get_drvdata(pdev); | |
903 | ||
904 | if (nic->flags & NIC_SRIOV_ENABLED) | |
905 | pci_disable_sriov(pdev); | |
906 | ||
907 | if (nic->check_link) { | |
908 | /* Destroy work Queue */ | |
909 | cancel_delayed_work(&nic->dwork); | |
910 | flush_workqueue(nic->check_link); | |
911 | destroy_workqueue(nic->check_link); | |
912 | } | |
913 | ||
914 | nic_unregister_interrupts(nic); | |
915 | pci_release_regions(pdev); | |
916 | pci_disable_device(pdev); | |
917 | pci_set_drvdata(pdev, NULL); | |
918 | } | |
919 | ||
920 | static struct pci_driver nic_driver = { | |
921 | .name = DRV_NAME, | |
922 | .id_table = nic_id_table, | |
923 | .probe = nic_probe, | |
924 | .remove = nic_remove, | |
925 | }; | |
926 | ||
927 | static int __init nic_init_module(void) | |
928 | { | |
929 | pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION); | |
930 | ||
931 | return pci_register_driver(&nic_driver); | |
932 | } | |
933 | ||
934 | static void __exit nic_cleanup_module(void) | |
935 | { | |
936 | pci_unregister_driver(&nic_driver); | |
937 | } | |
938 | ||
939 | module_init(nic_init_module); | |
940 | module_exit(nic_cleanup_module); |