]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2014-2018 Chelsio Communications. | |
3 | * All rights reserved. | |
7c673cae FG |
4 | */ |
5 | ||
6 | #include <sys/queue.h> | |
7 | #include <stdio.h> | |
8 | #include <errno.h> | |
9 | #include <stdint.h> | |
10 | #include <string.h> | |
11 | #include <unistd.h> | |
12 | #include <stdarg.h> | |
13 | #include <inttypes.h> | |
14 | #include <netinet/in.h> | |
15 | ||
16 | #include <rte_byteorder.h> | |
17 | #include <rte_common.h> | |
18 | #include <rte_cycles.h> | |
19 | #include <rte_interrupts.h> | |
20 | #include <rte_log.h> | |
21 | #include <rte_debug.h> | |
22 | #include <rte_pci.h> | |
11fdf7f2 | 23 | #include <rte_bus_pci.h> |
7c673cae FG |
24 | #include <rte_atomic.h> |
25 | #include <rte_branch_prediction.h> | |
26 | #include <rte_memory.h> | |
7c673cae FG |
27 | #include <rte_tailq.h> |
28 | #include <rte_eal.h> | |
29 | #include <rte_alarm.h> | |
30 | #include <rte_ether.h> | |
11fdf7f2 TL |
31 | #include <rte_ethdev_driver.h> |
32 | #include <rte_ethdev_pci.h> | |
7c673cae FG |
33 | #include <rte_malloc.h> |
34 | #include <rte_random.h> | |
35 | #include <rte_dev.h> | |
36 | ||
37 | #include "cxgbe.h" | |
11fdf7f2 TL |
38 | #include "cxgbe_pfvf.h" |
39 | #include "cxgbe_flow.h" | |
7c673cae FG |
40 | |
41 | /* | |
42 | * Macros needed to support the PCI Device ID Table ... | |
43 | */ | |
44 | #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ | |
11fdf7f2 | 45 | static const struct rte_pci_id cxgb4_pci_tbl[] = { |
7c673cae FG |
46 | #define CH_PCI_DEVICE_ID_FUNCTION 0x4 |
47 | ||
48 | #define PCI_VENDOR_ID_CHELSIO 0x1425 | |
49 | ||
50 | #define CH_PCI_ID_TABLE_ENTRY(devid) \ | |
51 | { RTE_PCI_DEVICE(PCI_VENDOR_ID_CHELSIO, (devid)) } | |
52 | ||
53 | #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ | |
54 | { .vendor_id = 0, } \ | |
55 | } | |
56 | ||
57 | /* | |
58 | *... and the PCI ID Table itself ... | |
59 | */ | |
9f95a23c | 60 | #include "base/t4_pci_id_tbl.h" |
11fdf7f2 TL |
61 | |
62 | uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, | |
63 | uint16_t nb_pkts) | |
7c673cae FG |
64 | { |
65 | struct sge_eth_txq *txq = (struct sge_eth_txq *)tx_queue; | |
66 | uint16_t pkts_sent, pkts_remain; | |
67 | uint16_t total_sent = 0; | |
68 | int ret = 0; | |
69 | ||
70 | CXGBE_DEBUG_TX(adapter, "%s: txq = %p; tx_pkts = %p; nb_pkts = %d\n", | |
71 | __func__, txq, tx_pkts, nb_pkts); | |
72 | ||
73 | t4_os_lock(&txq->txq_lock); | |
74 | /* free up desc from already completed tx */ | |
75 | reclaim_completed_tx(&txq->q); | |
76 | while (total_sent < nb_pkts) { | |
77 | pkts_remain = nb_pkts - total_sent; | |
78 | ||
79 | for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) { | |
11fdf7f2 TL |
80 | ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent], |
81 | nb_pkts); | |
7c673cae FG |
82 | if (ret < 0) |
83 | break; | |
84 | } | |
85 | if (!pkts_sent) | |
86 | break; | |
87 | total_sent += pkts_sent; | |
88 | /* reclaim as much as possible */ | |
89 | reclaim_completed_tx(&txq->q); | |
90 | } | |
91 | ||
92 | t4_os_unlock(&txq->txq_lock); | |
93 | return total_sent; | |
94 | } | |
95 | ||
11fdf7f2 TL |
96 | uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, |
97 | uint16_t nb_pkts) | |
7c673cae FG |
98 | { |
99 | struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue; | |
100 | unsigned int work_done; | |
101 | ||
102 | CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n", | |
103 | __func__, rxq->rspq.cntxt_id, nb_pkts); | |
104 | ||
105 | if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done)) | |
106 | dev_err(adapter, "error in cxgbe poll\n"); | |
107 | ||
108 | CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done); | |
109 | return work_done; | |
110 | } | |
111 | ||
11fdf7f2 TL |
112 | void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev, |
113 | struct rte_eth_dev_info *device_info) | |
7c673cae FG |
114 | { |
115 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
116 | struct adapter *adapter = pi->adapter; | |
117 | int max_queues = adapter->sge.max_ethqsets / adapter->params.nports; | |
118 | ||
119 | static const struct rte_eth_desc_lim cxgbe_desc_lim = { | |
120 | .nb_max = CXGBE_MAX_RING_DESC_SIZE, | |
121 | .nb_min = CXGBE_MIN_RING_DESC_SIZE, | |
122 | .nb_align = 1, | |
123 | }; | |
124 | ||
125 | device_info->min_rx_bufsize = CXGBE_MIN_RX_BUFSIZE; | |
126 | device_info->max_rx_pktlen = CXGBE_MAX_RX_PKTLEN; | |
127 | device_info->max_rx_queues = max_queues; | |
128 | device_info->max_tx_queues = max_queues; | |
129 | device_info->max_mac_addrs = 1; | |
130 | /* XXX: For now we support one MAC/port */ | |
131 | device_info->max_vfs = adapter->params.arch.vfcount; | |
132 | device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */ | |
133 | ||
11fdf7f2 TL |
134 | device_info->rx_queue_offload_capa = 0UL; |
135 | device_info->rx_offload_capa = CXGBE_RX_OFFLOADS; | |
7c673cae | 136 | |
11fdf7f2 TL |
137 | device_info->tx_queue_offload_capa = 0UL; |
138 | device_info->tx_offload_capa = CXGBE_TX_OFFLOADS; | |
7c673cae FG |
139 | |
140 | device_info->reta_size = pi->rss_size; | |
11fdf7f2 TL |
141 | device_info->hash_key_size = CXGBE_DEFAULT_RSS_KEY_LEN; |
142 | device_info->flow_type_rss_offloads = CXGBE_RSS_HF_ALL; | |
7c673cae FG |
143 | |
144 | device_info->rx_desc_lim = cxgbe_desc_lim; | |
145 | device_info->tx_desc_lim = cxgbe_desc_lim; | |
11fdf7f2 | 146 | cxgbe_get_speed_caps(pi, &device_info->speed_capa); |
7c673cae FG |
147 | } |
148 | ||
11fdf7f2 | 149 | void cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) |
7c673cae FG |
150 | { |
151 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
152 | struct adapter *adapter = pi->adapter; | |
153 | ||
154 | t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1, | |
155 | 1, -1, 1, -1, false); | |
156 | } | |
157 | ||
11fdf7f2 | 158 | void cxgbe_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) |
7c673cae FG |
159 | { |
160 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
161 | struct adapter *adapter = pi->adapter; | |
162 | ||
163 | t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1, | |
164 | 0, -1, 1, -1, false); | |
165 | } | |
166 | ||
11fdf7f2 | 167 | void cxgbe_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) |
7c673cae FG |
168 | { |
169 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
170 | struct adapter *adapter = pi->adapter; | |
171 | ||
172 | /* TODO: address filters ?? */ | |
173 | ||
174 | t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1, | |
175 | -1, 1, 1, -1, false); | |
176 | } | |
177 | ||
11fdf7f2 | 178 | void cxgbe_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) |
7c673cae FG |
179 | { |
180 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
181 | struct adapter *adapter = pi->adapter; | |
182 | ||
183 | /* TODO: address filters ?? */ | |
184 | ||
185 | t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1, | |
186 | -1, 0, 1, -1, false); | |
187 | } | |
188 | ||
11fdf7f2 TL |
189 | int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev, |
190 | int wait_to_complete) | |
7c673cae FG |
191 | { |
192 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
193 | struct adapter *adapter = pi->adapter; | |
194 | struct sge *s = &adapter->sge; | |
11fdf7f2 TL |
195 | struct rte_eth_link new_link = { 0 }; |
196 | unsigned int i, work_done, budget = 32; | |
197 | u8 old_link = pi->link_cfg.link_ok; | |
198 | ||
199 | for (i = 0; i < CXGBE_LINK_STATUS_POLL_CNT; i++) { | |
200 | cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done); | |
201 | ||
202 | /* Exit if link status changed or always forced up */ | |
9f95a23c TL |
203 | if (pi->link_cfg.link_ok != old_link || |
204 | cxgbe_force_linkup(adapter)) | |
11fdf7f2 TL |
205 | break; |
206 | ||
207 | if (!wait_to_complete) | |
208 | break; | |
209 | ||
210 | rte_delay_ms(CXGBE_LINK_STATUS_POLL_MS); | |
211 | } | |
212 | ||
9f95a23c | 213 | new_link.link_status = cxgbe_force_linkup(adapter) ? |
11fdf7f2 TL |
214 | ETH_LINK_UP : pi->link_cfg.link_ok; |
215 | new_link.link_autoneg = pi->link_cfg.autoneg; | |
216 | new_link.link_duplex = ETH_LINK_FULL_DUPLEX; | |
217 | new_link.link_speed = pi->link_cfg.speed; | |
218 | ||
219 | return rte_eth_linkstatus_set(eth_dev, &new_link); | |
220 | } | |
221 | ||
222 | /** | |
223 | * Set device link up. | |
224 | */ | |
225 | int cxgbe_dev_set_link_up(struct rte_eth_dev *dev) | |
226 | { | |
227 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
228 | struct adapter *adapter = pi->adapter; | |
229 | unsigned int work_done, budget = 32; | |
230 | struct sge *s = &adapter->sge; | |
231 | int ret; | |
232 | ||
233 | /* Flush all link events */ | |
234 | cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done); | |
235 | ||
236 | /* If link already up, nothing to do */ | |
237 | if (pi->link_cfg.link_ok) | |
238 | return 0; | |
239 | ||
240 | ret = cxgbe_set_link_status(pi, true); | |
241 | if (ret) | |
242 | return ret; | |
243 | ||
244 | cxgbe_dev_link_update(dev, 1); | |
245 | return 0; | |
246 | } | |
247 | ||
248 | /** | |
249 | * Set device link down. | |
250 | */ | |
251 | int cxgbe_dev_set_link_down(struct rte_eth_dev *dev) | |
252 | { | |
253 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
254 | struct adapter *adapter = pi->adapter; | |
255 | unsigned int work_done, budget = 32; | |
256 | struct sge *s = &adapter->sge; | |
257 | int ret; | |
7c673cae | 258 | |
11fdf7f2 | 259 | /* Flush all link events */ |
7c673cae | 260 | cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done); |
7c673cae | 261 | |
11fdf7f2 TL |
262 | /* If link already down, nothing to do */ |
263 | if (!pi->link_cfg.link_ok) | |
264 | return 0; | |
265 | ||
266 | ret = cxgbe_set_link_status(pi, false); | |
267 | if (ret) | |
268 | return ret; | |
7c673cae | 269 | |
11fdf7f2 | 270 | cxgbe_dev_link_update(dev, 0); |
7c673cae FG |
271 | return 0; |
272 | } | |
273 | ||
11fdf7f2 | 274 | int cxgbe_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) |
7c673cae FG |
275 | { |
276 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
277 | struct adapter *adapter = pi->adapter; | |
278 | struct rte_eth_dev_info dev_info; | |
279 | int err; | |
280 | uint16_t new_mtu = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; | |
281 | ||
282 | cxgbe_dev_info_get(eth_dev, &dev_info); | |
283 | ||
284 | /* Must accommodate at least ETHER_MIN_MTU */ | |
285 | if ((new_mtu < ETHER_MIN_MTU) || (new_mtu > dev_info.max_rx_pktlen)) | |
286 | return -EINVAL; | |
287 | ||
288 | /* set to jumbo mode if needed */ | |
289 | if (new_mtu > ETHER_MAX_LEN) | |
11fdf7f2 TL |
290 | eth_dev->data->dev_conf.rxmode.offloads |= |
291 | DEV_RX_OFFLOAD_JUMBO_FRAME; | |
7c673cae | 292 | else |
11fdf7f2 TL |
293 | eth_dev->data->dev_conf.rxmode.offloads &= |
294 | ~DEV_RX_OFFLOAD_JUMBO_FRAME; | |
7c673cae FG |
295 | |
296 | err = t4_set_rxmode(adapter, adapter->mbox, pi->viid, new_mtu, -1, -1, | |
297 | -1, -1, true); | |
298 | if (!err) | |
299 | eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_mtu; | |
300 | ||
301 | return err; | |
302 | } | |
303 | ||
7c673cae FG |
304 | /* |
305 | * Stop device. | |
306 | */ | |
11fdf7f2 | 307 | void cxgbe_dev_close(struct rte_eth_dev *eth_dev) |
7c673cae FG |
308 | { |
309 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
310 | struct adapter *adapter = pi->adapter; | |
7c673cae FG |
311 | |
312 | CXGBE_FUNC_TRACE(); | |
313 | ||
314 | if (!(adapter->flags & FULL_INIT_DONE)) | |
315 | return; | |
316 | ||
317 | cxgbe_down(pi); | |
318 | ||
319 | /* | |
320 | * We clear queues only if both tx and rx path of the port | |
321 | * have been disabled | |
322 | */ | |
323 | t4_sge_eth_clear_queues(pi); | |
7c673cae FG |
324 | } |
325 | ||
326 | /* Start the device. | |
327 | * It returns 0 on success. | |
328 | */ | |
11fdf7f2 | 329 | int cxgbe_dev_start(struct rte_eth_dev *eth_dev) |
7c673cae FG |
330 | { |
331 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
9f95a23c | 332 | struct rte_eth_rxmode *rx_conf = ð_dev->data->dev_conf.rxmode; |
7c673cae FG |
333 | struct adapter *adapter = pi->adapter; |
334 | int err = 0, i; | |
335 | ||
336 | CXGBE_FUNC_TRACE(); | |
337 | ||
338 | /* | |
339 | * If we don't have a connection to the firmware there's nothing we | |
340 | * can do. | |
341 | */ | |
342 | if (!(adapter->flags & FW_OK)) { | |
343 | err = -ENXIO; | |
344 | goto out; | |
345 | } | |
346 | ||
347 | if (!(adapter->flags & FULL_INIT_DONE)) { | |
348 | err = cxgbe_up(adapter); | |
349 | if (err < 0) | |
350 | goto out; | |
351 | } | |
352 | ||
9f95a23c TL |
353 | if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) |
354 | eth_dev->data->scattered_rx = 1; | |
355 | else | |
356 | eth_dev->data->scattered_rx = 0; | |
357 | ||
11fdf7f2 TL |
358 | cxgbe_enable_rx_queues(pi); |
359 | ||
9f95a23c | 360 | err = cxgbe_setup_rss(pi); |
7c673cae FG |
361 | if (err) |
362 | goto out; | |
363 | ||
364 | for (i = 0; i < pi->n_tx_qsets; i++) { | |
365 | err = cxgbe_dev_tx_queue_start(eth_dev, i); | |
366 | if (err) | |
367 | goto out; | |
368 | } | |
369 | ||
370 | for (i = 0; i < pi->n_rx_qsets; i++) { | |
371 | err = cxgbe_dev_rx_queue_start(eth_dev, i); | |
372 | if (err) | |
373 | goto out; | |
374 | } | |
375 | ||
9f95a23c | 376 | err = cxgbe_link_start(pi); |
7c673cae FG |
377 | if (err) |
378 | goto out; | |
379 | ||
380 | out: | |
381 | return err; | |
382 | } | |
383 | ||
384 | /* | |
385 | * Stop device: disable rx and tx functions to allow for reconfiguring. | |
386 | */ | |
11fdf7f2 | 387 | void cxgbe_dev_stop(struct rte_eth_dev *eth_dev) |
7c673cae FG |
388 | { |
389 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
390 | struct adapter *adapter = pi->adapter; | |
391 | ||
392 | CXGBE_FUNC_TRACE(); | |
393 | ||
394 | if (!(adapter->flags & FULL_INIT_DONE)) | |
395 | return; | |
396 | ||
397 | cxgbe_down(pi); | |
398 | ||
399 | /* | |
400 | * We clear queues only if both tx and rx path of the port | |
401 | * have been disabled | |
402 | */ | |
403 | t4_sge_eth_clear_queues(pi); | |
9f95a23c | 404 | eth_dev->data->scattered_rx = 0; |
7c673cae FG |
405 | } |
406 | ||
11fdf7f2 | 407 | int cxgbe_dev_configure(struct rte_eth_dev *eth_dev) |
7c673cae FG |
408 | { |
409 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
410 | struct adapter *adapter = pi->adapter; | |
411 | int err; | |
412 | ||
413 | CXGBE_FUNC_TRACE(); | |
414 | ||
415 | if (!(adapter->flags & FW_QUEUE_BOUND)) { | |
9f95a23c | 416 | err = cxgbe_setup_sge_fwevtq(adapter); |
7c673cae FG |
417 | if (err) |
418 | return err; | |
419 | adapter->flags |= FW_QUEUE_BOUND; | |
11fdf7f2 | 420 | if (is_pf4(adapter)) { |
9f95a23c | 421 | err = cxgbe_setup_sge_ctrl_txq(adapter); |
11fdf7f2 TL |
422 | if (err) |
423 | return err; | |
424 | } | |
7c673cae FG |
425 | } |
426 | ||
9f95a23c | 427 | err = cxgbe_cfg_queue_count(eth_dev); |
7c673cae FG |
428 | if (err) |
429 | return err; | |
430 | ||
431 | return 0; | |
432 | } | |
433 | ||
11fdf7f2 | 434 | int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) |
7c673cae FG |
435 | { |
436 | int ret; | |
437 | struct sge_eth_txq *txq = (struct sge_eth_txq *) | |
438 | (eth_dev->data->tx_queues[tx_queue_id]); | |
439 | ||
440 | dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id); | |
441 | ||
442 | ret = t4_sge_eth_txq_start(txq); | |
443 | if (ret == 0) | |
444 | eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; | |
445 | ||
446 | return ret; | |
447 | } | |
448 | ||
11fdf7f2 | 449 | int cxgbe_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) |
7c673cae FG |
450 | { |
451 | int ret; | |
452 | struct sge_eth_txq *txq = (struct sge_eth_txq *) | |
453 | (eth_dev->data->tx_queues[tx_queue_id]); | |
454 | ||
455 | dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id); | |
456 | ||
457 | ret = t4_sge_eth_txq_stop(txq); | |
458 | if (ret == 0) | |
459 | eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; | |
460 | ||
461 | return ret; | |
462 | } | |
463 | ||
11fdf7f2 TL |
464 | int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, |
465 | uint16_t queue_idx, uint16_t nb_desc, | |
466 | unsigned int socket_id, | |
467 | const struct rte_eth_txconf *tx_conf __rte_unused) | |
7c673cae FG |
468 | { |
469 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
470 | struct adapter *adapter = pi->adapter; | |
471 | struct sge *s = &adapter->sge; | |
472 | struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset + queue_idx]; | |
473 | int err = 0; | |
474 | unsigned int temp_nb_desc; | |
475 | ||
7c673cae FG |
476 | dev_debug(adapter, "%s: eth_dev->data->nb_tx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; pi->first_qset = %u\n", |
477 | __func__, eth_dev->data->nb_tx_queues, queue_idx, nb_desc, | |
478 | socket_id, pi->first_qset); | |
479 | ||
480 | /* Free up the existing queue */ | |
481 | if (eth_dev->data->tx_queues[queue_idx]) { | |
482 | cxgbe_dev_tx_queue_release(eth_dev->data->tx_queues[queue_idx]); | |
483 | eth_dev->data->tx_queues[queue_idx] = NULL; | |
484 | } | |
485 | ||
486 | eth_dev->data->tx_queues[queue_idx] = (void *)txq; | |
487 | ||
488 | /* Sanity Checking | |
489 | * | |
490 | * nb_desc should be > 1023 and <= CXGBE_MAX_RING_DESC_SIZE | |
491 | */ | |
492 | temp_nb_desc = nb_desc; | |
493 | if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) { | |
494 | dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n", | |
495 | __func__, CXGBE_MIN_RING_DESC_SIZE, | |
496 | CXGBE_DEFAULT_TX_DESC_SIZE); | |
497 | temp_nb_desc = CXGBE_DEFAULT_TX_DESC_SIZE; | |
498 | } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) { | |
499 | dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n", | |
500 | __func__, CXGBE_MIN_RING_DESC_SIZE, | |
501 | CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_TX_DESC_SIZE); | |
502 | return -(EINVAL); | |
503 | } | |
504 | ||
505 | txq->q.size = temp_nb_desc; | |
506 | ||
507 | err = t4_sge_alloc_eth_txq(adapter, txq, eth_dev, queue_idx, | |
508 | s->fw_evtq.cntxt_id, socket_id); | |
509 | ||
11fdf7f2 TL |
510 | dev_debug(adapter, "%s: txq->q.cntxt_id= %u txq->q.abs_id= %u err = %d\n", |
511 | __func__, txq->q.cntxt_id, txq->q.abs_id, err); | |
7c673cae FG |
512 | return err; |
513 | } | |
514 | ||
11fdf7f2 | 515 | void cxgbe_dev_tx_queue_release(void *q) |
7c673cae FG |
516 | { |
517 | struct sge_eth_txq *txq = (struct sge_eth_txq *)q; | |
518 | ||
519 | if (txq) { | |
520 | struct port_info *pi = (struct port_info *) | |
521 | (txq->eth_dev->data->dev_private); | |
522 | struct adapter *adap = pi->adapter; | |
523 | ||
524 | dev_debug(adapter, "%s: pi->port_id = %d; tx_queue_id = %d\n", | |
525 | __func__, pi->port_id, txq->q.cntxt_id); | |
526 | ||
527 | t4_sge_eth_txq_release(adap, txq); | |
528 | } | |
529 | } | |
530 | ||
11fdf7f2 | 531 | int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) |
7c673cae FG |
532 | { |
533 | int ret; | |
534 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
535 | struct adapter *adap = pi->adapter; | |
536 | struct sge_rspq *q; | |
537 | ||
538 | dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n", | |
539 | __func__, pi->port_id, rx_queue_id); | |
540 | ||
541 | q = eth_dev->data->rx_queues[rx_queue_id]; | |
542 | ||
543 | ret = t4_sge_eth_rxq_start(adap, q); | |
544 | if (ret == 0) | |
545 | eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; | |
546 | ||
547 | return ret; | |
548 | } | |
549 | ||
11fdf7f2 | 550 | int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) |
7c673cae FG |
551 | { |
552 | int ret; | |
553 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
554 | struct adapter *adap = pi->adapter; | |
555 | struct sge_rspq *q; | |
556 | ||
557 | dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n", | |
558 | __func__, pi->port_id, rx_queue_id); | |
559 | ||
560 | q = eth_dev->data->rx_queues[rx_queue_id]; | |
561 | ret = t4_sge_eth_rxq_stop(adap, q); | |
562 | if (ret == 0) | |
563 | eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; | |
564 | ||
565 | return ret; | |
566 | } | |
567 | ||
11fdf7f2 TL |
568 | int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, |
569 | uint16_t queue_idx, uint16_t nb_desc, | |
570 | unsigned int socket_id, | |
571 | const struct rte_eth_rxconf *rx_conf __rte_unused, | |
572 | struct rte_mempool *mp) | |
7c673cae FG |
573 | { |
574 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
575 | struct adapter *adapter = pi->adapter; | |
576 | struct sge *s = &adapter->sge; | |
577 | struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx]; | |
578 | int err = 0; | |
579 | int msi_idx = 0; | |
580 | unsigned int temp_nb_desc; | |
581 | struct rte_eth_dev_info dev_info; | |
582 | unsigned int pkt_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; | |
583 | ||
7c673cae FG |
584 | dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n", |
585 | __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc, | |
586 | socket_id, mp); | |
587 | ||
588 | cxgbe_dev_info_get(eth_dev, &dev_info); | |
589 | ||
590 | /* Must accommodate at least ETHER_MIN_MTU */ | |
591 | if ((pkt_len < dev_info.min_rx_bufsize) || | |
592 | (pkt_len > dev_info.max_rx_pktlen)) { | |
593 | dev_err(adap, "%s: max pkt len must be > %d and <= %d\n", | |
594 | __func__, dev_info.min_rx_bufsize, | |
595 | dev_info.max_rx_pktlen); | |
596 | return -EINVAL; | |
597 | } | |
598 | ||
599 | /* Free up the existing queue */ | |
600 | if (eth_dev->data->rx_queues[queue_idx]) { | |
601 | cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]); | |
602 | eth_dev->data->rx_queues[queue_idx] = NULL; | |
603 | } | |
604 | ||
605 | eth_dev->data->rx_queues[queue_idx] = (void *)rxq; | |
606 | ||
607 | /* Sanity Checking | |
608 | * | |
609 | * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE | |
610 | */ | |
611 | temp_nb_desc = nb_desc; | |
612 | if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) { | |
613 | dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n", | |
614 | __func__, CXGBE_MIN_RING_DESC_SIZE, | |
615 | CXGBE_DEFAULT_RX_DESC_SIZE); | |
616 | temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE; | |
617 | } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) { | |
618 | dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n", | |
619 | __func__, CXGBE_MIN_RING_DESC_SIZE, | |
620 | CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE); | |
621 | return -(EINVAL); | |
622 | } | |
623 | ||
624 | rxq->rspq.size = temp_nb_desc; | |
625 | if ((&rxq->fl) != NULL) | |
626 | rxq->fl.size = temp_nb_desc; | |
627 | ||
628 | /* Set to jumbo mode if necessary */ | |
629 | if (pkt_len > ETHER_MAX_LEN) | |
11fdf7f2 TL |
630 | eth_dev->data->dev_conf.rxmode.offloads |= |
631 | DEV_RX_OFFLOAD_JUMBO_FRAME; | |
7c673cae | 632 | else |
11fdf7f2 TL |
633 | eth_dev->data->dev_conf.rxmode.offloads &= |
634 | ~DEV_RX_OFFLOAD_JUMBO_FRAME; | |
7c673cae FG |
635 | |
636 | err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx, | |
9f95a23c | 637 | &rxq->fl, NULL, |
11fdf7f2 TL |
638 | is_pf4(adapter) ? |
639 | t4_get_tp_ch_map(adapter, pi->tx_chan) : 0, mp, | |
7c673cae FG |
640 | queue_idx, socket_id); |
641 | ||
11fdf7f2 TL |
642 | dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u; abs_id = %u\n", |
643 | __func__, err, pi->port_id, rxq->rspq.cntxt_id, | |
644 | rxq->rspq.abs_id); | |
7c673cae FG |
645 | return err; |
646 | } | |
647 | ||
11fdf7f2 | 648 | void cxgbe_dev_rx_queue_release(void *q) |
7c673cae FG |
649 | { |
650 | struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q; | |
651 | struct sge_rspq *rq = &rxq->rspq; | |
652 | ||
653 | if (rq) { | |
654 | struct port_info *pi = (struct port_info *) | |
655 | (rq->eth_dev->data->dev_private); | |
656 | struct adapter *adap = pi->adapter; | |
657 | ||
658 | dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n", | |
659 | __func__, pi->port_id, rxq->rspq.cntxt_id); | |
660 | ||
661 | t4_sge_eth_rxq_release(adap, rxq); | |
662 | } | |
663 | } | |
664 | ||
665 | /* | |
666 | * Get port statistics. | |
667 | */ | |
11fdf7f2 | 668 | static int cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev, |
7c673cae FG |
669 | struct rte_eth_stats *eth_stats) |
670 | { | |
671 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
672 | struct adapter *adapter = pi->adapter; | |
673 | struct sge *s = &adapter->sge; | |
674 | struct port_stats ps; | |
675 | unsigned int i; | |
676 | ||
677 | cxgbe_stats_get(pi, &ps); | |
678 | ||
679 | /* RX Stats */ | |
7c673cae FG |
680 | eth_stats->imissed = ps.rx_ovflow0 + ps.rx_ovflow1 + |
681 | ps.rx_ovflow2 + ps.rx_ovflow3 + | |
682 | ps.rx_trunc0 + ps.rx_trunc1 + | |
683 | ps.rx_trunc2 + ps.rx_trunc3; | |
684 | eth_stats->ierrors = ps.rx_symbol_err + ps.rx_fcs_err + | |
685 | ps.rx_jabber + ps.rx_too_long + ps.rx_runt + | |
686 | ps.rx_len_err; | |
687 | ||
688 | /* TX Stats */ | |
689 | eth_stats->opackets = ps.tx_frames; | |
690 | eth_stats->obytes = ps.tx_octets; | |
691 | eth_stats->oerrors = ps.tx_error_frames; | |
692 | ||
693 | for (i = 0; i < pi->n_rx_qsets; i++) { | |
694 | struct sge_eth_rxq *rxq = | |
695 | &s->ethrxq[pi->first_qset + i]; | |
696 | ||
697 | eth_stats->q_ipackets[i] = rxq->stats.pkts; | |
698 | eth_stats->q_ibytes[i] = rxq->stats.rx_bytes; | |
11fdf7f2 TL |
699 | eth_stats->ipackets += eth_stats->q_ipackets[i]; |
700 | eth_stats->ibytes += eth_stats->q_ibytes[i]; | |
7c673cae FG |
701 | } |
702 | ||
703 | for (i = 0; i < pi->n_tx_qsets; i++) { | |
704 | struct sge_eth_txq *txq = | |
705 | &s->ethtxq[pi->first_qset + i]; | |
706 | ||
707 | eth_stats->q_opackets[i] = txq->stats.pkts; | |
708 | eth_stats->q_obytes[i] = txq->stats.tx_bytes; | |
709 | eth_stats->q_errors[i] = txq->stats.mapping_err; | |
710 | } | |
11fdf7f2 | 711 | return 0; |
7c673cae FG |
712 | } |
713 | ||
714 | /* | |
715 | * Reset port statistics. | |
716 | */ | |
717 | static void cxgbe_dev_stats_reset(struct rte_eth_dev *eth_dev) | |
718 | { | |
719 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
720 | struct adapter *adapter = pi->adapter; | |
721 | struct sge *s = &adapter->sge; | |
722 | unsigned int i; | |
723 | ||
724 | cxgbe_stats_reset(pi); | |
725 | for (i = 0; i < pi->n_rx_qsets; i++) { | |
726 | struct sge_eth_rxq *rxq = | |
727 | &s->ethrxq[pi->first_qset + i]; | |
728 | ||
729 | rxq->stats.pkts = 0; | |
730 | rxq->stats.rx_bytes = 0; | |
731 | } | |
732 | for (i = 0; i < pi->n_tx_qsets; i++) { | |
733 | struct sge_eth_txq *txq = | |
734 | &s->ethtxq[pi->first_qset + i]; | |
735 | ||
736 | txq->stats.pkts = 0; | |
737 | txq->stats.tx_bytes = 0; | |
738 | txq->stats.mapping_err = 0; | |
739 | } | |
740 | } | |
741 | ||
742 | static int cxgbe_flow_ctrl_get(struct rte_eth_dev *eth_dev, | |
743 | struct rte_eth_fc_conf *fc_conf) | |
744 | { | |
745 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
746 | struct link_config *lc = &pi->link_cfg; | |
747 | int rx_pause, tx_pause; | |
748 | ||
749 | fc_conf->autoneg = lc->fc & PAUSE_AUTONEG; | |
750 | rx_pause = lc->fc & PAUSE_RX; | |
751 | tx_pause = lc->fc & PAUSE_TX; | |
752 | ||
753 | if (rx_pause && tx_pause) | |
754 | fc_conf->mode = RTE_FC_FULL; | |
755 | else if (rx_pause) | |
756 | fc_conf->mode = RTE_FC_RX_PAUSE; | |
757 | else if (tx_pause) | |
758 | fc_conf->mode = RTE_FC_TX_PAUSE; | |
759 | else | |
760 | fc_conf->mode = RTE_FC_NONE; | |
761 | return 0; | |
762 | } | |
763 | ||
764 | static int cxgbe_flow_ctrl_set(struct rte_eth_dev *eth_dev, | |
765 | struct rte_eth_fc_conf *fc_conf) | |
766 | { | |
767 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
768 | struct adapter *adapter = pi->adapter; | |
769 | struct link_config *lc = &pi->link_cfg; | |
770 | ||
11fdf7f2 | 771 | if (lc->pcaps & FW_PORT_CAP32_ANEG) { |
7c673cae FG |
772 | if (fc_conf->autoneg) |
773 | lc->requested_fc |= PAUSE_AUTONEG; | |
774 | else | |
775 | lc->requested_fc &= ~PAUSE_AUTONEG; | |
776 | } | |
777 | ||
778 | if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || | |
779 | (fc_conf->mode & RTE_FC_RX_PAUSE)) | |
780 | lc->requested_fc |= PAUSE_RX; | |
781 | else | |
782 | lc->requested_fc &= ~PAUSE_RX; | |
783 | ||
784 | if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || | |
785 | (fc_conf->mode & RTE_FC_TX_PAUSE)) | |
786 | lc->requested_fc |= PAUSE_TX; | |
787 | else | |
788 | lc->requested_fc &= ~PAUSE_TX; | |
789 | ||
790 | return t4_link_l1cfg(adapter, adapter->mbox, pi->tx_chan, | |
791 | &pi->link_cfg); | |
792 | } | |
793 | ||
11fdf7f2 | 794 | const uint32_t * |
7c673cae FG |
795 | cxgbe_dev_supported_ptypes_get(struct rte_eth_dev *eth_dev) |
796 | { | |
797 | static const uint32_t ptypes[] = { | |
798 | RTE_PTYPE_L3_IPV4, | |
799 | RTE_PTYPE_L3_IPV6, | |
800 | RTE_PTYPE_UNKNOWN | |
801 | }; | |
802 | ||
803 | if (eth_dev->rx_pkt_burst == cxgbe_recv_pkts) | |
804 | return ptypes; | |
805 | return NULL; | |
806 | } | |
807 | ||
11fdf7f2 TL |
808 | /* Update RSS hash configuration |
809 | */ | |
810 | static int cxgbe_dev_rss_hash_update(struct rte_eth_dev *dev, | |
811 | struct rte_eth_rss_conf *rss_conf) | |
812 | { | |
813 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
814 | struct adapter *adapter = pi->adapter; | |
815 | int err; | |
816 | ||
817 | err = cxgbe_write_rss_conf(pi, rss_conf->rss_hf); | |
818 | if (err) | |
819 | return err; | |
820 | ||
821 | pi->rss_hf = rss_conf->rss_hf; | |
822 | ||
823 | if (rss_conf->rss_key) { | |
824 | u32 key[10], mod_key[10]; | |
825 | int i, j; | |
826 | ||
827 | memcpy(key, rss_conf->rss_key, CXGBE_DEFAULT_RSS_KEY_LEN); | |
828 | ||
829 | for (i = 9, j = 0; i >= 0; i--, j++) | |
830 | mod_key[j] = cpu_to_be32(key[i]); | |
831 | ||
832 | t4_write_rss_key(adapter, mod_key, -1); | |
833 | } | |
834 | ||
835 | return 0; | |
836 | } | |
837 | ||
838 | /* Get RSS hash configuration | |
839 | */ | |
840 | static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, | |
841 | struct rte_eth_rss_conf *rss_conf) | |
842 | { | |
843 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
844 | struct adapter *adapter = pi->adapter; | |
845 | u64 rss_hf = 0; | |
846 | u64 flags = 0; | |
847 | int err; | |
848 | ||
849 | err = t4_read_config_vi_rss(adapter, adapter->mbox, pi->viid, | |
850 | &flags, NULL); | |
851 | ||
852 | if (err) | |
853 | return err; | |
854 | ||
855 | if (flags & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) { | |
856 | rss_hf |= CXGBE_RSS_HF_TCP_IPV6_MASK; | |
857 | if (flags & F_FW_RSS_VI_CONFIG_CMD_UDPEN) | |
858 | rss_hf |= CXGBE_RSS_HF_UDP_IPV6_MASK; | |
859 | } | |
860 | ||
861 | if (flags & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) | |
862 | rss_hf |= CXGBE_RSS_HF_IPV6_MASK; | |
863 | ||
864 | if (flags & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) { | |
865 | rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; | |
866 | if (flags & F_FW_RSS_VI_CONFIG_CMD_UDPEN) | |
867 | rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; | |
868 | } | |
869 | ||
870 | if (flags & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) | |
871 | rss_hf |= CXGBE_RSS_HF_IPV4_MASK; | |
872 | ||
873 | rss_conf->rss_hf = rss_hf; | |
874 | ||
875 | if (rss_conf->rss_key) { | |
876 | u32 key[10], mod_key[10]; | |
877 | int i, j; | |
878 | ||
879 | t4_read_rss_key(adapter, key); | |
880 | ||
881 | for (i = 9, j = 0; i >= 0; i--, j++) | |
882 | mod_key[j] = be32_to_cpu(key[i]); | |
883 | ||
884 | memcpy(rss_conf->rss_key, mod_key, CXGBE_DEFAULT_RSS_KEY_LEN); | |
885 | } | |
886 | ||
887 | return 0; | |
888 | } | |
889 | ||
7c673cae FG |
890 | static int cxgbe_get_eeprom_length(struct rte_eth_dev *dev) |
891 | { | |
892 | RTE_SET_USED(dev); | |
893 | return EEPROMSIZE; | |
894 | } | |
895 | ||
896 | /** | |
897 | * eeprom_ptov - translate a physical EEPROM address to virtual | |
898 | * @phys_addr: the physical EEPROM address | |
899 | * @fn: the PCI function number | |
900 | * @sz: size of function-specific area | |
901 | * | |
902 | * Translate a physical EEPROM address to virtual. The first 1K is | |
903 | * accessed through virtual addresses starting at 31K, the rest is | |
904 | * accessed through virtual addresses starting at 0. | |
905 | * | |
906 | * The mapping is as follows: | |
907 | * [0..1K) -> [31K..32K) | |
908 | * [1K..1K+A) -> [31K-A..31K) | |
909 | * [1K+A..ES) -> [0..ES-A-1K) | |
910 | * | |
911 | * where A = @fn * @sz, and ES = EEPROM size. | |
912 | */ | |
913 | static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz) | |
914 | { | |
915 | fn *= sz; | |
916 | if (phys_addr < 1024) | |
917 | return phys_addr + (31 << 10); | |
918 | if (phys_addr < 1024 + fn) | |
919 | return fn + phys_addr - 1024; | |
920 | if (phys_addr < EEPROMSIZE) | |
921 | return phys_addr - 1024 - fn; | |
922 | if (phys_addr < EEPROMVSIZE) | |
923 | return phys_addr - 1024; | |
924 | return -EINVAL; | |
925 | } | |
926 | ||
927 | /* The next two routines implement eeprom read/write from physical addresses. | |
928 | */ | |
929 | static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v) | |
930 | { | |
931 | int vaddr = eeprom_ptov(phys_addr, adap->pf, EEPROMPFSIZE); | |
932 | ||
933 | if (vaddr >= 0) | |
934 | vaddr = t4_seeprom_read(adap, vaddr, v); | |
935 | return vaddr < 0 ? vaddr : 0; | |
936 | } | |
937 | ||
938 | static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v) | |
939 | { | |
940 | int vaddr = eeprom_ptov(phys_addr, adap->pf, EEPROMPFSIZE); | |
941 | ||
942 | if (vaddr >= 0) | |
943 | vaddr = t4_seeprom_write(adap, vaddr, v); | |
944 | return vaddr < 0 ? vaddr : 0; | |
945 | } | |
946 | ||
947 | #define EEPROM_MAGIC 0x38E2F10C | |
948 | ||
949 | static int cxgbe_get_eeprom(struct rte_eth_dev *dev, | |
950 | struct rte_dev_eeprom_info *e) | |
951 | { | |
952 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
953 | struct adapter *adapter = pi->adapter; | |
954 | u32 i, err = 0; | |
955 | u8 *buf = rte_zmalloc(NULL, EEPROMSIZE, 0); | |
956 | ||
957 | if (!buf) | |
958 | return -ENOMEM; | |
959 | ||
960 | e->magic = EEPROM_MAGIC; | |
961 | for (i = e->offset & ~3; !err && i < e->offset + e->length; i += 4) | |
962 | err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]); | |
963 | ||
964 | if (!err) | |
965 | rte_memcpy(e->data, buf + e->offset, e->length); | |
966 | rte_free(buf); | |
967 | return err; | |
968 | } | |
969 | ||
970 | static int cxgbe_set_eeprom(struct rte_eth_dev *dev, | |
971 | struct rte_dev_eeprom_info *eeprom) | |
972 | { | |
973 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
974 | struct adapter *adapter = pi->adapter; | |
975 | u8 *buf; | |
976 | int err = 0; | |
977 | u32 aligned_offset, aligned_len, *p; | |
978 | ||
979 | if (eeprom->magic != EEPROM_MAGIC) | |
980 | return -EINVAL; | |
981 | ||
982 | aligned_offset = eeprom->offset & ~3; | |
983 | aligned_len = (eeprom->length + (eeprom->offset & 3) + 3) & ~3; | |
984 | ||
985 | if (adapter->pf > 0) { | |
986 | u32 start = 1024 + adapter->pf * EEPROMPFSIZE; | |
987 | ||
988 | if (aligned_offset < start || | |
989 | aligned_offset + aligned_len > start + EEPROMPFSIZE) | |
990 | return -EPERM; | |
991 | } | |
992 | ||
993 | if (aligned_offset != eeprom->offset || aligned_len != eeprom->length) { | |
994 | /* RMW possibly needed for first or last words. | |
995 | */ | |
996 | buf = rte_zmalloc(NULL, aligned_len, 0); | |
997 | if (!buf) | |
998 | return -ENOMEM; | |
999 | err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf); | |
1000 | if (!err && aligned_len > 4) | |
1001 | err = eeprom_rd_phys(adapter, | |
1002 | aligned_offset + aligned_len - 4, | |
1003 | (u32 *)&buf[aligned_len - 4]); | |
1004 | if (err) | |
1005 | goto out; | |
1006 | rte_memcpy(buf + (eeprom->offset & 3), eeprom->data, | |
1007 | eeprom->length); | |
1008 | } else { | |
1009 | buf = eeprom->data; | |
1010 | } | |
1011 | ||
1012 | err = t4_seeprom_wp(adapter, false); | |
1013 | if (err) | |
1014 | goto out; | |
1015 | ||
1016 | for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { | |
1017 | err = eeprom_wr_phys(adapter, aligned_offset, *p); | |
1018 | aligned_offset += 4; | |
1019 | } | |
1020 | ||
1021 | if (!err) | |
1022 | err = t4_seeprom_wp(adapter, true); | |
1023 | out: | |
1024 | if (buf != eeprom->data) | |
1025 | rte_free(buf); | |
1026 | return err; | |
1027 | } | |
1028 | ||
1029 | static int cxgbe_get_regs_len(struct rte_eth_dev *eth_dev) | |
1030 | { | |
1031 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
1032 | struct adapter *adapter = pi->adapter; | |
1033 | ||
1034 | return t4_get_regs_len(adapter) / sizeof(uint32_t); | |
1035 | } | |
1036 | ||
1037 | static int cxgbe_get_regs(struct rte_eth_dev *eth_dev, | |
1038 | struct rte_dev_reg_info *regs) | |
1039 | { | |
1040 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
1041 | struct adapter *adapter = pi->adapter; | |
1042 | ||
1043 | regs->version = CHELSIO_CHIP_VERSION(adapter->params.chip) | | |
1044 | (CHELSIO_CHIP_RELEASE(adapter->params.chip) << 10) | | |
1045 | (1 << 16); | |
1046 | ||
1047 | if (regs->data == NULL) { | |
1048 | regs->length = cxgbe_get_regs_len(eth_dev); | |
1049 | regs->width = sizeof(uint32_t); | |
1050 | ||
1051 | return 0; | |
1052 | } | |
1053 | ||
1054 | t4_get_regs(adapter, regs->data, (regs->length * sizeof(uint32_t))); | |
1055 | ||
1056 | return 0; | |
1057 | } | |
1058 | ||
11fdf7f2 TL |
1059 | int cxgbe_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr) |
1060 | { | |
1061 | struct port_info *pi = (struct port_info *)(dev->data->dev_private); | |
11fdf7f2 TL |
1062 | int ret; |
1063 | ||
9f95a23c | 1064 | ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt, (u8 *)addr); |
11fdf7f2 TL |
1065 | if (ret < 0) { |
1066 | dev_err(adapter, "failed to set mac addr; err = %d\n", | |
1067 | ret); | |
1068 | return ret; | |
1069 | } | |
1070 | pi->xact_addr_filt = ret; | |
1071 | return 0; | |
1072 | } | |
1073 | ||
7c673cae FG |
1074 | static const struct eth_dev_ops cxgbe_eth_dev_ops = { |
1075 | .dev_start = cxgbe_dev_start, | |
1076 | .dev_stop = cxgbe_dev_stop, | |
1077 | .dev_close = cxgbe_dev_close, | |
1078 | .promiscuous_enable = cxgbe_dev_promiscuous_enable, | |
1079 | .promiscuous_disable = cxgbe_dev_promiscuous_disable, | |
1080 | .allmulticast_enable = cxgbe_dev_allmulticast_enable, | |
1081 | .allmulticast_disable = cxgbe_dev_allmulticast_disable, | |
1082 | .dev_configure = cxgbe_dev_configure, | |
1083 | .dev_infos_get = cxgbe_dev_info_get, | |
1084 | .dev_supported_ptypes_get = cxgbe_dev_supported_ptypes_get, | |
1085 | .link_update = cxgbe_dev_link_update, | |
11fdf7f2 TL |
1086 | .dev_set_link_up = cxgbe_dev_set_link_up, |
1087 | .dev_set_link_down = cxgbe_dev_set_link_down, | |
7c673cae FG |
1088 | .mtu_set = cxgbe_dev_mtu_set, |
1089 | .tx_queue_setup = cxgbe_dev_tx_queue_setup, | |
1090 | .tx_queue_start = cxgbe_dev_tx_queue_start, | |
1091 | .tx_queue_stop = cxgbe_dev_tx_queue_stop, | |
1092 | .tx_queue_release = cxgbe_dev_tx_queue_release, | |
1093 | .rx_queue_setup = cxgbe_dev_rx_queue_setup, | |
1094 | .rx_queue_start = cxgbe_dev_rx_queue_start, | |
1095 | .rx_queue_stop = cxgbe_dev_rx_queue_stop, | |
1096 | .rx_queue_release = cxgbe_dev_rx_queue_release, | |
11fdf7f2 | 1097 | .filter_ctrl = cxgbe_dev_filter_ctrl, |
7c673cae FG |
1098 | .stats_get = cxgbe_dev_stats_get, |
1099 | .stats_reset = cxgbe_dev_stats_reset, | |
1100 | .flow_ctrl_get = cxgbe_flow_ctrl_get, | |
1101 | .flow_ctrl_set = cxgbe_flow_ctrl_set, | |
1102 | .get_eeprom_length = cxgbe_get_eeprom_length, | |
1103 | .get_eeprom = cxgbe_get_eeprom, | |
1104 | .set_eeprom = cxgbe_set_eeprom, | |
1105 | .get_reg = cxgbe_get_regs, | |
11fdf7f2 TL |
1106 | .rss_hash_update = cxgbe_dev_rss_hash_update, |
1107 | .rss_hash_conf_get = cxgbe_dev_rss_hash_conf_get, | |
1108 | .mac_addr_set = cxgbe_mac_addr_set, | |
7c673cae FG |
1109 | }; |
1110 | ||
1111 | /* | |
1112 | * Initialize driver | |
1113 | * It returns 0 on success. | |
1114 | */ | |
1115 | static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev) | |
1116 | { | |
1117 | struct rte_pci_device *pci_dev; | |
1118 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
1119 | struct adapter *adapter = NULL; | |
1120 | char name[RTE_ETH_NAME_MAX_LEN]; | |
1121 | int err = 0; | |
1122 | ||
1123 | CXGBE_FUNC_TRACE(); | |
1124 | ||
1125 | eth_dev->dev_ops = &cxgbe_eth_dev_ops; | |
1126 | eth_dev->rx_pkt_burst = &cxgbe_recv_pkts; | |
1127 | eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts; | |
11fdf7f2 | 1128 | pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); |
7c673cae | 1129 | |
11fdf7f2 TL |
1130 | /* for secondary processes, we attach to ethdevs allocated by primary |
1131 | * and do minimal initialization. | |
7c673cae | 1132 | */ |
11fdf7f2 TL |
1133 | if (rte_eal_process_type() != RTE_PROC_PRIMARY) { |
1134 | int i; | |
1135 | ||
1136 | for (i = 1; i < MAX_NPORTS; i++) { | |
1137 | struct rte_eth_dev *rest_eth_dev; | |
1138 | char namei[RTE_ETH_NAME_MAX_LEN]; | |
1139 | ||
1140 | snprintf(namei, sizeof(namei), "%s_%d", | |
1141 | pci_dev->device.name, i); | |
1142 | rest_eth_dev = rte_eth_dev_attach_secondary(namei); | |
1143 | if (rest_eth_dev) { | |
1144 | rest_eth_dev->device = &pci_dev->device; | |
1145 | rest_eth_dev->dev_ops = | |
1146 | eth_dev->dev_ops; | |
1147 | rest_eth_dev->rx_pkt_burst = | |
1148 | eth_dev->rx_pkt_burst; | |
1149 | rest_eth_dev->tx_pkt_burst = | |
1150 | eth_dev->tx_pkt_burst; | |
1151 | rte_eth_dev_probing_finish(rest_eth_dev); | |
1152 | } | |
1153 | } | |
7c673cae | 1154 | return 0; |
11fdf7f2 | 1155 | } |
7c673cae FG |
1156 | |
1157 | snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id); | |
1158 | adapter = rte_zmalloc(name, sizeof(*adapter), 0); | |
1159 | if (!adapter) | |
1160 | return -1; | |
1161 | ||
1162 | adapter->use_unpacked_mode = 1; | |
1163 | adapter->regs = (void *)pci_dev->mem_resource[0].addr; | |
1164 | if (!adapter->regs) { | |
1165 | dev_err(adapter, "%s: cannot map device registers\n", __func__); | |
1166 | err = -ENOMEM; | |
1167 | goto out_free_adapter; | |
1168 | } | |
1169 | adapter->pdev = pci_dev; | |
1170 | adapter->eth_dev = eth_dev; | |
1171 | pi->adapter = adapter; | |
1172 | ||
1173 | err = cxgbe_probe(adapter); | |
1174 | if (err) { | |
1175 | dev_err(adapter, "%s: cxgbe probe failed with err %d\n", | |
1176 | __func__, err); | |
1177 | goto out_free_adapter; | |
1178 | } | |
1179 | ||
1180 | return 0; | |
1181 | ||
1182 | out_free_adapter: | |
1183 | rte_free(adapter); | |
1184 | return err; | |
1185 | } | |
1186 | ||
11fdf7f2 TL |
1187 | static int eth_cxgbe_dev_uninit(struct rte_eth_dev *eth_dev) |
1188 | { | |
1189 | struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); | |
1190 | struct adapter *adap = pi->adapter; | |
1191 | ||
1192 | /* Free up other ports and all resources */ | |
1193 | cxgbe_close(adap); | |
1194 | return 0; | |
1195 | } | |
1196 | ||
1197 | static int eth_cxgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, | |
1198 | struct rte_pci_device *pci_dev) | |
1199 | { | |
1200 | return rte_eth_dev_pci_generic_probe(pci_dev, | |
1201 | sizeof(struct port_info), eth_cxgbe_dev_init); | |
1202 | } | |
1203 | ||
1204 | static int eth_cxgbe_pci_remove(struct rte_pci_device *pci_dev) | |
1205 | { | |
1206 | return rte_eth_dev_pci_generic_remove(pci_dev, eth_cxgbe_dev_uninit); | |
1207 | } | |
1208 | ||
1209 | static struct rte_pci_driver rte_cxgbe_pmd = { | |
1210 | .id_table = cxgb4_pci_tbl, | |
1211 | .drv_flags = RTE_PCI_DRV_NEED_MAPPING, | |
1212 | .probe = eth_cxgbe_pci_probe, | |
1213 | .remove = eth_cxgbe_pci_remove, | |
7c673cae FG |
1214 | }; |
1215 | ||
11fdf7f2 | 1216 | RTE_PMD_REGISTER_PCI(net_cxgbe, rte_cxgbe_pmd); |
7c673cae | 1217 | RTE_PMD_REGISTER_PCI_TABLE(net_cxgbe, cxgb4_pci_tbl); |
11fdf7f2 TL |
1218 | RTE_PMD_REGISTER_KMOD_DEP(net_cxgbe, "* igb_uio | uio_pci_generic | vfio-pci"); |
1219 | RTE_PMD_REGISTER_PARAM_STRING(net_cxgbe, | |
1220 | CXGBE_DEVARG_KEEP_OVLAN "=<0|1> " | |
1221 | CXGBE_DEVARG_FORCE_LINK_UP "=<0|1> "); |