]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
netdevice: add queue selection fallback handler for ndo_select_queue
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2013 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/tcp.h>
25 #include <net/ipv6.h>
26 #include <net/ip6_checksum.h>
27 #include <net/busy_poll.h>
28 #include <linux/prefetch.h>
29 #include "bnx2x_cmn.h"
30 #include "bnx2x_init.h"
31 #include "bnx2x_sp.h"
32
33 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
34 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
35 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
36 static int bnx2x_poll(struct napi_struct *napi, int budget);
37
38 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
39 {
40 int i;
41
42 /* Add NAPI objects */
43 for_each_rx_queue_cnic(bp, i) {
44 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
45 bnx2x_poll, NAPI_POLL_WEIGHT);
46 napi_hash_add(&bnx2x_fp(bp, i, napi));
47 }
48 }
49
50 static void bnx2x_add_all_napi(struct bnx2x *bp)
51 {
52 int i;
53
54 /* Add NAPI objects */
55 for_each_eth_queue(bp, i) {
56 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
57 bnx2x_poll, NAPI_POLL_WEIGHT);
58 napi_hash_add(&bnx2x_fp(bp, i, napi));
59 }
60 }
61
62 static int bnx2x_calc_num_queues(struct bnx2x *bp)
63 {
64 return bnx2x_num_queues ?
65 min_t(int, bnx2x_num_queues, BNX2X_MAX_QUEUES(bp)) :
66 min_t(int, netif_get_num_default_rss_queues(),
67 BNX2X_MAX_QUEUES(bp));
68 }
69
70 /**
71 * bnx2x_move_fp - move content of the fastpath structure.
72 *
73 * @bp: driver handle
74 * @from: source FP index
75 * @to: destination FP index
76 *
77 * Makes sure the contents of the bp->fp[to].napi is kept
78 * intact. This is done by first copying the napi struct from
79 * the target to the source, and then mem copying the entire
80 * source onto the target. Update txdata pointers and related
81 * content.
82 */
83 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
84 {
85 struct bnx2x_fastpath *from_fp = &bp->fp[from];
86 struct bnx2x_fastpath *to_fp = &bp->fp[to];
87 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
88 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
89 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
90 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
91 int old_max_eth_txqs, new_max_eth_txqs;
92 int old_txdata_index = 0, new_txdata_index = 0;
93 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
94
95 /* Copy the NAPI object as it has been already initialized */
96 from_fp->napi = to_fp->napi;
97
98 /* Move bnx2x_fastpath contents */
99 memcpy(to_fp, from_fp, sizeof(*to_fp));
100 to_fp->index = to;
101
102 /* Retain the tpa_info of the original `to' version as we don't want
103 * 2 FPs to contain the same tpa_info pointer.
104 */
105 to_fp->tpa_info = old_tpa_info;
106
107 /* move sp_objs contents as well, as their indices match fp ones */
108 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
109
110 /* move fp_stats contents as well, as their indices match fp ones */
111 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
112
113 /* Update txdata pointers in fp and move txdata content accordingly:
114 * Each fp consumes 'max_cos' txdata structures, so the index should be
115 * decremented by max_cos x delta.
116 */
117
118 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
119 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
120 (bp)->max_cos;
121 if (from == FCOE_IDX(bp)) {
122 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
123 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
124 }
125
126 memcpy(&bp->bnx2x_txq[new_txdata_index],
127 &bp->bnx2x_txq[old_txdata_index],
128 sizeof(struct bnx2x_fp_txdata));
129 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
130 }
131
132 /**
133 * bnx2x_fill_fw_str - Fill buffer with FW version string.
134 *
135 * @bp: driver handle
136 * @buf: character buffer to fill with the fw name
137 * @buf_len: length of the above buffer
138 *
139 */
140 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
141 {
142 if (IS_PF(bp)) {
143 u8 phy_fw_ver[PHY_FW_VER_LEN];
144
145 phy_fw_ver[0] = '\0';
146 bnx2x_get_ext_phy_fw_version(&bp->link_params,
147 phy_fw_ver, PHY_FW_VER_LEN);
148 strlcpy(buf, bp->fw_ver, buf_len);
149 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
150 "bc %d.%d.%d%s%s",
151 (bp->common.bc_ver & 0xff0000) >> 16,
152 (bp->common.bc_ver & 0xff00) >> 8,
153 (bp->common.bc_ver & 0xff),
154 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
155 } else {
156 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
157 }
158 }
159
160 /**
161 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
162 *
163 * @bp: driver handle
164 * @delta: number of eth queues which were not allocated
165 */
166 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
167 {
168 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
169
170 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
171 * backward along the array could cause memory to be overridden
172 */
173 for (cos = 1; cos < bp->max_cos; cos++) {
174 for (i = 0; i < old_eth_num - delta; i++) {
175 struct bnx2x_fastpath *fp = &bp->fp[i];
176 int new_idx = cos * (old_eth_num - delta) + i;
177
178 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
179 sizeof(struct bnx2x_fp_txdata));
180 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
181 }
182 }
183 }
184
185 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
186
187 /* free skb in the packet ring at pos idx
188 * return idx of last bd freed
189 */
190 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
191 u16 idx, unsigned int *pkts_compl,
192 unsigned int *bytes_compl)
193 {
194 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
195 struct eth_tx_start_bd *tx_start_bd;
196 struct eth_tx_bd *tx_data_bd;
197 struct sk_buff *skb = tx_buf->skb;
198 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
199 int nbd;
200 u16 split_bd_len = 0;
201
202 /* prefetch skb end pointer to speedup dev_kfree_skb() */
203 prefetch(&skb->end);
204
205 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
206 txdata->txq_index, idx, tx_buf, skb);
207
208 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
209
210 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
211 #ifdef BNX2X_STOP_ON_ERROR
212 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
213 BNX2X_ERR("BAD nbd!\n");
214 bnx2x_panic();
215 }
216 #endif
217 new_cons = nbd + tx_buf->first_bd;
218
219 /* Get the next bd */
220 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
221
222 /* Skip a parse bd... */
223 --nbd;
224 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
225
226 /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
227 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
228 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
229 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
230 --nbd;
231 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
232 }
233
234 /* unmap first bd */
235 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
236 BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
237 DMA_TO_DEVICE);
238
239 /* now free frags */
240 while (nbd > 0) {
241
242 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
243 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
244 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
245 if (--nbd)
246 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
247 }
248
249 /* release skb */
250 WARN_ON(!skb);
251 if (likely(skb)) {
252 (*pkts_compl)++;
253 (*bytes_compl) += skb->len;
254 }
255
256 dev_kfree_skb_any(skb);
257 tx_buf->first_bd = 0;
258 tx_buf->skb = NULL;
259
260 return new_cons;
261 }
262
263 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
264 {
265 struct netdev_queue *txq;
266 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
267 unsigned int pkts_compl = 0, bytes_compl = 0;
268
269 #ifdef BNX2X_STOP_ON_ERROR
270 if (unlikely(bp->panic))
271 return -1;
272 #endif
273
274 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
275 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
276 sw_cons = txdata->tx_pkt_cons;
277
278 while (sw_cons != hw_cons) {
279 u16 pkt_cons;
280
281 pkt_cons = TX_BD(sw_cons);
282
283 DP(NETIF_MSG_TX_DONE,
284 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
285 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
286
287 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
288 &pkts_compl, &bytes_compl);
289
290 sw_cons++;
291 }
292
293 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
294
295 txdata->tx_pkt_cons = sw_cons;
296 txdata->tx_bd_cons = bd_cons;
297
298 /* Need to make the tx_bd_cons update visible to start_xmit()
299 * before checking for netif_tx_queue_stopped(). Without the
300 * memory barrier, there is a small possibility that
301 * start_xmit() will miss it and cause the queue to be stopped
302 * forever.
303 * On the other hand we need an rmb() here to ensure the proper
304 * ordering of bit testing in the following
305 * netif_tx_queue_stopped(txq) call.
306 */
307 smp_mb();
308
309 if (unlikely(netif_tx_queue_stopped(txq))) {
310 /* Taking tx_lock() is needed to prevent re-enabling the queue
311 * while it's empty. This could have happen if rx_action() gets
312 * suspended in bnx2x_tx_int() after the condition before
313 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
314 *
315 * stops the queue->sees fresh tx_bd_cons->releases the queue->
316 * sends some packets consuming the whole queue again->
317 * stops the queue
318 */
319
320 __netif_tx_lock(txq, smp_processor_id());
321
322 if ((netif_tx_queue_stopped(txq)) &&
323 (bp->state == BNX2X_STATE_OPEN) &&
324 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
325 netif_tx_wake_queue(txq);
326
327 __netif_tx_unlock(txq);
328 }
329 return 0;
330 }
331
332 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
333 u16 idx)
334 {
335 u16 last_max = fp->last_max_sge;
336
337 if (SUB_S16(idx, last_max) > 0)
338 fp->last_max_sge = idx;
339 }
340
341 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
342 u16 sge_len,
343 struct eth_end_agg_rx_cqe *cqe)
344 {
345 struct bnx2x *bp = fp->bp;
346 u16 last_max, last_elem, first_elem;
347 u16 delta = 0;
348 u16 i;
349
350 if (!sge_len)
351 return;
352
353 /* First mark all used pages */
354 for (i = 0; i < sge_len; i++)
355 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
356 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
357
358 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
359 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
360
361 /* Here we assume that the last SGE index is the biggest */
362 prefetch((void *)(fp->sge_mask));
363 bnx2x_update_last_max_sge(fp,
364 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
365
366 last_max = RX_SGE(fp->last_max_sge);
367 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
368 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
369
370 /* If ring is not full */
371 if (last_elem + 1 != first_elem)
372 last_elem++;
373
374 /* Now update the prod */
375 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
376 if (likely(fp->sge_mask[i]))
377 break;
378
379 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
380 delta += BIT_VEC64_ELEM_SZ;
381 }
382
383 if (delta > 0) {
384 fp->rx_sge_prod += delta;
385 /* clear page-end entries */
386 bnx2x_clear_sge_mask_next_elems(fp);
387 }
388
389 DP(NETIF_MSG_RX_STATUS,
390 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
391 fp->last_max_sge, fp->rx_sge_prod);
392 }
393
394 /* Get Toeplitz hash value in the skb using the value from the
395 * CQE (calculated by HW).
396 */
397 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
398 const struct eth_fast_path_rx_cqe *cqe,
399 enum pkt_hash_types *rxhash_type)
400 {
401 /* Get Toeplitz hash from CQE */
402 if ((bp->dev->features & NETIF_F_RXHASH) &&
403 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
404 enum eth_rss_hash_type htype;
405
406 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
407 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
408 (htype == TCP_IPV6_HASH_TYPE)) ?
409 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
410
411 return le32_to_cpu(cqe->rss_hash_result);
412 }
413 *rxhash_type = PKT_HASH_TYPE_NONE;
414 return 0;
415 }
416
417 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
418 u16 cons, u16 prod,
419 struct eth_fast_path_rx_cqe *cqe)
420 {
421 struct bnx2x *bp = fp->bp;
422 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
423 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
424 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
425 dma_addr_t mapping;
426 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
427 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
428
429 /* print error if current state != stop */
430 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
431 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
432
433 /* Try to map an empty data buffer from the aggregation info */
434 mapping = dma_map_single(&bp->pdev->dev,
435 first_buf->data + NET_SKB_PAD,
436 fp->rx_buf_size, DMA_FROM_DEVICE);
437 /*
438 * ...if it fails - move the skb from the consumer to the producer
439 * and set the current aggregation state as ERROR to drop it
440 * when TPA_STOP arrives.
441 */
442
443 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
444 /* Move the BD from the consumer to the producer */
445 bnx2x_reuse_rx_data(fp, cons, prod);
446 tpa_info->tpa_state = BNX2X_TPA_ERROR;
447 return;
448 }
449
450 /* move empty data from pool to prod */
451 prod_rx_buf->data = first_buf->data;
452 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
453 /* point prod_bd to new data */
454 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
455 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
456
457 /* move partial skb from cons to pool (don't unmap yet) */
458 *first_buf = *cons_rx_buf;
459
460 /* mark bin state as START */
461 tpa_info->parsing_flags =
462 le16_to_cpu(cqe->pars_flags.flags);
463 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
464 tpa_info->tpa_state = BNX2X_TPA_START;
465 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
466 tpa_info->placement_offset = cqe->placement_offset;
467 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
468 if (fp->mode == TPA_MODE_GRO) {
469 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
470 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
471 tpa_info->gro_size = gro_size;
472 }
473
474 #ifdef BNX2X_STOP_ON_ERROR
475 fp->tpa_queue_used |= (1 << queue);
476 #ifdef _ASM_GENERIC_INT_L64_H
477 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
478 #else
479 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
480 #endif
481 fp->tpa_queue_used);
482 #endif
483 }
484
485 /* Timestamp option length allowed for TPA aggregation:
486 *
487 * nop nop kind length echo val
488 */
489 #define TPA_TSTAMP_OPT_LEN 12
490 /**
491 * bnx2x_set_gro_params - compute GRO values
492 *
493 * @skb: packet skb
494 * @parsing_flags: parsing flags from the START CQE
495 * @len_on_bd: total length of the first packet for the
496 * aggregation.
497 * @pkt_len: length of all segments
498 *
499 * Approximate value of the MSS for this aggregation calculated using
500 * the first packet of it.
501 * Compute number of aggregated segments, and gso_type.
502 */
503 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
504 u16 len_on_bd, unsigned int pkt_len,
505 u16 num_of_coalesced_segs)
506 {
507 /* TPA aggregation won't have either IP options or TCP options
508 * other than timestamp or IPv6 extension headers.
509 */
510 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
511
512 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
513 PRS_FLAG_OVERETH_IPV6) {
514 hdrs_len += sizeof(struct ipv6hdr);
515 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
516 } else {
517 hdrs_len += sizeof(struct iphdr);
518 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
519 }
520
521 /* Check if there was a TCP timestamp, if there is it's will
522 * always be 12 bytes length: nop nop kind length echo val.
523 *
524 * Otherwise FW would close the aggregation.
525 */
526 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
527 hdrs_len += TPA_TSTAMP_OPT_LEN;
528
529 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
530
531 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
532 * to skb_shinfo(skb)->gso_segs
533 */
534 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
535 }
536
537 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
538 u16 index, gfp_t gfp_mask)
539 {
540 struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
541 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
542 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
543 dma_addr_t mapping;
544
545 if (unlikely(page == NULL)) {
546 BNX2X_ERR("Can't alloc sge\n");
547 return -ENOMEM;
548 }
549
550 mapping = dma_map_page(&bp->pdev->dev, page, 0,
551 SGE_PAGES, DMA_FROM_DEVICE);
552 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
553 __free_pages(page, PAGES_PER_SGE_SHIFT);
554 BNX2X_ERR("Can't map sge\n");
555 return -ENOMEM;
556 }
557
558 sw_buf->page = page;
559 dma_unmap_addr_set(sw_buf, mapping, mapping);
560
561 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
562 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
563
564 return 0;
565 }
566
567 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
568 struct bnx2x_agg_info *tpa_info,
569 u16 pages,
570 struct sk_buff *skb,
571 struct eth_end_agg_rx_cqe *cqe,
572 u16 cqe_idx)
573 {
574 struct sw_rx_page *rx_pg, old_rx_pg;
575 u32 i, frag_len, frag_size;
576 int err, j, frag_id = 0;
577 u16 len_on_bd = tpa_info->len_on_bd;
578 u16 full_page = 0, gro_size = 0;
579
580 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
581
582 if (fp->mode == TPA_MODE_GRO) {
583 gro_size = tpa_info->gro_size;
584 full_page = tpa_info->full_page;
585 }
586
587 /* This is needed in order to enable forwarding support */
588 if (frag_size)
589 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
590 le16_to_cpu(cqe->pkt_len),
591 le16_to_cpu(cqe->num_of_coalesced_segs));
592
593 #ifdef BNX2X_STOP_ON_ERROR
594 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
595 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
596 pages, cqe_idx);
597 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
598 bnx2x_panic();
599 return -EINVAL;
600 }
601 #endif
602
603 /* Run through the SGL and compose the fragmented skb */
604 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
605 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
606
607 /* FW gives the indices of the SGE as if the ring is an array
608 (meaning that "next" element will consume 2 indices) */
609 if (fp->mode == TPA_MODE_GRO)
610 frag_len = min_t(u32, frag_size, (u32)full_page);
611 else /* LRO */
612 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
613
614 rx_pg = &fp->rx_page_ring[sge_idx];
615 old_rx_pg = *rx_pg;
616
617 /* If we fail to allocate a substitute page, we simply stop
618 where we are and drop the whole packet */
619 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
620 if (unlikely(err)) {
621 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
622 return err;
623 }
624
625 /* Unmap the page as we're going to pass it to the stack */
626 dma_unmap_page(&bp->pdev->dev,
627 dma_unmap_addr(&old_rx_pg, mapping),
628 SGE_PAGES, DMA_FROM_DEVICE);
629 /* Add one frag and update the appropriate fields in the skb */
630 if (fp->mode == TPA_MODE_LRO)
631 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
632 else { /* GRO */
633 int rem;
634 int offset = 0;
635 for (rem = frag_len; rem > 0; rem -= gro_size) {
636 int len = rem > gro_size ? gro_size : rem;
637 skb_fill_page_desc(skb, frag_id++,
638 old_rx_pg.page, offset, len);
639 if (offset)
640 get_page(old_rx_pg.page);
641 offset += len;
642 }
643 }
644
645 skb->data_len += frag_len;
646 skb->truesize += SGE_PAGES;
647 skb->len += frag_len;
648
649 frag_size -= frag_len;
650 }
651
652 return 0;
653 }
654
655 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
656 {
657 if (fp->rx_frag_size)
658 put_page(virt_to_head_page(data));
659 else
660 kfree(data);
661 }
662
663 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
664 {
665 if (fp->rx_frag_size) {
666 /* GFP_KERNEL allocations are used only during initialization */
667 if (unlikely(gfp_mask & __GFP_WAIT))
668 return (void *)__get_free_page(gfp_mask);
669
670 return netdev_alloc_frag(fp->rx_frag_size);
671 }
672
673 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
674 }
675
676 #ifdef CONFIG_INET
677 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
678 {
679 const struct iphdr *iph = ip_hdr(skb);
680 struct tcphdr *th;
681
682 skb_set_transport_header(skb, sizeof(struct iphdr));
683 th = tcp_hdr(skb);
684
685 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
686 iph->saddr, iph->daddr, 0);
687 }
688
689 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
690 {
691 struct ipv6hdr *iph = ipv6_hdr(skb);
692 struct tcphdr *th;
693
694 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
695 th = tcp_hdr(skb);
696
697 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
698 &iph->saddr, &iph->daddr, 0);
699 }
700
701 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
702 void (*gro_func)(struct bnx2x*, struct sk_buff*))
703 {
704 skb_set_network_header(skb, 0);
705 gro_func(bp, skb);
706 tcp_gro_complete(skb);
707 }
708 #endif
709
710 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
711 struct sk_buff *skb)
712 {
713 #ifdef CONFIG_INET
714 if (skb_shinfo(skb)->gso_size) {
715 switch (be16_to_cpu(skb->protocol)) {
716 case ETH_P_IP:
717 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
718 break;
719 case ETH_P_IPV6:
720 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
721 break;
722 default:
723 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
724 be16_to_cpu(skb->protocol));
725 }
726 }
727 #endif
728 skb_record_rx_queue(skb, fp->rx_queue);
729 napi_gro_receive(&fp->napi, skb);
730 }
731
732 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
733 struct bnx2x_agg_info *tpa_info,
734 u16 pages,
735 struct eth_end_agg_rx_cqe *cqe,
736 u16 cqe_idx)
737 {
738 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
739 u8 pad = tpa_info->placement_offset;
740 u16 len = tpa_info->len_on_bd;
741 struct sk_buff *skb = NULL;
742 u8 *new_data, *data = rx_buf->data;
743 u8 old_tpa_state = tpa_info->tpa_state;
744
745 tpa_info->tpa_state = BNX2X_TPA_STOP;
746
747 /* If we there was an error during the handling of the TPA_START -
748 * drop this aggregation.
749 */
750 if (old_tpa_state == BNX2X_TPA_ERROR)
751 goto drop;
752
753 /* Try to allocate the new data */
754 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
755 /* Unmap skb in the pool anyway, as we are going to change
756 pool entry status to BNX2X_TPA_STOP even if new skb allocation
757 fails. */
758 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
759 fp->rx_buf_size, DMA_FROM_DEVICE);
760 if (likely(new_data))
761 skb = build_skb(data, fp->rx_frag_size);
762
763 if (likely(skb)) {
764 #ifdef BNX2X_STOP_ON_ERROR
765 if (pad + len > fp->rx_buf_size) {
766 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
767 pad, len, fp->rx_buf_size);
768 bnx2x_panic();
769 return;
770 }
771 #endif
772
773 skb_reserve(skb, pad + NET_SKB_PAD);
774 skb_put(skb, len);
775 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
776
777 skb->protocol = eth_type_trans(skb, bp->dev);
778 skb->ip_summed = CHECKSUM_UNNECESSARY;
779
780 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
781 skb, cqe, cqe_idx)) {
782 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
783 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
784 bnx2x_gro_receive(bp, fp, skb);
785 } else {
786 DP(NETIF_MSG_RX_STATUS,
787 "Failed to allocate new pages - dropping packet!\n");
788 dev_kfree_skb_any(skb);
789 }
790
791 /* put new data in bin */
792 rx_buf->data = new_data;
793
794 return;
795 }
796 bnx2x_frag_free(fp, new_data);
797 drop:
798 /* drop the packet and keep the buffer in the bin */
799 DP(NETIF_MSG_RX_STATUS,
800 "Failed to allocate or map a new skb - dropping packet!\n");
801 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
802 }
803
804 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
805 u16 index, gfp_t gfp_mask)
806 {
807 u8 *data;
808 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
809 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
810 dma_addr_t mapping;
811
812 data = bnx2x_frag_alloc(fp, gfp_mask);
813 if (unlikely(data == NULL))
814 return -ENOMEM;
815
816 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
817 fp->rx_buf_size,
818 DMA_FROM_DEVICE);
819 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
820 bnx2x_frag_free(fp, data);
821 BNX2X_ERR("Can't map rx data\n");
822 return -ENOMEM;
823 }
824
825 rx_buf->data = data;
826 dma_unmap_addr_set(rx_buf, mapping, mapping);
827
828 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
829 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
830
831 return 0;
832 }
833
834 static
835 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
836 struct bnx2x_fastpath *fp,
837 struct bnx2x_eth_q_stats *qstats)
838 {
839 /* Do nothing if no L4 csum validation was done.
840 * We do not check whether IP csum was validated. For IPv4 we assume
841 * that if the card got as far as validating the L4 csum, it also
842 * validated the IP csum. IPv6 has no IP csum.
843 */
844 if (cqe->fast_path_cqe.status_flags &
845 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
846 return;
847
848 /* If L4 validation was done, check if an error was found. */
849
850 if (cqe->fast_path_cqe.type_error_flags &
851 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
852 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
853 qstats->hw_csum_err++;
854 else
855 skb->ip_summed = CHECKSUM_UNNECESSARY;
856 }
857
858 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
859 {
860 struct bnx2x *bp = fp->bp;
861 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
862 u16 sw_comp_cons, sw_comp_prod;
863 int rx_pkt = 0;
864 union eth_rx_cqe *cqe;
865 struct eth_fast_path_rx_cqe *cqe_fp;
866
867 #ifdef BNX2X_STOP_ON_ERROR
868 if (unlikely(bp->panic))
869 return 0;
870 #endif
871
872 bd_cons = fp->rx_bd_cons;
873 bd_prod = fp->rx_bd_prod;
874 bd_prod_fw = bd_prod;
875 sw_comp_cons = fp->rx_comp_cons;
876 sw_comp_prod = fp->rx_comp_prod;
877
878 comp_ring_cons = RCQ_BD(sw_comp_cons);
879 cqe = &fp->rx_comp_ring[comp_ring_cons];
880 cqe_fp = &cqe->fast_path_cqe;
881
882 DP(NETIF_MSG_RX_STATUS,
883 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
884
885 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
886 struct sw_rx_bd *rx_buf = NULL;
887 struct sk_buff *skb;
888 u8 cqe_fp_flags;
889 enum eth_rx_cqe_type cqe_fp_type;
890 u16 len, pad, queue;
891 u8 *data;
892 u32 rxhash;
893 enum pkt_hash_types rxhash_type;
894
895 #ifdef BNX2X_STOP_ON_ERROR
896 if (unlikely(bp->panic))
897 return 0;
898 #endif
899
900 bd_prod = RX_BD(bd_prod);
901 bd_cons = RX_BD(bd_cons);
902
903 cqe_fp_flags = cqe_fp->type_error_flags;
904 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
905
906 DP(NETIF_MSG_RX_STATUS,
907 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
908 CQE_TYPE(cqe_fp_flags),
909 cqe_fp_flags, cqe_fp->status_flags,
910 le32_to_cpu(cqe_fp->rss_hash_result),
911 le16_to_cpu(cqe_fp->vlan_tag),
912 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
913
914 /* is this a slowpath msg? */
915 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
916 bnx2x_sp_event(fp, cqe);
917 goto next_cqe;
918 }
919
920 rx_buf = &fp->rx_buf_ring[bd_cons];
921 data = rx_buf->data;
922
923 if (!CQE_TYPE_FAST(cqe_fp_type)) {
924 struct bnx2x_agg_info *tpa_info;
925 u16 frag_size, pages;
926 #ifdef BNX2X_STOP_ON_ERROR
927 /* sanity check */
928 if (fp->disable_tpa &&
929 (CQE_TYPE_START(cqe_fp_type) ||
930 CQE_TYPE_STOP(cqe_fp_type)))
931 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
932 CQE_TYPE(cqe_fp_type));
933 #endif
934
935 if (CQE_TYPE_START(cqe_fp_type)) {
936 u16 queue = cqe_fp->queue_index;
937 DP(NETIF_MSG_RX_STATUS,
938 "calling tpa_start on queue %d\n",
939 queue);
940
941 bnx2x_tpa_start(fp, queue,
942 bd_cons, bd_prod,
943 cqe_fp);
944
945 goto next_rx;
946 }
947 queue = cqe->end_agg_cqe.queue_index;
948 tpa_info = &fp->tpa_info[queue];
949 DP(NETIF_MSG_RX_STATUS,
950 "calling tpa_stop on queue %d\n",
951 queue);
952
953 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
954 tpa_info->len_on_bd;
955
956 if (fp->mode == TPA_MODE_GRO)
957 pages = (frag_size + tpa_info->full_page - 1) /
958 tpa_info->full_page;
959 else
960 pages = SGE_PAGE_ALIGN(frag_size) >>
961 SGE_PAGE_SHIFT;
962
963 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
964 &cqe->end_agg_cqe, comp_ring_cons);
965 #ifdef BNX2X_STOP_ON_ERROR
966 if (bp->panic)
967 return 0;
968 #endif
969
970 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
971 goto next_cqe;
972 }
973 /* non TPA */
974 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
975 pad = cqe_fp->placement_offset;
976 dma_sync_single_for_cpu(&bp->pdev->dev,
977 dma_unmap_addr(rx_buf, mapping),
978 pad + RX_COPY_THRESH,
979 DMA_FROM_DEVICE);
980 pad += NET_SKB_PAD;
981 prefetch(data + pad); /* speedup eth_type_trans() */
982 /* is this an error packet? */
983 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
984 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
985 "ERROR flags %x rx packet %u\n",
986 cqe_fp_flags, sw_comp_cons);
987 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
988 goto reuse_rx;
989 }
990
991 /* Since we don't have a jumbo ring
992 * copy small packets if mtu > 1500
993 */
994 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
995 (len <= RX_COPY_THRESH)) {
996 skb = netdev_alloc_skb_ip_align(bp->dev, len);
997 if (skb == NULL) {
998 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
999 "ERROR packet dropped because of alloc failure\n");
1000 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1001 goto reuse_rx;
1002 }
1003 memcpy(skb->data, data + pad, len);
1004 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1005 } else {
1006 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1007 GFP_ATOMIC) == 0)) {
1008 dma_unmap_single(&bp->pdev->dev,
1009 dma_unmap_addr(rx_buf, mapping),
1010 fp->rx_buf_size,
1011 DMA_FROM_DEVICE);
1012 skb = build_skb(data, fp->rx_frag_size);
1013 if (unlikely(!skb)) {
1014 bnx2x_frag_free(fp, data);
1015 bnx2x_fp_qstats(bp, fp)->
1016 rx_skb_alloc_failed++;
1017 goto next_rx;
1018 }
1019 skb_reserve(skb, pad);
1020 } else {
1021 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1022 "ERROR packet dropped because of alloc failure\n");
1023 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1024 reuse_rx:
1025 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1026 goto next_rx;
1027 }
1028 }
1029
1030 skb_put(skb, len);
1031 skb->protocol = eth_type_trans(skb, bp->dev);
1032
1033 /* Set Toeplitz hash for a none-LRO skb */
1034 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1035 skb_set_hash(skb, rxhash, rxhash_type);
1036
1037 skb_checksum_none_assert(skb);
1038
1039 if (bp->dev->features & NETIF_F_RXCSUM)
1040 bnx2x_csum_validate(skb, cqe, fp,
1041 bnx2x_fp_qstats(bp, fp));
1042
1043 skb_record_rx_queue(skb, fp->rx_queue);
1044
1045 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1046 PARSING_FLAGS_VLAN)
1047 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1048 le16_to_cpu(cqe_fp->vlan_tag));
1049
1050 skb_mark_napi_id(skb, &fp->napi);
1051
1052 if (bnx2x_fp_ll_polling(fp))
1053 netif_receive_skb(skb);
1054 else
1055 napi_gro_receive(&fp->napi, skb);
1056 next_rx:
1057 rx_buf->data = NULL;
1058
1059 bd_cons = NEXT_RX_IDX(bd_cons);
1060 bd_prod = NEXT_RX_IDX(bd_prod);
1061 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1062 rx_pkt++;
1063 next_cqe:
1064 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1065 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1066
1067 /* mark CQE as free */
1068 BNX2X_SEED_CQE(cqe_fp);
1069
1070 if (rx_pkt == budget)
1071 break;
1072
1073 comp_ring_cons = RCQ_BD(sw_comp_cons);
1074 cqe = &fp->rx_comp_ring[comp_ring_cons];
1075 cqe_fp = &cqe->fast_path_cqe;
1076 } /* while */
1077
1078 fp->rx_bd_cons = bd_cons;
1079 fp->rx_bd_prod = bd_prod_fw;
1080 fp->rx_comp_cons = sw_comp_cons;
1081 fp->rx_comp_prod = sw_comp_prod;
1082
1083 /* Update producers */
1084 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1085 fp->rx_sge_prod);
1086
1087 fp->rx_pkt += rx_pkt;
1088 fp->rx_calls++;
1089
1090 return rx_pkt;
1091 }
1092
1093 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1094 {
1095 struct bnx2x_fastpath *fp = fp_cookie;
1096 struct bnx2x *bp = fp->bp;
1097 u8 cos;
1098
1099 DP(NETIF_MSG_INTR,
1100 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1101 fp->index, fp->fw_sb_id, fp->igu_sb_id);
1102
1103 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1104
1105 #ifdef BNX2X_STOP_ON_ERROR
1106 if (unlikely(bp->panic))
1107 return IRQ_HANDLED;
1108 #endif
1109
1110 /* Handle Rx and Tx according to MSI-X vector */
1111 for_each_cos_in_tx_queue(fp, cos)
1112 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1113
1114 prefetch(&fp->sb_running_index[SM_RX_ID]);
1115 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1116
1117 return IRQ_HANDLED;
1118 }
1119
1120 /* HW Lock for shared dual port PHYs */
1121 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1122 {
1123 mutex_lock(&bp->port.phy_mutex);
1124
1125 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1126 }
1127
1128 void bnx2x_release_phy_lock(struct bnx2x *bp)
1129 {
1130 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1131
1132 mutex_unlock(&bp->port.phy_mutex);
1133 }
1134
1135 /* calculates MF speed according to current linespeed and MF configuration */
1136 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1137 {
1138 u16 line_speed = bp->link_vars.line_speed;
1139 if (IS_MF(bp)) {
1140 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1141 bp->mf_config[BP_VN(bp)]);
1142
1143 /* Calculate the current MAX line speed limit for the MF
1144 * devices
1145 */
1146 if (IS_MF_SI(bp))
1147 line_speed = (line_speed * maxCfg) / 100;
1148 else { /* SD mode */
1149 u16 vn_max_rate = maxCfg * 100;
1150
1151 if (vn_max_rate < line_speed)
1152 line_speed = vn_max_rate;
1153 }
1154 }
1155
1156 return line_speed;
1157 }
1158
1159 /**
1160 * bnx2x_fill_report_data - fill link report data to report
1161 *
1162 * @bp: driver handle
1163 * @data: link state to update
1164 *
1165 * It uses a none-atomic bit operations because is called under the mutex.
1166 */
1167 static void bnx2x_fill_report_data(struct bnx2x *bp,
1168 struct bnx2x_link_report_data *data)
1169 {
1170 u16 line_speed = bnx2x_get_mf_speed(bp);
1171
1172 memset(data, 0, sizeof(*data));
1173
1174 /* Fill the report data: effective line speed */
1175 data->line_speed = line_speed;
1176
1177 /* Link is down */
1178 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1179 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1180 &data->link_report_flags);
1181
1182 /* Full DUPLEX */
1183 if (bp->link_vars.duplex == DUPLEX_FULL)
1184 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1185
1186 /* Rx Flow Control is ON */
1187 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1188 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1189
1190 /* Tx Flow Control is ON */
1191 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1192 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1193 }
1194
1195 /**
1196 * bnx2x_link_report - report link status to OS.
1197 *
1198 * @bp: driver handle
1199 *
1200 * Calls the __bnx2x_link_report() under the same locking scheme
1201 * as a link/PHY state managing code to ensure a consistent link
1202 * reporting.
1203 */
1204
1205 void bnx2x_link_report(struct bnx2x *bp)
1206 {
1207 bnx2x_acquire_phy_lock(bp);
1208 __bnx2x_link_report(bp);
1209 bnx2x_release_phy_lock(bp);
1210 }
1211
1212 /**
1213 * __bnx2x_link_report - report link status to OS.
1214 *
1215 * @bp: driver handle
1216 *
1217 * None atomic implementation.
1218 * Should be called under the phy_lock.
1219 */
1220 void __bnx2x_link_report(struct bnx2x *bp)
1221 {
1222 struct bnx2x_link_report_data cur_data;
1223
1224 /* reread mf_cfg */
1225 if (IS_PF(bp) && !CHIP_IS_E1(bp))
1226 bnx2x_read_mf_cfg(bp);
1227
1228 /* Read the current link report info */
1229 bnx2x_fill_report_data(bp, &cur_data);
1230
1231 /* Don't report link down or exactly the same link status twice */
1232 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1233 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1234 &bp->last_reported_link.link_report_flags) &&
1235 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1236 &cur_data.link_report_flags)))
1237 return;
1238
1239 bp->link_cnt++;
1240
1241 /* We are going to report a new link parameters now -
1242 * remember the current data for the next time.
1243 */
1244 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1245
1246 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1247 &cur_data.link_report_flags)) {
1248 netif_carrier_off(bp->dev);
1249 netdev_err(bp->dev, "NIC Link is Down\n");
1250 return;
1251 } else {
1252 const char *duplex;
1253 const char *flow;
1254
1255 netif_carrier_on(bp->dev);
1256
1257 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1258 &cur_data.link_report_flags))
1259 duplex = "full";
1260 else
1261 duplex = "half";
1262
1263 /* Handle the FC at the end so that only these flags would be
1264 * possibly set. This way we may easily check if there is no FC
1265 * enabled.
1266 */
1267 if (cur_data.link_report_flags) {
1268 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1269 &cur_data.link_report_flags)) {
1270 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1271 &cur_data.link_report_flags))
1272 flow = "ON - receive & transmit";
1273 else
1274 flow = "ON - receive";
1275 } else {
1276 flow = "ON - transmit";
1277 }
1278 } else {
1279 flow = "none";
1280 }
1281 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1282 cur_data.line_speed, duplex, flow);
1283 }
1284 }
1285
1286 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1287 {
1288 int i;
1289
1290 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1291 struct eth_rx_sge *sge;
1292
1293 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1294 sge->addr_hi =
1295 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1296 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1297
1298 sge->addr_lo =
1299 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1300 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1301 }
1302 }
1303
1304 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1305 struct bnx2x_fastpath *fp, int last)
1306 {
1307 int i;
1308
1309 for (i = 0; i < last; i++) {
1310 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1311 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1312 u8 *data = first_buf->data;
1313
1314 if (data == NULL) {
1315 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1316 continue;
1317 }
1318 if (tpa_info->tpa_state == BNX2X_TPA_START)
1319 dma_unmap_single(&bp->pdev->dev,
1320 dma_unmap_addr(first_buf, mapping),
1321 fp->rx_buf_size, DMA_FROM_DEVICE);
1322 bnx2x_frag_free(fp, data);
1323 first_buf->data = NULL;
1324 }
1325 }
1326
1327 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1328 {
1329 int j;
1330
1331 for_each_rx_queue_cnic(bp, j) {
1332 struct bnx2x_fastpath *fp = &bp->fp[j];
1333
1334 fp->rx_bd_cons = 0;
1335
1336 /* Activate BD ring */
1337 /* Warning!
1338 * this will generate an interrupt (to the TSTORM)
1339 * must only be done after chip is initialized
1340 */
1341 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1342 fp->rx_sge_prod);
1343 }
1344 }
1345
1346 void bnx2x_init_rx_rings(struct bnx2x *bp)
1347 {
1348 int func = BP_FUNC(bp);
1349 u16 ring_prod;
1350 int i, j;
1351
1352 /* Allocate TPA resources */
1353 for_each_eth_queue(bp, j) {
1354 struct bnx2x_fastpath *fp = &bp->fp[j];
1355
1356 DP(NETIF_MSG_IFUP,
1357 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1358
1359 if (!fp->disable_tpa) {
1360 /* Fill the per-aggregation pool */
1361 for (i = 0; i < MAX_AGG_QS(bp); i++) {
1362 struct bnx2x_agg_info *tpa_info =
1363 &fp->tpa_info[i];
1364 struct sw_rx_bd *first_buf =
1365 &tpa_info->first_buf;
1366
1367 first_buf->data =
1368 bnx2x_frag_alloc(fp, GFP_KERNEL);
1369 if (!first_buf->data) {
1370 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1371 j);
1372 bnx2x_free_tpa_pool(bp, fp, i);
1373 fp->disable_tpa = 1;
1374 break;
1375 }
1376 dma_unmap_addr_set(first_buf, mapping, 0);
1377 tpa_info->tpa_state = BNX2X_TPA_STOP;
1378 }
1379
1380 /* "next page" elements initialization */
1381 bnx2x_set_next_page_sgl(fp);
1382
1383 /* set SGEs bit mask */
1384 bnx2x_init_sge_ring_bit_mask(fp);
1385
1386 /* Allocate SGEs and initialize the ring elements */
1387 for (i = 0, ring_prod = 0;
1388 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1389
1390 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1391 GFP_KERNEL) < 0) {
1392 BNX2X_ERR("was only able to allocate %d rx sges\n",
1393 i);
1394 BNX2X_ERR("disabling TPA for queue[%d]\n",
1395 j);
1396 /* Cleanup already allocated elements */
1397 bnx2x_free_rx_sge_range(bp, fp,
1398 ring_prod);
1399 bnx2x_free_tpa_pool(bp, fp,
1400 MAX_AGG_QS(bp));
1401 fp->disable_tpa = 1;
1402 ring_prod = 0;
1403 break;
1404 }
1405 ring_prod = NEXT_SGE_IDX(ring_prod);
1406 }
1407
1408 fp->rx_sge_prod = ring_prod;
1409 }
1410 }
1411
1412 for_each_eth_queue(bp, j) {
1413 struct bnx2x_fastpath *fp = &bp->fp[j];
1414
1415 fp->rx_bd_cons = 0;
1416
1417 /* Activate BD ring */
1418 /* Warning!
1419 * this will generate an interrupt (to the TSTORM)
1420 * must only be done after chip is initialized
1421 */
1422 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1423 fp->rx_sge_prod);
1424
1425 if (j != 0)
1426 continue;
1427
1428 if (CHIP_IS_E1(bp)) {
1429 REG_WR(bp, BAR_USTRORM_INTMEM +
1430 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1431 U64_LO(fp->rx_comp_mapping));
1432 REG_WR(bp, BAR_USTRORM_INTMEM +
1433 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1434 U64_HI(fp->rx_comp_mapping));
1435 }
1436 }
1437 }
1438
1439 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1440 {
1441 u8 cos;
1442 struct bnx2x *bp = fp->bp;
1443
1444 for_each_cos_in_tx_queue(fp, cos) {
1445 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1446 unsigned pkts_compl = 0, bytes_compl = 0;
1447
1448 u16 sw_prod = txdata->tx_pkt_prod;
1449 u16 sw_cons = txdata->tx_pkt_cons;
1450
1451 while (sw_cons != sw_prod) {
1452 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1453 &pkts_compl, &bytes_compl);
1454 sw_cons++;
1455 }
1456
1457 netdev_tx_reset_queue(
1458 netdev_get_tx_queue(bp->dev,
1459 txdata->txq_index));
1460 }
1461 }
1462
1463 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1464 {
1465 int i;
1466
1467 for_each_tx_queue_cnic(bp, i) {
1468 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1469 }
1470 }
1471
1472 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1473 {
1474 int i;
1475
1476 for_each_eth_queue(bp, i) {
1477 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1478 }
1479 }
1480
1481 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1482 {
1483 struct bnx2x *bp = fp->bp;
1484 int i;
1485
1486 /* ring wasn't allocated */
1487 if (fp->rx_buf_ring == NULL)
1488 return;
1489
1490 for (i = 0; i < NUM_RX_BD; i++) {
1491 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1492 u8 *data = rx_buf->data;
1493
1494 if (data == NULL)
1495 continue;
1496 dma_unmap_single(&bp->pdev->dev,
1497 dma_unmap_addr(rx_buf, mapping),
1498 fp->rx_buf_size, DMA_FROM_DEVICE);
1499
1500 rx_buf->data = NULL;
1501 bnx2x_frag_free(fp, data);
1502 }
1503 }
1504
1505 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1506 {
1507 int j;
1508
1509 for_each_rx_queue_cnic(bp, j) {
1510 bnx2x_free_rx_bds(&bp->fp[j]);
1511 }
1512 }
1513
1514 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1515 {
1516 int j;
1517
1518 for_each_eth_queue(bp, j) {
1519 struct bnx2x_fastpath *fp = &bp->fp[j];
1520
1521 bnx2x_free_rx_bds(fp);
1522
1523 if (!fp->disable_tpa)
1524 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1525 }
1526 }
1527
1528 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1529 {
1530 bnx2x_free_tx_skbs_cnic(bp);
1531 bnx2x_free_rx_skbs_cnic(bp);
1532 }
1533
1534 void bnx2x_free_skbs(struct bnx2x *bp)
1535 {
1536 bnx2x_free_tx_skbs(bp);
1537 bnx2x_free_rx_skbs(bp);
1538 }
1539
1540 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1541 {
1542 /* load old values */
1543 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1544
1545 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1546 /* leave all but MAX value */
1547 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1548
1549 /* set new MAX value */
1550 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1551 & FUNC_MF_CFG_MAX_BW_MASK;
1552
1553 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1554 }
1555 }
1556
1557 /**
1558 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1559 *
1560 * @bp: driver handle
1561 * @nvecs: number of vectors to be released
1562 */
1563 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1564 {
1565 int i, offset = 0;
1566
1567 if (nvecs == offset)
1568 return;
1569
1570 /* VFs don't have a default SB */
1571 if (IS_PF(bp)) {
1572 free_irq(bp->msix_table[offset].vector, bp->dev);
1573 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1574 bp->msix_table[offset].vector);
1575 offset++;
1576 }
1577
1578 if (CNIC_SUPPORT(bp)) {
1579 if (nvecs == offset)
1580 return;
1581 offset++;
1582 }
1583
1584 for_each_eth_queue(bp, i) {
1585 if (nvecs == offset)
1586 return;
1587 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1588 i, bp->msix_table[offset].vector);
1589
1590 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1591 }
1592 }
1593
1594 void bnx2x_free_irq(struct bnx2x *bp)
1595 {
1596 if (bp->flags & USING_MSIX_FLAG &&
1597 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1598 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1599
1600 /* vfs don't have a default status block */
1601 if (IS_PF(bp))
1602 nvecs++;
1603
1604 bnx2x_free_msix_irqs(bp, nvecs);
1605 } else {
1606 free_irq(bp->dev->irq, bp->dev);
1607 }
1608 }
1609
1610 int bnx2x_enable_msix(struct bnx2x *bp)
1611 {
1612 int msix_vec = 0, i, rc;
1613
1614 /* VFs don't have a default status block */
1615 if (IS_PF(bp)) {
1616 bp->msix_table[msix_vec].entry = msix_vec;
1617 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1618 bp->msix_table[0].entry);
1619 msix_vec++;
1620 }
1621
1622 /* Cnic requires an msix vector for itself */
1623 if (CNIC_SUPPORT(bp)) {
1624 bp->msix_table[msix_vec].entry = msix_vec;
1625 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1626 msix_vec, bp->msix_table[msix_vec].entry);
1627 msix_vec++;
1628 }
1629
1630 /* We need separate vectors for ETH queues only (not FCoE) */
1631 for_each_eth_queue(bp, i) {
1632 bp->msix_table[msix_vec].entry = msix_vec;
1633 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1634 msix_vec, msix_vec, i);
1635 msix_vec++;
1636 }
1637
1638 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1639 msix_vec);
1640
1641 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1642
1643 /*
1644 * reconfigure number of tx/rx queues according to available
1645 * MSI-X vectors
1646 */
1647 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1648 /* how less vectors we will have? */
1649 int diff = msix_vec - rc;
1650
1651 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1652
1653 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1654
1655 if (rc) {
1656 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1657 goto no_msix;
1658 }
1659 /*
1660 * decrease number of queues by number of unallocated entries
1661 */
1662 bp->num_ethernet_queues -= diff;
1663 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1664
1665 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1666 bp->num_queues);
1667 } else if (rc > 0) {
1668 /* Get by with single vector */
1669 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1670 if (rc) {
1671 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1672 rc);
1673 goto no_msix;
1674 }
1675
1676 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1677 bp->flags |= USING_SINGLE_MSIX_FLAG;
1678
1679 BNX2X_DEV_INFO("set number of queues to 1\n");
1680 bp->num_ethernet_queues = 1;
1681 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1682 } else if (rc < 0) {
1683 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1684 goto no_msix;
1685 }
1686
1687 bp->flags |= USING_MSIX_FLAG;
1688
1689 return 0;
1690
1691 no_msix:
1692 /* fall to INTx if not enough memory */
1693 if (rc == -ENOMEM)
1694 bp->flags |= DISABLE_MSI_FLAG;
1695
1696 return rc;
1697 }
1698
1699 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1700 {
1701 int i, rc, offset = 0;
1702
1703 /* no default status block for vf */
1704 if (IS_PF(bp)) {
1705 rc = request_irq(bp->msix_table[offset++].vector,
1706 bnx2x_msix_sp_int, 0,
1707 bp->dev->name, bp->dev);
1708 if (rc) {
1709 BNX2X_ERR("request sp irq failed\n");
1710 return -EBUSY;
1711 }
1712 }
1713
1714 if (CNIC_SUPPORT(bp))
1715 offset++;
1716
1717 for_each_eth_queue(bp, i) {
1718 struct bnx2x_fastpath *fp = &bp->fp[i];
1719 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1720 bp->dev->name, i);
1721
1722 rc = request_irq(bp->msix_table[offset].vector,
1723 bnx2x_msix_fp_int, 0, fp->name, fp);
1724 if (rc) {
1725 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1726 bp->msix_table[offset].vector, rc);
1727 bnx2x_free_msix_irqs(bp, offset);
1728 return -EBUSY;
1729 }
1730
1731 offset++;
1732 }
1733
1734 i = BNX2X_NUM_ETH_QUEUES(bp);
1735 if (IS_PF(bp)) {
1736 offset = 1 + CNIC_SUPPORT(bp);
1737 netdev_info(bp->dev,
1738 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1739 bp->msix_table[0].vector,
1740 0, bp->msix_table[offset].vector,
1741 i - 1, bp->msix_table[offset + i - 1].vector);
1742 } else {
1743 offset = CNIC_SUPPORT(bp);
1744 netdev_info(bp->dev,
1745 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n",
1746 0, bp->msix_table[offset].vector,
1747 i - 1, bp->msix_table[offset + i - 1].vector);
1748 }
1749 return 0;
1750 }
1751
1752 int bnx2x_enable_msi(struct bnx2x *bp)
1753 {
1754 int rc;
1755
1756 rc = pci_enable_msi(bp->pdev);
1757 if (rc) {
1758 BNX2X_DEV_INFO("MSI is not attainable\n");
1759 return -1;
1760 }
1761 bp->flags |= USING_MSI_FLAG;
1762
1763 return 0;
1764 }
1765
1766 static int bnx2x_req_irq(struct bnx2x *bp)
1767 {
1768 unsigned long flags;
1769 unsigned int irq;
1770
1771 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1772 flags = 0;
1773 else
1774 flags = IRQF_SHARED;
1775
1776 if (bp->flags & USING_MSIX_FLAG)
1777 irq = bp->msix_table[0].vector;
1778 else
1779 irq = bp->pdev->irq;
1780
1781 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1782 }
1783
1784 static int bnx2x_setup_irqs(struct bnx2x *bp)
1785 {
1786 int rc = 0;
1787 if (bp->flags & USING_MSIX_FLAG &&
1788 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1789 rc = bnx2x_req_msix_irqs(bp);
1790 if (rc)
1791 return rc;
1792 } else {
1793 rc = bnx2x_req_irq(bp);
1794 if (rc) {
1795 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1796 return rc;
1797 }
1798 if (bp->flags & USING_MSI_FLAG) {
1799 bp->dev->irq = bp->pdev->irq;
1800 netdev_info(bp->dev, "using MSI IRQ %d\n",
1801 bp->dev->irq);
1802 }
1803 if (bp->flags & USING_MSIX_FLAG) {
1804 bp->dev->irq = bp->msix_table[0].vector;
1805 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1806 bp->dev->irq);
1807 }
1808 }
1809
1810 return 0;
1811 }
1812
1813 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1814 {
1815 int i;
1816
1817 for_each_rx_queue_cnic(bp, i) {
1818 bnx2x_fp_init_lock(&bp->fp[i]);
1819 napi_enable(&bnx2x_fp(bp, i, napi));
1820 }
1821 }
1822
1823 static void bnx2x_napi_enable(struct bnx2x *bp)
1824 {
1825 int i;
1826
1827 for_each_eth_queue(bp, i) {
1828 bnx2x_fp_init_lock(&bp->fp[i]);
1829 napi_enable(&bnx2x_fp(bp, i, napi));
1830 }
1831 }
1832
1833 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1834 {
1835 int i;
1836
1837 for_each_rx_queue_cnic(bp, i) {
1838 napi_disable(&bnx2x_fp(bp, i, napi));
1839 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1840 usleep_range(1000, 2000);
1841 }
1842 }
1843
1844 static void bnx2x_napi_disable(struct bnx2x *bp)
1845 {
1846 int i;
1847
1848 for_each_eth_queue(bp, i) {
1849 napi_disable(&bnx2x_fp(bp, i, napi));
1850 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1851 usleep_range(1000, 2000);
1852 }
1853 }
1854
1855 void bnx2x_netif_start(struct bnx2x *bp)
1856 {
1857 if (netif_running(bp->dev)) {
1858 bnx2x_napi_enable(bp);
1859 if (CNIC_LOADED(bp))
1860 bnx2x_napi_enable_cnic(bp);
1861 bnx2x_int_enable(bp);
1862 if (bp->state == BNX2X_STATE_OPEN)
1863 netif_tx_wake_all_queues(bp->dev);
1864 }
1865 }
1866
1867 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1868 {
1869 bnx2x_int_disable_sync(bp, disable_hw);
1870 bnx2x_napi_disable(bp);
1871 if (CNIC_LOADED(bp))
1872 bnx2x_napi_disable_cnic(bp);
1873 }
1874
1875 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1876 void *accel_priv, select_queue_fallback_t fallback)
1877 {
1878 struct bnx2x *bp = netdev_priv(dev);
1879
1880 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1881 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1882 u16 ether_type = ntohs(hdr->h_proto);
1883
1884 /* Skip VLAN tag if present */
1885 if (ether_type == ETH_P_8021Q) {
1886 struct vlan_ethhdr *vhdr =
1887 (struct vlan_ethhdr *)skb->data;
1888
1889 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1890 }
1891
1892 /* If ethertype is FCoE or FIP - use FCoE ring */
1893 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1894 return bnx2x_fcoe_tx(bp, txq_index);
1895 }
1896
1897 /* select a non-FCoE queue */
1898 return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1899 }
1900
1901 void bnx2x_set_num_queues(struct bnx2x *bp)
1902 {
1903 /* RSS queues */
1904 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1905
1906 /* override in STORAGE SD modes */
1907 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1908 bp->num_ethernet_queues = 1;
1909
1910 /* Add special queues */
1911 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1912 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1913
1914 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1915 }
1916
1917 /**
1918 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1919 *
1920 * @bp: Driver handle
1921 *
1922 * We currently support for at most 16 Tx queues for each CoS thus we will
1923 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1924 * bp->max_cos.
1925 *
1926 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1927 * index after all ETH L2 indices.
1928 *
1929 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1930 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1931 * 16..31,...) with indices that are not coupled with any real Tx queue.
1932 *
1933 * The proper configuration of skb->queue_mapping is handled by
1934 * bnx2x_select_queue() and __skb_tx_hash().
1935 *
1936 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1937 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1938 */
1939 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1940 {
1941 int rc, tx, rx;
1942
1943 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1944 rx = BNX2X_NUM_ETH_QUEUES(bp);
1945
1946 /* account for fcoe queue */
1947 if (include_cnic && !NO_FCOE(bp)) {
1948 rx++;
1949 tx++;
1950 }
1951
1952 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1953 if (rc) {
1954 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1955 return rc;
1956 }
1957 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1958 if (rc) {
1959 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1960 return rc;
1961 }
1962
1963 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1964 tx, rx);
1965
1966 return rc;
1967 }
1968
1969 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1970 {
1971 int i;
1972
1973 for_each_queue(bp, i) {
1974 struct bnx2x_fastpath *fp = &bp->fp[i];
1975 u32 mtu;
1976
1977 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1978 if (IS_FCOE_IDX(i))
1979 /*
1980 * Although there are no IP frames expected to arrive to
1981 * this ring we still want to add an
1982 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1983 * overrun attack.
1984 */
1985 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1986 else
1987 mtu = bp->dev->mtu;
1988 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1989 IP_HEADER_ALIGNMENT_PADDING +
1990 ETH_OVREHEAD +
1991 mtu +
1992 BNX2X_FW_RX_ALIGN_END;
1993 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
1994 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1995 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1996 else
1997 fp->rx_frag_size = 0;
1998 }
1999 }
2000
2001 static int bnx2x_init_rss(struct bnx2x *bp)
2002 {
2003 int i;
2004 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2005
2006 /* Prepare the initial contents for the indirection table if RSS is
2007 * enabled
2008 */
2009 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2010 bp->rss_conf_obj.ind_table[i] =
2011 bp->fp->cl_id +
2012 ethtool_rxfh_indir_default(i, num_eth_queues);
2013
2014 /*
2015 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2016 * per-port, so if explicit configuration is needed , do it only
2017 * for a PMF.
2018 *
2019 * For 57712 and newer on the other hand it's a per-function
2020 * configuration.
2021 */
2022 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2023 }
2024
2025 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2026 bool config_hash, bool enable)
2027 {
2028 struct bnx2x_config_rss_params params = {NULL};
2029
2030 /* Although RSS is meaningless when there is a single HW queue we
2031 * still need it enabled in order to have HW Rx hash generated.
2032 *
2033 * if (!is_eth_multi(bp))
2034 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
2035 */
2036
2037 params.rss_obj = rss_obj;
2038
2039 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2040
2041 if (enable) {
2042 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2043
2044 /* RSS configuration */
2045 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2046 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2047 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2048 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2049 if (rss_obj->udp_rss_v4)
2050 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2051 if (rss_obj->udp_rss_v6)
2052 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2053 } else {
2054 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2055 }
2056
2057 /* Hash bits */
2058 params.rss_result_mask = MULTI_MASK;
2059
2060 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2061
2062 if (config_hash) {
2063 /* RSS keys */
2064 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2065 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2066 }
2067
2068 if (IS_PF(bp))
2069 return bnx2x_config_rss(bp, &params);
2070 else
2071 return bnx2x_vfpf_config_rss(bp, &params);
2072 }
2073
2074 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2075 {
2076 struct bnx2x_func_state_params func_params = {NULL};
2077
2078 /* Prepare parameters for function state transitions */
2079 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2080
2081 func_params.f_obj = &bp->func_obj;
2082 func_params.cmd = BNX2X_F_CMD_HW_INIT;
2083
2084 func_params.params.hw_init.load_phase = load_code;
2085
2086 return bnx2x_func_state_change(bp, &func_params);
2087 }
2088
2089 /*
2090 * Cleans the object that have internal lists without sending
2091 * ramrods. Should be run when interrupts are disabled.
2092 */
2093 void bnx2x_squeeze_objects(struct bnx2x *bp)
2094 {
2095 int rc;
2096 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2097 struct bnx2x_mcast_ramrod_params rparam = {NULL};
2098 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2099
2100 /***************** Cleanup MACs' object first *************************/
2101
2102 /* Wait for completion of requested */
2103 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2104 /* Perform a dry cleanup */
2105 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2106
2107 /* Clean ETH primary MAC */
2108 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2109 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2110 &ramrod_flags);
2111 if (rc != 0)
2112 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2113
2114 /* Cleanup UC list */
2115 vlan_mac_flags = 0;
2116 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2117 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2118 &ramrod_flags);
2119 if (rc != 0)
2120 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2121
2122 /***************** Now clean mcast object *****************************/
2123 rparam.mcast_obj = &bp->mcast_obj;
2124 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2125
2126 /* Add a DEL command... - Since we're doing a driver cleanup only,
2127 * we take a lock surrounding both the initial send and the CONTs,
2128 * as we don't want a true completion to disrupt us in the middle.
2129 */
2130 netif_addr_lock_bh(bp->dev);
2131 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2132 if (rc < 0)
2133 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2134 rc);
2135
2136 /* ...and wait until all pending commands are cleared */
2137 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2138 while (rc != 0) {
2139 if (rc < 0) {
2140 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2141 rc);
2142 netif_addr_unlock_bh(bp->dev);
2143 return;
2144 }
2145
2146 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2147 }
2148 netif_addr_unlock_bh(bp->dev);
2149 }
2150
2151 #ifndef BNX2X_STOP_ON_ERROR
2152 #define LOAD_ERROR_EXIT(bp, label) \
2153 do { \
2154 (bp)->state = BNX2X_STATE_ERROR; \
2155 goto label; \
2156 } while (0)
2157
2158 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2159 do { \
2160 bp->cnic_loaded = false; \
2161 goto label; \
2162 } while (0)
2163 #else /*BNX2X_STOP_ON_ERROR*/
2164 #define LOAD_ERROR_EXIT(bp, label) \
2165 do { \
2166 (bp)->state = BNX2X_STATE_ERROR; \
2167 (bp)->panic = 1; \
2168 return -EBUSY; \
2169 } while (0)
2170 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2171 do { \
2172 bp->cnic_loaded = false; \
2173 (bp)->panic = 1; \
2174 return -EBUSY; \
2175 } while (0)
2176 #endif /*BNX2X_STOP_ON_ERROR*/
2177
2178 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2179 {
2180 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2181 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2182 return;
2183 }
2184
2185 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2186 {
2187 int num_groups, vf_headroom = 0;
2188 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2189
2190 /* number of queues for statistics is number of eth queues + FCoE */
2191 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2192
2193 /* Total number of FW statistics requests =
2194 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2195 * and fcoe l2 queue) stats + num of queues (which includes another 1
2196 * for fcoe l2 queue if applicable)
2197 */
2198 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2199
2200 /* vf stats appear in the request list, but their data is allocated by
2201 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2202 * it is used to determine where to place the vf stats queries in the
2203 * request struct
2204 */
2205 if (IS_SRIOV(bp))
2206 vf_headroom = bnx2x_vf_headroom(bp);
2207
2208 /* Request is built from stats_query_header and an array of
2209 * stats_query_cmd_group each of which contains
2210 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2211 * configured in the stats_query_header.
2212 */
2213 num_groups =
2214 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2215 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2216 1 : 0));
2217
2218 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2219 bp->fw_stats_num, vf_headroom, num_groups);
2220 bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2221 num_groups * sizeof(struct stats_query_cmd_group);
2222
2223 /* Data for statistics requests + stats_counter
2224 * stats_counter holds per-STORM counters that are incremented
2225 * when STORM has finished with the current request.
2226 * memory for FCoE offloaded statistics are counted anyway,
2227 * even if they will not be sent.
2228 * VF stats are not accounted for here as the data of VF stats is stored
2229 * in memory allocated by the VF, not here.
2230 */
2231 bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2232 sizeof(struct per_pf_stats) +
2233 sizeof(struct fcoe_statistics_params) +
2234 sizeof(struct per_queue_stats) * num_queue_stats +
2235 sizeof(struct stats_counter);
2236
2237 BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2238 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2239
2240 /* Set shortcuts */
2241 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2242 bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2243 bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2244 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2245 bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2246 bp->fw_stats_req_sz;
2247
2248 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2249 U64_HI(bp->fw_stats_req_mapping),
2250 U64_LO(bp->fw_stats_req_mapping));
2251 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2252 U64_HI(bp->fw_stats_data_mapping),
2253 U64_LO(bp->fw_stats_data_mapping));
2254 return 0;
2255
2256 alloc_mem_err:
2257 bnx2x_free_fw_stats_mem(bp);
2258 BNX2X_ERR("Can't allocate FW stats memory\n");
2259 return -ENOMEM;
2260 }
2261
2262 /* send load request to mcp and analyze response */
2263 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2264 {
2265 u32 param;
2266
2267 /* init fw_seq */
2268 bp->fw_seq =
2269 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2270 DRV_MSG_SEQ_NUMBER_MASK);
2271 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2272
2273 /* Get current FW pulse sequence */
2274 bp->fw_drv_pulse_wr_seq =
2275 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2276 DRV_PULSE_SEQ_MASK);
2277 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2278
2279 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2280
2281 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2282 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2283
2284 /* load request */
2285 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2286
2287 /* if mcp fails to respond we must abort */
2288 if (!(*load_code)) {
2289 BNX2X_ERR("MCP response failure, aborting\n");
2290 return -EBUSY;
2291 }
2292
2293 /* If mcp refused (e.g. other port is in diagnostic mode) we
2294 * must abort
2295 */
2296 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2297 BNX2X_ERR("MCP refused load request, aborting\n");
2298 return -EBUSY;
2299 }
2300 return 0;
2301 }
2302
2303 /* check whether another PF has already loaded FW to chip. In
2304 * virtualized environments a pf from another VM may have already
2305 * initialized the device including loading FW
2306 */
2307 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2308 {
2309 /* is another pf loaded on this engine? */
2310 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2311 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2312 /* build my FW version dword */
2313 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2314 (BCM_5710_FW_MINOR_VERSION << 8) +
2315 (BCM_5710_FW_REVISION_VERSION << 16) +
2316 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2317
2318 /* read loaded FW from chip */
2319 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2320
2321 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2322 loaded_fw, my_fw);
2323
2324 /* abort nic load if version mismatch */
2325 if (my_fw != loaded_fw) {
2326 if (print_err)
2327 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2328 loaded_fw, my_fw);
2329 else
2330 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2331 loaded_fw, my_fw);
2332 return -EBUSY;
2333 }
2334 }
2335 return 0;
2336 }
2337
2338 /* returns the "mcp load_code" according to global load_count array */
2339 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2340 {
2341 int path = BP_PATH(bp);
2342
2343 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2344 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2345 bnx2x_load_count[path][2]);
2346 bnx2x_load_count[path][0]++;
2347 bnx2x_load_count[path][1 + port]++;
2348 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2349 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2350 bnx2x_load_count[path][2]);
2351 if (bnx2x_load_count[path][0] == 1)
2352 return FW_MSG_CODE_DRV_LOAD_COMMON;
2353 else if (bnx2x_load_count[path][1 + port] == 1)
2354 return FW_MSG_CODE_DRV_LOAD_PORT;
2355 else
2356 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2357 }
2358
2359 /* mark PMF if applicable */
2360 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2361 {
2362 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2363 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2364 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2365 bp->port.pmf = 1;
2366 /* We need the barrier to ensure the ordering between the
2367 * writing to bp->port.pmf here and reading it from the
2368 * bnx2x_periodic_task().
2369 */
2370 smp_mb();
2371 } else {
2372 bp->port.pmf = 0;
2373 }
2374
2375 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2376 }
2377
2378 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2379 {
2380 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2381 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2382 (bp->common.shmem2_base)) {
2383 if (SHMEM2_HAS(bp, dcc_support))
2384 SHMEM2_WR(bp, dcc_support,
2385 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2386 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2387 if (SHMEM2_HAS(bp, afex_driver_support))
2388 SHMEM2_WR(bp, afex_driver_support,
2389 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2390 }
2391
2392 /* Set AFEX default VLAN tag to an invalid value */
2393 bp->afex_def_vlan_tag = -1;
2394 }
2395
2396 /**
2397 * bnx2x_bz_fp - zero content of the fastpath structure.
2398 *
2399 * @bp: driver handle
2400 * @index: fastpath index to be zeroed
2401 *
2402 * Makes sure the contents of the bp->fp[index].napi is kept
2403 * intact.
2404 */
2405 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2406 {
2407 struct bnx2x_fastpath *fp = &bp->fp[index];
2408 int cos;
2409 struct napi_struct orig_napi = fp->napi;
2410 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2411
2412 /* bzero bnx2x_fastpath contents */
2413 if (fp->tpa_info)
2414 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2415 sizeof(struct bnx2x_agg_info));
2416 memset(fp, 0, sizeof(*fp));
2417
2418 /* Restore the NAPI object as it has been already initialized */
2419 fp->napi = orig_napi;
2420 fp->tpa_info = orig_tpa_info;
2421 fp->bp = bp;
2422 fp->index = index;
2423 if (IS_ETH_FP(fp))
2424 fp->max_cos = bp->max_cos;
2425 else
2426 /* Special queues support only one CoS */
2427 fp->max_cos = 1;
2428
2429 /* Init txdata pointers */
2430 if (IS_FCOE_FP(fp))
2431 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2432 if (IS_ETH_FP(fp))
2433 for_each_cos_in_tx_queue(fp, cos)
2434 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2435 BNX2X_NUM_ETH_QUEUES(bp) + index];
2436
2437 /* set the tpa flag for each queue. The tpa flag determines the queue
2438 * minimal size so it must be set prior to queue memory allocation
2439 */
2440 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2441 (bp->flags & GRO_ENABLE_FLAG &&
2442 bnx2x_mtu_allows_gro(bp->dev->mtu)));
2443 if (bp->flags & TPA_ENABLE_FLAG)
2444 fp->mode = TPA_MODE_LRO;
2445 else if (bp->flags & GRO_ENABLE_FLAG)
2446 fp->mode = TPA_MODE_GRO;
2447
2448 /* We don't want TPA on an FCoE L2 ring */
2449 if (IS_FCOE_FP(fp))
2450 fp->disable_tpa = 1;
2451 }
2452
2453 int bnx2x_load_cnic(struct bnx2x *bp)
2454 {
2455 int i, rc, port = BP_PORT(bp);
2456
2457 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2458
2459 mutex_init(&bp->cnic_mutex);
2460
2461 if (IS_PF(bp)) {
2462 rc = bnx2x_alloc_mem_cnic(bp);
2463 if (rc) {
2464 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2465 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2466 }
2467 }
2468
2469 rc = bnx2x_alloc_fp_mem_cnic(bp);
2470 if (rc) {
2471 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2472 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2473 }
2474
2475 /* Update the number of queues with the cnic queues */
2476 rc = bnx2x_set_real_num_queues(bp, 1);
2477 if (rc) {
2478 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2479 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2480 }
2481
2482 /* Add all CNIC NAPI objects */
2483 bnx2x_add_all_napi_cnic(bp);
2484 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2485 bnx2x_napi_enable_cnic(bp);
2486
2487 rc = bnx2x_init_hw_func_cnic(bp);
2488 if (rc)
2489 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2490
2491 bnx2x_nic_init_cnic(bp);
2492
2493 if (IS_PF(bp)) {
2494 /* Enable Timer scan */
2495 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2496
2497 /* setup cnic queues */
2498 for_each_cnic_queue(bp, i) {
2499 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2500 if (rc) {
2501 BNX2X_ERR("Queue setup failed\n");
2502 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2503 }
2504 }
2505 }
2506
2507 /* Initialize Rx filter. */
2508 bnx2x_set_rx_mode_inner(bp);
2509
2510 /* re-read iscsi info */
2511 bnx2x_get_iscsi_info(bp);
2512 bnx2x_setup_cnic_irq_info(bp);
2513 bnx2x_setup_cnic_info(bp);
2514 bp->cnic_loaded = true;
2515 if (bp->state == BNX2X_STATE_OPEN)
2516 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2517
2518 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2519
2520 return 0;
2521
2522 #ifndef BNX2X_STOP_ON_ERROR
2523 load_error_cnic2:
2524 /* Disable Timer scan */
2525 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2526
2527 load_error_cnic1:
2528 bnx2x_napi_disable_cnic(bp);
2529 /* Update the number of queues without the cnic queues */
2530 if (bnx2x_set_real_num_queues(bp, 0))
2531 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2532 load_error_cnic0:
2533 BNX2X_ERR("CNIC-related load failed\n");
2534 bnx2x_free_fp_mem_cnic(bp);
2535 bnx2x_free_mem_cnic(bp);
2536 return rc;
2537 #endif /* ! BNX2X_STOP_ON_ERROR */
2538 }
2539
2540 /* must be called with rtnl_lock */
2541 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2542 {
2543 int port = BP_PORT(bp);
2544 int i, rc = 0, load_code = 0;
2545
2546 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2547 DP(NETIF_MSG_IFUP,
2548 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2549
2550 #ifdef BNX2X_STOP_ON_ERROR
2551 if (unlikely(bp->panic)) {
2552 BNX2X_ERR("Can't load NIC when there is panic\n");
2553 return -EPERM;
2554 }
2555 #endif
2556
2557 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2558
2559 /* zero the structure w/o any lock, before SP handler is initialized */
2560 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2561 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2562 &bp->last_reported_link.link_report_flags);
2563
2564 if (IS_PF(bp))
2565 /* must be called before memory allocation and HW init */
2566 bnx2x_ilt_set_info(bp);
2567
2568 /*
2569 * Zero fastpath structures preserving invariants like napi, which are
2570 * allocated only once, fp index, max_cos, bp pointer.
2571 * Also set fp->disable_tpa and txdata_ptr.
2572 */
2573 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2574 for_each_queue(bp, i)
2575 bnx2x_bz_fp(bp, i);
2576 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2577 bp->num_cnic_queues) *
2578 sizeof(struct bnx2x_fp_txdata));
2579
2580 bp->fcoe_init = false;
2581
2582 /* Set the receive queues buffer size */
2583 bnx2x_set_rx_buf_size(bp);
2584
2585 if (IS_PF(bp)) {
2586 rc = bnx2x_alloc_mem(bp);
2587 if (rc) {
2588 BNX2X_ERR("Unable to allocate bp memory\n");
2589 return rc;
2590 }
2591 }
2592
2593 /* need to be done after alloc mem, since it's self adjusting to amount
2594 * of memory available for RSS queues
2595 */
2596 rc = bnx2x_alloc_fp_mem(bp);
2597 if (rc) {
2598 BNX2X_ERR("Unable to allocate memory for fps\n");
2599 LOAD_ERROR_EXIT(bp, load_error0);
2600 }
2601
2602 /* Allocated memory for FW statistics */
2603 if (bnx2x_alloc_fw_stats_mem(bp))
2604 LOAD_ERROR_EXIT(bp, load_error0);
2605
2606 /* request pf to initialize status blocks */
2607 if (IS_VF(bp)) {
2608 rc = bnx2x_vfpf_init(bp);
2609 if (rc)
2610 LOAD_ERROR_EXIT(bp, load_error0);
2611 }
2612
2613 /* As long as bnx2x_alloc_mem() may possibly update
2614 * bp->num_queues, bnx2x_set_real_num_queues() should always
2615 * come after it. At this stage cnic queues are not counted.
2616 */
2617 rc = bnx2x_set_real_num_queues(bp, 0);
2618 if (rc) {
2619 BNX2X_ERR("Unable to set real_num_queues\n");
2620 LOAD_ERROR_EXIT(bp, load_error0);
2621 }
2622
2623 /* configure multi cos mappings in kernel.
2624 * this configuration may be overridden by a multi class queue
2625 * discipline or by a dcbx negotiation result.
2626 */
2627 bnx2x_setup_tc(bp->dev, bp->max_cos);
2628
2629 /* Add all NAPI objects */
2630 bnx2x_add_all_napi(bp);
2631 DP(NETIF_MSG_IFUP, "napi added\n");
2632 bnx2x_napi_enable(bp);
2633
2634 if (IS_PF(bp)) {
2635 /* set pf load just before approaching the MCP */
2636 bnx2x_set_pf_load(bp);
2637
2638 /* if mcp exists send load request and analyze response */
2639 if (!BP_NOMCP(bp)) {
2640 /* attempt to load pf */
2641 rc = bnx2x_nic_load_request(bp, &load_code);
2642 if (rc)
2643 LOAD_ERROR_EXIT(bp, load_error1);
2644
2645 /* what did mcp say? */
2646 rc = bnx2x_compare_fw_ver(bp, load_code, true);
2647 if (rc) {
2648 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2649 LOAD_ERROR_EXIT(bp, load_error2);
2650 }
2651 } else {
2652 load_code = bnx2x_nic_load_no_mcp(bp, port);
2653 }
2654
2655 /* mark pmf if applicable */
2656 bnx2x_nic_load_pmf(bp, load_code);
2657
2658 /* Init Function state controlling object */
2659 bnx2x__init_func_obj(bp);
2660
2661 /* Initialize HW */
2662 rc = bnx2x_init_hw(bp, load_code);
2663 if (rc) {
2664 BNX2X_ERR("HW init failed, aborting\n");
2665 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2666 LOAD_ERROR_EXIT(bp, load_error2);
2667 }
2668 }
2669
2670 bnx2x_pre_irq_nic_init(bp);
2671
2672 /* Connect to IRQs */
2673 rc = bnx2x_setup_irqs(bp);
2674 if (rc) {
2675 BNX2X_ERR("setup irqs failed\n");
2676 if (IS_PF(bp))
2677 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2678 LOAD_ERROR_EXIT(bp, load_error2);
2679 }
2680
2681 /* Init per-function objects */
2682 if (IS_PF(bp)) {
2683 /* Setup NIC internals and enable interrupts */
2684 bnx2x_post_irq_nic_init(bp, load_code);
2685
2686 bnx2x_init_bp_objs(bp);
2687 bnx2x_iov_nic_init(bp);
2688
2689 /* Set AFEX default VLAN tag to an invalid value */
2690 bp->afex_def_vlan_tag = -1;
2691 bnx2x_nic_load_afex_dcc(bp, load_code);
2692 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2693 rc = bnx2x_func_start(bp);
2694 if (rc) {
2695 BNX2X_ERR("Function start failed!\n");
2696 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2697
2698 LOAD_ERROR_EXIT(bp, load_error3);
2699 }
2700
2701 /* Send LOAD_DONE command to MCP */
2702 if (!BP_NOMCP(bp)) {
2703 load_code = bnx2x_fw_command(bp,
2704 DRV_MSG_CODE_LOAD_DONE, 0);
2705 if (!load_code) {
2706 BNX2X_ERR("MCP response failure, aborting\n");
2707 rc = -EBUSY;
2708 LOAD_ERROR_EXIT(bp, load_error3);
2709 }
2710 }
2711
2712 /* initialize FW coalescing state machines in RAM */
2713 bnx2x_update_coalesce(bp);
2714 }
2715
2716 /* setup the leading queue */
2717 rc = bnx2x_setup_leading(bp);
2718 if (rc) {
2719 BNX2X_ERR("Setup leading failed!\n");
2720 LOAD_ERROR_EXIT(bp, load_error3);
2721 }
2722
2723 /* set up the rest of the queues */
2724 for_each_nondefault_eth_queue(bp, i) {
2725 if (IS_PF(bp))
2726 rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2727 else /* VF */
2728 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2729 if (rc) {
2730 BNX2X_ERR("Queue %d setup failed\n", i);
2731 LOAD_ERROR_EXIT(bp, load_error3);
2732 }
2733 }
2734
2735 /* setup rss */
2736 rc = bnx2x_init_rss(bp);
2737 if (rc) {
2738 BNX2X_ERR("PF RSS init failed\n");
2739 LOAD_ERROR_EXIT(bp, load_error3);
2740 }
2741
2742 /* Now when Clients are configured we are ready to work */
2743 bp->state = BNX2X_STATE_OPEN;
2744
2745 /* Configure a ucast MAC */
2746 if (IS_PF(bp))
2747 rc = bnx2x_set_eth_mac(bp, true);
2748 else /* vf */
2749 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2750 true);
2751 if (rc) {
2752 BNX2X_ERR("Setting Ethernet MAC failed\n");
2753 LOAD_ERROR_EXIT(bp, load_error3);
2754 }
2755
2756 if (IS_PF(bp) && bp->pending_max) {
2757 bnx2x_update_max_mf_config(bp, bp->pending_max);
2758 bp->pending_max = 0;
2759 }
2760
2761 if (bp->port.pmf) {
2762 rc = bnx2x_initial_phy_init(bp, load_mode);
2763 if (rc)
2764 LOAD_ERROR_EXIT(bp, load_error3);
2765 }
2766 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2767
2768 /* Start fast path */
2769
2770 /* Initialize Rx filter. */
2771 bnx2x_set_rx_mode_inner(bp);
2772
2773 /* Start the Tx */
2774 switch (load_mode) {
2775 case LOAD_NORMAL:
2776 /* Tx queue should be only re-enabled */
2777 netif_tx_wake_all_queues(bp->dev);
2778 break;
2779
2780 case LOAD_OPEN:
2781 netif_tx_start_all_queues(bp->dev);
2782 smp_mb__after_clear_bit();
2783 break;
2784
2785 case LOAD_DIAG:
2786 case LOAD_LOOPBACK_EXT:
2787 bp->state = BNX2X_STATE_DIAG;
2788 break;
2789
2790 default:
2791 break;
2792 }
2793
2794 if (bp->port.pmf)
2795 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2796 else
2797 bnx2x__link_status_update(bp);
2798
2799 /* start the timer */
2800 mod_timer(&bp->timer, jiffies + bp->current_interval);
2801
2802 if (CNIC_ENABLED(bp))
2803 bnx2x_load_cnic(bp);
2804
2805 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2806 /* mark driver is loaded in shmem2 */
2807 u32 val;
2808 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2809 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2810 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2811 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2812 }
2813
2814 /* Wait for all pending SP commands to complete */
2815 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2816 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2817 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2818 return -EBUSY;
2819 }
2820
2821 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2822 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2823 bnx2x_dcbx_init(bp, false);
2824
2825 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2826
2827 return 0;
2828
2829 #ifndef BNX2X_STOP_ON_ERROR
2830 load_error3:
2831 if (IS_PF(bp)) {
2832 bnx2x_int_disable_sync(bp, 1);
2833
2834 /* Clean queueable objects */
2835 bnx2x_squeeze_objects(bp);
2836 }
2837
2838 /* Free SKBs, SGEs, TPA pool and driver internals */
2839 bnx2x_free_skbs(bp);
2840 for_each_rx_queue(bp, i)
2841 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2842
2843 /* Release IRQs */
2844 bnx2x_free_irq(bp);
2845 load_error2:
2846 if (IS_PF(bp) && !BP_NOMCP(bp)) {
2847 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2848 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2849 }
2850
2851 bp->port.pmf = 0;
2852 load_error1:
2853 bnx2x_napi_disable(bp);
2854 bnx2x_del_all_napi(bp);
2855
2856 /* clear pf_load status, as it was already set */
2857 if (IS_PF(bp))
2858 bnx2x_clear_pf_load(bp);
2859 load_error0:
2860 bnx2x_free_fw_stats_mem(bp);
2861 bnx2x_free_fp_mem(bp);
2862 bnx2x_free_mem(bp);
2863
2864 return rc;
2865 #endif /* ! BNX2X_STOP_ON_ERROR */
2866 }
2867
2868 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2869 {
2870 u8 rc = 0, cos, i;
2871
2872 /* Wait until tx fastpath tasks complete */
2873 for_each_tx_queue(bp, i) {
2874 struct bnx2x_fastpath *fp = &bp->fp[i];
2875
2876 for_each_cos_in_tx_queue(fp, cos)
2877 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2878 if (rc)
2879 return rc;
2880 }
2881 return 0;
2882 }
2883
2884 /* must be called with rtnl_lock */
2885 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2886 {
2887 int i;
2888 bool global = false;
2889
2890 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2891
2892 /* mark driver is unloaded in shmem2 */
2893 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2894 u32 val;
2895 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2896 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2897 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2898 }
2899
2900 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2901 (bp->state == BNX2X_STATE_CLOSED ||
2902 bp->state == BNX2X_STATE_ERROR)) {
2903 /* We can get here if the driver has been unloaded
2904 * during parity error recovery and is either waiting for a
2905 * leader to complete or for other functions to unload and
2906 * then ifdown has been issued. In this case we want to
2907 * unload and let other functions to complete a recovery
2908 * process.
2909 */
2910 bp->recovery_state = BNX2X_RECOVERY_DONE;
2911 bp->is_leader = 0;
2912 bnx2x_release_leader_lock(bp);
2913 smp_mb();
2914
2915 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2916 BNX2X_ERR("Can't unload in closed or error state\n");
2917 return -EINVAL;
2918 }
2919
2920 /* Nothing to do during unload if previous bnx2x_nic_load()
2921 * have not completed successfully - all resources are released.
2922 *
2923 * we can get here only after unsuccessful ndo_* callback, during which
2924 * dev->IFF_UP flag is still on.
2925 */
2926 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2927 return 0;
2928
2929 /* It's important to set the bp->state to the value different from
2930 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2931 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2932 */
2933 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2934 smp_mb();
2935
2936 /* indicate to VFs that the PF is going down */
2937 bnx2x_iov_channel_down(bp);
2938
2939 if (CNIC_LOADED(bp))
2940 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2941
2942 /* Stop Tx */
2943 bnx2x_tx_disable(bp);
2944 netdev_reset_tc(bp->dev);
2945
2946 bp->rx_mode = BNX2X_RX_MODE_NONE;
2947
2948 del_timer_sync(&bp->timer);
2949
2950 if (IS_PF(bp)) {
2951 /* Set ALWAYS_ALIVE bit in shmem */
2952 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2953 bnx2x_drv_pulse(bp);
2954 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2955 bnx2x_save_statistics(bp);
2956 }
2957
2958 /* wait till consumers catch up with producers in all queues */
2959 bnx2x_drain_tx_queues(bp);
2960
2961 /* if VF indicate to PF this function is going down (PF will delete sp
2962 * elements and clear initializations
2963 */
2964 if (IS_VF(bp))
2965 bnx2x_vfpf_close_vf(bp);
2966 else if (unload_mode != UNLOAD_RECOVERY)
2967 /* if this is a normal/close unload need to clean up chip*/
2968 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2969 else {
2970 /* Send the UNLOAD_REQUEST to the MCP */
2971 bnx2x_send_unload_req(bp, unload_mode);
2972
2973 /* Prevent transactions to host from the functions on the
2974 * engine that doesn't reset global blocks in case of global
2975 * attention once global blocks are reset and gates are opened
2976 * (the engine which leader will perform the recovery
2977 * last).
2978 */
2979 if (!CHIP_IS_E1x(bp))
2980 bnx2x_pf_disable(bp);
2981
2982 /* Disable HW interrupts, NAPI */
2983 bnx2x_netif_stop(bp, 1);
2984 /* Delete all NAPI objects */
2985 bnx2x_del_all_napi(bp);
2986 if (CNIC_LOADED(bp))
2987 bnx2x_del_all_napi_cnic(bp);
2988 /* Release IRQs */
2989 bnx2x_free_irq(bp);
2990
2991 /* Report UNLOAD_DONE to MCP */
2992 bnx2x_send_unload_done(bp, false);
2993 }
2994
2995 /*
2996 * At this stage no more interrupts will arrive so we may safely clean
2997 * the queueable objects here in case they failed to get cleaned so far.
2998 */
2999 if (IS_PF(bp))
3000 bnx2x_squeeze_objects(bp);
3001
3002 /* There should be no more pending SP commands at this stage */
3003 bp->sp_state = 0;
3004
3005 bp->port.pmf = 0;
3006
3007 /* clear pending work in rtnl task */
3008 bp->sp_rtnl_state = 0;
3009 smp_mb();
3010
3011 /* Free SKBs, SGEs, TPA pool and driver internals */
3012 bnx2x_free_skbs(bp);
3013 if (CNIC_LOADED(bp))
3014 bnx2x_free_skbs_cnic(bp);
3015 for_each_rx_queue(bp, i)
3016 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3017
3018 bnx2x_free_fp_mem(bp);
3019 if (CNIC_LOADED(bp))
3020 bnx2x_free_fp_mem_cnic(bp);
3021
3022 if (IS_PF(bp)) {
3023 if (CNIC_LOADED(bp))
3024 bnx2x_free_mem_cnic(bp);
3025 }
3026 bnx2x_free_mem(bp);
3027
3028 bp->state = BNX2X_STATE_CLOSED;
3029 bp->cnic_loaded = false;
3030
3031 /* Check if there are pending parity attentions. If there are - set
3032 * RECOVERY_IN_PROGRESS.
3033 */
3034 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3035 bnx2x_set_reset_in_progress(bp);
3036
3037 /* Set RESET_IS_GLOBAL if needed */
3038 if (global)
3039 bnx2x_set_reset_global(bp);
3040 }
3041
3042 /* The last driver must disable a "close the gate" if there is no
3043 * parity attention or "process kill" pending.
3044 */
3045 if (IS_PF(bp) &&
3046 !bnx2x_clear_pf_load(bp) &&
3047 bnx2x_reset_is_done(bp, BP_PATH(bp)))
3048 bnx2x_disable_close_the_gate(bp);
3049
3050 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3051
3052 return 0;
3053 }
3054
3055 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3056 {
3057 u16 pmcsr;
3058
3059 /* If there is no power capability, silently succeed */
3060 if (!bp->pdev->pm_cap) {
3061 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3062 return 0;
3063 }
3064
3065 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3066
3067 switch (state) {
3068 case PCI_D0:
3069 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3070 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3071 PCI_PM_CTRL_PME_STATUS));
3072
3073 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3074 /* delay required during transition out of D3hot */
3075 msleep(20);
3076 break;
3077
3078 case PCI_D3hot:
3079 /* If there are other clients above don't
3080 shut down the power */
3081 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3082 return 0;
3083 /* Don't shut down the power for emulation and FPGA */
3084 if (CHIP_REV_IS_SLOW(bp))
3085 return 0;
3086
3087 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3088 pmcsr |= 3;
3089
3090 if (bp->wol)
3091 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3092
3093 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3094 pmcsr);
3095
3096 /* No more memory access after this point until
3097 * device is brought back to D0.
3098 */
3099 break;
3100
3101 default:
3102 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3103 return -EINVAL;
3104 }
3105 return 0;
3106 }
3107
3108 /*
3109 * net_device service functions
3110 */
3111 static int bnx2x_poll(struct napi_struct *napi, int budget)
3112 {
3113 int work_done = 0;
3114 u8 cos;
3115 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3116 napi);
3117 struct bnx2x *bp = fp->bp;
3118
3119 while (1) {
3120 #ifdef BNX2X_STOP_ON_ERROR
3121 if (unlikely(bp->panic)) {
3122 napi_complete(napi);
3123 return 0;
3124 }
3125 #endif
3126 if (!bnx2x_fp_lock_napi(fp))
3127 return work_done;
3128
3129 for_each_cos_in_tx_queue(fp, cos)
3130 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3131 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3132
3133 if (bnx2x_has_rx_work(fp)) {
3134 work_done += bnx2x_rx_int(fp, budget - work_done);
3135
3136 /* must not complete if we consumed full budget */
3137 if (work_done >= budget) {
3138 bnx2x_fp_unlock_napi(fp);
3139 break;
3140 }
3141 }
3142
3143 /* Fall out from the NAPI loop if needed */
3144 if (!bnx2x_fp_unlock_napi(fp) &&
3145 !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3146
3147 /* No need to update SB for FCoE L2 ring as long as
3148 * it's connected to the default SB and the SB
3149 * has been updated when NAPI was scheduled.
3150 */
3151 if (IS_FCOE_FP(fp)) {
3152 napi_complete(napi);
3153 break;
3154 }
3155 bnx2x_update_fpsb_idx(fp);
3156 /* bnx2x_has_rx_work() reads the status block,
3157 * thus we need to ensure that status block indices
3158 * have been actually read (bnx2x_update_fpsb_idx)
3159 * prior to this check (bnx2x_has_rx_work) so that
3160 * we won't write the "newer" value of the status block
3161 * to IGU (if there was a DMA right after
3162 * bnx2x_has_rx_work and if there is no rmb, the memory
3163 * reading (bnx2x_update_fpsb_idx) may be postponed
3164 * to right before bnx2x_ack_sb). In this case there
3165 * will never be another interrupt until there is
3166 * another update of the status block, while there
3167 * is still unhandled work.
3168 */
3169 rmb();
3170
3171 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3172 napi_complete(napi);
3173 /* Re-enable interrupts */
3174 DP(NETIF_MSG_RX_STATUS,
3175 "Update index to %d\n", fp->fp_hc_idx);
3176 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3177 le16_to_cpu(fp->fp_hc_idx),
3178 IGU_INT_ENABLE, 1);
3179 break;
3180 }
3181 }
3182 }
3183
3184 return work_done;
3185 }
3186
3187 #ifdef CONFIG_NET_RX_BUSY_POLL
3188 /* must be called with local_bh_disable()d */
3189 int bnx2x_low_latency_recv(struct napi_struct *napi)
3190 {
3191 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3192 napi);
3193 struct bnx2x *bp = fp->bp;
3194 int found = 0;
3195
3196 if ((bp->state == BNX2X_STATE_CLOSED) ||
3197 (bp->state == BNX2X_STATE_ERROR) ||
3198 (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3199 return LL_FLUSH_FAILED;
3200
3201 if (!bnx2x_fp_lock_poll(fp))
3202 return LL_FLUSH_BUSY;
3203
3204 if (bnx2x_has_rx_work(fp))
3205 found = bnx2x_rx_int(fp, 4);
3206
3207 bnx2x_fp_unlock_poll(fp);
3208
3209 return found;
3210 }
3211 #endif
3212
3213 /* we split the first BD into headers and data BDs
3214 * to ease the pain of our fellow microcode engineers
3215 * we use one mapping for both BDs
3216 */
3217 static u16 bnx2x_tx_split(struct bnx2x *bp,
3218 struct bnx2x_fp_txdata *txdata,
3219 struct sw_tx_bd *tx_buf,
3220 struct eth_tx_start_bd **tx_bd, u16 hlen,
3221 u16 bd_prod)
3222 {
3223 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3224 struct eth_tx_bd *d_tx_bd;
3225 dma_addr_t mapping;
3226 int old_len = le16_to_cpu(h_tx_bd->nbytes);
3227
3228 /* first fix first BD */
3229 h_tx_bd->nbytes = cpu_to_le16(hlen);
3230
3231 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3232 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3233
3234 /* now get a new data BD
3235 * (after the pbd) and fill it */
3236 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3237 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3238
3239 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3240 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3241
3242 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3243 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3244 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3245
3246 /* this marks the BD as one that has no individual mapping */
3247 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3248
3249 DP(NETIF_MSG_TX_QUEUED,
3250 "TSO split data size is %d (%x:%x)\n",
3251 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3252
3253 /* update tx_bd */
3254 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3255
3256 return bd_prod;
3257 }
3258
3259 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3260 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3261 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3262 {
3263 __sum16 tsum = (__force __sum16) csum;
3264
3265 if (fix > 0)
3266 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3267 csum_partial(t_header - fix, fix, 0)));
3268
3269 else if (fix < 0)
3270 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3271 csum_partial(t_header, -fix, 0)));
3272
3273 return bswab16(tsum);
3274 }
3275
3276 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3277 {
3278 u32 rc;
3279 __u8 prot = 0;
3280 __be16 protocol;
3281
3282 if (skb->ip_summed != CHECKSUM_PARTIAL)
3283 return XMIT_PLAIN;
3284
3285 protocol = vlan_get_protocol(skb);
3286 if (protocol == htons(ETH_P_IPV6)) {
3287 rc = XMIT_CSUM_V6;
3288 prot = ipv6_hdr(skb)->nexthdr;
3289 } else {
3290 rc = XMIT_CSUM_V4;
3291 prot = ip_hdr(skb)->protocol;
3292 }
3293
3294 if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3295 if (inner_ip_hdr(skb)->version == 6) {
3296 rc |= XMIT_CSUM_ENC_V6;
3297 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3298 rc |= XMIT_CSUM_TCP;
3299 } else {
3300 rc |= XMIT_CSUM_ENC_V4;
3301 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3302 rc |= XMIT_CSUM_TCP;
3303 }
3304 }
3305 if (prot == IPPROTO_TCP)
3306 rc |= XMIT_CSUM_TCP;
3307
3308 if (skb_is_gso(skb)) {
3309 if (skb_is_gso_v6(skb)) {
3310 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3311 if (rc & XMIT_CSUM_ENC)
3312 rc |= XMIT_GSO_ENC_V6;
3313 } else {
3314 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3315 if (rc & XMIT_CSUM_ENC)
3316 rc |= XMIT_GSO_ENC_V4;
3317 }
3318 }
3319
3320 return rc;
3321 }
3322
3323 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3324 /* check if packet requires linearization (packet is too fragmented)
3325 no need to check fragmentation if page size > 8K (there will be no
3326 violation to FW restrictions) */
3327 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3328 u32 xmit_type)
3329 {
3330 int to_copy = 0;
3331 int hlen = 0;
3332 int first_bd_sz = 0;
3333
3334 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3335 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3336
3337 if (xmit_type & XMIT_GSO) {
3338 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3339 /* Check if LSO packet needs to be copied:
3340 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3341 int wnd_size = MAX_FETCH_BD - 3;
3342 /* Number of windows to check */
3343 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3344 int wnd_idx = 0;
3345 int frag_idx = 0;
3346 u32 wnd_sum = 0;
3347
3348 /* Headers length */
3349 hlen = (int)(skb_transport_header(skb) - skb->data) +
3350 tcp_hdrlen(skb);
3351
3352 /* Amount of data (w/o headers) on linear part of SKB*/
3353 first_bd_sz = skb_headlen(skb) - hlen;
3354
3355 wnd_sum = first_bd_sz;
3356
3357 /* Calculate the first sum - it's special */
3358 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3359 wnd_sum +=
3360 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3361
3362 /* If there was data on linear skb data - check it */
3363 if (first_bd_sz > 0) {
3364 if (unlikely(wnd_sum < lso_mss)) {
3365 to_copy = 1;
3366 goto exit_lbl;
3367 }
3368
3369 wnd_sum -= first_bd_sz;
3370 }
3371
3372 /* Others are easier: run through the frag list and
3373 check all windows */
3374 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3375 wnd_sum +=
3376 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3377
3378 if (unlikely(wnd_sum < lso_mss)) {
3379 to_copy = 1;
3380 break;
3381 }
3382 wnd_sum -=
3383 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3384 }
3385 } else {
3386 /* in non-LSO too fragmented packet should always
3387 be linearized */
3388 to_copy = 1;
3389 }
3390 }
3391
3392 exit_lbl:
3393 if (unlikely(to_copy))
3394 DP(NETIF_MSG_TX_QUEUED,
3395 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
3396 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3397 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3398
3399 return to_copy;
3400 }
3401 #endif
3402
3403 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3404 u32 xmit_type)
3405 {
3406 struct ipv6hdr *ipv6;
3407
3408 *parsing_data |= (skb_shinfo(skb)->gso_size <<
3409 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3410 ETH_TX_PARSE_BD_E2_LSO_MSS;
3411
3412 if (xmit_type & XMIT_GSO_ENC_V6)
3413 ipv6 = inner_ipv6_hdr(skb);
3414 else if (xmit_type & XMIT_GSO_V6)
3415 ipv6 = ipv6_hdr(skb);
3416 else
3417 ipv6 = NULL;
3418
3419 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3420 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3421 }
3422
3423 /**
3424 * bnx2x_set_pbd_gso - update PBD in GSO case.
3425 *
3426 * @skb: packet skb
3427 * @pbd: parse BD
3428 * @xmit_type: xmit flags
3429 */
3430 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3431 struct eth_tx_parse_bd_e1x *pbd,
3432 struct eth_tx_start_bd *tx_start_bd,
3433 u32 xmit_type)
3434 {
3435 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3436 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3437 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3438
3439 if (xmit_type & XMIT_GSO_V4) {
3440 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3441 pbd->tcp_pseudo_csum =
3442 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3443 ip_hdr(skb)->daddr,
3444 0, IPPROTO_TCP, 0));
3445
3446 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3447 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3448 } else {
3449 pbd->tcp_pseudo_csum =
3450 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3451 &ipv6_hdr(skb)->daddr,
3452 0, IPPROTO_TCP, 0));
3453 }
3454
3455 pbd->global_data |=
3456 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3457 }
3458
3459 /**
3460 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3461 *
3462 * @bp: driver handle
3463 * @skb: packet skb
3464 * @parsing_data: data to be updated
3465 * @xmit_type: xmit flags
3466 *
3467 * 57712/578xx related, when skb has encapsulation
3468 */
3469 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3470 u32 *parsing_data, u32 xmit_type)
3471 {
3472 *parsing_data |=
3473 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3474 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3475 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3476
3477 if (xmit_type & XMIT_CSUM_TCP) {
3478 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3479 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3480 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3481
3482 return skb_inner_transport_header(skb) +
3483 inner_tcp_hdrlen(skb) - skb->data;
3484 }
3485
3486 /* We support checksum offload for TCP and UDP only.
3487 * No need to pass the UDP header length - it's a constant.
3488 */
3489 return skb_inner_transport_header(skb) +
3490 sizeof(struct udphdr) - skb->data;
3491 }
3492
3493 /**
3494 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3495 *
3496 * @bp: driver handle
3497 * @skb: packet skb
3498 * @parsing_data: data to be updated
3499 * @xmit_type: xmit flags
3500 *
3501 * 57712/578xx related
3502 */
3503 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3504 u32 *parsing_data, u32 xmit_type)
3505 {
3506 *parsing_data |=
3507 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3508 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3509 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3510
3511 if (xmit_type & XMIT_CSUM_TCP) {
3512 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3513 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3514 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3515
3516 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3517 }
3518 /* We support checksum offload for TCP and UDP only.
3519 * No need to pass the UDP header length - it's a constant.
3520 */
3521 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3522 }
3523
3524 /* set FW indication according to inner or outer protocols if tunneled */
3525 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3526 struct eth_tx_start_bd *tx_start_bd,
3527 u32 xmit_type)
3528 {
3529 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3530
3531 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3532 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3533
3534 if (!(xmit_type & XMIT_CSUM_TCP))
3535 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3536 }
3537
3538 /**
3539 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3540 *
3541 * @bp: driver handle
3542 * @skb: packet skb
3543 * @pbd: parse BD to be updated
3544 * @xmit_type: xmit flags
3545 */
3546 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3547 struct eth_tx_parse_bd_e1x *pbd,
3548 u32 xmit_type)
3549 {
3550 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3551
3552 /* for now NS flag is not used in Linux */
3553 pbd->global_data =
3554 cpu_to_le16(hlen |
3555 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3556 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3557
3558 pbd->ip_hlen_w = (skb_transport_header(skb) -
3559 skb_network_header(skb)) >> 1;
3560
3561 hlen += pbd->ip_hlen_w;
3562
3563 /* We support checksum offload for TCP and UDP only */
3564 if (xmit_type & XMIT_CSUM_TCP)
3565 hlen += tcp_hdrlen(skb) / 2;
3566 else
3567 hlen += sizeof(struct udphdr) / 2;
3568
3569 pbd->total_hlen_w = cpu_to_le16(hlen);
3570 hlen = hlen*2;
3571
3572 if (xmit_type & XMIT_CSUM_TCP) {
3573 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3574
3575 } else {
3576 s8 fix = SKB_CS_OFF(skb); /* signed! */
3577
3578 DP(NETIF_MSG_TX_QUEUED,
3579 "hlen %d fix %d csum before fix %x\n",
3580 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3581
3582 /* HW bug: fixup the CSUM */
3583 pbd->tcp_pseudo_csum =
3584 bnx2x_csum_fix(skb_transport_header(skb),
3585 SKB_CS(skb), fix);
3586
3587 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3588 pbd->tcp_pseudo_csum);
3589 }
3590
3591 return hlen;
3592 }
3593
3594 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3595 struct eth_tx_parse_bd_e2 *pbd_e2,
3596 struct eth_tx_parse_2nd_bd *pbd2,
3597 u16 *global_data,
3598 u32 xmit_type)
3599 {
3600 u16 hlen_w = 0;
3601 u8 outerip_off, outerip_len = 0;
3602
3603 /* from outer IP to transport */
3604 hlen_w = (skb_inner_transport_header(skb) -
3605 skb_network_header(skb)) >> 1;
3606
3607 /* transport len */
3608 hlen_w += inner_tcp_hdrlen(skb) >> 1;
3609
3610 pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3611
3612 /* outer IP header info */
3613 if (xmit_type & XMIT_CSUM_V4) {
3614 struct iphdr *iph = ip_hdr(skb);
3615 u32 csum = (__force u32)(~iph->check) -
3616 (__force u32)iph->tot_len -
3617 (__force u32)iph->frag_off;
3618
3619 pbd2->fw_ip_csum_wo_len_flags_frag =
3620 bswab16(csum_fold((__force __wsum)csum));
3621 } else {
3622 pbd2->fw_ip_hdr_to_payload_w =
3623 hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3624 }
3625
3626 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3627
3628 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3629
3630 if (xmit_type & XMIT_GSO_V4) {
3631 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3632
3633 pbd_e2->data.tunnel_data.pseudo_csum =
3634 bswab16(~csum_tcpudp_magic(
3635 inner_ip_hdr(skb)->saddr,
3636 inner_ip_hdr(skb)->daddr,
3637 0, IPPROTO_TCP, 0));
3638
3639 outerip_len = ip_hdr(skb)->ihl << 1;
3640 } else {
3641 pbd_e2->data.tunnel_data.pseudo_csum =
3642 bswab16(~csum_ipv6_magic(
3643 &inner_ipv6_hdr(skb)->saddr,
3644 &inner_ipv6_hdr(skb)->daddr,
3645 0, IPPROTO_TCP, 0));
3646 }
3647
3648 outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3649
3650 *global_data |=
3651 outerip_off |
3652 (!!(xmit_type & XMIT_CSUM_V6) <<
3653 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3654 (outerip_len <<
3655 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3656 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3657 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3658
3659 if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3660 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3661 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3662 }
3663 }
3664
3665 /* called with netif_tx_lock
3666 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3667 * netif_wake_queue()
3668 */
3669 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3670 {
3671 struct bnx2x *bp = netdev_priv(dev);
3672
3673 struct netdev_queue *txq;
3674 struct bnx2x_fp_txdata *txdata;
3675 struct sw_tx_bd *tx_buf;
3676 struct eth_tx_start_bd *tx_start_bd, *first_bd;
3677 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3678 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3679 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3680 struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3681 u32 pbd_e2_parsing_data = 0;
3682 u16 pkt_prod, bd_prod;
3683 int nbd, txq_index;
3684 dma_addr_t mapping;
3685 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3686 int i;
3687 u8 hlen = 0;
3688 __le16 pkt_size = 0;
3689 struct ethhdr *eth;
3690 u8 mac_type = UNICAST_ADDRESS;
3691
3692 #ifdef BNX2X_STOP_ON_ERROR
3693 if (unlikely(bp->panic))
3694 return NETDEV_TX_BUSY;
3695 #endif
3696
3697 txq_index = skb_get_queue_mapping(skb);
3698 txq = netdev_get_tx_queue(dev, txq_index);
3699
3700 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3701
3702 txdata = &bp->bnx2x_txq[txq_index];
3703
3704 /* enable this debug print to view the transmission queue being used
3705 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3706 txq_index, fp_index, txdata_index); */
3707
3708 /* enable this debug print to view the transmission details
3709 DP(NETIF_MSG_TX_QUEUED,
3710 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3711 txdata->cid, fp_index, txdata_index, txdata, fp); */
3712
3713 if (unlikely(bnx2x_tx_avail(bp, txdata) <
3714 skb_shinfo(skb)->nr_frags +
3715 BDS_PER_TX_PKT +
3716 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3717 /* Handle special storage cases separately */
3718 if (txdata->tx_ring_size == 0) {
3719 struct bnx2x_eth_q_stats *q_stats =
3720 bnx2x_fp_qstats(bp, txdata->parent_fp);
3721 q_stats->driver_filtered_tx_pkt++;
3722 dev_kfree_skb(skb);
3723 return NETDEV_TX_OK;
3724 }
3725 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3726 netif_tx_stop_queue(txq);
3727 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3728
3729 return NETDEV_TX_BUSY;
3730 }
3731
3732 DP(NETIF_MSG_TX_QUEUED,
3733 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n",
3734 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3735 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3736 skb->len);
3737
3738 eth = (struct ethhdr *)skb->data;
3739
3740 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3741 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3742 if (is_broadcast_ether_addr(eth->h_dest))
3743 mac_type = BROADCAST_ADDRESS;
3744 else
3745 mac_type = MULTICAST_ADDRESS;
3746 }
3747
3748 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3749 /* First, check if we need to linearize the skb (due to FW
3750 restrictions). No need to check fragmentation if page size > 8K
3751 (there will be no violation to FW restrictions) */
3752 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3753 /* Statistics of linearization */
3754 bp->lin_cnt++;
3755 if (skb_linearize(skb) != 0) {
3756 DP(NETIF_MSG_TX_QUEUED,
3757 "SKB linearization failed - silently dropping this SKB\n");
3758 dev_kfree_skb_any(skb);
3759 return NETDEV_TX_OK;
3760 }
3761 }
3762 #endif
3763 /* Map skb linear data for DMA */
3764 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3765 skb_headlen(skb), DMA_TO_DEVICE);
3766 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3767 DP(NETIF_MSG_TX_QUEUED,
3768 "SKB mapping failed - silently dropping this SKB\n");
3769 dev_kfree_skb_any(skb);
3770 return NETDEV_TX_OK;
3771 }
3772 /*
3773 Please read carefully. First we use one BD which we mark as start,
3774 then we have a parsing info BD (used for TSO or xsum),
3775 and only then we have the rest of the TSO BDs.
3776 (don't forget to mark the last one as last,
3777 and to unmap only AFTER you write to the BD ...)
3778 And above all, all pdb sizes are in words - NOT DWORDS!
3779 */
3780
3781 /* get current pkt produced now - advance it just before sending packet
3782 * since mapping of pages may fail and cause packet to be dropped
3783 */
3784 pkt_prod = txdata->tx_pkt_prod;
3785 bd_prod = TX_BD(txdata->tx_bd_prod);
3786
3787 /* get a tx_buf and first BD
3788 * tx_start_bd may be changed during SPLIT,
3789 * but first_bd will always stay first
3790 */
3791 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3792 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3793 first_bd = tx_start_bd;
3794
3795 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3796
3797 /* header nbd: indirectly zero other flags! */
3798 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3799
3800 /* remember the first BD of the packet */
3801 tx_buf->first_bd = txdata->tx_bd_prod;
3802 tx_buf->skb = skb;
3803 tx_buf->flags = 0;
3804
3805 DP(NETIF_MSG_TX_QUEUED,
3806 "sending pkt %u @%p next_idx %u bd %u @%p\n",
3807 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3808
3809 if (vlan_tx_tag_present(skb)) {
3810 tx_start_bd->vlan_or_ethertype =
3811 cpu_to_le16(vlan_tx_tag_get(skb));
3812 tx_start_bd->bd_flags.as_bitfield |=
3813 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3814 } else {
3815 /* when transmitting in a vf, start bd must hold the ethertype
3816 * for fw to enforce it
3817 */
3818 if (IS_VF(bp))
3819 tx_start_bd->vlan_or_ethertype =
3820 cpu_to_le16(ntohs(eth->h_proto));
3821 else
3822 /* used by FW for packet accounting */
3823 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3824 }
3825
3826 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3827
3828 /* turn on parsing and get a BD */
3829 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3830
3831 if (xmit_type & XMIT_CSUM)
3832 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3833
3834 if (!CHIP_IS_E1x(bp)) {
3835 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3836 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3837
3838 if (xmit_type & XMIT_CSUM_ENC) {
3839 u16 global_data = 0;
3840
3841 /* Set PBD in enc checksum offload case */
3842 hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3843 &pbd_e2_parsing_data,
3844 xmit_type);
3845
3846 /* turn on 2nd parsing and get a BD */
3847 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3848
3849 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3850
3851 memset(pbd2, 0, sizeof(*pbd2));
3852
3853 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3854 (skb_inner_network_header(skb) -
3855 skb->data) >> 1;
3856
3857 if (xmit_type & XMIT_GSO_ENC)
3858 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3859 &global_data,
3860 xmit_type);
3861
3862 pbd2->global_data = cpu_to_le16(global_data);
3863
3864 /* add addition parse BD indication to start BD */
3865 SET_FLAG(tx_start_bd->general_data,
3866 ETH_TX_START_BD_PARSE_NBDS, 1);
3867 /* set encapsulation flag in start BD */
3868 SET_FLAG(tx_start_bd->general_data,
3869 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3870 nbd++;
3871 } else if (xmit_type & XMIT_CSUM) {
3872 /* Set PBD in checksum offload case w/o encapsulation */
3873 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3874 &pbd_e2_parsing_data,
3875 xmit_type);
3876 }
3877
3878 /* Add the macs to the parsing BD this is a vf */
3879 if (IS_VF(bp)) {
3880 /* override GRE parameters in BD */
3881 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3882 &pbd_e2->data.mac_addr.src_mid,
3883 &pbd_e2->data.mac_addr.src_lo,
3884 eth->h_source);
3885
3886 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3887 &pbd_e2->data.mac_addr.dst_mid,
3888 &pbd_e2->data.mac_addr.dst_lo,
3889 eth->h_dest);
3890 }
3891
3892 SET_FLAG(pbd_e2_parsing_data,
3893 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3894 } else {
3895 u16 global_data = 0;
3896 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3897 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3898 /* Set PBD in checksum offload case */
3899 if (xmit_type & XMIT_CSUM)
3900 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3901
3902 SET_FLAG(global_data,
3903 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3904 pbd_e1x->global_data |= cpu_to_le16(global_data);
3905 }
3906
3907 /* Setup the data pointer of the first BD of the packet */
3908 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3909 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3910 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3911 pkt_size = tx_start_bd->nbytes;
3912
3913 DP(NETIF_MSG_TX_QUEUED,
3914 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n",
3915 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3916 le16_to_cpu(tx_start_bd->nbytes),
3917 tx_start_bd->bd_flags.as_bitfield,
3918 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3919
3920 if (xmit_type & XMIT_GSO) {
3921
3922 DP(NETIF_MSG_TX_QUEUED,
3923 "TSO packet len %d hlen %d total len %d tso size %d\n",
3924 skb->len, hlen, skb_headlen(skb),
3925 skb_shinfo(skb)->gso_size);
3926
3927 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3928
3929 if (unlikely(skb_headlen(skb) > hlen)) {
3930 nbd++;
3931 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3932 &tx_start_bd, hlen,
3933 bd_prod);
3934 }
3935 if (!CHIP_IS_E1x(bp))
3936 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3937 xmit_type);
3938 else
3939 bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3940 }
3941
3942 /* Set the PBD's parsing_data field if not zero
3943 * (for the chips newer than 57711).
3944 */
3945 if (pbd_e2_parsing_data)
3946 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3947
3948 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3949
3950 /* Handle fragmented skb */
3951 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3952 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3953
3954 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3955 skb_frag_size(frag), DMA_TO_DEVICE);
3956 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3957 unsigned int pkts_compl = 0, bytes_compl = 0;
3958
3959 DP(NETIF_MSG_TX_QUEUED,
3960 "Unable to map page - dropping packet...\n");
3961
3962 /* we need unmap all buffers already mapped
3963 * for this SKB;
3964 * first_bd->nbd need to be properly updated
3965 * before call to bnx2x_free_tx_pkt
3966 */
3967 first_bd->nbd = cpu_to_le16(nbd);
3968 bnx2x_free_tx_pkt(bp, txdata,
3969 TX_BD(txdata->tx_pkt_prod),
3970 &pkts_compl, &bytes_compl);
3971 return NETDEV_TX_OK;
3972 }
3973
3974 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3975 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3976 if (total_pkt_bd == NULL)
3977 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3978
3979 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3980 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3981 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3982 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3983 nbd++;
3984
3985 DP(NETIF_MSG_TX_QUEUED,
3986 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
3987 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3988 le16_to_cpu(tx_data_bd->nbytes));
3989 }
3990
3991 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3992
3993 /* update with actual num BDs */
3994 first_bd->nbd = cpu_to_le16(nbd);
3995
3996 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3997
3998 /* now send a tx doorbell, counting the next BD
3999 * if the packet contains or ends with it
4000 */
4001 if (TX_BD_POFF(bd_prod) < nbd)
4002 nbd++;
4003
4004 /* total_pkt_bytes should be set on the first data BD if
4005 * it's not an LSO packet and there is more than one
4006 * data BD. In this case pkt_size is limited by an MTU value.
4007 * However we prefer to set it for an LSO packet (while we don't
4008 * have to) in order to save some CPU cycles in a none-LSO
4009 * case, when we much more care about them.
4010 */
4011 if (total_pkt_bd != NULL)
4012 total_pkt_bd->total_pkt_bytes = pkt_size;
4013
4014 if (pbd_e1x)
4015 DP(NETIF_MSG_TX_QUEUED,
4016 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
4017 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4018 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4019 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4020 le16_to_cpu(pbd_e1x->total_hlen_w));
4021 if (pbd_e2)
4022 DP(NETIF_MSG_TX_QUEUED,
4023 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
4024 pbd_e2,
4025 pbd_e2->data.mac_addr.dst_hi,
4026 pbd_e2->data.mac_addr.dst_mid,
4027 pbd_e2->data.mac_addr.dst_lo,
4028 pbd_e2->data.mac_addr.src_hi,
4029 pbd_e2->data.mac_addr.src_mid,
4030 pbd_e2->data.mac_addr.src_lo,
4031 pbd_e2->parsing_data);
4032 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
4033
4034 netdev_tx_sent_queue(txq, skb->len);
4035
4036 skb_tx_timestamp(skb);
4037
4038 txdata->tx_pkt_prod++;
4039 /*
4040 * Make sure that the BD data is updated before updating the producer
4041 * since FW might read the BD right after the producer is updated.
4042 * This is only applicable for weak-ordered memory model archs such
4043 * as IA-64. The following barrier is also mandatory since FW will
4044 * assumes packets must have BDs.
4045 */
4046 wmb();
4047
4048 txdata->tx_db.data.prod += nbd;
4049 barrier();
4050
4051 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4052
4053 mmiowb();
4054
4055 txdata->tx_bd_prod += nbd;
4056
4057 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4058 netif_tx_stop_queue(txq);
4059
4060 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4061 * ordering of set_bit() in netif_tx_stop_queue() and read of
4062 * fp->bd_tx_cons */
4063 smp_mb();
4064
4065 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4066 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4067 netif_tx_wake_queue(txq);
4068 }
4069 txdata->tx_pkt++;
4070
4071 return NETDEV_TX_OK;
4072 }
4073
4074 /**
4075 * bnx2x_setup_tc - routine to configure net_device for multi tc
4076 *
4077 * @netdev: net device to configure
4078 * @tc: number of traffic classes to enable
4079 *
4080 * callback connected to the ndo_setup_tc function pointer
4081 */
4082 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4083 {
4084 int cos, prio, count, offset;
4085 struct bnx2x *bp = netdev_priv(dev);
4086
4087 /* setup tc must be called under rtnl lock */
4088 ASSERT_RTNL();
4089
4090 /* no traffic classes requested. Aborting */
4091 if (!num_tc) {
4092 netdev_reset_tc(dev);
4093 return 0;
4094 }
4095
4096 /* requested to support too many traffic classes */
4097 if (num_tc > bp->max_cos) {
4098 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4099 num_tc, bp->max_cos);
4100 return -EINVAL;
4101 }
4102
4103 /* declare amount of supported traffic classes */
4104 if (netdev_set_num_tc(dev, num_tc)) {
4105 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4106 return -EINVAL;
4107 }
4108
4109 /* configure priority to traffic class mapping */
4110 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4111 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4112 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4113 "mapping priority %d to tc %d\n",
4114 prio, bp->prio_to_cos[prio]);
4115 }
4116
4117 /* Use this configuration to differentiate tc0 from other COSes
4118 This can be used for ets or pfc, and save the effort of setting
4119 up a multio class queue disc or negotiating DCBX with a switch
4120 netdev_set_prio_tc_map(dev, 0, 0);
4121 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4122 for (prio = 1; prio < 16; prio++) {
4123 netdev_set_prio_tc_map(dev, prio, 1);
4124 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4125 } */
4126
4127 /* configure traffic class to transmission queue mapping */
4128 for (cos = 0; cos < bp->max_cos; cos++) {
4129 count = BNX2X_NUM_ETH_QUEUES(bp);
4130 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4131 netdev_set_tc_queue(dev, cos, count, offset);
4132 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4133 "mapping tc %d to offset %d count %d\n",
4134 cos, offset, count);
4135 }
4136
4137 return 0;
4138 }
4139
4140 /* called with rtnl_lock */
4141 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4142 {
4143 struct sockaddr *addr = p;
4144 struct bnx2x *bp = netdev_priv(dev);
4145 int rc = 0;
4146
4147 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4148 BNX2X_ERR("Requested MAC address is not valid\n");
4149 return -EINVAL;
4150 }
4151
4152 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4153 !is_zero_ether_addr(addr->sa_data)) {
4154 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4155 return -EINVAL;
4156 }
4157
4158 if (netif_running(dev)) {
4159 rc = bnx2x_set_eth_mac(bp, false);
4160 if (rc)
4161 return rc;
4162 }
4163
4164 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4165
4166 if (netif_running(dev))
4167 rc = bnx2x_set_eth_mac(bp, true);
4168
4169 return rc;
4170 }
4171
4172 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4173 {
4174 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4175 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4176 u8 cos;
4177
4178 /* Common */
4179
4180 if (IS_FCOE_IDX(fp_index)) {
4181 memset(sb, 0, sizeof(union host_hc_status_block));
4182 fp->status_blk_mapping = 0;
4183 } else {
4184 /* status blocks */
4185 if (!CHIP_IS_E1x(bp))
4186 BNX2X_PCI_FREE(sb->e2_sb,
4187 bnx2x_fp(bp, fp_index,
4188 status_blk_mapping),
4189 sizeof(struct host_hc_status_block_e2));
4190 else
4191 BNX2X_PCI_FREE(sb->e1x_sb,
4192 bnx2x_fp(bp, fp_index,
4193 status_blk_mapping),
4194 sizeof(struct host_hc_status_block_e1x));
4195 }
4196
4197 /* Rx */
4198 if (!skip_rx_queue(bp, fp_index)) {
4199 bnx2x_free_rx_bds(fp);
4200
4201 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4202 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4203 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4204 bnx2x_fp(bp, fp_index, rx_desc_mapping),
4205 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4206
4207 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4208 bnx2x_fp(bp, fp_index, rx_comp_mapping),
4209 sizeof(struct eth_fast_path_rx_cqe) *
4210 NUM_RCQ_BD);
4211
4212 /* SGE ring */
4213 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4214 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4215 bnx2x_fp(bp, fp_index, rx_sge_mapping),
4216 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4217 }
4218
4219 /* Tx */
4220 if (!skip_tx_queue(bp, fp_index)) {
4221 /* fastpath tx rings: tx_buf tx_desc */
4222 for_each_cos_in_tx_queue(fp, cos) {
4223 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4224
4225 DP(NETIF_MSG_IFDOWN,
4226 "freeing tx memory of fp %d cos %d cid %d\n",
4227 fp_index, cos, txdata->cid);
4228
4229 BNX2X_FREE(txdata->tx_buf_ring);
4230 BNX2X_PCI_FREE(txdata->tx_desc_ring,
4231 txdata->tx_desc_mapping,
4232 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4233 }
4234 }
4235 /* end of fastpath */
4236 }
4237
4238 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4239 {
4240 int i;
4241 for_each_cnic_queue(bp, i)
4242 bnx2x_free_fp_mem_at(bp, i);
4243 }
4244
4245 void bnx2x_free_fp_mem(struct bnx2x *bp)
4246 {
4247 int i;
4248 for_each_eth_queue(bp, i)
4249 bnx2x_free_fp_mem_at(bp, i);
4250 }
4251
4252 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4253 {
4254 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4255 if (!CHIP_IS_E1x(bp)) {
4256 bnx2x_fp(bp, index, sb_index_values) =
4257 (__le16 *)status_blk.e2_sb->sb.index_values;
4258 bnx2x_fp(bp, index, sb_running_index) =
4259 (__le16 *)status_blk.e2_sb->sb.running_index;
4260 } else {
4261 bnx2x_fp(bp, index, sb_index_values) =
4262 (__le16 *)status_blk.e1x_sb->sb.index_values;
4263 bnx2x_fp(bp, index, sb_running_index) =
4264 (__le16 *)status_blk.e1x_sb->sb.running_index;
4265 }
4266 }
4267
4268 /* Returns the number of actually allocated BDs */
4269 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4270 int rx_ring_size)
4271 {
4272 struct bnx2x *bp = fp->bp;
4273 u16 ring_prod, cqe_ring_prod;
4274 int i, failure_cnt = 0;
4275
4276 fp->rx_comp_cons = 0;
4277 cqe_ring_prod = ring_prod = 0;
4278
4279 /* This routine is called only during fo init so
4280 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4281 */
4282 for (i = 0; i < rx_ring_size; i++) {
4283 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4284 failure_cnt++;
4285 continue;
4286 }
4287 ring_prod = NEXT_RX_IDX(ring_prod);
4288 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4289 WARN_ON(ring_prod <= (i - failure_cnt));
4290 }
4291
4292 if (failure_cnt)
4293 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4294 i - failure_cnt, fp->index);
4295
4296 fp->rx_bd_prod = ring_prod;
4297 /* Limit the CQE producer by the CQE ring size */
4298 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4299 cqe_ring_prod);
4300 fp->rx_pkt = fp->rx_calls = 0;
4301
4302 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4303
4304 return i - failure_cnt;
4305 }
4306
4307 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4308 {
4309 int i;
4310
4311 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4312 struct eth_rx_cqe_next_page *nextpg;
4313
4314 nextpg = (struct eth_rx_cqe_next_page *)
4315 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4316 nextpg->addr_hi =
4317 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4318 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4319 nextpg->addr_lo =
4320 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4321 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4322 }
4323 }
4324
4325 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4326 {
4327 union host_hc_status_block *sb;
4328 struct bnx2x_fastpath *fp = &bp->fp[index];
4329 int ring_size = 0;
4330 u8 cos;
4331 int rx_ring_size = 0;
4332
4333 if (!bp->rx_ring_size &&
4334 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4335 rx_ring_size = MIN_RX_SIZE_NONTPA;
4336 bp->rx_ring_size = rx_ring_size;
4337 } else if (!bp->rx_ring_size) {
4338 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4339
4340 if (CHIP_IS_E3(bp)) {
4341 u32 cfg = SHMEM_RD(bp,
4342 dev_info.port_hw_config[BP_PORT(bp)].
4343 default_cfg);
4344
4345 /* Decrease ring size for 1G functions */
4346 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4347 PORT_HW_CFG_NET_SERDES_IF_SGMII)
4348 rx_ring_size /= 10;
4349 }
4350
4351 /* allocate at least number of buffers required by FW */
4352 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4353 MIN_RX_SIZE_TPA, rx_ring_size);
4354
4355 bp->rx_ring_size = rx_ring_size;
4356 } else /* if rx_ring_size specified - use it */
4357 rx_ring_size = bp->rx_ring_size;
4358
4359 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4360
4361 /* Common */
4362 sb = &bnx2x_fp(bp, index, status_blk);
4363
4364 if (!IS_FCOE_IDX(index)) {
4365 /* status blocks */
4366 if (!CHIP_IS_E1x(bp))
4367 BNX2X_PCI_ALLOC(sb->e2_sb,
4368 &bnx2x_fp(bp, index, status_blk_mapping),
4369 sizeof(struct host_hc_status_block_e2));
4370 else
4371 BNX2X_PCI_ALLOC(sb->e1x_sb,
4372 &bnx2x_fp(bp, index, status_blk_mapping),
4373 sizeof(struct host_hc_status_block_e1x));
4374 }
4375
4376 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4377 * set shortcuts for it.
4378 */
4379 if (!IS_FCOE_IDX(index))
4380 set_sb_shortcuts(bp, index);
4381
4382 /* Tx */
4383 if (!skip_tx_queue(bp, index)) {
4384 /* fastpath tx rings: tx_buf tx_desc */
4385 for_each_cos_in_tx_queue(fp, cos) {
4386 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4387
4388 DP(NETIF_MSG_IFUP,
4389 "allocating tx memory of fp %d cos %d\n",
4390 index, cos);
4391
4392 BNX2X_ALLOC(txdata->tx_buf_ring,
4393 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4394 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4395 &txdata->tx_desc_mapping,
4396 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4397 }
4398 }
4399
4400 /* Rx */
4401 if (!skip_rx_queue(bp, index)) {
4402 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4403 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4404 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4405 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4406 &bnx2x_fp(bp, index, rx_desc_mapping),
4407 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4408
4409 /* Seed all CQEs by 1s */
4410 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4411 &bnx2x_fp(bp, index, rx_comp_mapping),
4412 sizeof(struct eth_fast_path_rx_cqe) *
4413 NUM_RCQ_BD);
4414
4415 /* SGE ring */
4416 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4417 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4418 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4419 &bnx2x_fp(bp, index, rx_sge_mapping),
4420 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4421 /* RX BD ring */
4422 bnx2x_set_next_page_rx_bd(fp);
4423
4424 /* CQ ring */
4425 bnx2x_set_next_page_rx_cq(fp);
4426
4427 /* BDs */
4428 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4429 if (ring_size < rx_ring_size)
4430 goto alloc_mem_err;
4431 }
4432
4433 return 0;
4434
4435 /* handles low memory cases */
4436 alloc_mem_err:
4437 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4438 index, ring_size);
4439 /* FW will drop all packets if queue is not big enough,
4440 * In these cases we disable the queue
4441 * Min size is different for OOO, TPA and non-TPA queues
4442 */
4443 if (ring_size < (fp->disable_tpa ?
4444 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4445 /* release memory allocated for this queue */
4446 bnx2x_free_fp_mem_at(bp, index);
4447 return -ENOMEM;
4448 }
4449 return 0;
4450 }
4451
4452 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4453 {
4454 if (!NO_FCOE(bp))
4455 /* FCoE */
4456 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4457 /* we will fail load process instead of mark
4458 * NO_FCOE_FLAG
4459 */
4460 return -ENOMEM;
4461
4462 return 0;
4463 }
4464
4465 static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4466 {
4467 int i;
4468
4469 /* 1. Allocate FP for leading - fatal if error
4470 * 2. Allocate RSS - fix number of queues if error
4471 */
4472
4473 /* leading */
4474 if (bnx2x_alloc_fp_mem_at(bp, 0))
4475 return -ENOMEM;
4476
4477 /* RSS */
4478 for_each_nondefault_eth_queue(bp, i)
4479 if (bnx2x_alloc_fp_mem_at(bp, i))
4480 break;
4481
4482 /* handle memory failures */
4483 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4484 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4485
4486 WARN_ON(delta < 0);
4487 bnx2x_shrink_eth_fp(bp, delta);
4488 if (CNIC_SUPPORT(bp))
4489 /* move non eth FPs next to last eth FP
4490 * must be done in that order
4491 * FCOE_IDX < FWD_IDX < OOO_IDX
4492 */
4493
4494 /* move FCoE fp even NO_FCOE_FLAG is on */
4495 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4496 bp->num_ethernet_queues -= delta;
4497 bp->num_queues = bp->num_ethernet_queues +
4498 bp->num_cnic_queues;
4499 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4500 bp->num_queues + delta, bp->num_queues);
4501 }
4502
4503 return 0;
4504 }
4505
4506 void bnx2x_free_mem_bp(struct bnx2x *bp)
4507 {
4508 int i;
4509
4510 for (i = 0; i < bp->fp_array_size; i++)
4511 kfree(bp->fp[i].tpa_info);
4512 kfree(bp->fp);
4513 kfree(bp->sp_objs);
4514 kfree(bp->fp_stats);
4515 kfree(bp->bnx2x_txq);
4516 kfree(bp->msix_table);
4517 kfree(bp->ilt);
4518 }
4519
4520 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4521 {
4522 struct bnx2x_fastpath *fp;
4523 struct msix_entry *tbl;
4524 struct bnx2x_ilt *ilt;
4525 int msix_table_size = 0;
4526 int fp_array_size, txq_array_size;
4527 int i;
4528
4529 /*
4530 * The biggest MSI-X table we might need is as a maximum number of fast
4531 * path IGU SBs plus default SB (for PF only).
4532 */
4533 msix_table_size = bp->igu_sb_cnt;
4534 if (IS_PF(bp))
4535 msix_table_size++;
4536 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4537
4538 /* fp array: RSS plus CNIC related L2 queues */
4539 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4540 bp->fp_array_size = fp_array_size;
4541 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4542
4543 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4544 if (!fp)
4545 goto alloc_err;
4546 for (i = 0; i < bp->fp_array_size; i++) {
4547 fp[i].tpa_info =
4548 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4549 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4550 if (!(fp[i].tpa_info))
4551 goto alloc_err;
4552 }
4553
4554 bp->fp = fp;
4555
4556 /* allocate sp objs */
4557 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4558 GFP_KERNEL);
4559 if (!bp->sp_objs)
4560 goto alloc_err;
4561
4562 /* allocate fp_stats */
4563 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4564 GFP_KERNEL);
4565 if (!bp->fp_stats)
4566 goto alloc_err;
4567
4568 /* Allocate memory for the transmission queues array */
4569 txq_array_size =
4570 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4571 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4572
4573 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4574 GFP_KERNEL);
4575 if (!bp->bnx2x_txq)
4576 goto alloc_err;
4577
4578 /* msix table */
4579 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4580 if (!tbl)
4581 goto alloc_err;
4582 bp->msix_table = tbl;
4583
4584 /* ilt */
4585 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4586 if (!ilt)
4587 goto alloc_err;
4588 bp->ilt = ilt;
4589
4590 return 0;
4591 alloc_err:
4592 bnx2x_free_mem_bp(bp);
4593 return -ENOMEM;
4594 }
4595
4596 int bnx2x_reload_if_running(struct net_device *dev)
4597 {
4598 struct bnx2x *bp = netdev_priv(dev);
4599
4600 if (unlikely(!netif_running(dev)))
4601 return 0;
4602
4603 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4604 return bnx2x_nic_load(bp, LOAD_NORMAL);
4605 }
4606
4607 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4608 {
4609 u32 sel_phy_idx = 0;
4610 if (bp->link_params.num_phys <= 1)
4611 return INT_PHY;
4612
4613 if (bp->link_vars.link_up) {
4614 sel_phy_idx = EXT_PHY1;
4615 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4616 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4617 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4618 sel_phy_idx = EXT_PHY2;
4619 } else {
4620
4621 switch (bnx2x_phy_selection(&bp->link_params)) {
4622 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4623 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4624 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4625 sel_phy_idx = EXT_PHY1;
4626 break;
4627 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4628 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4629 sel_phy_idx = EXT_PHY2;
4630 break;
4631 }
4632 }
4633
4634 return sel_phy_idx;
4635 }
4636 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4637 {
4638 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4639 /*
4640 * The selected activated PHY is always after swapping (in case PHY
4641 * swapping is enabled). So when swapping is enabled, we need to reverse
4642 * the configuration
4643 */
4644
4645 if (bp->link_params.multi_phy_config &
4646 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4647 if (sel_phy_idx == EXT_PHY1)
4648 sel_phy_idx = EXT_PHY2;
4649 else if (sel_phy_idx == EXT_PHY2)
4650 sel_phy_idx = EXT_PHY1;
4651 }
4652 return LINK_CONFIG_IDX(sel_phy_idx);
4653 }
4654
4655 #ifdef NETDEV_FCOE_WWNN
4656 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4657 {
4658 struct bnx2x *bp = netdev_priv(dev);
4659 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4660
4661 switch (type) {
4662 case NETDEV_FCOE_WWNN:
4663 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4664 cp->fcoe_wwn_node_name_lo);
4665 break;
4666 case NETDEV_FCOE_WWPN:
4667 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4668 cp->fcoe_wwn_port_name_lo);
4669 break;
4670 default:
4671 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4672 return -EINVAL;
4673 }
4674
4675 return 0;
4676 }
4677 #endif
4678
4679 /* called with rtnl_lock */
4680 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4681 {
4682 struct bnx2x *bp = netdev_priv(dev);
4683
4684 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4685 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4686 return -EAGAIN;
4687 }
4688
4689 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4690 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4691 BNX2X_ERR("Can't support requested MTU size\n");
4692 return -EINVAL;
4693 }
4694
4695 /* This does not race with packet allocation
4696 * because the actual alloc size is
4697 * only updated as part of load
4698 */
4699 dev->mtu = new_mtu;
4700
4701 return bnx2x_reload_if_running(dev);
4702 }
4703
4704 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4705 netdev_features_t features)
4706 {
4707 struct bnx2x *bp = netdev_priv(dev);
4708
4709 /* TPA requires Rx CSUM offloading */
4710 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4711 features &= ~NETIF_F_LRO;
4712 features &= ~NETIF_F_GRO;
4713 }
4714
4715 return features;
4716 }
4717
4718 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4719 {
4720 struct bnx2x *bp = netdev_priv(dev);
4721 u32 flags = bp->flags;
4722 u32 changes;
4723 bool bnx2x_reload = false;
4724
4725 if (features & NETIF_F_LRO)
4726 flags |= TPA_ENABLE_FLAG;
4727 else
4728 flags &= ~TPA_ENABLE_FLAG;
4729
4730 if (features & NETIF_F_GRO)
4731 flags |= GRO_ENABLE_FLAG;
4732 else
4733 flags &= ~GRO_ENABLE_FLAG;
4734
4735 if (features & NETIF_F_LOOPBACK) {
4736 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4737 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4738 bnx2x_reload = true;
4739 }
4740 } else {
4741 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4742 bp->link_params.loopback_mode = LOOPBACK_NONE;
4743 bnx2x_reload = true;
4744 }
4745 }
4746
4747 changes = flags ^ bp->flags;
4748
4749 /* if GRO is changed while LRO is enabled, don't force a reload */
4750 if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4751 changes &= ~GRO_ENABLE_FLAG;
4752
4753 if (changes)
4754 bnx2x_reload = true;
4755
4756 bp->flags = flags;
4757
4758 if (bnx2x_reload) {
4759 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4760 return bnx2x_reload_if_running(dev);
4761 /* else: bnx2x_nic_load() will be called at end of recovery */
4762 }
4763
4764 return 0;
4765 }
4766
4767 void bnx2x_tx_timeout(struct net_device *dev)
4768 {
4769 struct bnx2x *bp = netdev_priv(dev);
4770
4771 #ifdef BNX2X_STOP_ON_ERROR
4772 if (!bp->panic)
4773 bnx2x_panic();
4774 #endif
4775
4776 smp_mb__before_clear_bit();
4777 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4778 smp_mb__after_clear_bit();
4779
4780 /* This allows the netif to be shutdown gracefully before resetting */
4781 schedule_delayed_work(&bp->sp_rtnl_task, 0);
4782 }
4783
4784 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4785 {
4786 struct net_device *dev = pci_get_drvdata(pdev);
4787 struct bnx2x *bp;
4788
4789 if (!dev) {
4790 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4791 return -ENODEV;
4792 }
4793 bp = netdev_priv(dev);
4794
4795 rtnl_lock();
4796
4797 pci_save_state(pdev);
4798
4799 if (!netif_running(dev)) {
4800 rtnl_unlock();
4801 return 0;
4802 }
4803
4804 netif_device_detach(dev);
4805
4806 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4807
4808 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4809
4810 rtnl_unlock();
4811
4812 return 0;
4813 }
4814
4815 int bnx2x_resume(struct pci_dev *pdev)
4816 {
4817 struct net_device *dev = pci_get_drvdata(pdev);
4818 struct bnx2x *bp;
4819 int rc;
4820
4821 if (!dev) {
4822 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4823 return -ENODEV;
4824 }
4825 bp = netdev_priv(dev);
4826
4827 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4828 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4829 return -EAGAIN;
4830 }
4831
4832 rtnl_lock();
4833
4834 pci_restore_state(pdev);
4835
4836 if (!netif_running(dev)) {
4837 rtnl_unlock();
4838 return 0;
4839 }
4840
4841 bnx2x_set_power_state(bp, PCI_D0);
4842 netif_device_attach(dev);
4843
4844 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4845
4846 rtnl_unlock();
4847
4848 return rc;
4849 }
4850
4851 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4852 u32 cid)
4853 {
4854 if (!cxt) {
4855 BNX2X_ERR("bad context pointer %p\n", cxt);
4856 return;
4857 }
4858
4859 /* ustorm cxt validation */
4860 cxt->ustorm_ag_context.cdu_usage =
4861 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4862 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4863 /* xcontext validation */
4864 cxt->xstorm_ag_context.cdu_reserved =
4865 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4866 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4867 }
4868
4869 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4870 u8 fw_sb_id, u8 sb_index,
4871 u8 ticks)
4872 {
4873 u32 addr = BAR_CSTRORM_INTMEM +
4874 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4875 REG_WR8(bp, addr, ticks);
4876 DP(NETIF_MSG_IFUP,
4877 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4878 port, fw_sb_id, sb_index, ticks);
4879 }
4880
4881 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4882 u16 fw_sb_id, u8 sb_index,
4883 u8 disable)
4884 {
4885 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4886 u32 addr = BAR_CSTRORM_INTMEM +
4887 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4888 u8 flags = REG_RD8(bp, addr);
4889 /* clear and set */
4890 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4891 flags |= enable_flag;
4892 REG_WR8(bp, addr, flags);
4893 DP(NETIF_MSG_IFUP,
4894 "port %x fw_sb_id %d sb_index %d disable %d\n",
4895 port, fw_sb_id, sb_index, disable);
4896 }
4897
4898 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4899 u8 sb_index, u8 disable, u16 usec)
4900 {
4901 int port = BP_PORT(bp);
4902 u8 ticks = usec / BNX2X_BTR;
4903
4904 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4905
4906 disable = disable ? 1 : (usec ? 0 : 1);
4907 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4908 }
ubuntu-zesty-kernel mirror