]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/cxgb4/cxgb4_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4: fix TSO descriptors
[mirror_ubuntu-zesty-kernel.git] / drivers / net / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
44#include <linux/if_vlan.h>
45#include <linux/init.h>
46#include <linux/log2.h>
47#include <linux/mdio.h>
48#include <linux/module.h>
49#include <linux/moduleparam.h>
50#include <linux/mutex.h>
51#include <linux/netdevice.h>
52#include <linux/pci.h>
53#include <linux/aer.h>
54#include <linux/rtnetlink.h>
55#include <linux/sched.h>
56#include <linux/seq_file.h>
57#include <linux/sockios.h>
58#include <linux/vmalloc.h>
59#include <linux/workqueue.h>
60#include <net/neighbour.h>
61#include <net/netevent.h>
62#include <asm/uaccess.h>
63
64#include "cxgb4.h"
65#include "t4_regs.h"
66#include "t4_msg.h"
67#include "t4fw_api.h"
68#include "l2t.h"
69
70#define DRV_VERSION "1.0.0-ko"
71#define DRV_DESC "Chelsio T4 Network Driver"
72
73/*
74 * Max interrupt hold-off timer value in us. Queues fall back to this value
75 * under extreme memory pressure so it's largish to give the system time to
76 * recover.
77 */
78#define MAX_SGE_TIMERVAL 200U
79
7ee9ff94
CL		 80#ifdef CONFIG_PCI_IOV
81/*
82 * Virtual Function provisioning constants. We need two extra Ingress Queues
83 * with Interrupt capability to serve as the VF's Firmware Event Queue and
84 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
85 * Lists associated with them). For each Ethernet/Control Egress Queue and
86 * for each Free List, we need an Egress Context.
87 */
88enum {
89 VFRES_NPORTS = 1, /* # of "ports" per VF */
90 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
91
92 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
93 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
94 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
95 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
96 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
97 VFRES_TC = 0, /* PCI-E traffic class */
98 VFRES_NEXACTF = 16, /* # of exact MPS filters */
99
100 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
101 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
102};
103
104/*
105 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
106 * static and likely not to be useful in the long run. We really need to
107 * implement some form of persistent configuration which the firmware
108 * controls.
109 */
110static unsigned int pfvfres_pmask(struct adapter *adapter,
111 unsigned int pf, unsigned int vf)
112{
113 unsigned int portn, portvec;
114
115 /*
116 * Give PF's access to all of the ports.
117 */
118 if (vf == 0)
119 return FW_PFVF_CMD_PMASK_MASK;
120
121 /*
122 * For VFs, we'll assign them access to the ports based purely on the
123 * PF. We assign active ports in order, wrapping around if there are
124 * fewer active ports than PFs: e.g. active port[pf % nports].
125 * Unfortunately the adapter's port_info structs haven't been
126 * initialized yet so we have to compute this.
127 */
128 if (adapter->params.nports == 0)
129 return 0;
130
131 portn = pf % adapter->params.nports;
132 portvec = adapter->params.portvec;
133 for (;;) {
134 /*
135 * Isolate the lowest set bit in the port vector. If we're at
136 * the port number that we want, return that as the pmask.
137 * otherwise mask that bit out of the port vector and
138 * decrement our port number ...
139 */
140 unsigned int pmask = portvec ^ (portvec & (portvec-1));
141 if (portn == 0)
142 return pmask;
143 portn--;
144 portvec &= ~pmask;
145 }
146 /*NOTREACHED*/
147}
148#endif
149
b8ff05a9
DM		 150enum {
151 MEMWIN0_APERTURE = 65536,
152 MEMWIN0_BASE = 0x30000,
153 MEMWIN1_APERTURE = 32768,
154 MEMWIN1_BASE = 0x28000,
155 MEMWIN2_APERTURE = 2048,
156 MEMWIN2_BASE = 0x1b800,
157};
158
159enum {
160 MAX_TXQ_ENTRIES = 16384,
161 MAX_CTRL_TXQ_ENTRIES = 1024,
162 MAX_RSPQ_ENTRIES = 16384,
163 MAX_RX_BUFFERS = 16384,
164 MIN_TXQ_ENTRIES = 32,
165 MIN_CTRL_TXQ_ENTRIES = 32,
166 MIN_RSPQ_ENTRIES = 128,
167 MIN_FL_ENTRIES = 16
168};
169
170#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
171 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
172 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
173
174#define CH_DEVICE(devid) { PCI_VDEVICE(CHELSIO, devid), 0 }
175
176static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
177 CH_DEVICE(0xa000), /* PE10K */
178 { 0, }
179};
180
181#define FW_FNAME "cxgb4/t4fw.bin"
182
183MODULE_DESCRIPTION(DRV_DESC);
184MODULE_AUTHOR("Chelsio Communications");
185MODULE_LICENSE("Dual BSD/GPL");
186MODULE_VERSION(DRV_VERSION);
187MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
188MODULE_FIRMWARE(FW_FNAME);
189
190static int dflt_msg_enable = DFLT_MSG_ENABLE;
191
192module_param(dflt_msg_enable, int, 0644);
193MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
194
195/*
196 * The driver uses the best interrupt scheme available on a platform in the
197 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
198 * of these schemes the driver may consider as follows:
199 *
200 * msi = 2: choose from among all three options
201 * msi = 1: only consider MSI and INTx interrupts
202 * msi = 0: force INTx interrupts
203 */
204static int msi = 2;
205
206module_param(msi, int, 0644);
207MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
208
209/*
210 * Queue interrupt hold-off timer values. Queues default to the first of these
211 * upon creation.
212 */
213static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
214
215module_param_array(intr_holdoff, uint, NULL, 0644);
216MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
217 "0..4 in microseconds");
218
219static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
220
221module_param_array(intr_cnt, uint, NULL, 0644);
222MODULE_PARM_DESC(intr_cnt,
223 "thresholds 1..3 for queue interrupt packet counters");
224
225static int vf_acls;
226
227#ifdef CONFIG_PCI_IOV
228module_param(vf_acls, bool, 0644);
229MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
230
231static unsigned int num_vf[4];
232
233module_param_array(num_vf, uint, NULL, 0644);
234MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
235#endif
236
237static struct dentry *cxgb4_debugfs_root;
238
239static LIST_HEAD(adapter_list);
240static DEFINE_MUTEX(uld_mutex);
241static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
242static const char *uld_str[] = { "RDMA", "iSCSI" };
243
244static void link_report(struct net_device *dev)
245{
246 if (!netif_carrier_ok(dev))
247 netdev_info(dev, "link down\n");
248 else {
249 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
250
251 const char *s = "10Mbps";
252 const struct port_info *p = netdev_priv(dev);
253
254 switch (p->link_cfg.speed) {
255 case SPEED_10000:
256 s = "10Gbps";
257 break;
258 case SPEED_1000:
259 s = "1000Mbps";
260 break;
261 case SPEED_100:
262 s = "100Mbps";
263 break;
264 }
265
266 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
267 fc[p->link_cfg.fc]);
268 }
269}
270
271void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
272{
273 struct net_device *dev = adapter->port[port_id];
274
275 /* Skip changes from disabled ports. */
276 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
277 if (link_stat)
278 netif_carrier_on(dev);
279 else
280 netif_carrier_off(dev);
281
282 link_report(dev);
283 }
284}
285
286void t4_os_portmod_changed(const struct adapter *adap, int port_id)
287{
288 static const char *mod_str[] = {
a0881cab 289 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 290 };
291
292 const struct net_device *dev = adap->port[port_id];
293 const struct port_info *pi = netdev_priv(dev);
294
295 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
296 netdev_info(dev, "port module unplugged\n");
a0881cab 297 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 298 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
299}
300
301/*
302 * Configure the exact and hash address filters to handle a port's multicast
303 * and secondary unicast MAC addresses.
304 */
305static int set_addr_filters(const struct net_device *dev, bool sleep)
306{
307 u64 mhash = 0;
308 u64 uhash = 0;
309 bool free = true;
310 u16 filt_idx[7];
311 const u8 *addr[7];
312 int ret, naddr = 0;
b8ff05a9
DM		 313 const struct netdev_hw_addr *ha;
314 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 315 int mc_cnt = netdev_mc_count(dev);
b8ff05a9
DM		 316 const struct port_info *pi = netdev_priv(dev);
317
318 /* first do the secondary unicast addresses */
319 netdev_for_each_uc_addr(ha, dev) {
320 addr[naddr++] = ha->addr;
321 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
322 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
323 naddr, addr, filt_idx, &uhash, sleep);
324 if (ret < 0)
325 return ret;
326
327 free = false;
328 naddr = 0;
329 }
330 }
331
332 /* next set up the multicast addresses */
4a35ecf8
DM		 333 netdev_for_each_mc_addr(ha, dev) {
334 addr[naddr++] = ha->addr;
335 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
b8ff05a9
DM		 336 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
337 naddr, addr, filt_idx, &mhash, sleep);
338 if (ret < 0)
339 return ret;
340
341 free = false;
342 naddr = 0;
343 }
344 }
345
346 return t4_set_addr_hash(pi->adapter, 0, pi->viid, uhash != 0,
347 uhash | mhash, sleep);
348}
349
350/*
351 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
352 * If @mtu is -1 it is left unchanged.
353 */
354static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
355{
356 int ret;
357 struct port_info *pi = netdev_priv(dev);
358
359 ret = set_addr_filters(dev, sleep_ok);
360 if (ret == 0)
361 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, mtu,
362 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 363 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 364 sleep_ok);
365 return ret;
366}
367
368/**
369 * link_start - enable a port
370 * @dev: the port to enable
371 *
372 * Performs the MAC and PHY actions needed to enable a port.
373 */
374static int link_start(struct net_device *dev)
375{
376 int ret;
377 struct port_info *pi = netdev_priv(dev);
378
379 /*
380 * We do not set address filters and promiscuity here, the stack does
381 * that step explicitly.
382 */
383 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, dev->mtu, -1, -1, -1,
f8f5aafa 384 pi->vlan_grp != NULL, true);
b8ff05a9
DM		 385 if (ret == 0) {
386 ret = t4_change_mac(pi->adapter, 0, pi->viid,
387 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 388 true);
b8ff05a9
DM		 389 if (ret >= 0) {
390 pi->xact_addr_filt = ret;
391 ret = 0;
392 }
393 }
394 if (ret == 0)
395 ret = t4_link_start(pi->adapter, 0, pi->tx_chan, &pi->link_cfg);
396 if (ret == 0)
397 ret = t4_enable_vi(pi->adapter, 0, pi->viid, true, true);
398 return ret;
399}
400
401/*
402 * Response queue handler for the FW event queue.
403 */
404static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
405 const struct pkt_gl *gl)
406{
407 u8 opcode = ((const struct rss_header *)rsp)->opcode;
408
409 rsp++; /* skip RSS header */
410 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
411 const struct cpl_sge_egr_update *p = (void *)rsp;
412 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
413 struct sge_txq *txq = q->adap->sge.egr_map[qid];
414
415 txq->restarts++;
416 if ((u8 *)txq < (u8 *)q->adap->sge.ethrxq) {
417 struct sge_eth_txq *eq;
418
419 eq = container_of(txq, struct sge_eth_txq, q);
420 netif_tx_wake_queue(eq->txq);
421 } else {
422 struct sge_ofld_txq *oq;
423
424 oq = container_of(txq, struct sge_ofld_txq, q);
425 tasklet_schedule(&oq->qresume_tsk);
426 }
427 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
428 const struct cpl_fw6_msg *p = (void *)rsp;
429
430 if (p->type == 0)
431 t4_handle_fw_rpl(q->adap, p->data);
432 } else if (opcode == CPL_L2T_WRITE_RPL) {
433 const struct cpl_l2t_write_rpl *p = (void *)rsp;
434
435 do_l2t_write_rpl(q->adap, p);
436 } else
437 dev_err(q->adap->pdev_dev,
438 "unexpected CPL %#x on FW event queue\n", opcode);
439 return 0;
440}
441
442/**
443 * uldrx_handler - response queue handler for ULD queues
444 * @q: the response queue that received the packet
445 * @rsp: the response queue descriptor holding the offload message
446 * @gl: the gather list of packet fragments
447 *
448 * Deliver an ingress offload packet to a ULD. All processing is done by
449 * the ULD, we just maintain statistics.
450 */
451static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
452 const struct pkt_gl *gl)
453{
454 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
455
456 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
457 rxq->stats.nomem++;
458 return -1;
459 }
460 if (gl == NULL)
461 rxq->stats.imm++;
462 else if (gl == CXGB4_MSG_AN)
463 rxq->stats.an++;
464 else
465 rxq->stats.pkts++;
466 return 0;
467}
468
469static void disable_msi(struct adapter *adapter)
470{
471 if (adapter->flags & USING_MSIX) {
472 pci_disable_msix(adapter->pdev);
473 adapter->flags &= ~USING_MSIX;
474 } else if (adapter->flags & USING_MSI) {
475 pci_disable_msi(adapter->pdev);
476 adapter->flags &= ~USING_MSI;
477 }
478}
479
480/*
481 * Interrupt handler for non-data events used with MSI-X.
482 */
483static irqreturn_t t4_nondata_intr(int irq, void *cookie)
484{
485 struct adapter *adap = cookie;
486
487 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
488 if (v & PFSW) {
489 adap->swintr = 1;
490 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
491 }
492 t4_slow_intr_handler(adap);
493 return IRQ_HANDLED;
494}
495
496/*
497 * Name the MSI-X interrupts.
498 */
499static void name_msix_vecs(struct adapter *adap)
500{
501 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
502
503 /* non-data interrupts */
504 snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
505 adap->msix_info[0].desc[n] = 0;
506
507 /* FW events */
508 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
509 adap->msix_info[1].desc[n] = 0;
510
511 /* Ethernet queues */
512 for_each_port(adap, j) {
513 struct net_device *d = adap->port[j];
514 const struct port_info *pi = netdev_priv(d);
515
516 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
517 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
518 d->name, i);
519 adap->msix_info[msi_idx].desc[n] = 0;
520 }
521 }
522
523 /* offload queues */
524 for_each_ofldrxq(&adap->sge, i) {
525 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
526 adap->name, i);
527 adap->msix_info[msi_idx++].desc[n] = 0;
528 }
529 for_each_rdmarxq(&adap->sge, i) {
530 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
531 adap->name, i);
532 adap->msix_info[msi_idx++].desc[n] = 0;
533 }
534}
535
536static int request_msix_queue_irqs(struct adapter *adap)
537{
538 struct sge *s = &adap->sge;
539 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
540
541 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
542 adap->msix_info[1].desc, &s->fw_evtq);
543 if (err)
544 return err;
545
546 for_each_ethrxq(s, ethqidx) {
547 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
548 adap->msix_info[msi].desc,
549 &s->ethrxq[ethqidx].rspq);
550 if (err)
551 goto unwind;
552 msi++;
553 }
554 for_each_ofldrxq(s, ofldqidx) {
555 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
556 adap->msix_info[msi].desc,
557 &s->ofldrxq[ofldqidx].rspq);
558 if (err)
559 goto unwind;
560 msi++;
561 }
562 for_each_rdmarxq(s, rdmaqidx) {
563 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
564 adap->msix_info[msi].desc,
565 &s->rdmarxq[rdmaqidx].rspq);
566 if (err)
567 goto unwind;
568 msi++;
569 }
570 return 0;
571
572unwind:
573 while (--rdmaqidx >= 0)
574 free_irq(adap->msix_info[--msi].vec,
575 &s->rdmarxq[rdmaqidx].rspq);
576 while (--ofldqidx >= 0)
577 free_irq(adap->msix_info[--msi].vec,
578 &s->ofldrxq[ofldqidx].rspq);
579 while (--ethqidx >= 0)
580 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
581 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
582 return err;
583}
584
585static void free_msix_queue_irqs(struct adapter *adap)
586{
587 int i, msi = 2;
588 struct sge *s = &adap->sge;
589
590 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
591 for_each_ethrxq(s, i)
592 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
593 for_each_ofldrxq(s, i)
594 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
595 for_each_rdmarxq(s, i)
596 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
597}
598
671b0060
DM		 599/**
600 * write_rss - write the RSS table for a given port
601 * @pi: the port
602 * @queues: array of queue indices for RSS
603 *
604 * Sets up the portion of the HW RSS table for the port's VI to distribute
605 * packets to the Rx queues in @queues.
606 */
607static int write_rss(const struct port_info *pi, const u16 *queues)
608{
609 u16 *rss;
610 int i, err;
611 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
612
613 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
614 if (!rss)
615 return -ENOMEM;
616
617 /* map the queue indices to queue ids */
618 for (i = 0; i < pi->rss_size; i++, queues++)
619 rss[i] = q[*queues].rspq.abs_id;
620
621 err = t4_config_rss_range(pi->adapter, 0, pi->viid, 0, pi->rss_size,
622 rss, pi->rss_size);
623 kfree(rss);
624 return err;
625}
626
b8ff05a9
DM		 627/**
628 * setup_rss - configure RSS
629 * @adap: the adapter
630 *
671b0060 631 * Sets up RSS for each port.
b8ff05a9
DM		 632 */
633static int setup_rss(struct adapter *adap)
634{
671b0060 635 int i, err;
b8ff05a9
DM		 636
637 for_each_port(adap, i) {
638 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 639
671b0060 640 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 641 if (err)
642 return err;
643 }
644 return 0;
645}
646
647/*
648 * Wait until all NAPI handlers are descheduled.
649 */
650static void quiesce_rx(struct adapter *adap)
651{
652 int i;
653
654 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
655 struct sge_rspq *q = adap->sge.ingr_map[i];
656
657 if (q && q->handler)
658 napi_disable(&q->napi);
659 }
660}
661
662/*
663 * Enable NAPI scheduling and interrupt generation for all Rx queues.
664 */
665static void enable_rx(struct adapter *adap)
666{
667 int i;
668
669 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
670 struct sge_rspq *q = adap->sge.ingr_map[i];
671
672 if (!q)
673 continue;
674 if (q->handler)
675 napi_enable(&q->napi);
676 /* 0-increment GTS to start the timer and enable interrupts */
677 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
678 SEINTARM(q->intr_params) |
679 INGRESSQID(q->cntxt_id));
680 }
681}
682
683/**
684 * setup_sge_queues - configure SGE Tx/Rx/response queues
685 * @adap: the adapter
686 *
687 * Determines how many sets of SGE queues to use and initializes them.
688 * We support multiple queue sets per port if we have MSI-X, otherwise
689 * just one queue set per port.
690 */
691static int setup_sge_queues(struct adapter *adap)
692{
693 int err, msi_idx, i, j;
694 struct sge *s = &adap->sge;
695
696 bitmap_zero(s->starving_fl, MAX_EGRQ);
697 bitmap_zero(s->txq_maperr, MAX_EGRQ);
698
699 if (adap->flags & USING_MSIX)
700 msi_idx = 1; /* vector 0 is for non-queue interrupts */
701 else {
702 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
703 NULL, NULL);
704 if (err)
705 return err;
706 msi_idx = -((int)s->intrq.abs_id + 1);
707 }
708
709 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
710 msi_idx, NULL, fwevtq_handler);
711 if (err) {
712freeout: t4_free_sge_resources(adap);
713 return err;
714 }
715
716 for_each_port(adap, i) {
717 struct net_device *dev = adap->port[i];
718 struct port_info *pi = netdev_priv(dev);
719 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
720 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
721
722 for (j = 0; j < pi->nqsets; j++, q++) {
723 if (msi_idx > 0)
724 msi_idx++;
725 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
726 msi_idx, &q->fl,
727 t4_ethrx_handler);
728 if (err)
729 goto freeout;
730 q->rspq.idx = j;
731 memset(&q->stats, 0, sizeof(q->stats));
732 }
733 for (j = 0; j < pi->nqsets; j++, t++) {
734 err = t4_sge_alloc_eth_txq(adap, t, dev,
735 netdev_get_tx_queue(dev, j),
736 s->fw_evtq.cntxt_id);
737 if (err)
738 goto freeout;
739 }
740 }
741
742 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
743 for_each_ofldrxq(s, i) {
744 struct sge_ofld_rxq *q = &s->ofldrxq[i];
745 struct net_device *dev = adap->port[i / j];
746
747 if (msi_idx > 0)
748 msi_idx++;
749 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
750 &q->fl, uldrx_handler);
751 if (err)
752 goto freeout;
753 memset(&q->stats, 0, sizeof(q->stats));
754 s->ofld_rxq[i] = q->rspq.abs_id;
755 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
756 s->fw_evtq.cntxt_id);
757 if (err)
758 goto freeout;
759 }
760
761 for_each_rdmarxq(s, i) {
762 struct sge_ofld_rxq *q = &s->rdmarxq[i];
763
764 if (msi_idx > 0)
765 msi_idx++;
766 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
767 msi_idx, &q->fl, uldrx_handler);
768 if (err)
769 goto freeout;
770 memset(&q->stats, 0, sizeof(q->stats));
771 s->rdma_rxq[i] = q->rspq.abs_id;
772 }
773
774 for_each_port(adap, i) {
775 /*
776 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
777 * have RDMA queues, and that's the right value.
778 */
779 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
780 s->fw_evtq.cntxt_id,
781 s->rdmarxq[i].rspq.cntxt_id);
782 if (err)
783 goto freeout;
784 }
785
786 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
787 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
788 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
789 return 0;
790}
791
792/*
793 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
794 * started but failed, and a negative errno if flash load couldn't start.
795 */
796static int upgrade_fw(struct adapter *adap)
797{
798 int ret;
799 u32 vers;
800 const struct fw_hdr *hdr;
801 const struct firmware *fw;
802 struct device *dev = adap->pdev_dev;
803
804 ret = request_firmware(&fw, FW_FNAME, dev);
805 if (ret < 0) {
806 dev_err(dev, "unable to load firmware image " FW_FNAME
807 ", error %d\n", ret);
808 return ret;
809 }
810
811 hdr = (const struct fw_hdr *)fw->data;
812 vers = ntohl(hdr->fw_ver);
813 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
814 ret = -EINVAL; /* wrong major version, won't do */
815 goto out;
816 }
817
818 /*
819 * If the flash FW is unusable or we found something newer, load it.
820 */
821 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
822 vers > adap->params.fw_vers) {
823 ret = -t4_load_fw(adap, fw->data, fw->size);
824 if (!ret)
825 dev_info(dev, "firmware upgraded to version %pI4 from "
826 FW_FNAME "\n", &hdr->fw_ver);
827 }
828out: release_firmware(fw);
829 return ret;
830}
831
832/*
833 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
834 * The allocated memory is cleared.
835 */
836void *t4_alloc_mem(size_t size)
837{
838 void *p = kmalloc(size, GFP_KERNEL);
839
840 if (!p)
841 p = vmalloc(size);
842 if (p)
843 memset(p, 0, size);
844 return p;
845}
846
847/*
848 * Free memory allocated through alloc_mem().
849 */
850void t4_free_mem(void *addr)
851{
852 if (is_vmalloc_addr(addr))
853 vfree(addr);
854 else
855 kfree(addr);
856}
857
858static inline int is_offload(const struct adapter *adap)
859{
860 return adap->params.offload;
861}
862
863/*
864 * Implementation of ethtool operations.
865 */
866
867static u32 get_msglevel(struct net_device *dev)
868{
869 return netdev2adap(dev)->msg_enable;
870}
871
872static void set_msglevel(struct net_device *dev, u32 val)
873{
874 netdev2adap(dev)->msg_enable = val;
875}
876
877static char stats_strings[][ETH_GSTRING_LEN] = {
878 "TxOctetsOK ",
879 "TxFramesOK ",
880 "TxBroadcastFrames ",
881 "TxMulticastFrames ",
882 "TxUnicastFrames ",
883 "TxErrorFrames ",
884
885 "TxFrames64 ",
886 "TxFrames65To127 ",
887 "TxFrames128To255 ",
888 "TxFrames256To511 ",
889 "TxFrames512To1023 ",
890 "TxFrames1024To1518 ",
891 "TxFrames1519ToMax ",
892
893 "TxFramesDropped ",
894 "TxPauseFrames ",
895 "TxPPP0Frames ",
896 "TxPPP1Frames ",
897 "TxPPP2Frames ",
898 "TxPPP3Frames ",
899 "TxPPP4Frames ",
900 "TxPPP5Frames ",
901 "TxPPP6Frames ",
902 "TxPPP7Frames ",
903
904 "RxOctetsOK ",
905 "RxFramesOK ",
906 "RxBroadcastFrames ",
907 "RxMulticastFrames ",
908 "RxUnicastFrames ",
909
910 "RxFramesTooLong ",
911 "RxJabberErrors ",
912 "RxFCSErrors ",
913 "RxLengthErrors ",
914 "RxSymbolErrors ",
915 "RxRuntFrames ",
916
917 "RxFrames64 ",
918 "RxFrames65To127 ",
919 "RxFrames128To255 ",
920 "RxFrames256To511 ",
921 "RxFrames512To1023 ",
922 "RxFrames1024To1518 ",
923 "RxFrames1519ToMax ",
924
925 "RxPauseFrames ",
926 "RxPPP0Frames ",
927 "RxPPP1Frames ",
928 "RxPPP2Frames ",
929 "RxPPP3Frames ",
930 "RxPPP4Frames ",
931 "RxPPP5Frames ",
932 "RxPPP6Frames ",
933 "RxPPP7Frames ",
934
935 "RxBG0FramesDropped ",
936 "RxBG1FramesDropped ",
937 "RxBG2FramesDropped ",
938 "RxBG3FramesDropped ",
939 "RxBG0FramesTrunc ",
940 "RxBG1FramesTrunc ",
941 "RxBG2FramesTrunc ",
942 "RxBG3FramesTrunc ",
943
944 "TSO ",
945 "TxCsumOffload ",
946 "RxCsumGood ",
947 "VLANextractions ",
948 "VLANinsertions ",
4a6346d4
DM		 949 "GROpackets ",
950 "GROmerged ",
b8ff05a9
DM		 951};
952
953static int get_sset_count(struct net_device *dev, int sset)
954{
955 switch (sset) {
956 case ETH_SS_STATS:
957 return ARRAY_SIZE(stats_strings);
958 default:
959 return -EOPNOTSUPP;
960 }
961}
962
963#define T4_REGMAP_SIZE (160 * 1024)
964
965static int get_regs_len(struct net_device *dev)
966{
967 return T4_REGMAP_SIZE;
968}
969
970static int get_eeprom_len(struct net_device *dev)
971{
972 return EEPROMSIZE;
973}
974
975static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
976{
977 struct adapter *adapter = netdev2adap(dev);
978
979 strcpy(info->driver, KBUILD_MODNAME);
980 strcpy(info->version, DRV_VERSION);
981 strcpy(info->bus_info, pci_name(adapter->pdev));
982
983 if (!adapter->params.fw_vers)
984 strcpy(info->fw_version, "N/A");
985 else
986 snprintf(info->fw_version, sizeof(info->fw_version),
987 "%u.%u.%u.%u, TP %u.%u.%u.%u",
988 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
989 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
990 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
991 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
992 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
993 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
994 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
995 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
996}
997
998static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
999{
1000 if (stringset == ETH_SS_STATS)
1001 memcpy(data, stats_strings, sizeof(stats_strings));
1002}
1003
1004/*
1005 * port stats maintained per queue of the port. They should be in the same
1006 * order as in stats_strings above.
1007 */
1008struct queue_port_stats {
1009 u64 tso;
1010 u64 tx_csum;
1011 u64 rx_csum;
1012 u64 vlan_ex;
1013 u64 vlan_ins;
4a6346d4
DM		 1014 u64 gro_pkts;
1015 u64 gro_merged;
b8ff05a9
DM		 1016};
1017
1018static void collect_sge_port_stats(const struct adapter *adap,
1019 const struct port_info *p, struct queue_port_stats *s)
1020{
1021 int i;
1022 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1023 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1024
1025 memset(s, 0, sizeof(*s));
1026 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1027 s->tso += tx->tso;
1028 s->tx_csum += tx->tx_cso;
1029 s->rx_csum += rx->stats.rx_cso;
1030 s->vlan_ex += rx->stats.vlan_ex;
1031 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1032 s->gro_pkts += rx->stats.lro_pkts;
1033 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1034 }
1035}
1036
1037static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1038 u64 *data)
1039{
1040 struct port_info *pi = netdev_priv(dev);
1041 struct adapter *adapter = pi->adapter;
1042
1043 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1044
1045 data += sizeof(struct port_stats) / sizeof(u64);
1046 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1047}
1048
1049/*
1050 * Return a version number to identify the type of adapter. The scheme is:
1051 * - bits 0..9: chip version
1052 * - bits 10..15: chip revision
835bb606 1053 * - bits 16..23: register dump version
b8ff05a9
DM		 1054 */
1055static inline unsigned int mk_adap_vers(const struct adapter *ap)
1056{
835bb606 1057 return 4 | (ap->params.rev << 10) | (1 << 16);
b8ff05a9
DM		 1058}
1059
1060static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1061 unsigned int end)
1062{
1063 u32 *p = buf + start;
1064
1065 for ( ; start <= end; start += sizeof(u32))
1066 *p++ = t4_read_reg(ap, start);
1067}
1068
1069static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1070 void *buf)
1071{
1072 static const unsigned int reg_ranges[] = {
1073 0x1008, 0x1108,
1074 0x1180, 0x11b4,
1075 0x11fc, 0x123c,
1076 0x1300, 0x173c,
1077 0x1800, 0x18fc,
1078 0x3000, 0x30d8,
1079 0x30e0, 0x5924,
1080 0x5960, 0x59d4,
1081 0x5a00, 0x5af8,
1082 0x6000, 0x6098,
1083 0x6100, 0x6150,
1084 0x6200, 0x6208,
1085 0x6240, 0x6248,
1086 0x6280, 0x6338,
1087 0x6370, 0x638c,
1088 0x6400, 0x643c,
1089 0x6500, 0x6524,
1090 0x6a00, 0x6a38,
1091 0x6a60, 0x6a78,
1092 0x6b00, 0x6b84,
1093 0x6bf0, 0x6c84,
1094 0x6cf0, 0x6d84,
1095 0x6df0, 0x6e84,
1096 0x6ef0, 0x6f84,
1097 0x6ff0, 0x7084,
1098 0x70f0, 0x7184,
1099 0x71f0, 0x7284,
1100 0x72f0, 0x7384,
1101 0x73f0, 0x7450,
1102 0x7500, 0x7530,
1103 0x7600, 0x761c,
1104 0x7680, 0x76cc,
1105 0x7700, 0x7798,
1106 0x77c0, 0x77fc,
1107 0x7900, 0x79fc,
1108 0x7b00, 0x7c38,
1109 0x7d00, 0x7efc,
1110 0x8dc0, 0x8e1c,
1111 0x8e30, 0x8e78,
1112 0x8ea0, 0x8f6c,
1113 0x8fc0, 0x9074,
1114 0x90fc, 0x90fc,
1115 0x9400, 0x9458,
1116 0x9600, 0x96bc,
1117 0x9800, 0x9808,
1118 0x9820, 0x983c,
1119 0x9850, 0x9864,
1120 0x9c00, 0x9c6c,
1121 0x9c80, 0x9cec,
1122 0x9d00, 0x9d6c,
1123 0x9d80, 0x9dec,
1124 0x9e00, 0x9e6c,
1125 0x9e80, 0x9eec,
1126 0x9f00, 0x9f6c,
1127 0x9f80, 0x9fec,
1128 0xd004, 0xd03c,
1129 0xdfc0, 0xdfe0,
1130 0xe000, 0xea7c,
1131 0xf000, 0x11190,
835bb606
DM		 1132 0x19040, 0x1906c,
1133 0x19078, 0x19080,
1134 0x1908c, 0x19124,
b8ff05a9
DM		 1135 0x19150, 0x191b0,
1136 0x191d0, 0x191e8,
1137 0x19238, 0x1924c,
1138 0x193f8, 0x19474,
1139 0x19490, 0x194f8,
1140 0x19800, 0x19f30,
1141 0x1a000, 0x1a06c,
1142 0x1a0b0, 0x1a120,
1143 0x1a128, 0x1a138,
1144 0x1a190, 0x1a1c4,
1145 0x1a1fc, 0x1a1fc,
1146 0x1e040, 0x1e04c,
835bb606 1147 0x1e284, 0x1e28c,
b8ff05a9
DM		 1148 0x1e2c0, 0x1e2c0,
1149 0x1e2e0, 0x1e2e0,
1150 0x1e300, 0x1e384,
1151 0x1e3c0, 0x1e3c8,
1152 0x1e440, 0x1e44c,
835bb606 1153 0x1e684, 0x1e68c,
b8ff05a9
DM		 1154 0x1e6c0, 0x1e6c0,
1155 0x1e6e0, 0x1e6e0,
1156 0x1e700, 0x1e784,
1157 0x1e7c0, 0x1e7c8,
1158 0x1e840, 0x1e84c,
835bb606 1159 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1160 0x1eac0, 0x1eac0,
1161 0x1eae0, 0x1eae0,
1162 0x1eb00, 0x1eb84,
1163 0x1ebc0, 0x1ebc8,
1164 0x1ec40, 0x1ec4c,
835bb606 1165 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1166 0x1eec0, 0x1eec0,
1167 0x1eee0, 0x1eee0,
1168 0x1ef00, 0x1ef84,
1169 0x1efc0, 0x1efc8,
1170 0x1f040, 0x1f04c,
835bb606 1171 0x1f284, 0x1f28c,
b8ff05a9
DM		 1172 0x1f2c0, 0x1f2c0,
1173 0x1f2e0, 0x1f2e0,
1174 0x1f300, 0x1f384,
1175 0x1f3c0, 0x1f3c8,
1176 0x1f440, 0x1f44c,
835bb606 1177 0x1f684, 0x1f68c,
b8ff05a9
DM		 1178 0x1f6c0, 0x1f6c0,
1179 0x1f6e0, 0x1f6e0,
1180 0x1f700, 0x1f784,
1181 0x1f7c0, 0x1f7c8,
1182 0x1f840, 0x1f84c,
835bb606 1183 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1184 0x1fac0, 0x1fac0,
1185 0x1fae0, 0x1fae0,
1186 0x1fb00, 0x1fb84,
1187 0x1fbc0, 0x1fbc8,
1188 0x1fc40, 0x1fc4c,
835bb606 1189 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1190 0x1fec0, 0x1fec0,
1191 0x1fee0, 0x1fee0,
1192 0x1ff00, 0x1ff84,
1193 0x1ffc0, 0x1ffc8,
1194 0x20000, 0x2002c,
1195 0x20100, 0x2013c,
1196 0x20190, 0x201c8,
1197 0x20200, 0x20318,
1198 0x20400, 0x20528,
1199 0x20540, 0x20614,
1200 0x21000, 0x21040,
1201 0x2104c, 0x21060,
1202 0x210c0, 0x210ec,
1203 0x21200, 0x21268,
1204 0x21270, 0x21284,
1205 0x212fc, 0x21388,
1206 0x21400, 0x21404,
1207 0x21500, 0x21518,
1208 0x2152c, 0x2153c,
1209 0x21550, 0x21554,
1210 0x21600, 0x21600,
1211 0x21608, 0x21628,
1212 0x21630, 0x2163c,
1213 0x21700, 0x2171c,
1214 0x21780, 0x2178c,
1215 0x21800, 0x21c38,
1216 0x21c80, 0x21d7c,
1217 0x21e00, 0x21e04,
1218 0x22000, 0x2202c,
1219 0x22100, 0x2213c,
1220 0x22190, 0x221c8,
1221 0x22200, 0x22318,
1222 0x22400, 0x22528,
1223 0x22540, 0x22614,
1224 0x23000, 0x23040,
1225 0x2304c, 0x23060,
1226 0x230c0, 0x230ec,
1227 0x23200, 0x23268,
1228 0x23270, 0x23284,
1229 0x232fc, 0x23388,
1230 0x23400, 0x23404,
1231 0x23500, 0x23518,
1232 0x2352c, 0x2353c,
1233 0x23550, 0x23554,
1234 0x23600, 0x23600,
1235 0x23608, 0x23628,
1236 0x23630, 0x2363c,
1237 0x23700, 0x2371c,
1238 0x23780, 0x2378c,
1239 0x23800, 0x23c38,
1240 0x23c80, 0x23d7c,
1241 0x23e00, 0x23e04,
1242 0x24000, 0x2402c,
1243 0x24100, 0x2413c,
1244 0x24190, 0x241c8,
1245 0x24200, 0x24318,
1246 0x24400, 0x24528,
1247 0x24540, 0x24614,
1248 0x25000, 0x25040,
1249 0x2504c, 0x25060,
1250 0x250c0, 0x250ec,
1251 0x25200, 0x25268,
1252 0x25270, 0x25284,
1253 0x252fc, 0x25388,
1254 0x25400, 0x25404,
1255 0x25500, 0x25518,
1256 0x2552c, 0x2553c,
1257 0x25550, 0x25554,
1258 0x25600, 0x25600,
1259 0x25608, 0x25628,
1260 0x25630, 0x2563c,
1261 0x25700, 0x2571c,
1262 0x25780, 0x2578c,
1263 0x25800, 0x25c38,
1264 0x25c80, 0x25d7c,
1265 0x25e00, 0x25e04,
1266 0x26000, 0x2602c,
1267 0x26100, 0x2613c,
1268 0x26190, 0x261c8,
1269 0x26200, 0x26318,
1270 0x26400, 0x26528,
1271 0x26540, 0x26614,
1272 0x27000, 0x27040,
1273 0x2704c, 0x27060,
1274 0x270c0, 0x270ec,
1275 0x27200, 0x27268,
1276 0x27270, 0x27284,
1277 0x272fc, 0x27388,
1278 0x27400, 0x27404,
1279 0x27500, 0x27518,
1280 0x2752c, 0x2753c,
1281 0x27550, 0x27554,
1282 0x27600, 0x27600,
1283 0x27608, 0x27628,
1284 0x27630, 0x2763c,
1285 0x27700, 0x2771c,
1286 0x27780, 0x2778c,
1287 0x27800, 0x27c38,
1288 0x27c80, 0x27d7c,
1289 0x27e00, 0x27e04
1290 };
1291
1292 int i;
1293 struct adapter *ap = netdev2adap(dev);
1294
1295 regs->version = mk_adap_vers(ap);
1296
1297 memset(buf, 0, T4_REGMAP_SIZE);
1298 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1299 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1300}
1301
1302static int restart_autoneg(struct net_device *dev)
1303{
1304 struct port_info *p = netdev_priv(dev);
1305
1306 if (!netif_running(dev))
1307 return -EAGAIN;
1308 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1309 return -EINVAL;
1310 t4_restart_aneg(p->adapter, 0, p->tx_chan);
1311 return 0;
1312}
1313
1314static int identify_port(struct net_device *dev, u32 data)
1315{
1316 if (data == 0)
1317 data = 2; /* default to 2 seconds */
1318
1319 return t4_identify_port(netdev2adap(dev), 0, netdev2pinfo(dev)->viid,
1320 data * 5);
1321}
1322
1323static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1324{
1325 unsigned int v = 0;
1326
a0881cab
DM		 1327 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1328 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 1329 v |= SUPPORTED_TP;
1330 if (caps & FW_PORT_CAP_SPEED_100M)
1331 v |= SUPPORTED_100baseT_Full;
1332 if (caps & FW_PORT_CAP_SPEED_1G)
1333 v |= SUPPORTED_1000baseT_Full;
1334 if (caps & FW_PORT_CAP_SPEED_10G)
1335 v |= SUPPORTED_10000baseT_Full;
1336 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1337 v |= SUPPORTED_Backplane;
1338 if (caps & FW_PORT_CAP_SPEED_1G)
1339 v |= SUPPORTED_1000baseKX_Full;
1340 if (caps & FW_PORT_CAP_SPEED_10G)
1341 v |= SUPPORTED_10000baseKX4_Full;
1342 } else if (type == FW_PORT_TYPE_KR)
1343 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab
DM		 1344 else if (type == FW_PORT_TYPE_BP_AP)
1345 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1346 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1347 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9
DM		 1348 v |= SUPPORTED_FIBRE;
1349
1350 if (caps & FW_PORT_CAP_ANEG)
1351 v |= SUPPORTED_Autoneg;
1352 return v;
1353}
1354
1355static unsigned int to_fw_linkcaps(unsigned int caps)
1356{
1357 unsigned int v = 0;
1358
1359 if (caps & ADVERTISED_100baseT_Full)
1360 v |= FW_PORT_CAP_SPEED_100M;
1361 if (caps & ADVERTISED_1000baseT_Full)
1362 v |= FW_PORT_CAP_SPEED_1G;
1363 if (caps & ADVERTISED_10000baseT_Full)
1364 v |= FW_PORT_CAP_SPEED_10G;
1365 return v;
1366}
1367
1368static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1369{
1370 const struct port_info *p = netdev_priv(dev);
1371
1372 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 1373 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 1374 p->port_type == FW_PORT_TYPE_BT_XAUI)
1375 cmd->port = PORT_TP;
a0881cab
DM		 1376 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1377 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 1378 cmd->port = PORT_FIBRE;
a0881cab
DM		 1379 else if (p->port_type == FW_PORT_TYPE_SFP) {
1380 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1381 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1382 cmd->port = PORT_DA;
1383 else
1384 cmd->port = PORT_FIBRE;
1385 } else
b8ff05a9
DM		 1386 cmd->port = PORT_OTHER;
1387
1388 if (p->mdio_addr >= 0) {
1389 cmd->phy_address = p->mdio_addr;
1390 cmd->transceiver = XCVR_EXTERNAL;
1391 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1392 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1393 } else {
1394 cmd->phy_address = 0; /* not really, but no better option */
1395 cmd->transceiver = XCVR_INTERNAL;
1396 cmd->mdio_support = 0;
1397 }
1398
1399 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1400 cmd->advertising = from_fw_linkcaps(p->port_type,
1401 p->link_cfg.advertising);
1402 cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1403 cmd->duplex = DUPLEX_FULL;
1404 cmd->autoneg = p->link_cfg.autoneg;
1405 cmd->maxtxpkt = 0;
1406 cmd->maxrxpkt = 0;
1407 return 0;
1408}
1409
1410static unsigned int speed_to_caps(int speed)
1411{
1412 if (speed == SPEED_100)
1413 return FW_PORT_CAP_SPEED_100M;
1414 if (speed == SPEED_1000)
1415 return FW_PORT_CAP_SPEED_1G;
1416 if (speed == SPEED_10000)
1417 return FW_PORT_CAP_SPEED_10G;
1418 return 0;
1419}
1420
1421static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1422{
1423 unsigned int cap;
1424 struct port_info *p = netdev_priv(dev);
1425 struct link_config *lc = &p->link_cfg;
1426
1427 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1428 return -EINVAL;
1429
1430 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1431 /*
1432 * PHY offers a single speed. See if that's what's
1433 * being requested.
1434 */
1435 if (cmd->autoneg == AUTONEG_DISABLE &&
1436 (lc->supported & speed_to_caps(cmd->speed)))
1437 return 0;
1438 return -EINVAL;
1439 }
1440
1441 if (cmd->autoneg == AUTONEG_DISABLE) {
1442 cap = speed_to_caps(cmd->speed);
1443
1444 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1445 cmd->speed == SPEED_10000)
1446 return -EINVAL;
1447 lc->requested_speed = cap;
1448 lc->advertising = 0;
1449 } else {
1450 cap = to_fw_linkcaps(cmd->advertising);
1451 if (!(lc->supported & cap))
1452 return -EINVAL;
1453 lc->requested_speed = 0;
1454 lc->advertising = cap | FW_PORT_CAP_ANEG;
1455 }
1456 lc->autoneg = cmd->autoneg;
1457
1458 if (netif_running(dev))
1459 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1460 return 0;
1461}
1462
1463static void get_pauseparam(struct net_device *dev,
1464 struct ethtool_pauseparam *epause)
1465{
1466 struct port_info *p = netdev_priv(dev);
1467
1468 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1469 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1470 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1471}
1472
1473static int set_pauseparam(struct net_device *dev,
1474 struct ethtool_pauseparam *epause)
1475{
1476 struct port_info *p = netdev_priv(dev);
1477 struct link_config *lc = &p->link_cfg;
1478
1479 if (epause->autoneg == AUTONEG_DISABLE)
1480 lc->requested_fc = 0;
1481 else if (lc->supported & FW_PORT_CAP_ANEG)
1482 lc->requested_fc = PAUSE_AUTONEG;
1483 else
1484 return -EINVAL;
1485
1486 if (epause->rx_pause)
1487 lc->requested_fc |= PAUSE_RX;
1488 if (epause->tx_pause)
1489 lc->requested_fc |= PAUSE_TX;
1490 if (netif_running(dev))
1491 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1492 return 0;
1493}
1494
1495static u32 get_rx_csum(struct net_device *dev)
1496{
1497 struct port_info *p = netdev_priv(dev);
1498
1499 return p->rx_offload & RX_CSO;
1500}
1501
1502static int set_rx_csum(struct net_device *dev, u32 data)
1503{
1504 struct port_info *p = netdev_priv(dev);
1505
1506 if (data)
1507 p->rx_offload |= RX_CSO;
1508 else
1509 p->rx_offload &= ~RX_CSO;
1510 return 0;
1511}
1512
1513static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1514{
1515 const struct port_info *pi = netdev_priv(dev);
1516 const struct sge *s = &pi->adapter->sge;
1517
1518 e->rx_max_pending = MAX_RX_BUFFERS;
1519 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1520 e->rx_jumbo_max_pending = 0;
1521 e->tx_max_pending = MAX_TXQ_ENTRIES;
1522
1523 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1524 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1525 e->rx_jumbo_pending = 0;
1526 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1527}
1528
1529static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1530{
1531 int i;
1532 const struct port_info *pi = netdev_priv(dev);
1533 struct adapter *adapter = pi->adapter;
1534 struct sge *s = &adapter->sge;
1535
1536 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1537 e->tx_pending > MAX_TXQ_ENTRIES ||
1538 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1539 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1540 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1541 return -EINVAL;
1542
1543 if (adapter->flags & FULL_INIT_DONE)
1544 return -EBUSY;
1545
1546 for (i = 0; i < pi->nqsets; ++i) {
1547 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1548 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1549 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1550 }
1551 return 0;
1552}
1553
1554static int closest_timer(const struct sge *s, int time)
1555{
1556 int i, delta, match = 0, min_delta = INT_MAX;
1557
1558 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1559 delta = time - s->timer_val[i];
1560 if (delta < 0)
1561 delta = -delta;
1562 if (delta < min_delta) {
1563 min_delta = delta;
1564 match = i;
1565 }
1566 }
1567 return match;
1568}
1569
1570static int closest_thres(const struct sge *s, int thres)
1571{
1572 int i, delta, match = 0, min_delta = INT_MAX;
1573
1574 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1575 delta = thres - s->counter_val[i];
1576 if (delta < 0)
1577 delta = -delta;
1578 if (delta < min_delta) {
1579 min_delta = delta;
1580 match = i;
1581 }
1582 }
1583 return match;
1584}
1585
1586/*
1587 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1588 */
1589static unsigned int qtimer_val(const struct adapter *adap,
1590 const struct sge_rspq *q)
1591{
1592 unsigned int idx = q->intr_params >> 1;
1593
1594 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1595}
1596
1597/**
1598 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1599 * @adap: the adapter
1600 * @q: the Rx queue
1601 * @us: the hold-off time in us, or 0 to disable timer
1602 * @cnt: the hold-off packet count, or 0 to disable counter
1603 *
1604 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1605 * one of the two needs to be enabled for the queue to generate interrupts.
1606 */
1607static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1608 unsigned int us, unsigned int cnt)
1609{
1610 if ((us | cnt) == 0)
1611 cnt = 1;
1612
1613 if (cnt) {
1614 int err;
1615 u32 v, new_idx;
1616
1617 new_idx = closest_thres(&adap->sge, cnt);
1618 if (q->desc && q->pktcnt_idx != new_idx) {
1619 /* the queue has already been created, update it */
1620 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1621 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1622 FW_PARAMS_PARAM_YZ(q->cntxt_id);
1623 err = t4_set_params(adap, 0, 0, 0, 1, &v, &new_idx);
1624 if (err)
1625 return err;
1626 }
1627 q->pktcnt_idx = new_idx;
1628 }
1629
1630 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1631 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1632 return 0;
1633}
1634
1635static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1636{
1637 const struct port_info *pi = netdev_priv(dev);
1638 struct adapter *adap = pi->adapter;
1639
1640 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1641 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1642}
1643
1644static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1645{
1646 const struct port_info *pi = netdev_priv(dev);
1647 const struct adapter *adap = pi->adapter;
1648 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1649
1650 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1651 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1652 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1653 return 0;
1654}
1655
1656/*
1657 * Translate a physical EEPROM address to virtual. The first 1K is accessed
1658 * through virtual addresses starting at 31K, the rest is accessed through
1659 * virtual addresses starting at 0. This mapping is correct only for PF0.
1660 */
1661static int eeprom_ptov(unsigned int phys_addr)
1662{
1663 if (phys_addr < 1024)
1664 return phys_addr + (31 << 10);
1665 if (phys_addr < EEPROMSIZE)
1666 return phys_addr - 1024;
1667 return -EINVAL;
1668}
1669
1670/*
1671 * The next two routines implement eeprom read/write from physical addresses.
1672 * The physical->virtual translation is correct only for PF0.
1673 */
1674static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1675{
1676 int vaddr = eeprom_ptov(phys_addr);
1677
1678 if (vaddr >= 0)
1679 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1680 return vaddr < 0 ? vaddr : 0;
1681}
1682
1683static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1684{
1685 int vaddr = eeprom_ptov(phys_addr);
1686
1687 if (vaddr >= 0)
1688 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1689 return vaddr < 0 ? vaddr : 0;
1690}
1691
1692#define EEPROM_MAGIC 0x38E2F10C
1693
1694static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1695 u8 *data)
1696{
1697 int i, err = 0;
1698 struct adapter *adapter = netdev2adap(dev);
1699
1700 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1701 if (!buf)
1702 return -ENOMEM;
1703
1704 e->magic = EEPROM_MAGIC;
1705 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1706 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1707
1708 if (!err)
1709 memcpy(data, buf + e->offset, e->len);
1710 kfree(buf);
1711 return err;
1712}
1713
1714static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1715 u8 *data)
1716{
1717 u8 *buf;
1718 int err = 0;
1719 u32 aligned_offset, aligned_len, *p;
1720 struct adapter *adapter = netdev2adap(dev);
1721
1722 if (eeprom->magic != EEPROM_MAGIC)
1723 return -EINVAL;
1724
1725 aligned_offset = eeprom->offset & ~3;
1726 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1727
1728 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1729 /*
1730 * RMW possibly needed for first or last words.
1731 */
1732 buf = kmalloc(aligned_len, GFP_KERNEL);
1733 if (!buf)
1734 return -ENOMEM;
1735 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1736 if (!err && aligned_len > 4)
1737 err = eeprom_rd_phys(adapter,
1738 aligned_offset + aligned_len - 4,
1739 (u32 *)&buf[aligned_len - 4]);
1740 if (err)
1741 goto out;
1742 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1743 } else
1744 buf = data;
1745
1746 err = t4_seeprom_wp(adapter, false);
1747 if (err)
1748 goto out;
1749
1750 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1751 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1752 aligned_offset += 4;
1753 }
1754
1755 if (!err)
1756 err = t4_seeprom_wp(adapter, true);
1757out:
1758 if (buf != data)
1759 kfree(buf);
1760 return err;
1761}
1762
1763static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1764{
1765 int ret;
1766 const struct firmware *fw;
1767 struct adapter *adap = netdev2adap(netdev);
1768
1769 ef->data[sizeof(ef->data) - 1] = '\0';
1770 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1771 if (ret < 0)
1772 return ret;
1773
1774 ret = t4_load_fw(adap, fw->data, fw->size);
1775 release_firmware(fw);
1776 if (!ret)
1777 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1778 return ret;
1779}
1780
1781#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1782#define BCAST_CRC 0xa0ccc1a6
1783
1784static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1785{
1786 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1787 wol->wolopts = netdev2adap(dev)->wol;
1788 memset(&wol->sopass, 0, sizeof(wol->sopass));
1789}
1790
1791static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1792{
1793 int err = 0;
1794 struct port_info *pi = netdev_priv(dev);
1795
1796 if (wol->wolopts & ~WOL_SUPPORTED)
1797 return -EINVAL;
1798 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1799 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1800 if (wol->wolopts & WAKE_BCAST) {
1801 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1802 ~0ULL, 0, false);
1803 if (!err)
1804 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1805 ~6ULL, ~0ULL, BCAST_CRC, true);
1806 } else
1807 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1808 return err;
1809}
1810
1811static int set_tso(struct net_device *dev, u32 value)
1812{
1813 if (value)
1814 dev->features |= NETIF_F_TSO | NETIF_F_TSO6;
1815 else
1816 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
1817 return 0;
1818}
1819
87b6cf51
DM		 1820static int set_flags(struct net_device *dev, u32 flags)
1821{
1437ce39 1822 return ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH);
87b6cf51
DM		 1823}
1824
671b0060
DM		 1825static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1826{
1827 const struct port_info *pi = netdev_priv(dev);
1828 unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1829
1830 p->size = pi->rss_size;
1831 while (n--)
1832 p->ring_index[n] = pi->rss[n];
1833 return 0;
1834}
1835
1836static int set_rss_table(struct net_device *dev,
1837 const struct ethtool_rxfh_indir *p)
1838{
1839 unsigned int i;
1840 struct port_info *pi = netdev_priv(dev);
1841
1842 if (p->size != pi->rss_size)
1843 return -EINVAL;
1844 for (i = 0; i < p->size; i++)
1845 if (p->ring_index[i] >= pi->nqsets)
1846 return -EINVAL;
1847 for (i = 0; i < p->size; i++)
1848 pi->rss[i] = p->ring_index[i];
1849 if (pi->adapter->flags & FULL_INIT_DONE)
1850 return write_rss(pi, pi->rss);
1851 return 0;
1852}
1853
1854static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1855 void *rules)
1856{
f796564a
DM		 1857 const struct port_info *pi = netdev_priv(dev);
1858
671b0060 1859 switch (info->cmd) {
f796564a
DM		 1860 case ETHTOOL_GRXFH: {
1861 unsigned int v = pi->rss_mode;
1862
1863 info->data = 0;
1864 switch (info->flow_type) {
1865 case TCP_V4_FLOW:
1866 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1867 info->data = RXH_IP_SRC | RXH_IP_DST |
1868 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1869 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1870 info->data = RXH_IP_SRC | RXH_IP_DST;
1871 break;
1872 case UDP_V4_FLOW:
1873 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1874 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1875 info->data = RXH_IP_SRC | RXH_IP_DST |
1876 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1877 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1878 info->data = RXH_IP_SRC | RXH_IP_DST;
1879 break;
1880 case SCTP_V4_FLOW:
1881 case AH_ESP_V4_FLOW:
1882 case IPV4_FLOW:
1883 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1884 info->data = RXH_IP_SRC | RXH_IP_DST;
1885 break;
1886 case TCP_V6_FLOW:
1887 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1888 info->data = RXH_IP_SRC | RXH_IP_DST |
1889 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1890 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1891 info->data = RXH_IP_SRC | RXH_IP_DST;
1892 break;
1893 case UDP_V6_FLOW:
1894 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1895 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1896 info->data = RXH_IP_SRC | RXH_IP_DST |
1897 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1898 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1899 info->data = RXH_IP_SRC | RXH_IP_DST;
1900 break;
1901 case SCTP_V6_FLOW:
1902 case AH_ESP_V6_FLOW:
1903 case IPV6_FLOW:
1904 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1905 info->data = RXH_IP_SRC | RXH_IP_DST;
1906 break;
1907 }
1908 return 0;
1909 }
671b0060 1910 case ETHTOOL_GRXRINGS:
f796564a 1911 info->data = pi->nqsets;
671b0060
DM		 1912 return 0;
1913 }
1914 return -EOPNOTSUPP;
1915}
1916
b8ff05a9
DM		 1917static struct ethtool_ops cxgb_ethtool_ops = {
1918 .get_settings = get_settings,
1919 .set_settings = set_settings,
1920 .get_drvinfo = get_drvinfo,
1921 .get_msglevel = get_msglevel,
1922 .set_msglevel = set_msglevel,
1923 .get_ringparam = get_sge_param,
1924 .set_ringparam = set_sge_param,
1925 .get_coalesce = get_coalesce,
1926 .set_coalesce = set_coalesce,
1927 .get_eeprom_len = get_eeprom_len,
1928 .get_eeprom = get_eeprom,
1929 .set_eeprom = set_eeprom,
1930 .get_pauseparam = get_pauseparam,
1931 .set_pauseparam = set_pauseparam,
1932 .get_rx_csum = get_rx_csum,
1933 .set_rx_csum = set_rx_csum,
1934 .set_tx_csum = ethtool_op_set_tx_ipv6_csum,
1935 .set_sg = ethtool_op_set_sg,
1936 .get_link = ethtool_op_get_link,
1937 .get_strings = get_strings,
1938 .phys_id = identify_port,
1939 .nway_reset = restart_autoneg,
1940 .get_sset_count = get_sset_count,
1941 .get_ethtool_stats = get_stats,
1942 .get_regs_len = get_regs_len,
1943 .get_regs = get_regs,
1944 .get_wol = get_wol,
1945 .set_wol = set_wol,
1946 .set_tso = set_tso,
87b6cf51 1947 .set_flags = set_flags,
671b0060
DM		 1948 .get_rxnfc = get_rxnfc,
1949 .get_rxfh_indir = get_rss_table,
1950 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 1951 .flash_device = set_flash,
1952};
1953
1954/*
1955 * debugfs support
1956 */
1957
1958static int mem_open(struct inode *inode, struct file *file)
1959{
1960 file->private_data = inode->i_private;
1961 return 0;
1962}
1963
1964static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
1965 loff_t *ppos)
1966{
1967 loff_t pos = *ppos;
1968 loff_t avail = file->f_path.dentry->d_inode->i_size;
1969 unsigned int mem = (uintptr_t)file->private_data & 3;
1970 struct adapter *adap = file->private_data - mem;
1971
1972 if (pos < 0)
1973 return -EINVAL;
1974 if (pos >= avail)
1975 return 0;
1976 if (count > avail - pos)
1977 count = avail - pos;
1978
1979 while (count) {
1980 size_t len;
1981 int ret, ofst;
1982 __be32 data[16];
1983
1984 if (mem == MEM_MC)
1985 ret = t4_mc_read(adap, pos, data, NULL);
1986 else
1987 ret = t4_edc_read(adap, mem, pos, data, NULL);
1988 if (ret)
1989 return ret;
1990
1991 ofst = pos % sizeof(data);
1992 len = min(count, sizeof(data) - ofst);
1993 if (copy_to_user(buf, (u8 *)data + ofst, len))
1994 return -EFAULT;
1995
1996 buf += len;
1997 pos += len;
1998 count -= len;
1999 }
2000 count = pos - *ppos;
2001 *ppos = pos;
2002 return count;
2003}
2004
2005static const struct file_operations mem_debugfs_fops = {
2006 .owner = THIS_MODULE,
2007 .open = mem_open,
2008 .read = mem_read,
2009};
2010
2011static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2012 unsigned int idx, unsigned int size_mb)
2013{
2014 struct dentry *de;
2015
2016 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2017 (void *)adap + idx, &mem_debugfs_fops);
2018 if (de && de->d_inode)
2019 de->d_inode->i_size = size_mb << 20;
2020}
2021
2022static int __devinit setup_debugfs(struct adapter *adap)
2023{
2024 int i;
2025
2026 if (IS_ERR_OR_NULL(adap->debugfs_root))
2027 return -1;
2028
2029 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2030 if (i & EDRAM0_ENABLE)
2031 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2032 if (i & EDRAM1_ENABLE)
2033 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2034 if (i & EXT_MEM_ENABLE)
2035 add_debugfs_mem(adap, "mc", MEM_MC,
2036 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2037 if (adap->l2t)
2038 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2039 &t4_l2t_fops);
2040 return 0;
2041}
2042
2043/*
2044 * upper-layer driver support
2045 */
2046
2047/*
2048 * Allocate an active-open TID and set it to the supplied value.
2049 */
2050int cxgb4_alloc_atid(struct tid_info *t, void *data)
2051{
2052 int atid = -1;
2053
2054 spin_lock_bh(&t->atid_lock);
2055 if (t->afree) {
2056 union aopen_entry *p = t->afree;
2057
2058 atid = p - t->atid_tab;
2059 t->afree = p->next;
2060 p->data = data;
2061 t->atids_in_use++;
2062 }
2063 spin_unlock_bh(&t->atid_lock);
2064 return atid;
2065}
2066EXPORT_SYMBOL(cxgb4_alloc_atid);
2067
2068/*
2069 * Release an active-open TID.
2070 */
2071void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2072{
2073 union aopen_entry *p = &t->atid_tab[atid];
2074
2075 spin_lock_bh(&t->atid_lock);
2076 p->next = t->afree;
2077 t->afree = p;
2078 t->atids_in_use--;
2079 spin_unlock_bh(&t->atid_lock);
2080}
2081EXPORT_SYMBOL(cxgb4_free_atid);
2082
2083/*
2084 * Allocate a server TID and set it to the supplied value.
2085 */
2086int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2087{
2088 int stid;
2089
2090 spin_lock_bh(&t->stid_lock);
2091 if (family == PF_INET) {
2092 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2093 if (stid < t->nstids)
2094 __set_bit(stid, t->stid_bmap);
2095 else
2096 stid = -1;
2097 } else {
2098 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2099 if (stid < 0)
2100 stid = -1;
2101 }
2102 if (stid >= 0) {
2103 t->stid_tab[stid].data = data;
2104 stid += t->stid_base;
2105 t->stids_in_use++;
2106 }
2107 spin_unlock_bh(&t->stid_lock);
2108 return stid;
2109}
2110EXPORT_SYMBOL(cxgb4_alloc_stid);
2111
2112/*
2113 * Release a server TID.
2114 */
2115void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2116{
2117 stid -= t->stid_base;
2118 spin_lock_bh(&t->stid_lock);
2119 if (family == PF_INET)
2120 __clear_bit(stid, t->stid_bmap);
2121 else
2122 bitmap_release_region(t->stid_bmap, stid, 2);
2123 t->stid_tab[stid].data = NULL;
2124 t->stids_in_use--;
2125 spin_unlock_bh(&t->stid_lock);
2126}
2127EXPORT_SYMBOL(cxgb4_free_stid);
2128
2129/*
2130 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2131 */
2132static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2133 unsigned int tid)
2134{
2135 struct cpl_tid_release *req;
2136
2137 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2138 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2139 INIT_TP_WR(req, tid);
2140 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2141}
2142
2143/*
2144 * Queue a TID release request and if necessary schedule a work queue to
2145 * process it.
2146 */
2147void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2148 unsigned int tid)
2149{
2150 void **p = &t->tid_tab[tid];
2151 struct adapter *adap = container_of(t, struct adapter, tids);
2152
2153 spin_lock_bh(&adap->tid_release_lock);
2154 *p = adap->tid_release_head;
2155 /* Low 2 bits encode the Tx channel number */
2156 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2157 if (!adap->tid_release_task_busy) {
2158 adap->tid_release_task_busy = true;
2159 schedule_work(&adap->tid_release_task);
2160 }
2161 spin_unlock_bh(&adap->tid_release_lock);
2162}
2163EXPORT_SYMBOL(cxgb4_queue_tid_release);
2164
2165/*
2166 * Process the list of pending TID release requests.
2167 */
2168static void process_tid_release_list(struct work_struct *work)
2169{
2170 struct sk_buff *skb;
2171 struct adapter *adap;
2172
2173 adap = container_of(work, struct adapter, tid_release_task);
2174
2175 spin_lock_bh(&adap->tid_release_lock);
2176 while (adap->tid_release_head) {
2177 void **p = adap->tid_release_head;
2178 unsigned int chan = (uintptr_t)p & 3;
2179 p = (void *)p - chan;
2180
2181 adap->tid_release_head = *p;
2182 *p = NULL;
2183 spin_unlock_bh(&adap->tid_release_lock);
2184
2185 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2186 GFP_KERNEL)))
2187 schedule_timeout_uninterruptible(1);
2188
2189 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2190 t4_ofld_send(adap, skb);
2191 spin_lock_bh(&adap->tid_release_lock);
2192 }
2193 adap->tid_release_task_busy = false;
2194 spin_unlock_bh(&adap->tid_release_lock);
2195}
2196
2197/*
2198 * Release a TID and inform HW. If we are unable to allocate the release
2199 * message we defer to a work queue.
2200 */
2201void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2202{
2203 void *old;
2204 struct sk_buff *skb;
2205 struct adapter *adap = container_of(t, struct adapter, tids);
2206
2207 old = t->tid_tab[tid];
2208 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2209 if (likely(skb)) {
2210 t->tid_tab[tid] = NULL;
2211 mk_tid_release(skb, chan, tid);
2212 t4_ofld_send(adap, skb);
2213 } else
2214 cxgb4_queue_tid_release(t, chan, tid);
2215 if (old)
2216 atomic_dec(&t->tids_in_use);
2217}
2218EXPORT_SYMBOL(cxgb4_remove_tid);
2219
2220/*
2221 * Allocate and initialize the TID tables. Returns 0 on success.
2222 */
2223static int tid_init(struct tid_info *t)
2224{
2225 size_t size;
2226 unsigned int natids = t->natids;
2227
2228 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2229 t->nstids * sizeof(*t->stid_tab) +
2230 BITS_TO_LONGS(t->nstids) * sizeof(long);
2231 t->tid_tab = t4_alloc_mem(size);
2232 if (!t->tid_tab)
2233 return -ENOMEM;
2234
2235 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2236 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2237 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2238 spin_lock_init(&t->stid_lock);
2239 spin_lock_init(&t->atid_lock);
2240
2241 t->stids_in_use = 0;
2242 t->afree = NULL;
2243 t->atids_in_use = 0;
2244 atomic_set(&t->tids_in_use, 0);
2245
2246 /* Setup the free list for atid_tab and clear the stid bitmap. */
2247 if (natids) {
2248 while (--natids)
2249 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2250 t->afree = t->atid_tab;
2251 }
2252 bitmap_zero(t->stid_bmap, t->nstids);
2253 return 0;
2254}
2255
2256/**
2257 * cxgb4_create_server - create an IP server
2258 * @dev: the device
2259 * @stid: the server TID
2260 * @sip: local IP address to bind server to
2261 * @sport: the server's TCP port
2262 * @queue: queue to direct messages from this server to
2263 *
2264 * Create an IP server for the given port and address.
2265 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2266 */
2267int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2268 __be32 sip, __be16 sport, unsigned int queue)
2269{
2270 unsigned int chan;
2271 struct sk_buff *skb;
2272 struct adapter *adap;
2273 struct cpl_pass_open_req *req;
2274
2275 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2276 if (!skb)
2277 return -ENOMEM;
2278
2279 adap = netdev2adap(dev);
2280 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2281 INIT_TP_WR(req, 0);
2282 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2283 req->local_port = sport;
2284 req->peer_port = htons(0);
2285 req->local_ip = sip;
2286 req->peer_ip = htonl(0);
2287 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2288 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2289 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2290 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2291 return t4_mgmt_tx(adap, skb);
2292}
2293EXPORT_SYMBOL(cxgb4_create_server);
2294
2295/**
2296 * cxgb4_create_server6 - create an IPv6 server
2297 * @dev: the device
2298 * @stid: the server TID
2299 * @sip: local IPv6 address to bind server to
2300 * @sport: the server's TCP port
2301 * @queue: queue to direct messages from this server to
2302 *
2303 * Create an IPv6 server for the given port and address.
2304 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2305 */
2306int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2307 const struct in6_addr *sip, __be16 sport,
2308 unsigned int queue)
2309{
2310 unsigned int chan;
2311 struct sk_buff *skb;
2312 struct adapter *adap;
2313 struct cpl_pass_open_req6 *req;
2314
2315 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2316 if (!skb)
2317 return -ENOMEM;
2318
2319 adap = netdev2adap(dev);
2320 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2321 INIT_TP_WR(req, 0);
2322 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2323 req->local_port = sport;
2324 req->peer_port = htons(0);
2325 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2326 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2327 req->peer_ip_hi = cpu_to_be64(0);
2328 req->peer_ip_lo = cpu_to_be64(0);
2329 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2330 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2331 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2332 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2333 return t4_mgmt_tx(adap, skb);
2334}
2335EXPORT_SYMBOL(cxgb4_create_server6);
2336
2337/**
2338 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2339 * @mtus: the HW MTU table
2340 * @mtu: the target MTU
2341 * @idx: index of selected entry in the MTU table
2342 *
2343 * Returns the index and the value in the HW MTU table that is closest to
2344 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2345 * table, in which case that smallest available value is selected.
2346 */
2347unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2348 unsigned int *idx)
2349{
2350 unsigned int i = 0;
2351
2352 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2353 ++i;
2354 if (idx)
2355 *idx = i;
2356 return mtus[i];
2357}
2358EXPORT_SYMBOL(cxgb4_best_mtu);
2359
2360/**
2361 * cxgb4_port_chan - get the HW channel of a port
2362 * @dev: the net device for the port
2363 *
2364 * Return the HW Tx channel of the given port.
2365 */
2366unsigned int cxgb4_port_chan(const struct net_device *dev)
2367{
2368 return netdev2pinfo(dev)->tx_chan;
2369}
2370EXPORT_SYMBOL(cxgb4_port_chan);
2371
2372/**
2373 * cxgb4_port_viid - get the VI id of a port
2374 * @dev: the net device for the port
2375 *
2376 * Return the VI id of the given port.
2377 */
2378unsigned int cxgb4_port_viid(const struct net_device *dev)
2379{
2380 return netdev2pinfo(dev)->viid;
2381}
2382EXPORT_SYMBOL(cxgb4_port_viid);
2383
2384/**
2385 * cxgb4_port_idx - get the index of a port
2386 * @dev: the net device for the port
2387 *
2388 * Return the index of the given port.
2389 */
2390unsigned int cxgb4_port_idx(const struct net_device *dev)
2391{
2392 return netdev2pinfo(dev)->port_id;
2393}
2394EXPORT_SYMBOL(cxgb4_port_idx);
2395
2396/**
2397 * cxgb4_netdev_by_hwid - return the net device of a HW port
2398 * @pdev: identifies the adapter
2399 * @id: the HW port id
2400 *
2401 * Return the net device associated with the interface with the given HW
2402 * id.
2403 */
2404struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2405{
2406 const struct adapter *adap = pci_get_drvdata(pdev);
2407
2408 if (!adap || id >= NCHAN)
2409 return NULL;
2410 id = adap->chan_map[id];
2411 return id < MAX_NPORTS ? adap->port[id] : NULL;
2412}
2413EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2414
2415void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2416 struct tp_tcp_stats *v6)
2417{
2418 struct adapter *adap = pci_get_drvdata(pdev);
2419
2420 spin_lock(&adap->stats_lock);
2421 t4_tp_get_tcp_stats(adap, v4, v6);
2422 spin_unlock(&adap->stats_lock);
2423}
2424EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2425
2426void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2427 const unsigned int *pgsz_order)
2428{
2429 struct adapter *adap = netdev2adap(dev);
2430
2431 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2432 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2433 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2434 HPZ3(pgsz_order[3]));
2435}
2436EXPORT_SYMBOL(cxgb4_iscsi_init);
2437
2438static struct pci_driver cxgb4_driver;
2439
2440static void check_neigh_update(struct neighbour *neigh)
2441{
2442 const struct device *parent;
2443 const struct net_device *netdev = neigh->dev;
2444
2445 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2446 netdev = vlan_dev_real_dev(netdev);
2447 parent = netdev->dev.parent;
2448 if (parent && parent->driver == &cxgb4_driver.driver)
2449 t4_l2t_update(dev_get_drvdata(parent), neigh);
2450}
2451
2452static int netevent_cb(struct notifier_block *nb, unsigned long event,
2453 void *data)
2454{
2455 switch (event) {
2456 case NETEVENT_NEIGH_UPDATE:
2457 check_neigh_update(data);
2458 break;
2459 case NETEVENT_PMTU_UPDATE:
2460 case NETEVENT_REDIRECT:
2461 default:
2462 break;
2463 }
2464 return 0;
2465}
2466
2467static bool netevent_registered;
2468static struct notifier_block cxgb4_netevent_nb = {
2469 .notifier_call = netevent_cb
2470};
2471
2472static void uld_attach(struct adapter *adap, unsigned int uld)
2473{
2474 void *handle;
2475 struct cxgb4_lld_info lli;
2476
2477 lli.pdev = adap->pdev;
2478 lli.l2t = adap->l2t;
2479 lli.tids = &adap->tids;
2480 lli.ports = adap->port;
2481 lli.vr = &adap->vres;
2482 lli.mtus = adap->params.mtus;
2483 if (uld == CXGB4_ULD_RDMA) {
2484 lli.rxq_ids = adap->sge.rdma_rxq;
2485 lli.nrxq = adap->sge.rdmaqs;
2486 } else if (uld == CXGB4_ULD_ISCSI) {
2487 lli.rxq_ids = adap->sge.ofld_rxq;
2488 lli.nrxq = adap->sge.ofldqsets;
2489 }
2490 lli.ntxq = adap->sge.ofldqsets;
2491 lli.nchan = adap->params.nports;
2492 lli.nports = adap->params.nports;
2493 lli.wr_cred = adap->params.ofldq_wr_cred;
2494 lli.adapter_type = adap->params.rev;
2495 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2496 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2497 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF));
2498 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2499 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF));
2500 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2501 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2502 lli.fw_vers = adap->params.fw_vers;
2503
2504 handle = ulds[uld].add(&lli);
2505 if (IS_ERR(handle)) {
2506 dev_warn(adap->pdev_dev,
2507 "could not attach to the %s driver, error %ld\n",
2508 uld_str[uld], PTR_ERR(handle));
2509 return;
2510 }
2511
2512 adap->uld_handle[uld] = handle;
2513
2514 if (!netevent_registered) {
2515 register_netevent_notifier(&cxgb4_netevent_nb);
2516 netevent_registered = true;
2517 }
e29f5dbc
DM		 2518
2519 if (adap->flags & FULL_INIT_DONE)
2520 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 2521}
2522
2523static void attach_ulds(struct adapter *adap)
2524{
2525 unsigned int i;
2526
2527 mutex_lock(&uld_mutex);
2528 list_add_tail(&adap->list_node, &adapter_list);
2529 for (i = 0; i < CXGB4_ULD_MAX; i++)
2530 if (ulds[i].add)
2531 uld_attach(adap, i);
2532 mutex_unlock(&uld_mutex);
2533}
2534
2535static void detach_ulds(struct adapter *adap)
2536{
2537 unsigned int i;
2538
2539 mutex_lock(&uld_mutex);
2540 list_del(&adap->list_node);
2541 for (i = 0; i < CXGB4_ULD_MAX; i++)
2542 if (adap->uld_handle[i]) {
2543 ulds[i].state_change(adap->uld_handle[i],
2544 CXGB4_STATE_DETACH);
2545 adap->uld_handle[i] = NULL;
2546 }
2547 if (netevent_registered && list_empty(&adapter_list)) {
2548 unregister_netevent_notifier(&cxgb4_netevent_nb);
2549 netevent_registered = false;
2550 }
2551 mutex_unlock(&uld_mutex);
2552}
2553
2554static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2555{
2556 unsigned int i;
2557
2558 mutex_lock(&uld_mutex);
2559 for (i = 0; i < CXGB4_ULD_MAX; i++)
2560 if (adap->uld_handle[i])
2561 ulds[i].state_change(adap->uld_handle[i], new_state);
2562 mutex_unlock(&uld_mutex);
2563}
2564
2565/**
2566 * cxgb4_register_uld - register an upper-layer driver
2567 * @type: the ULD type
2568 * @p: the ULD methods
2569 *
2570 * Registers an upper-layer driver with this driver and notifies the ULD
2571 * about any presently available devices that support its type. Returns
2572 * %-EBUSY if a ULD of the same type is already registered.
2573 */
2574int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2575{
2576 int ret = 0;
2577 struct adapter *adap;
2578
2579 if (type >= CXGB4_ULD_MAX)
2580 return -EINVAL;
2581 mutex_lock(&uld_mutex);
2582 if (ulds[type].add) {
2583 ret = -EBUSY;
2584 goto out;
2585 }
2586 ulds[type] = *p;
2587 list_for_each_entry(adap, &adapter_list, list_node)
2588 uld_attach(adap, type);
2589out: mutex_unlock(&uld_mutex);
2590 return ret;
2591}
2592EXPORT_SYMBOL(cxgb4_register_uld);
2593
2594/**
2595 * cxgb4_unregister_uld - unregister an upper-layer driver
2596 * @type: the ULD type
2597 *
2598 * Unregisters an existing upper-layer driver.
2599 */
2600int cxgb4_unregister_uld(enum cxgb4_uld type)
2601{
2602 struct adapter *adap;
2603
2604 if (type >= CXGB4_ULD_MAX)
2605 return -EINVAL;
2606 mutex_lock(&uld_mutex);
2607 list_for_each_entry(adap, &adapter_list, list_node)
2608 adap->uld_handle[type] = NULL;
2609 ulds[type].add = NULL;
2610 mutex_unlock(&uld_mutex);
2611 return 0;
2612}
2613EXPORT_SYMBOL(cxgb4_unregister_uld);
2614
2615/**
2616 * cxgb_up - enable the adapter
2617 * @adap: adapter being enabled
2618 *
2619 * Called when the first port is enabled, this function performs the
2620 * actions necessary to make an adapter operational, such as completing
2621 * the initialization of HW modules, and enabling interrupts.
2622 *
2623 * Must be called with the rtnl lock held.
2624 */
2625static int cxgb_up(struct adapter *adap)
2626{
aaefae9b 2627 int err;
b8ff05a9 2628
aaefae9b
DM		 2629 err = setup_sge_queues(adap);
2630 if (err)
2631 goto out;
2632 err = setup_rss(adap);
2633 if (err)
2634 goto freeq;
b8ff05a9
DM		 2635
2636 if (adap->flags & USING_MSIX) {
aaefae9b 2637 name_msix_vecs(adap);
b8ff05a9
DM		 2638 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2639 adap->msix_info[0].desc, adap);
2640 if (err)
2641 goto irq_err;
2642
2643 err = request_msix_queue_irqs(adap);
2644 if (err) {
2645 free_irq(adap->msix_info[0].vec, adap);
2646 goto irq_err;
2647 }
2648 } else {
2649 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2650 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2651 adap->name, adap);
2652 if (err)
2653 goto irq_err;
2654 }
2655 enable_rx(adap);
2656 t4_sge_start(adap);
2657 t4_intr_enable(adap);
aaefae9b 2658 adap->flags |= FULL_INIT_DONE;
b8ff05a9
DM		 2659 notify_ulds(adap, CXGB4_STATE_UP);
2660 out:
2661 return err;
2662 irq_err:
2663 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 2664 freeq:
2665 t4_free_sge_resources(adap);
b8ff05a9
DM		 2666 goto out;
2667}
2668
2669static void cxgb_down(struct adapter *adapter)
2670{
2671 t4_intr_disable(adapter);
2672 cancel_work_sync(&adapter->tid_release_task);
2673 adapter->tid_release_task_busy = false;
204dc3c0 2674 adapter->tid_release_head = NULL;
b8ff05a9
DM		 2675
2676 if (adapter->flags & USING_MSIX) {
2677 free_msix_queue_irqs(adapter);
2678 free_irq(adapter->msix_info[0].vec, adapter);
2679 } else
2680 free_irq(adapter->pdev->irq, adapter);
2681 quiesce_rx(adapter);
aaefae9b
DM		 2682 t4_sge_stop(adapter);
2683 t4_free_sge_resources(adapter);
2684 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 2685}
2686
2687/*
2688 * net_device operations
2689 */
2690static int cxgb_open(struct net_device *dev)
2691{
2692 int err;
2693 struct port_info *pi = netdev_priv(dev);
2694 struct adapter *adapter = pi->adapter;
2695
aaefae9b
DM		 2696 if (!(adapter->flags & FULL_INIT_DONE)) {
2697 err = cxgb_up(adapter);
2698 if (err < 0)
2699 return err;
2700 }
b8ff05a9
DM		 2701
2702 dev->real_num_tx_queues = pi->nqsets;
f68707b8
DM		 2703 err = link_start(dev);
2704 if (!err)
2705 netif_tx_start_all_queues(dev);
2706 return err;
b8ff05a9
DM		 2707}
2708
2709static int cxgb_close(struct net_device *dev)
2710{
b8ff05a9
DM		 2711 struct port_info *pi = netdev_priv(dev);
2712 struct adapter *adapter = pi->adapter;
2713
2714 netif_tx_stop_all_queues(dev);
2715 netif_carrier_off(dev);
aaefae9b 2716 return t4_enable_vi(adapter, 0, pi->viid, false, false);
b8ff05a9
DM		 2717}
2718
f5152c90
DM		 2719static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2720 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 2721{
2722 struct port_stats stats;
2723 struct port_info *p = netdev_priv(dev);
2724 struct adapter *adapter = p->adapter;
b8ff05a9
DM		 2725
2726 spin_lock(&adapter->stats_lock);
2727 t4_get_port_stats(adapter, p->tx_chan, &stats);
2728 spin_unlock(&adapter->stats_lock);
2729
2730 ns->tx_bytes = stats.tx_octets;
2731 ns->tx_packets = stats.tx_frames;
2732 ns->rx_bytes = stats.rx_octets;
2733 ns->rx_packets = stats.rx_frames;
2734 ns->multicast = stats.rx_mcast_frames;
2735
2736 /* detailed rx_errors */
2737 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2738 stats.rx_runt;
2739 ns->rx_over_errors = 0;
2740 ns->rx_crc_errors = stats.rx_fcs_err;
2741 ns->rx_frame_errors = stats.rx_symbol_err;
2742 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2743 stats.rx_ovflow2 + stats.rx_ovflow3 +
2744 stats.rx_trunc0 + stats.rx_trunc1 +
2745 stats.rx_trunc2 + stats.rx_trunc3;
2746 ns->rx_missed_errors = 0;
2747
2748 /* detailed tx_errors */
2749 ns->tx_aborted_errors = 0;
2750 ns->tx_carrier_errors = 0;
2751 ns->tx_fifo_errors = 0;
2752 ns->tx_heartbeat_errors = 0;
2753 ns->tx_window_errors = 0;
2754
2755 ns->tx_errors = stats.tx_error_frames;
2756 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2757 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2758 return ns;
2759}
2760
2761static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2762{
2763 int ret = 0, prtad, devad;
2764 struct port_info *pi = netdev_priv(dev);
2765 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2766
2767 switch (cmd) {
2768 case SIOCGMIIPHY:
2769 if (pi->mdio_addr < 0)
2770 return -EOPNOTSUPP;
2771 data->phy_id = pi->mdio_addr;
2772 break;
2773 case SIOCGMIIREG:
2774 case SIOCSMIIREG:
2775 if (mdio_phy_id_is_c45(data->phy_id)) {
2776 prtad = mdio_phy_id_prtad(data->phy_id);
2777 devad = mdio_phy_id_devad(data->phy_id);
2778 } else if (data->phy_id < 32) {
2779 prtad = data->phy_id;
2780 devad = 0;
2781 data->reg_num &= 0x1f;
2782 } else
2783 return -EINVAL;
2784
2785 if (cmd == SIOCGMIIREG)
2786 ret = t4_mdio_rd(pi->adapter, 0, prtad, devad,
2787 data->reg_num, &data->val_out);
2788 else
2789 ret = t4_mdio_wr(pi->adapter, 0, prtad, devad,
2790 data->reg_num, data->val_in);
2791 break;
2792 default:
2793 return -EOPNOTSUPP;
2794 }
2795 return ret;
2796}
2797
2798static void cxgb_set_rxmode(struct net_device *dev)
2799{
2800 /* unfortunately we can't return errors to the stack */
2801 set_rxmode(dev, -1, false);
2802}
2803
2804static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2805{
2806 int ret;
2807 struct port_info *pi = netdev_priv(dev);
2808
2809 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2810 return -EINVAL;
f8f5aafa 2811 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, new_mtu, -1, -1, -1, -1,
b8ff05a9
DM		 2812 true);
2813 if (!ret)
2814 dev->mtu = new_mtu;
2815 return ret;
2816}
2817
2818static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2819{
2820 int ret;
2821 struct sockaddr *addr = p;
2822 struct port_info *pi = netdev_priv(dev);
2823
2824 if (!is_valid_ether_addr(addr->sa_data))
2825 return -EINVAL;
2826
2827 ret = t4_change_mac(pi->adapter, 0, pi->viid, pi->xact_addr_filt,
2828 addr->sa_data, true, true);
2829 if (ret < 0)
2830 return ret;
2831
2832 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2833 pi->xact_addr_filt = ret;
2834 return 0;
2835}
2836
2837static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2838{
2839 struct port_info *pi = netdev_priv(dev);
2840
2841 pi->vlan_grp = grp;
f8f5aafa
DM		 2842 t4_set_rxmode(pi->adapter, 0, pi->viid, -1, -1, -1, -1, grp != NULL,
2843 true);
b8ff05a9
DM		 2844}
2845
2846#ifdef CONFIG_NET_POLL_CONTROLLER
2847static void cxgb_netpoll(struct net_device *dev)
2848{
2849 struct port_info *pi = netdev_priv(dev);
2850 struct adapter *adap = pi->adapter;
2851
2852 if (adap->flags & USING_MSIX) {
2853 int i;
2854 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2855
2856 for (i = pi->nqsets; i; i--, rx++)
2857 t4_sge_intr_msix(0, &rx->rspq);
2858 } else
2859 t4_intr_handler(adap)(0, adap);
2860}
2861#endif
2862
2863static const struct net_device_ops cxgb4_netdev_ops = {
2864 .ndo_open = cxgb_open,
2865 .ndo_stop = cxgb_close,
2866 .ndo_start_xmit = t4_eth_xmit,
9be793bf 2867 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 2868 .ndo_set_rx_mode = cxgb_set_rxmode,
2869 .ndo_set_mac_address = cxgb_set_mac_addr,
2870 .ndo_validate_addr = eth_validate_addr,
2871 .ndo_do_ioctl = cxgb_ioctl,
2872 .ndo_change_mtu = cxgb_change_mtu,
2873 .ndo_vlan_rx_register = vlan_rx_register,
2874#ifdef CONFIG_NET_POLL_CONTROLLER
2875 .ndo_poll_controller = cxgb_netpoll,
2876#endif
2877};
2878
2879void t4_fatal_err(struct adapter *adap)
2880{
2881 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2882 t4_intr_disable(adap);
2883 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2884}
2885
2886static void setup_memwin(struct adapter *adap)
2887{
2888 u32 bar0;
2889
2890 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2891 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2892 (bar0 + MEMWIN0_BASE) | BIR(0) |
2893 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2894 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2895 (bar0 + MEMWIN1_BASE) | BIR(0) |
2896 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2897 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2898 (bar0 + MEMWIN2_BASE) | BIR(0) |
2899 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2900}
2901
02b5fb8e
DM		 2902static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2903{
2904 u32 v;
2905 int ret;
2906
2907 /* get device capabilities */
2908 memset(c, 0, sizeof(*c));
2909 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2910 FW_CMD_REQUEST | FW_CMD_READ);
2911 c->retval_len16 = htonl(FW_LEN16(*c));
2912 ret = t4_wr_mbox(adap, 0, c, sizeof(*c), c);
2913 if (ret < 0)
2914 return ret;
2915
2916 /* select capabilities we'll be using */
2917 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2918 if (!vf_acls)
2919 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2920 else
2921 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2922 } else if (vf_acls) {
2923 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2924 return ret;
2925 }
2926 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2927 FW_CMD_REQUEST | FW_CMD_WRITE);
2928 ret = t4_wr_mbox(adap, 0, c, sizeof(*c), NULL);
2929 if (ret < 0)
2930 return ret;
2931
2932 ret = t4_config_glbl_rss(adap, 0,
2933 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2934 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2935 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2936 if (ret < 0)
2937 return ret;
2938
20c0da65
DM		 2939 ret = t4_cfg_pfvf(adap, 0, 0, 0, MAX_EGRQ, 64, MAX_INGQ, 0, 0, 4,
2940 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 2941 if (ret < 0)
2942 return ret;
2943
2944 t4_sge_init(adap);
2945
2946 /* get basic stuff going */
2947 ret = t4_early_init(adap, 0);
2948 if (ret < 0)
2949 return ret;
2950
2951 /* tweak some settings */
2952 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2953 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2954 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2955 v = t4_read_reg(adap, TP_PIO_DATA);
2956 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2957 setup_memwin(adap);
2958 return 0;
2959}
2960
b8ff05a9
DM		 2961/*
2962 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2963 */
2964#define MAX_ATIDS 8192U
2965
2966/*
2967 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2968 */
2969static int adap_init0(struct adapter *adap)
2970{
2971 int ret;
2972 u32 v, port_vec;
2973 enum dev_state state;
2974 u32 params[7], val[7];
2975 struct fw_caps_config_cmd c;
2976
2977 ret = t4_check_fw_version(adap);
2978 if (ret == -EINVAL || ret > 0) {
2979 if (upgrade_fw(adap) >= 0) /* recache FW version */
2980 ret = t4_check_fw_version(adap);
2981 }
2982 if (ret < 0)
2983 return ret;
2984
2985 /* contact FW, request master */
2986 ret = t4_fw_hello(adap, 0, 0, MASTER_MUST, &state);
2987 if (ret < 0) {
2988 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2989 ret);
2990 return ret;
2991 }
2992
2993 /* reset device */
2994 ret = t4_fw_reset(adap, 0, PIORSTMODE | PIORST);
2995 if (ret < 0)
2996 goto bye;
2997
b8ff05a9
DM		 2998 for (v = 0; v < SGE_NTIMERS - 1; v++)
2999 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3000 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3001 adap->sge.counter_val[0] = 1;
3002 for (v = 1; v < SGE_NCOUNTERS; v++)
3003 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3004 THRESHOLD_3_MASK);
b8ff05a9
DM		 3005#define FW_PARAM_DEV(param) \
3006 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3007 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3008
a0881cab
DM		 3009 params[0] = FW_PARAM_DEV(CCLK);
3010 ret = t4_query_params(adap, 0, 0, 0, 1, params, val);
3011 if (ret < 0)
3012 goto bye;
3013 adap->params.vpd.cclk = val[0];
3014
3015 ret = adap_init1(adap, &c);
3016 if (ret < 0)
3017 goto bye;
3018
b8ff05a9
DM		 3019#define FW_PARAM_PFVF(param) \
3020 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3021 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
3022
3023 params[0] = FW_PARAM_DEV(PORTVEC);
3024 params[1] = FW_PARAM_PFVF(L2T_START);
3025 params[2] = FW_PARAM_PFVF(L2T_END);
3026 params[3] = FW_PARAM_PFVF(FILTER_START);
3027 params[4] = FW_PARAM_PFVF(FILTER_END);
3028 ret = t4_query_params(adap, 0, 0, 0, 5, params, val);
3029 if (ret < 0)
3030 goto bye;
3031 port_vec = val[0];
3032 adap->tids.ftid_base = val[3];
3033 adap->tids.nftids = val[4] - val[3] + 1;
3034
3035 if (c.ofldcaps) {
3036 /* query offload-related parameters */
3037 params[0] = FW_PARAM_DEV(NTID);
3038 params[1] = FW_PARAM_PFVF(SERVER_START);
3039 params[2] = FW_PARAM_PFVF(SERVER_END);
3040 params[3] = FW_PARAM_PFVF(TDDP_START);
3041 params[4] = FW_PARAM_PFVF(TDDP_END);
3042 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3043 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
3044 if (ret < 0)
3045 goto bye;
3046 adap->tids.ntids = val[0];
3047 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3048 adap->tids.stid_base = val[1];
3049 adap->tids.nstids = val[2] - val[1] + 1;
3050 adap->vres.ddp.start = val[3];
3051 adap->vres.ddp.size = val[4] - val[3] + 1;
3052 adap->params.ofldq_wr_cred = val[5];
3053 adap->params.offload = 1;
3054 }
3055 if (c.rdmacaps) {
3056 params[0] = FW_PARAM_PFVF(STAG_START);
3057 params[1] = FW_PARAM_PFVF(STAG_END);
3058 params[2] = FW_PARAM_PFVF(RQ_START);
3059 params[3] = FW_PARAM_PFVF(RQ_END);
3060 params[4] = FW_PARAM_PFVF(PBL_START);
3061 params[5] = FW_PARAM_PFVF(PBL_END);
3062 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
3063 if (ret < 0)
3064 goto bye;
3065 adap->vres.stag.start = val[0];
3066 adap->vres.stag.size = val[1] - val[0] + 1;
3067 adap->vres.rq.start = val[2];
3068 adap->vres.rq.size = val[3] - val[2] + 1;
3069 adap->vres.pbl.start = val[4];
3070 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 3071
3072 params[0] = FW_PARAM_PFVF(SQRQ_START);
3073 params[1] = FW_PARAM_PFVF(SQRQ_END);
3074 params[2] = FW_PARAM_PFVF(CQ_START);
3075 params[3] = FW_PARAM_PFVF(CQ_END);
3076 ret = t4_query_params(adap, 0, 0, 0, 4, params, val);
3077 if (ret < 0)
3078 goto bye;
3079 adap->vres.qp.start = val[0];
3080 adap->vres.qp.size = val[1] - val[0] + 1;
3081 adap->vres.cq.start = val[2];
3082 adap->vres.cq.size = val[3] - val[2] + 1;
b8ff05a9
DM		 3083 }
3084 if (c.iscsicaps) {
3085 params[0] = FW_PARAM_PFVF(ISCSI_START);
3086 params[1] = FW_PARAM_PFVF(ISCSI_END);
3087 ret = t4_query_params(adap, 0, 0, 0, 2, params, val);
3088 if (ret < 0)
3089 goto bye;
3090 adap->vres.iscsi.start = val[0];
3091 adap->vres.iscsi.size = val[1] - val[0] + 1;
3092 }
3093#undef FW_PARAM_PFVF
3094#undef FW_PARAM_DEV
3095
3096 adap->params.nports = hweight32(port_vec);
3097 adap->params.portvec = port_vec;
3098 adap->flags |= FW_OK;
3099
3100 /* These are finalized by FW initialization, load their values now */
3101 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3102 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3103 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3104 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3105 adap->params.b_wnd);
7ee9ff94
CL		 3106
3107#ifdef CONFIG_PCI_IOV
3108 /*
3109 * Provision resource limits for Virtual Functions. We currently
3110 * grant them all the same static resource limits except for the Port
3111 * Access Rights Mask which we're assigning based on the PF. All of
3112 * the static provisioning stuff for both the PF and VF really needs
3113 * to be managed in a persistent manner for each device which the
3114 * firmware controls.
3115 */
3116 {
3117 int pf, vf;
3118
3119 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3120 if (num_vf[pf] <= 0)
3121 continue;
3122
3123 /* VF numbering starts at 1! */
3124 for (vf = 1; vf <= num_vf[pf]; vf++) {
3125 ret = t4_cfg_pfvf(adap, 0, pf, vf,
3126 VFRES_NEQ, VFRES_NETHCTRL,
3127 VFRES_NIQFLINT, VFRES_NIQ,
3128 VFRES_TC, VFRES_NVI,
3129 FW_PFVF_CMD_CMASK_MASK,
3130 pfvfres_pmask(adap, pf, vf),
3131 VFRES_NEXACTF,
3132 VFRES_R_CAPS, VFRES_WX_CAPS);
3133 if (ret < 0)
3134 dev_warn(adap->pdev_dev, "failed to "
3135 "provision pf/vf=%d/%d; "
3136 "err=%d\n", pf, vf, ret);
3137 }
3138 }
3139 }
3140#endif
3141
b8ff05a9
DM		 3142 return 0;
3143
3144 /*
3145 * If a command timed out or failed with EIO FW does not operate within
3146 * its spec or something catastrophic happened to HW/FW, stop issuing
3147 * commands.
3148 */
3149bye: if (ret != -ETIMEDOUT && ret != -EIO)
3150 t4_fw_bye(adap, 0);
3151 return ret;
3152}
3153
204dc3c0
DM		 3154/* EEH callbacks */
3155
3156static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3157 pci_channel_state_t state)
3158{
3159 int i;
3160 struct adapter *adap = pci_get_drvdata(pdev);
3161
3162 if (!adap)
3163 goto out;
3164
3165 rtnl_lock();
3166 adap->flags &= ~FW_OK;
3167 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3168 for_each_port(adap, i) {
3169 struct net_device *dev = adap->port[i];
3170
3171 netif_device_detach(dev);
3172 netif_carrier_off(dev);
3173 }
3174 if (adap->flags & FULL_INIT_DONE)
3175 cxgb_down(adap);
3176 rtnl_unlock();
3177 pci_disable_device(pdev);
3178out: return state == pci_channel_io_perm_failure ?
3179 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3180}
3181
3182static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3183{
3184 int i, ret;
3185 struct fw_caps_config_cmd c;
3186 struct adapter *adap = pci_get_drvdata(pdev);
3187
3188 if (!adap) {
3189 pci_restore_state(pdev);
3190 pci_save_state(pdev);
3191 return PCI_ERS_RESULT_RECOVERED;
3192 }
3193
3194 if (pci_enable_device(pdev)) {
3195 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3196 return PCI_ERS_RESULT_DISCONNECT;
3197 }
3198
3199 pci_set_master(pdev);
3200 pci_restore_state(pdev);
3201 pci_save_state(pdev);
3202 pci_cleanup_aer_uncorrect_error_status(pdev);
3203
3204 if (t4_wait_dev_ready(adap) < 0)
3205 return PCI_ERS_RESULT_DISCONNECT;
3206 if (t4_fw_hello(adap, 0, 0, MASTER_MUST, NULL))
3207 return PCI_ERS_RESULT_DISCONNECT;
3208 adap->flags |= FW_OK;
3209 if (adap_init1(adap, &c))
3210 return PCI_ERS_RESULT_DISCONNECT;
3211
3212 for_each_port(adap, i) {
3213 struct port_info *p = adap2pinfo(adap, i);
3214
3215 ret = t4_alloc_vi(adap, 0, p->tx_chan, 0, 0, 1, NULL, NULL);
3216 if (ret < 0)
3217 return PCI_ERS_RESULT_DISCONNECT;
3218 p->viid = ret;
3219 p->xact_addr_filt = -1;
3220 }
3221
3222 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3223 adap->params.b_wnd);
3224 if (cxgb_up(adap))
3225 return PCI_ERS_RESULT_DISCONNECT;
3226 return PCI_ERS_RESULT_RECOVERED;
3227}
3228
3229static void eeh_resume(struct pci_dev *pdev)
3230{
3231 int i;
3232 struct adapter *adap = pci_get_drvdata(pdev);
3233
3234 if (!adap)
3235 return;
3236
3237 rtnl_lock();
3238 for_each_port(adap, i) {
3239 struct net_device *dev = adap->port[i];
3240
3241 if (netif_running(dev)) {
3242 link_start(dev);
3243 cxgb_set_rxmode(dev);
3244 }
3245 netif_device_attach(dev);
3246 }
3247 rtnl_unlock();
3248}
3249
3250static struct pci_error_handlers cxgb4_eeh = {
3251 .error_detected = eeh_err_detected,
3252 .slot_reset = eeh_slot_reset,
3253 .resume = eeh_resume,
3254};
3255
b8ff05a9
DM		 3256static inline bool is_10g_port(const struct link_config *lc)
3257{
3258 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3259}
3260
3261static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3262 unsigned int size, unsigned int iqe_size)
3263{
3264 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3265 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3266 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3267 q->iqe_len = iqe_size;
3268 q->size = size;
3269}
3270
3271/*
3272 * Perform default configuration of DMA queues depending on the number and type
3273 * of ports we found and the number of available CPUs. Most settings can be
3274 * modified by the admin prior to actual use.
3275 */
3276static void __devinit cfg_queues(struct adapter *adap)
3277{
3278 struct sge *s = &adap->sge;
3279 int i, q10g = 0, n10g = 0, qidx = 0;
3280
3281 for_each_port(adap, i)
3282 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3283
3284 /*
3285 * We default to 1 queue per non-10G port and up to # of cores queues
3286 * per 10G port.
3287 */
3288 if (n10g)
3289 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3290 if (q10g > num_online_cpus())
3291 q10g = num_online_cpus();
3292
3293 for_each_port(adap, i) {
3294 struct port_info *pi = adap2pinfo(adap, i);
3295
3296 pi->first_qset = qidx;
3297 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3298 qidx += pi->nqsets;
3299 }
3300
3301 s->ethqsets = qidx;
3302 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3303
3304 if (is_offload(adap)) {
3305 /*
3306 * For offload we use 1 queue/channel if all ports are up to 1G,
3307 * otherwise we divide all available queues amongst the channels
3308 * capped by the number of available cores.
3309 */
3310 if (n10g) {
3311 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3312 num_online_cpus());
3313 s->ofldqsets = roundup(i, adap->params.nports);
3314 } else
3315 s->ofldqsets = adap->params.nports;
3316 /* For RDMA one Rx queue per channel suffices */
3317 s->rdmaqs = adap->params.nports;
3318 }
3319
3320 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3321 struct sge_eth_rxq *r = &s->ethrxq[i];
3322
3323 init_rspq(&r->rspq, 0, 0, 1024, 64);
3324 r->fl.size = 72;
3325 }
3326
3327 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3328 s->ethtxq[i].q.size = 1024;
3329
3330 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3331 s->ctrlq[i].q.size = 512;
3332
3333 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3334 s->ofldtxq[i].q.size = 1024;
3335
3336 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3337 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3338
3339 init_rspq(&r->rspq, 0, 0, 1024, 64);
3340 r->rspq.uld = CXGB4_ULD_ISCSI;
3341 r->fl.size = 72;
3342 }
3343
3344 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3345 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3346
3347 init_rspq(&r->rspq, 0, 0, 511, 64);
3348 r->rspq.uld = CXGB4_ULD_RDMA;
3349 r->fl.size = 72;
3350 }
3351
3352 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3353 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3354}
3355
3356/*
3357 * Reduce the number of Ethernet queues across all ports to at most n.
3358 * n provides at least one queue per port.
3359 */
3360static void __devinit reduce_ethqs(struct adapter *adap, int n)
3361{
3362 int i;
3363 struct port_info *pi;
3364
3365 while (n < adap->sge.ethqsets)
3366 for_each_port(adap, i) {
3367 pi = adap2pinfo(adap, i);
3368 if (pi->nqsets > 1) {
3369 pi->nqsets--;
3370 adap->sge.ethqsets--;
3371 if (adap->sge.ethqsets <= n)
3372 break;
3373 }
3374 }
3375
3376 n = 0;
3377 for_each_port(adap, i) {
3378 pi = adap2pinfo(adap, i);
3379 pi->first_qset = n;
3380 n += pi->nqsets;
3381 }
3382}
3383
3384/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3385#define EXTRA_VECS 2
3386
3387static int __devinit enable_msix(struct adapter *adap)
3388{
3389 int ofld_need = 0;
3390 int i, err, want, need;
3391 struct sge *s = &adap->sge;
3392 unsigned int nchan = adap->params.nports;
3393 struct msix_entry entries[MAX_INGQ + 1];
3394
3395 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3396 entries[i].entry = i;
3397
3398 want = s->max_ethqsets + EXTRA_VECS;
3399 if (is_offload(adap)) {
3400 want += s->rdmaqs + s->ofldqsets;
3401 /* need nchan for each possible ULD */
3402 ofld_need = 2 * nchan;
3403 }
3404 need = adap->params.nports + EXTRA_VECS + ofld_need;
3405
3406 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3407 want = err;
3408
3409 if (!err) {
3410 /*
3411 * Distribute available vectors to the various queue groups.
3412 * Every group gets its minimum requirement and NIC gets top
3413 * priority for leftovers.
3414 */
3415 i = want - EXTRA_VECS - ofld_need;
3416 if (i < s->max_ethqsets) {
3417 s->max_ethqsets = i;
3418 if (i < s->ethqsets)
3419 reduce_ethqs(adap, i);
3420 }
3421 if (is_offload(adap)) {
3422 i = want - EXTRA_VECS - s->max_ethqsets;
3423 i -= ofld_need - nchan;
3424 s->ofldqsets = (i / nchan) * nchan; /* round down */
3425 }
3426 for (i = 0; i < want; ++i)
3427 adap->msix_info[i].vec = entries[i].vector;
3428 } else if (err > 0)
3429 dev_info(adap->pdev_dev,
3430 "only %d MSI-X vectors left, not using MSI-X\n", err);
3431 return err;
3432}
3433
3434#undef EXTRA_VECS
3435
671b0060
DM		 3436static int __devinit init_rss(struct adapter *adap)
3437{
3438 unsigned int i, j;
3439
3440 for_each_port(adap, i) {
3441 struct port_info *pi = adap2pinfo(adap, i);
3442
3443 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3444 if (!pi->rss)
3445 return -ENOMEM;
3446 for (j = 0; j < pi->rss_size; j++)
3447 pi->rss[j] = j % pi->nqsets;
3448 }
3449 return 0;
3450}
3451
b8ff05a9
DM		 3452static void __devinit print_port_info(struct adapter *adap)
3453{
3454 static const char *base[] = {
a0881cab
DM		 3455 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3456 "KX", "KR", "KR SFP+", "KR FEC"
b8ff05a9
DM		 3457 };
3458
3459 int i;
3460 char buf[80];
f1a051b9
DM		 3461 const char *spd = "";
3462
3463 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3464 spd = " 2.5 GT/s";
3465 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3466 spd = " 5 GT/s";
b8ff05a9
DM		 3467
3468 for_each_port(adap, i) {
3469 struct net_device *dev = adap->port[i];
3470 const struct port_info *pi = netdev_priv(dev);
3471 char *bufp = buf;
3472
3473 if (!test_bit(i, &adap->registered_device_map))
3474 continue;
3475
3476 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3477 bufp += sprintf(bufp, "100/");
3478 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3479 bufp += sprintf(bufp, "1000/");
3480 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3481 bufp += sprintf(bufp, "10G/");
3482 if (bufp != buf)
3483 --bufp;
3484 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3485
f1a051b9 3486 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
b8ff05a9
DM		 3487 adap->params.vpd.id, adap->params.rev,
3488 buf, is_offload(adap) ? "R" : "",
f1a051b9 3489 adap->params.pci.width, spd,
b8ff05a9
DM		 3490 (adap->flags & USING_MSIX) ? " MSI-X" :
3491 (adap->flags & USING_MSI) ? " MSI" : "");
3492 if (adap->name == dev->name)
3493 netdev_info(dev, "S/N: %s, E/C: %s\n",
3494 adap->params.vpd.sn, adap->params.vpd.ec);
3495 }
3496}
3497
06546391
DM		 3498/*
3499 * Free the following resources:
3500 * - memory used for tables
3501 * - MSI/MSI-X
3502 * - net devices
3503 * - resources FW is holding for us
3504 */
3505static void free_some_resources(struct adapter *adapter)
3506{
3507 unsigned int i;
3508
3509 t4_free_mem(adapter->l2t);
3510 t4_free_mem(adapter->tids.tid_tab);
3511 disable_msi(adapter);
3512
3513 for_each_port(adapter, i)
671b0060
DM		 3514 if (adapter->port[i]) {
3515 kfree(adap2pinfo(adapter, i)->rss);
06546391 3516 free_netdev(adapter->port[i]);
671b0060 3517 }
06546391
DM		 3518 if (adapter->flags & FW_OK)
3519 t4_fw_bye(adapter, 0);
3520}
3521
b8ff05a9
DM		 3522#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | NETIF_F_TSO6 |\
3523 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3524
3525static int __devinit init_one(struct pci_dev *pdev,
3526 const struct pci_device_id *ent)
3527{
3528 int func, i, err;
3529 struct port_info *pi;
3530 unsigned int highdma = 0;
3531 struct adapter *adapter = NULL;
3532
3533 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3534
3535 err = pci_request_regions(pdev, KBUILD_MODNAME);
3536 if (err) {
3537 /* Just info, some other driver may have claimed the device. */
3538 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3539 return err;
3540 }
3541
3542 /* We control everything through PF 0 */
3543 func = PCI_FUNC(pdev->devfn);
204dc3c0
DM		 3544 if (func > 0) {
3545 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 3546 goto sriov;
204dc3c0 3547 }
b8ff05a9
DM		 3548
3549 err = pci_enable_device(pdev);
3550 if (err) {
3551 dev_err(&pdev->dev, "cannot enable PCI device\n");
3552 goto out_release_regions;
3553 }
3554
3555 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3556 highdma = NETIF_F_HIGHDMA;
3557 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3558 if (err) {
3559 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3560 "coherent allocations\n");
3561 goto out_disable_device;
3562 }
3563 } else {
3564 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3565 if (err) {
3566 dev_err(&pdev->dev, "no usable DMA configuration\n");
3567 goto out_disable_device;
3568 }
3569 }
3570
3571 pci_enable_pcie_error_reporting(pdev);
3572 pci_set_master(pdev);
3573 pci_save_state(pdev);
3574
3575 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3576 if (!adapter) {
3577 err = -ENOMEM;
3578 goto out_disable_device;
3579 }
3580
3581 adapter->regs = pci_ioremap_bar(pdev, 0);
3582 if (!adapter->regs) {
3583 dev_err(&pdev->dev, "cannot map device registers\n");
3584 err = -ENOMEM;
3585 goto out_free_adapter;
3586 }
3587
3588 adapter->pdev = pdev;
3589 adapter->pdev_dev = &pdev->dev;
3590 adapter->name = pci_name(pdev);
3591 adapter->msg_enable = dflt_msg_enable;
3592 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3593
3594 spin_lock_init(&adapter->stats_lock);
3595 spin_lock_init(&adapter->tid_release_lock);
3596
3597 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3598
3599 err = t4_prep_adapter(adapter);
3600 if (err)
3601 goto out_unmap_bar;
3602 err = adap_init0(adapter);
3603 if (err)
3604 goto out_unmap_bar;
3605
3606 for_each_port(adapter, i) {
3607 struct net_device *netdev;
3608
3609 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3610 MAX_ETH_QSETS);
3611 if (!netdev) {
3612 err = -ENOMEM;
3613 goto out_free_dev;
3614 }
3615
3616 SET_NETDEV_DEV(netdev, &pdev->dev);
3617
3618 adapter->port[i] = netdev;
3619 pi = netdev_priv(netdev);
3620 pi->adapter = adapter;
3621 pi->xact_addr_filt = -1;
3622 pi->rx_offload = RX_CSO;
3623 pi->port_id = i;
3624 netif_carrier_off(netdev);
3625 netif_tx_stop_all_queues(netdev);
3626 netdev->irq = pdev->irq;
3627
3628 netdev->features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
3629 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
87b6cf51 3630 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
b8ff05a9
DM		 3631 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3632 netdev->vlan_features = netdev->features & VLAN_FEAT;
3633
3634 netdev->netdev_ops = &cxgb4_netdev_ops;
3635 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3636 }
3637
3638 pci_set_drvdata(pdev, adapter);
3639
3640 if (adapter->flags & FW_OK) {
3641 err = t4_port_init(adapter, 0, 0, 0);
3642 if (err)
3643 goto out_free_dev;
3644 }
3645
3646 /*
3647 * Configure queues and allocate tables now, they can be needed as
3648 * soon as the first register_netdev completes.
3649 */
3650 cfg_queues(adapter);
3651
3652 adapter->l2t = t4_init_l2t();
3653 if (!adapter->l2t) {
3654 /* We tolerate a lack of L2T, giving up some functionality */
3655 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3656 adapter->params.offload = 0;
3657 }
3658
3659 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3660 dev_warn(&pdev->dev, "could not allocate TID table, "
3661 "continuing\n");
3662 adapter->params.offload = 0;
3663 }
3664
f7cabcdd
DM		 3665 /* See what interrupts we'll be using */
3666 if (msi > 1 && enable_msix(adapter) == 0)
3667 adapter->flags |= USING_MSIX;
3668 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3669 adapter->flags |= USING_MSI;
3670
671b0060
DM		 3671 err = init_rss(adapter);
3672 if (err)
3673 goto out_free_dev;
3674
b8ff05a9
DM		 3675 /*
3676 * The card is now ready to go. If any errors occur during device
3677 * registration we do not fail the whole card but rather proceed only
3678 * with the ports we manage to register successfully. However we must
3679 * register at least one net device.
3680 */
3681 for_each_port(adapter, i) {
3682 err = register_netdev(adapter->port[i]);
3683 if (err)
3684 dev_warn(&pdev->dev,
3685 "cannot register net device %s, skipping\n",
3686 adapter->port[i]->name);
3687 else {
3688 /*
3689 * Change the name we use for messages to the name of
3690 * the first successfully registered interface.
3691 */
3692 if (!adapter->registered_device_map)
3693 adapter->name = adapter->port[i]->name;
3694
3695 __set_bit(i, &adapter->registered_device_map);
3696 adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3697 }
3698 }
3699 if (!adapter->registered_device_map) {
3700 dev_err(&pdev->dev, "could not register any net devices\n");
3701 goto out_free_dev;
3702 }
3703
3704 if (cxgb4_debugfs_root) {
3705 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3706 cxgb4_debugfs_root);
3707 setup_debugfs(adapter);
3708 }
3709
b8ff05a9
DM		 3710 if (is_offload(adapter))
3711 attach_ulds(adapter);
3712
3713 print_port_info(adapter);
3714
3715sriov:
3716#ifdef CONFIG_PCI_IOV
3717 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3718 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3719 dev_info(&pdev->dev,
3720 "instantiated %u virtual functions\n",
3721 num_vf[func]);
3722#endif
3723 return 0;
3724
3725 out_free_dev:
06546391 3726 free_some_resources(adapter);
b8ff05a9
DM		 3727 out_unmap_bar:
3728 iounmap(adapter->regs);
3729 out_free_adapter:
3730 kfree(adapter);
3731 out_disable_device:
3732 pci_disable_pcie_error_reporting(pdev);
3733 pci_disable_device(pdev);
3734 out_release_regions:
3735 pci_release_regions(pdev);
3736 pci_set_drvdata(pdev, NULL);
3737 return err;
3738}
3739
3740static void __devexit remove_one(struct pci_dev *pdev)
3741{
3742 struct adapter *adapter = pci_get_drvdata(pdev);
3743
3744 pci_disable_sriov(pdev);
3745
3746 if (adapter) {
3747 int i;
3748
3749 if (is_offload(adapter))
3750 detach_ulds(adapter);
3751
3752 for_each_port(adapter, i)
3753 if (test_bit(i, &adapter->registered_device_map))
3754 unregister_netdev(adapter->port[i]);
3755
3756 if (adapter->debugfs_root)
3757 debugfs_remove_recursive(adapter->debugfs_root);
3758
aaefae9b
DM		 3759 if (adapter->flags & FULL_INIT_DONE)
3760 cxgb_down(adapter);
b8ff05a9 3761
06546391 3762 free_some_resources(adapter);
b8ff05a9
DM		 3763 iounmap(adapter->regs);
3764 kfree(adapter);
3765 pci_disable_pcie_error_reporting(pdev);
3766 pci_disable_device(pdev);
3767 pci_release_regions(pdev);
3768 pci_set_drvdata(pdev, NULL);
3769 } else if (PCI_FUNC(pdev->devfn) > 0)
3770 pci_release_regions(pdev);
3771}
3772
3773static struct pci_driver cxgb4_driver = {
3774 .name = KBUILD_MODNAME,
3775 .id_table = cxgb4_pci_tbl,
3776 .probe = init_one,
3777 .remove = __devexit_p(remove_one),
204dc3c0 3778 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 3779};
3780
3781static int __init cxgb4_init_module(void)
3782{
3783 int ret;
3784
3785 /* Debugfs support is optional, just warn if this fails */
3786 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3787 if (!cxgb4_debugfs_root)
3788 pr_warning("could not create debugfs entry, continuing\n");
3789
3790 ret = pci_register_driver(&cxgb4_driver);
3791 if (ret < 0)
3792 debugfs_remove(cxgb4_debugfs_root);
3793 return ret;
3794}
3795
3796static void __exit cxgb4_cleanup_module(void)
3797{
3798 pci_unregister_driver(&cxgb4_driver);
3799 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3800}
3801
3802module_init(cxgb4_init_module);
3803module_exit(cxgb4_cleanup_module);
ubuntu-zesty-kernel mirror