]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ethtool: Explicitly state that RX NFC rule locations are priorities
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
63#include <asm/uaccess.h>
64
65#include "cxgb4.h"
66#include "t4_regs.h"
67#include "t4_msg.h"
68#include "t4fw_api.h"
69#include "l2t.h"
70
99e6d065 71#define DRV_VERSION "1.3.0-ko"
b8ff05a9
DM		 72#define DRV_DESC "Chelsio T4 Network Driver"
73
74/*
75 * Max interrupt hold-off timer value in us. Queues fall back to this value
76 * under extreme memory pressure so it's largish to give the system time to
77 * recover.
78 */
79#define MAX_SGE_TIMERVAL 200U
80
7ee9ff94
CL		 81#ifdef CONFIG_PCI_IOV
82/*
83 * Virtual Function provisioning constants. We need two extra Ingress Queues
84 * with Interrupt capability to serve as the VF's Firmware Event Queue and
85 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
86 * Lists associated with them). For each Ethernet/Control Egress Queue and
87 * for each Free List, we need an Egress Context.
88 */
89enum {
90 VFRES_NPORTS = 1, /* # of "ports" per VF */
91 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
92
93 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
94 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
95 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
96 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
97 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
98 VFRES_TC = 0, /* PCI-E traffic class */
99 VFRES_NEXACTF = 16, /* # of exact MPS filters */
100
101 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
102 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
103};
104
105/*
106 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
107 * static and likely not to be useful in the long run. We really need to
108 * implement some form of persistent configuration which the firmware
109 * controls.
110 */
111static unsigned int pfvfres_pmask(struct adapter *adapter,
112 unsigned int pf, unsigned int vf)
113{
114 unsigned int portn, portvec;
115
116 /*
117 * Give PF's access to all of the ports.
118 */
119 if (vf == 0)
120 return FW_PFVF_CMD_PMASK_MASK;
121
122 /*
123 * For VFs, we'll assign them access to the ports based purely on the
124 * PF. We assign active ports in order, wrapping around if there are
125 * fewer active ports than PFs: e.g. active port[pf % nports].
126 * Unfortunately the adapter's port_info structs haven't been
127 * initialized yet so we have to compute this.
128 */
129 if (adapter->params.nports == 0)
130 return 0;
131
132 portn = pf % adapter->params.nports;
133 portvec = adapter->params.portvec;
134 for (;;) {
135 /*
136 * Isolate the lowest set bit in the port vector. If we're at
137 * the port number that we want, return that as the pmask.
138 * otherwise mask that bit out of the port vector and
139 * decrement our port number ...
140 */
141 unsigned int pmask = portvec ^ (portvec & (portvec-1));
142 if (portn == 0)
143 return pmask;
144 portn--;
145 portvec &= ~pmask;
146 }
147 /*NOTREACHED*/
148}
149#endif
150
b8ff05a9
DM		 151enum {
152 MEMWIN0_APERTURE = 65536,
153 MEMWIN0_BASE = 0x30000,
154 MEMWIN1_APERTURE = 32768,
155 MEMWIN1_BASE = 0x28000,
156 MEMWIN2_APERTURE = 2048,
157 MEMWIN2_BASE = 0x1b800,
158};
159
160enum {
161 MAX_TXQ_ENTRIES = 16384,
162 MAX_CTRL_TXQ_ENTRIES = 1024,
163 MAX_RSPQ_ENTRIES = 16384,
164 MAX_RX_BUFFERS = 16384,
165 MIN_TXQ_ENTRIES = 32,
166 MIN_CTRL_TXQ_ENTRIES = 32,
167 MIN_RSPQ_ENTRIES = 128,
168 MIN_FL_ENTRIES = 16
169};
170
171#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
172 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
173 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
174
060e0c75 175#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
b8ff05a9
DM		 176
177static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
060e0c75 178 CH_DEVICE(0xa000, 0), /* PE10K */
ccea790e
DM		 179 CH_DEVICE(0x4001, -1),
180 CH_DEVICE(0x4002, -1),
181 CH_DEVICE(0x4003, -1),
182 CH_DEVICE(0x4004, -1),
183 CH_DEVICE(0x4005, -1),
184 CH_DEVICE(0x4006, -1),
185 CH_DEVICE(0x4007, -1),
186 CH_DEVICE(0x4008, -1),
187 CH_DEVICE(0x4009, -1),
188 CH_DEVICE(0x400a, -1),
189 CH_DEVICE(0x4401, 4),
190 CH_DEVICE(0x4402, 4),
191 CH_DEVICE(0x4403, 4),
192 CH_DEVICE(0x4404, 4),
193 CH_DEVICE(0x4405, 4),
194 CH_DEVICE(0x4406, 4),
195 CH_DEVICE(0x4407, 4),
196 CH_DEVICE(0x4408, 4),
197 CH_DEVICE(0x4409, 4),
198 CH_DEVICE(0x440a, 4),
b8ff05a9
DM		 199 { 0, }
200};
201
202#define FW_FNAME "cxgb4/t4fw.bin"
203
204MODULE_DESCRIPTION(DRV_DESC);
205MODULE_AUTHOR("Chelsio Communications");
206MODULE_LICENSE("Dual BSD/GPL");
207MODULE_VERSION(DRV_VERSION);
208MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
209MODULE_FIRMWARE(FW_FNAME);
210
211static int dflt_msg_enable = DFLT_MSG_ENABLE;
212
213module_param(dflt_msg_enable, int, 0644);
214MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
215
216/*
217 * The driver uses the best interrupt scheme available on a platform in the
218 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
219 * of these schemes the driver may consider as follows:
220 *
221 * msi = 2: choose from among all three options
222 * msi = 1: only consider MSI and INTx interrupts
223 * msi = 0: force INTx interrupts
224 */
225static int msi = 2;
226
227module_param(msi, int, 0644);
228MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
229
230/*
231 * Queue interrupt hold-off timer values. Queues default to the first of these
232 * upon creation.
233 */
234static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
235
236module_param_array(intr_holdoff, uint, NULL, 0644);
237MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
238 "0..4 in microseconds");
239
240static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
241
242module_param_array(intr_cnt, uint, NULL, 0644);
243MODULE_PARM_DESC(intr_cnt,
244 "thresholds 1..3 for queue interrupt packet counters");
245
246static int vf_acls;
247
248#ifdef CONFIG_PCI_IOV
249module_param(vf_acls, bool, 0644);
250MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
251
252static unsigned int num_vf[4];
253
254module_param_array(num_vf, uint, NULL, 0644);
255MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
256#endif
257
258static struct dentry *cxgb4_debugfs_root;
259
260static LIST_HEAD(adapter_list);
261static DEFINE_MUTEX(uld_mutex);
262static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
263static const char *uld_str[] = { "RDMA", "iSCSI" };
264
265static void link_report(struct net_device *dev)
266{
267 if (!netif_carrier_ok(dev))
268 netdev_info(dev, "link down\n");
269 else {
270 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
271
272 const char *s = "10Mbps";
273 const struct port_info *p = netdev_priv(dev);
274
275 switch (p->link_cfg.speed) {
276 case SPEED_10000:
277 s = "10Gbps";
278 break;
279 case SPEED_1000:
280 s = "1000Mbps";
281 break;
282 case SPEED_100:
283 s = "100Mbps";
284 break;
285 }
286
287 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
288 fc[p->link_cfg.fc]);
289 }
290}
291
292void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
293{
294 struct net_device *dev = adapter->port[port_id];
295
296 /* Skip changes from disabled ports. */
297 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
298 if (link_stat)
299 netif_carrier_on(dev);
300 else
301 netif_carrier_off(dev);
302
303 link_report(dev);
304 }
305}
306
307void t4_os_portmod_changed(const struct adapter *adap, int port_id)
308{
309 static const char *mod_str[] = {
a0881cab 310 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 311 };
312
313 const struct net_device *dev = adap->port[port_id];
314 const struct port_info *pi = netdev_priv(dev);
315
316 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
317 netdev_info(dev, "port module unplugged\n");
a0881cab 318 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 319 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
320}
321
322/*
323 * Configure the exact and hash address filters to handle a port's multicast
324 * and secondary unicast MAC addresses.
325 */
326static int set_addr_filters(const struct net_device *dev, bool sleep)
327{
328 u64 mhash = 0;
329 u64 uhash = 0;
330 bool free = true;
331 u16 filt_idx[7];
332 const u8 *addr[7];
333 int ret, naddr = 0;
b8ff05a9
DM		 334 const struct netdev_hw_addr *ha;
335 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 336 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 337 const struct port_info *pi = netdev_priv(dev);
060e0c75 338 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 339
340 /* first do the secondary unicast addresses */
341 netdev_for_each_uc_addr(ha, dev) {
342 addr[naddr++] = ha->addr;
343 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 344 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 345 naddr, addr, filt_idx, &uhash, sleep);
346 if (ret < 0)
347 return ret;
348
349 free = false;
350 naddr = 0;
351 }
352 }
353
354 /* next set up the multicast addresses */
4a35ecf8
DM		 355 netdev_for_each_mc_addr(ha, dev) {
356 addr[naddr++] = ha->addr;
357 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 358 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 359 naddr, addr, filt_idx, &mhash, sleep);
360 if (ret < 0)
361 return ret;
362
363 free = false;
364 naddr = 0;
365 }
366 }
367
060e0c75 368 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 369 uhash | mhash, sleep);
370}
371
372/*
373 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
374 * If @mtu is -1 it is left unchanged.
375 */
376static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
377{
378 int ret;
379 struct port_info *pi = netdev_priv(dev);
380
381 ret = set_addr_filters(dev, sleep_ok);
382 if (ret == 0)
060e0c75 383 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 384 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 385 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 386 sleep_ok);
387 return ret;
388}
389
390/**
391 * link_start - enable a port
392 * @dev: the port to enable
393 *
394 * Performs the MAC and PHY actions needed to enable a port.
395 */
396static int link_start(struct net_device *dev)
397{
398 int ret;
399 struct port_info *pi = netdev_priv(dev);
060e0c75 400 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 401
402 /*
403 * We do not set address filters and promiscuity here, the stack does
404 * that step explicitly.
405 */
060e0c75 406 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
19ecae2c 407 !!(dev->features & NETIF_F_HW_VLAN_RX), true);
b8ff05a9 408 if (ret == 0) {
060e0c75 409 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 410 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 411 true);
b8ff05a9
DM		 412 if (ret >= 0) {
413 pi->xact_addr_filt = ret;
414 ret = 0;
415 }
416 }
417 if (ret == 0)
060e0c75
DM		 418 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
419 &pi->link_cfg);
b8ff05a9 420 if (ret == 0)
060e0c75 421 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
b8ff05a9
DM		 422 return ret;
423}
424
425/*
426 * Response queue handler for the FW event queue.
427 */
428static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
429 const struct pkt_gl *gl)
430{
431 u8 opcode = ((const struct rss_header *)rsp)->opcode;
432
433 rsp++; /* skip RSS header */
434 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
435 const struct cpl_sge_egr_update *p = (void *)rsp;
436 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 437 struct sge_txq *txq;
b8ff05a9 438
e46dab4d 439 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 440 txq->restarts++;
e46dab4d 441 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 442 struct sge_eth_txq *eq;
443
444 eq = container_of(txq, struct sge_eth_txq, q);
445 netif_tx_wake_queue(eq->txq);
446 } else {
447 struct sge_ofld_txq *oq;
448
449 oq = container_of(txq, struct sge_ofld_txq, q);
450 tasklet_schedule(&oq->qresume_tsk);
451 }
452 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
453 const struct cpl_fw6_msg *p = (void *)rsp;
454
455 if (p->type == 0)
456 t4_handle_fw_rpl(q->adap, p->data);
457 } else if (opcode == CPL_L2T_WRITE_RPL) {
458 const struct cpl_l2t_write_rpl *p = (void *)rsp;
459
460 do_l2t_write_rpl(q->adap, p);
461 } else
462 dev_err(q->adap->pdev_dev,
463 "unexpected CPL %#x on FW event queue\n", opcode);
464 return 0;
465}
466
467/**
468 * uldrx_handler - response queue handler for ULD queues
469 * @q: the response queue that received the packet
470 * @rsp: the response queue descriptor holding the offload message
471 * @gl: the gather list of packet fragments
472 *
473 * Deliver an ingress offload packet to a ULD. All processing is done by
474 * the ULD, we just maintain statistics.
475 */
476static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
477 const struct pkt_gl *gl)
478{
479 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
480
481 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
482 rxq->stats.nomem++;
483 return -1;
484 }
485 if (gl == NULL)
486 rxq->stats.imm++;
487 else if (gl == CXGB4_MSG_AN)
488 rxq->stats.an++;
489 else
490 rxq->stats.pkts++;
491 return 0;
492}
493
494static void disable_msi(struct adapter *adapter)
495{
496 if (adapter->flags & USING_MSIX) {
497 pci_disable_msix(adapter->pdev);
498 adapter->flags &= ~USING_MSIX;
499 } else if (adapter->flags & USING_MSI) {
500 pci_disable_msi(adapter->pdev);
501 adapter->flags &= ~USING_MSI;
502 }
503}
504
505/*
506 * Interrupt handler for non-data events used with MSI-X.
507 */
508static irqreturn_t t4_nondata_intr(int irq, void *cookie)
509{
510 struct adapter *adap = cookie;
511
512 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
513 if (v & PFSW) {
514 adap->swintr = 1;
515 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
516 }
517 t4_slow_intr_handler(adap);
518 return IRQ_HANDLED;
519}
520
521/*
522 * Name the MSI-X interrupts.
523 */
524static void name_msix_vecs(struct adapter *adap)
525{
ba27816c 526 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 527
528 /* non-data interrupts */
b1a3c2b6 529 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 530
531 /* FW events */
b1a3c2b6
DM		 532 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
533 adap->port[0]->name);
b8ff05a9
DM		 534
535 /* Ethernet queues */
536 for_each_port(adap, j) {
537 struct net_device *d = adap->port[j];
538 const struct port_info *pi = netdev_priv(d);
539
ba27816c 540 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 541 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
542 d->name, i);
b8ff05a9
DM		 543 }
544
545 /* offload queues */
ba27816c
DM		 546 for_each_ofldrxq(&adap->sge, i)
547 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 548 adap->port[0]->name, i);
ba27816c
DM		 549
550 for_each_rdmarxq(&adap->sge, i)
551 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 552 adap->port[0]->name, i);
b8ff05a9
DM		 553}
554
555static int request_msix_queue_irqs(struct adapter *adap)
556{
557 struct sge *s = &adap->sge;
558 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
559
560 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
561 adap->msix_info[1].desc, &s->fw_evtq);
562 if (err)
563 return err;
564
565 for_each_ethrxq(s, ethqidx) {
566 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
567 adap->msix_info[msi].desc,
568 &s->ethrxq[ethqidx].rspq);
569 if (err)
570 goto unwind;
571 msi++;
572 }
573 for_each_ofldrxq(s, ofldqidx) {
574 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
575 adap->msix_info[msi].desc,
576 &s->ofldrxq[ofldqidx].rspq);
577 if (err)
578 goto unwind;
579 msi++;
580 }
581 for_each_rdmarxq(s, rdmaqidx) {
582 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
583 adap->msix_info[msi].desc,
584 &s->rdmarxq[rdmaqidx].rspq);
585 if (err)
586 goto unwind;
587 msi++;
588 }
589 return 0;
590
591unwind:
592 while (--rdmaqidx >= 0)
593 free_irq(adap->msix_info[--msi].vec,
594 &s->rdmarxq[rdmaqidx].rspq);
595 while (--ofldqidx >= 0)
596 free_irq(adap->msix_info[--msi].vec,
597 &s->ofldrxq[ofldqidx].rspq);
598 while (--ethqidx >= 0)
599 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
600 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
601 return err;
602}
603
604static void free_msix_queue_irqs(struct adapter *adap)
605{
606 int i, msi = 2;
607 struct sge *s = &adap->sge;
608
609 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
610 for_each_ethrxq(s, i)
611 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
612 for_each_ofldrxq(s, i)
613 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
614 for_each_rdmarxq(s, i)
615 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
616}
617
671b0060
DM		 618/**
619 * write_rss - write the RSS table for a given port
620 * @pi: the port
621 * @queues: array of queue indices for RSS
622 *
623 * Sets up the portion of the HW RSS table for the port's VI to distribute
624 * packets to the Rx queues in @queues.
625 */
626static int write_rss(const struct port_info *pi, const u16 *queues)
627{
628 u16 *rss;
629 int i, err;
630 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
631
632 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
633 if (!rss)
634 return -ENOMEM;
635
636 /* map the queue indices to queue ids */
637 for (i = 0; i < pi->rss_size; i++, queues++)
638 rss[i] = q[*queues].rspq.abs_id;
639
060e0c75
DM		 640 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
641 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 642 kfree(rss);
643 return err;
644}
645
b8ff05a9
DM		 646/**
647 * setup_rss - configure RSS
648 * @adap: the adapter
649 *
671b0060 650 * Sets up RSS for each port.
b8ff05a9
DM		 651 */
652static int setup_rss(struct adapter *adap)
653{
671b0060 654 int i, err;
b8ff05a9
DM		 655
656 for_each_port(adap, i) {
657 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 658
671b0060 659 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 660 if (err)
661 return err;
662 }
663 return 0;
664}
665
e46dab4d
DM		 666/*
667 * Return the channel of the ingress queue with the given qid.
668 */
669static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
670{
671 qid -= p->ingr_start;
672 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
673}
674
b8ff05a9
DM		 675/*
676 * Wait until all NAPI handlers are descheduled.
677 */
678static void quiesce_rx(struct adapter *adap)
679{
680 int i;
681
682 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
683 struct sge_rspq *q = adap->sge.ingr_map[i];
684
685 if (q && q->handler)
686 napi_disable(&q->napi);
687 }
688}
689
690/*
691 * Enable NAPI scheduling and interrupt generation for all Rx queues.
692 */
693static void enable_rx(struct adapter *adap)
694{
695 int i;
696
697 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
698 struct sge_rspq *q = adap->sge.ingr_map[i];
699
700 if (!q)
701 continue;
702 if (q->handler)
703 napi_enable(&q->napi);
704 /* 0-increment GTS to start the timer and enable interrupts */
705 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
706 SEINTARM(q->intr_params) |
707 INGRESSQID(q->cntxt_id));
708 }
709}
710
711/**
712 * setup_sge_queues - configure SGE Tx/Rx/response queues
713 * @adap: the adapter
714 *
715 * Determines how many sets of SGE queues to use and initializes them.
716 * We support multiple queue sets per port if we have MSI-X, otherwise
717 * just one queue set per port.
718 */
719static int setup_sge_queues(struct adapter *adap)
720{
721 int err, msi_idx, i, j;
722 struct sge *s = &adap->sge;
723
724 bitmap_zero(s->starving_fl, MAX_EGRQ);
725 bitmap_zero(s->txq_maperr, MAX_EGRQ);
726
727 if (adap->flags & USING_MSIX)
728 msi_idx = 1; /* vector 0 is for non-queue interrupts */
729 else {
730 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
731 NULL, NULL);
732 if (err)
733 return err;
734 msi_idx = -((int)s->intrq.abs_id + 1);
735 }
736
737 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
738 msi_idx, NULL, fwevtq_handler);
739 if (err) {
740freeout: t4_free_sge_resources(adap);
741 return err;
742 }
743
744 for_each_port(adap, i) {
745 struct net_device *dev = adap->port[i];
746 struct port_info *pi = netdev_priv(dev);
747 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
748 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
749
750 for (j = 0; j < pi->nqsets; j++, q++) {
751 if (msi_idx > 0)
752 msi_idx++;
753 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
754 msi_idx, &q->fl,
755 t4_ethrx_handler);
756 if (err)
757 goto freeout;
758 q->rspq.idx = j;
759 memset(&q->stats, 0, sizeof(q->stats));
760 }
761 for (j = 0; j < pi->nqsets; j++, t++) {
762 err = t4_sge_alloc_eth_txq(adap, t, dev,
763 netdev_get_tx_queue(dev, j),
764 s->fw_evtq.cntxt_id);
765 if (err)
766 goto freeout;
767 }
768 }
769
770 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
771 for_each_ofldrxq(s, i) {
772 struct sge_ofld_rxq *q = &s->ofldrxq[i];
773 struct net_device *dev = adap->port[i / j];
774
775 if (msi_idx > 0)
776 msi_idx++;
777 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
778 &q->fl, uldrx_handler);
779 if (err)
780 goto freeout;
781 memset(&q->stats, 0, sizeof(q->stats));
782 s->ofld_rxq[i] = q->rspq.abs_id;
783 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
784 s->fw_evtq.cntxt_id);
785 if (err)
786 goto freeout;
787 }
788
789 for_each_rdmarxq(s, i) {
790 struct sge_ofld_rxq *q = &s->rdmarxq[i];
791
792 if (msi_idx > 0)
793 msi_idx++;
794 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
795 msi_idx, &q->fl, uldrx_handler);
796 if (err)
797 goto freeout;
798 memset(&q->stats, 0, sizeof(q->stats));
799 s->rdma_rxq[i] = q->rspq.abs_id;
800 }
801
802 for_each_port(adap, i) {
803 /*
804 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
805 * have RDMA queues, and that's the right value.
806 */
807 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
808 s->fw_evtq.cntxt_id,
809 s->rdmarxq[i].rspq.cntxt_id);
810 if (err)
811 goto freeout;
812 }
813
814 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
815 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
816 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
817 return 0;
818}
819
820/*
821 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
822 * started but failed, and a negative errno if flash load couldn't start.
823 */
824static int upgrade_fw(struct adapter *adap)
825{
826 int ret;
827 u32 vers;
828 const struct fw_hdr *hdr;
829 const struct firmware *fw;
830 struct device *dev = adap->pdev_dev;
831
832 ret = request_firmware(&fw, FW_FNAME, dev);
833 if (ret < 0) {
834 dev_err(dev, "unable to load firmware image " FW_FNAME
835 ", error %d\n", ret);
836 return ret;
837 }
838
839 hdr = (const struct fw_hdr *)fw->data;
840 vers = ntohl(hdr->fw_ver);
841 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
842 ret = -EINVAL; /* wrong major version, won't do */
843 goto out;
844 }
845
846 /*
847 * If the flash FW is unusable or we found something newer, load it.
848 */
849 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
850 vers > adap->params.fw_vers) {
851 ret = -t4_load_fw(adap, fw->data, fw->size);
852 if (!ret)
853 dev_info(dev, "firmware upgraded to version %pI4 from "
854 FW_FNAME "\n", &hdr->fw_ver);
855 }
856out: release_firmware(fw);
857 return ret;
858}
859
860/*
861 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
862 * The allocated memory is cleared.
863 */
864void *t4_alloc_mem(size_t size)
865{
89bf67f1 866 void *p = kzalloc(size, GFP_KERNEL);
b8ff05a9
DM		 867
868 if (!p)
89bf67f1 869 p = vzalloc(size);
b8ff05a9
DM		 870 return p;
871}
872
873/*
874 * Free memory allocated through alloc_mem().
875 */
31b9c19b 876static void t4_free_mem(void *addr)
b8ff05a9
DM		 877{
878 if (is_vmalloc_addr(addr))
879 vfree(addr);
880 else
881 kfree(addr);
882}
883
884static inline int is_offload(const struct adapter *adap)
885{
886 return adap->params.offload;
887}
888
889/*
890 * Implementation of ethtool operations.
891 */
892
893static u32 get_msglevel(struct net_device *dev)
894{
895 return netdev2adap(dev)->msg_enable;
896}
897
898static void set_msglevel(struct net_device *dev, u32 val)
899{
900 netdev2adap(dev)->msg_enable = val;
901}
902
903static char stats_strings[][ETH_GSTRING_LEN] = {
904 "TxOctetsOK ",
905 "TxFramesOK ",
906 "TxBroadcastFrames ",
907 "TxMulticastFrames ",
908 "TxUnicastFrames ",
909 "TxErrorFrames ",
910
911 "TxFrames64 ",
912 "TxFrames65To127 ",
913 "TxFrames128To255 ",
914 "TxFrames256To511 ",
915 "TxFrames512To1023 ",
916 "TxFrames1024To1518 ",
917 "TxFrames1519ToMax ",
918
919 "TxFramesDropped ",
920 "TxPauseFrames ",
921 "TxPPP0Frames ",
922 "TxPPP1Frames ",
923 "TxPPP2Frames ",
924 "TxPPP3Frames ",
925 "TxPPP4Frames ",
926 "TxPPP5Frames ",
927 "TxPPP6Frames ",
928 "TxPPP7Frames ",
929
930 "RxOctetsOK ",
931 "RxFramesOK ",
932 "RxBroadcastFrames ",
933 "RxMulticastFrames ",
934 "RxUnicastFrames ",
935
936 "RxFramesTooLong ",
937 "RxJabberErrors ",
938 "RxFCSErrors ",
939 "RxLengthErrors ",
940 "RxSymbolErrors ",
941 "RxRuntFrames ",
942
943 "RxFrames64 ",
944 "RxFrames65To127 ",
945 "RxFrames128To255 ",
946 "RxFrames256To511 ",
947 "RxFrames512To1023 ",
948 "RxFrames1024To1518 ",
949 "RxFrames1519ToMax ",
950
951 "RxPauseFrames ",
952 "RxPPP0Frames ",
953 "RxPPP1Frames ",
954 "RxPPP2Frames ",
955 "RxPPP3Frames ",
956 "RxPPP4Frames ",
957 "RxPPP5Frames ",
958 "RxPPP6Frames ",
959 "RxPPP7Frames ",
960
961 "RxBG0FramesDropped ",
962 "RxBG1FramesDropped ",
963 "RxBG2FramesDropped ",
964 "RxBG3FramesDropped ",
965 "RxBG0FramesTrunc ",
966 "RxBG1FramesTrunc ",
967 "RxBG2FramesTrunc ",
968 "RxBG3FramesTrunc ",
969
970 "TSO ",
971 "TxCsumOffload ",
972 "RxCsumGood ",
973 "VLANextractions ",
974 "VLANinsertions ",
4a6346d4
DM		 975 "GROpackets ",
976 "GROmerged ",
b8ff05a9
DM		 977};
978
979static int get_sset_count(struct net_device *dev, int sset)
980{
981 switch (sset) {
982 case ETH_SS_STATS:
983 return ARRAY_SIZE(stats_strings);
984 default:
985 return -EOPNOTSUPP;
986 }
987}
988
989#define T4_REGMAP_SIZE (160 * 1024)
990
991static int get_regs_len(struct net_device *dev)
992{
993 return T4_REGMAP_SIZE;
994}
995
996static int get_eeprom_len(struct net_device *dev)
997{
998 return EEPROMSIZE;
999}
1000
1001static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1002{
1003 struct adapter *adapter = netdev2adap(dev);
1004
1005 strcpy(info->driver, KBUILD_MODNAME);
1006 strcpy(info->version, DRV_VERSION);
1007 strcpy(info->bus_info, pci_name(adapter->pdev));
1008
1009 if (!adapter->params.fw_vers)
1010 strcpy(info->fw_version, "N/A");
1011 else
1012 snprintf(info->fw_version, sizeof(info->fw_version),
1013 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1014 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1015 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1016 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1017 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1018 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1019 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1020 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1021 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1022}
1023
1024static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1025{
1026 if (stringset == ETH_SS_STATS)
1027 memcpy(data, stats_strings, sizeof(stats_strings));
1028}
1029
1030/*
1031 * port stats maintained per queue of the port. They should be in the same
1032 * order as in stats_strings above.
1033 */
1034struct queue_port_stats {
1035 u64 tso;
1036 u64 tx_csum;
1037 u64 rx_csum;
1038 u64 vlan_ex;
1039 u64 vlan_ins;
4a6346d4
DM		 1040 u64 gro_pkts;
1041 u64 gro_merged;
b8ff05a9
DM		 1042};
1043
1044static void collect_sge_port_stats(const struct adapter *adap,
1045 const struct port_info *p, struct queue_port_stats *s)
1046{
1047 int i;
1048 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1049 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1050
1051 memset(s, 0, sizeof(*s));
1052 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1053 s->tso += tx->tso;
1054 s->tx_csum += tx->tx_cso;
1055 s->rx_csum += rx->stats.rx_cso;
1056 s->vlan_ex += rx->stats.vlan_ex;
1057 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1058 s->gro_pkts += rx->stats.lro_pkts;
1059 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1060 }
1061}
1062
1063static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1064 u64 *data)
1065{
1066 struct port_info *pi = netdev_priv(dev);
1067 struct adapter *adapter = pi->adapter;
1068
1069 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1070
1071 data += sizeof(struct port_stats) / sizeof(u64);
1072 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1073}
1074
1075/*
1076 * Return a version number to identify the type of adapter. The scheme is:
1077 * - bits 0..9: chip version
1078 * - bits 10..15: chip revision
835bb606 1079 * - bits 16..23: register dump version
b8ff05a9
DM		 1080 */
1081static inline unsigned int mk_adap_vers(const struct adapter *ap)
1082{
835bb606 1083 return 4 | (ap->params.rev << 10) | (1 << 16);
b8ff05a9
DM		 1084}
1085
1086static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1087 unsigned int end)
1088{
1089 u32 *p = buf + start;
1090
1091 for ( ; start <= end; start += sizeof(u32))
1092 *p++ = t4_read_reg(ap, start);
1093}
1094
1095static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1096 void *buf)
1097{
1098 static const unsigned int reg_ranges[] = {
1099 0x1008, 0x1108,
1100 0x1180, 0x11b4,
1101 0x11fc, 0x123c,
1102 0x1300, 0x173c,
1103 0x1800, 0x18fc,
1104 0x3000, 0x30d8,
1105 0x30e0, 0x5924,
1106 0x5960, 0x59d4,
1107 0x5a00, 0x5af8,
1108 0x6000, 0x6098,
1109 0x6100, 0x6150,
1110 0x6200, 0x6208,
1111 0x6240, 0x6248,
1112 0x6280, 0x6338,
1113 0x6370, 0x638c,
1114 0x6400, 0x643c,
1115 0x6500, 0x6524,
1116 0x6a00, 0x6a38,
1117 0x6a60, 0x6a78,
1118 0x6b00, 0x6b84,
1119 0x6bf0, 0x6c84,
1120 0x6cf0, 0x6d84,
1121 0x6df0, 0x6e84,
1122 0x6ef0, 0x6f84,
1123 0x6ff0, 0x7084,
1124 0x70f0, 0x7184,
1125 0x71f0, 0x7284,
1126 0x72f0, 0x7384,
1127 0x73f0, 0x7450,
1128 0x7500, 0x7530,
1129 0x7600, 0x761c,
1130 0x7680, 0x76cc,
1131 0x7700, 0x7798,
1132 0x77c0, 0x77fc,
1133 0x7900, 0x79fc,
1134 0x7b00, 0x7c38,
1135 0x7d00, 0x7efc,
1136 0x8dc0, 0x8e1c,
1137 0x8e30, 0x8e78,
1138 0x8ea0, 0x8f6c,
1139 0x8fc0, 0x9074,
1140 0x90fc, 0x90fc,
1141 0x9400, 0x9458,
1142 0x9600, 0x96bc,
1143 0x9800, 0x9808,
1144 0x9820, 0x983c,
1145 0x9850, 0x9864,
1146 0x9c00, 0x9c6c,
1147 0x9c80, 0x9cec,
1148 0x9d00, 0x9d6c,
1149 0x9d80, 0x9dec,
1150 0x9e00, 0x9e6c,
1151 0x9e80, 0x9eec,
1152 0x9f00, 0x9f6c,
1153 0x9f80, 0x9fec,
1154 0xd004, 0xd03c,
1155 0xdfc0, 0xdfe0,
1156 0xe000, 0xea7c,
1157 0xf000, 0x11190,
835bb606
DM		 1158 0x19040, 0x1906c,
1159 0x19078, 0x19080,
1160 0x1908c, 0x19124,
b8ff05a9
DM		 1161 0x19150, 0x191b0,
1162 0x191d0, 0x191e8,
1163 0x19238, 0x1924c,
1164 0x193f8, 0x19474,
1165 0x19490, 0x194f8,
1166 0x19800, 0x19f30,
1167 0x1a000, 0x1a06c,
1168 0x1a0b0, 0x1a120,
1169 0x1a128, 0x1a138,
1170 0x1a190, 0x1a1c4,
1171 0x1a1fc, 0x1a1fc,
1172 0x1e040, 0x1e04c,
835bb606 1173 0x1e284, 0x1e28c,
b8ff05a9
DM		 1174 0x1e2c0, 0x1e2c0,
1175 0x1e2e0, 0x1e2e0,
1176 0x1e300, 0x1e384,
1177 0x1e3c0, 0x1e3c8,
1178 0x1e440, 0x1e44c,
835bb606 1179 0x1e684, 0x1e68c,
b8ff05a9
DM		 1180 0x1e6c0, 0x1e6c0,
1181 0x1e6e0, 0x1e6e0,
1182 0x1e700, 0x1e784,
1183 0x1e7c0, 0x1e7c8,
1184 0x1e840, 0x1e84c,
835bb606 1185 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1186 0x1eac0, 0x1eac0,
1187 0x1eae0, 0x1eae0,
1188 0x1eb00, 0x1eb84,
1189 0x1ebc0, 0x1ebc8,
1190 0x1ec40, 0x1ec4c,
835bb606 1191 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1192 0x1eec0, 0x1eec0,
1193 0x1eee0, 0x1eee0,
1194 0x1ef00, 0x1ef84,
1195 0x1efc0, 0x1efc8,
1196 0x1f040, 0x1f04c,
835bb606 1197 0x1f284, 0x1f28c,
b8ff05a9
DM		 1198 0x1f2c0, 0x1f2c0,
1199 0x1f2e0, 0x1f2e0,
1200 0x1f300, 0x1f384,
1201 0x1f3c0, 0x1f3c8,
1202 0x1f440, 0x1f44c,
835bb606 1203 0x1f684, 0x1f68c,
b8ff05a9
DM		 1204 0x1f6c0, 0x1f6c0,
1205 0x1f6e0, 0x1f6e0,
1206 0x1f700, 0x1f784,
1207 0x1f7c0, 0x1f7c8,
1208 0x1f840, 0x1f84c,
835bb606 1209 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1210 0x1fac0, 0x1fac0,
1211 0x1fae0, 0x1fae0,
1212 0x1fb00, 0x1fb84,
1213 0x1fbc0, 0x1fbc8,
1214 0x1fc40, 0x1fc4c,
835bb606 1215 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1216 0x1fec0, 0x1fec0,
1217 0x1fee0, 0x1fee0,
1218 0x1ff00, 0x1ff84,
1219 0x1ffc0, 0x1ffc8,
1220 0x20000, 0x2002c,
1221 0x20100, 0x2013c,
1222 0x20190, 0x201c8,
1223 0x20200, 0x20318,
1224 0x20400, 0x20528,
1225 0x20540, 0x20614,
1226 0x21000, 0x21040,
1227 0x2104c, 0x21060,
1228 0x210c0, 0x210ec,
1229 0x21200, 0x21268,
1230 0x21270, 0x21284,
1231 0x212fc, 0x21388,
1232 0x21400, 0x21404,
1233 0x21500, 0x21518,
1234 0x2152c, 0x2153c,
1235 0x21550, 0x21554,
1236 0x21600, 0x21600,
1237 0x21608, 0x21628,
1238 0x21630, 0x2163c,
1239 0x21700, 0x2171c,
1240 0x21780, 0x2178c,
1241 0x21800, 0x21c38,
1242 0x21c80, 0x21d7c,
1243 0x21e00, 0x21e04,
1244 0x22000, 0x2202c,
1245 0x22100, 0x2213c,
1246 0x22190, 0x221c8,
1247 0x22200, 0x22318,
1248 0x22400, 0x22528,
1249 0x22540, 0x22614,
1250 0x23000, 0x23040,
1251 0x2304c, 0x23060,
1252 0x230c0, 0x230ec,
1253 0x23200, 0x23268,
1254 0x23270, 0x23284,
1255 0x232fc, 0x23388,
1256 0x23400, 0x23404,
1257 0x23500, 0x23518,
1258 0x2352c, 0x2353c,
1259 0x23550, 0x23554,
1260 0x23600, 0x23600,
1261 0x23608, 0x23628,
1262 0x23630, 0x2363c,
1263 0x23700, 0x2371c,
1264 0x23780, 0x2378c,
1265 0x23800, 0x23c38,
1266 0x23c80, 0x23d7c,
1267 0x23e00, 0x23e04,
1268 0x24000, 0x2402c,
1269 0x24100, 0x2413c,
1270 0x24190, 0x241c8,
1271 0x24200, 0x24318,
1272 0x24400, 0x24528,
1273 0x24540, 0x24614,
1274 0x25000, 0x25040,
1275 0x2504c, 0x25060,
1276 0x250c0, 0x250ec,
1277 0x25200, 0x25268,
1278 0x25270, 0x25284,
1279 0x252fc, 0x25388,
1280 0x25400, 0x25404,
1281 0x25500, 0x25518,
1282 0x2552c, 0x2553c,
1283 0x25550, 0x25554,
1284 0x25600, 0x25600,
1285 0x25608, 0x25628,
1286 0x25630, 0x2563c,
1287 0x25700, 0x2571c,
1288 0x25780, 0x2578c,
1289 0x25800, 0x25c38,
1290 0x25c80, 0x25d7c,
1291 0x25e00, 0x25e04,
1292 0x26000, 0x2602c,
1293 0x26100, 0x2613c,
1294 0x26190, 0x261c8,
1295 0x26200, 0x26318,
1296 0x26400, 0x26528,
1297 0x26540, 0x26614,
1298 0x27000, 0x27040,
1299 0x2704c, 0x27060,
1300 0x270c0, 0x270ec,
1301 0x27200, 0x27268,
1302 0x27270, 0x27284,
1303 0x272fc, 0x27388,
1304 0x27400, 0x27404,
1305 0x27500, 0x27518,
1306 0x2752c, 0x2753c,
1307 0x27550, 0x27554,
1308 0x27600, 0x27600,
1309 0x27608, 0x27628,
1310 0x27630, 0x2763c,
1311 0x27700, 0x2771c,
1312 0x27780, 0x2778c,
1313 0x27800, 0x27c38,
1314 0x27c80, 0x27d7c,
1315 0x27e00, 0x27e04
1316 };
1317
1318 int i;
1319 struct adapter *ap = netdev2adap(dev);
1320
1321 regs->version = mk_adap_vers(ap);
1322
1323 memset(buf, 0, T4_REGMAP_SIZE);
1324 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1325 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1326}
1327
1328static int restart_autoneg(struct net_device *dev)
1329{
1330 struct port_info *p = netdev_priv(dev);
1331
1332 if (!netif_running(dev))
1333 return -EAGAIN;
1334 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1335 return -EINVAL;
060e0c75 1336 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 1337 return 0;
1338}
1339
c5e06360
DM		 1340static int identify_port(struct net_device *dev,
1341 enum ethtool_phys_id_state state)
b8ff05a9 1342{
c5e06360 1343 unsigned int val;
060e0c75
DM		 1344 struct adapter *adap = netdev2adap(dev);
1345
c5e06360
DM		 1346 if (state == ETHTOOL_ID_ACTIVE)
1347 val = 0xffff;
1348 else if (state == ETHTOOL_ID_INACTIVE)
1349 val = 0;
1350 else
1351 return -EINVAL;
b8ff05a9 1352
c5e06360 1353 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 1354}
1355
1356static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1357{
1358 unsigned int v = 0;
1359
a0881cab
DM		 1360 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1361 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 1362 v |= SUPPORTED_TP;
1363 if (caps & FW_PORT_CAP_SPEED_100M)
1364 v |= SUPPORTED_100baseT_Full;
1365 if (caps & FW_PORT_CAP_SPEED_1G)
1366 v |= SUPPORTED_1000baseT_Full;
1367 if (caps & FW_PORT_CAP_SPEED_10G)
1368 v |= SUPPORTED_10000baseT_Full;
1369 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1370 v |= SUPPORTED_Backplane;
1371 if (caps & FW_PORT_CAP_SPEED_1G)
1372 v |= SUPPORTED_1000baseKX_Full;
1373 if (caps & FW_PORT_CAP_SPEED_10G)
1374 v |= SUPPORTED_10000baseKX4_Full;
1375 } else if (type == FW_PORT_TYPE_KR)
1376 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 1377 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 1378 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1379 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1380 else if (type == FW_PORT_TYPE_BP4_AP)
1381 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1382 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1383 SUPPORTED_10000baseKX4_Full;
a0881cab
DM		 1384 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1385 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9
DM		 1386 v |= SUPPORTED_FIBRE;
1387
1388 if (caps & FW_PORT_CAP_ANEG)
1389 v |= SUPPORTED_Autoneg;
1390 return v;
1391}
1392
1393static unsigned int to_fw_linkcaps(unsigned int caps)
1394{
1395 unsigned int v = 0;
1396
1397 if (caps & ADVERTISED_100baseT_Full)
1398 v |= FW_PORT_CAP_SPEED_100M;
1399 if (caps & ADVERTISED_1000baseT_Full)
1400 v |= FW_PORT_CAP_SPEED_1G;
1401 if (caps & ADVERTISED_10000baseT_Full)
1402 v |= FW_PORT_CAP_SPEED_10G;
1403 return v;
1404}
1405
1406static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1407{
1408 const struct port_info *p = netdev_priv(dev);
1409
1410 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 1411 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 1412 p->port_type == FW_PORT_TYPE_BT_XAUI)
1413 cmd->port = PORT_TP;
a0881cab
DM		 1414 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1415 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 1416 cmd->port = PORT_FIBRE;
a0881cab
DM		 1417 else if (p->port_type == FW_PORT_TYPE_SFP) {
1418 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1419 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1420 cmd->port = PORT_DA;
1421 else
1422 cmd->port = PORT_FIBRE;
1423 } else
b8ff05a9
DM		 1424 cmd->port = PORT_OTHER;
1425
1426 if (p->mdio_addr >= 0) {
1427 cmd->phy_address = p->mdio_addr;
1428 cmd->transceiver = XCVR_EXTERNAL;
1429 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1430 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1431 } else {
1432 cmd->phy_address = 0; /* not really, but no better option */
1433 cmd->transceiver = XCVR_INTERNAL;
1434 cmd->mdio_support = 0;
1435 }
1436
1437 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1438 cmd->advertising = from_fw_linkcaps(p->port_type,
1439 p->link_cfg.advertising);
70739497
DD		 1440 ethtool_cmd_speed_set(cmd,
1441 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 1442 cmd->duplex = DUPLEX_FULL;
1443 cmd->autoneg = p->link_cfg.autoneg;
1444 cmd->maxtxpkt = 0;
1445 cmd->maxrxpkt = 0;
1446 return 0;
1447}
1448
1449static unsigned int speed_to_caps(int speed)
1450{
1451 if (speed == SPEED_100)
1452 return FW_PORT_CAP_SPEED_100M;
1453 if (speed == SPEED_1000)
1454 return FW_PORT_CAP_SPEED_1G;
1455 if (speed == SPEED_10000)
1456 return FW_PORT_CAP_SPEED_10G;
1457 return 0;
1458}
1459
1460static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1461{
1462 unsigned int cap;
1463 struct port_info *p = netdev_priv(dev);
1464 struct link_config *lc = &p->link_cfg;
25db0338 1465 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 1466
1467 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1468 return -EINVAL;
1469
1470 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1471 /*
1472 * PHY offers a single speed. See if that's what's
1473 * being requested.
1474 */
1475 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 1476 (lc->supported & speed_to_caps(speed)))
1477 return 0;
b8ff05a9
DM		 1478 return -EINVAL;
1479 }
1480
1481 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 1482 cap = speed_to_caps(speed);
b8ff05a9 1483
25db0338
DD		 1484 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1485 (speed == SPEED_10000))
b8ff05a9
DM		 1486 return -EINVAL;
1487 lc->requested_speed = cap;
1488 lc->advertising = 0;
1489 } else {
1490 cap = to_fw_linkcaps(cmd->advertising);
1491 if (!(lc->supported & cap))
1492 return -EINVAL;
1493 lc->requested_speed = 0;
1494 lc->advertising = cap | FW_PORT_CAP_ANEG;
1495 }
1496 lc->autoneg = cmd->autoneg;
1497
1498 if (netif_running(dev))
060e0c75
DM		 1499 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1500 lc);
b8ff05a9
DM		 1501 return 0;
1502}
1503
1504static void get_pauseparam(struct net_device *dev,
1505 struct ethtool_pauseparam *epause)
1506{
1507 struct port_info *p = netdev_priv(dev);
1508
1509 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1510 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1511 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1512}
1513
1514static int set_pauseparam(struct net_device *dev,
1515 struct ethtool_pauseparam *epause)
1516{
1517 struct port_info *p = netdev_priv(dev);
1518 struct link_config *lc = &p->link_cfg;
1519
1520 if (epause->autoneg == AUTONEG_DISABLE)
1521 lc->requested_fc = 0;
1522 else if (lc->supported & FW_PORT_CAP_ANEG)
1523 lc->requested_fc = PAUSE_AUTONEG;
1524 else
1525 return -EINVAL;
1526
1527 if (epause->rx_pause)
1528 lc->requested_fc |= PAUSE_RX;
1529 if (epause->tx_pause)
1530 lc->requested_fc |= PAUSE_TX;
1531 if (netif_running(dev))
060e0c75
DM		 1532 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1533 lc);
b8ff05a9
DM		 1534 return 0;
1535}
1536
b8ff05a9
DM		 1537static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1538{
1539 const struct port_info *pi = netdev_priv(dev);
1540 const struct sge *s = &pi->adapter->sge;
1541
1542 e->rx_max_pending = MAX_RX_BUFFERS;
1543 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1544 e->rx_jumbo_max_pending = 0;
1545 e->tx_max_pending = MAX_TXQ_ENTRIES;
1546
1547 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1548 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1549 e->rx_jumbo_pending = 0;
1550 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1551}
1552
1553static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1554{
1555 int i;
1556 const struct port_info *pi = netdev_priv(dev);
1557 struct adapter *adapter = pi->adapter;
1558 struct sge *s = &adapter->sge;
1559
1560 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1561 e->tx_pending > MAX_TXQ_ENTRIES ||
1562 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1563 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1564 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1565 return -EINVAL;
1566
1567 if (adapter->flags & FULL_INIT_DONE)
1568 return -EBUSY;
1569
1570 for (i = 0; i < pi->nqsets; ++i) {
1571 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1572 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1573 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1574 }
1575 return 0;
1576}
1577
1578static int closest_timer(const struct sge *s, int time)
1579{
1580 int i, delta, match = 0, min_delta = INT_MAX;
1581
1582 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1583 delta = time - s->timer_val[i];
1584 if (delta < 0)
1585 delta = -delta;
1586 if (delta < min_delta) {
1587 min_delta = delta;
1588 match = i;
1589 }
1590 }
1591 return match;
1592}
1593
1594static int closest_thres(const struct sge *s, int thres)
1595{
1596 int i, delta, match = 0, min_delta = INT_MAX;
1597
1598 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1599 delta = thres - s->counter_val[i];
1600 if (delta < 0)
1601 delta = -delta;
1602 if (delta < min_delta) {
1603 min_delta = delta;
1604 match = i;
1605 }
1606 }
1607 return match;
1608}
1609
1610/*
1611 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1612 */
1613static unsigned int qtimer_val(const struct adapter *adap,
1614 const struct sge_rspq *q)
1615{
1616 unsigned int idx = q->intr_params >> 1;
1617
1618 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1619}
1620
1621/**
1622 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1623 * @adap: the adapter
1624 * @q: the Rx queue
1625 * @us: the hold-off time in us, or 0 to disable timer
1626 * @cnt: the hold-off packet count, or 0 to disable counter
1627 *
1628 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1629 * one of the two needs to be enabled for the queue to generate interrupts.
1630 */
1631static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1632 unsigned int us, unsigned int cnt)
1633{
1634 if ((us | cnt) == 0)
1635 cnt = 1;
1636
1637 if (cnt) {
1638 int err;
1639 u32 v, new_idx;
1640
1641 new_idx = closest_thres(&adap->sge, cnt);
1642 if (q->desc && q->pktcnt_idx != new_idx) {
1643 /* the queue has already been created, update it */
1644 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1645 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1646 FW_PARAMS_PARAM_YZ(q->cntxt_id);
060e0c75
DM		 1647 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1648 &new_idx);
b8ff05a9
DM		 1649 if (err)
1650 return err;
1651 }
1652 q->pktcnt_idx = new_idx;
1653 }
1654
1655 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1656 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1657 return 0;
1658}
1659
1660static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1661{
1662 const struct port_info *pi = netdev_priv(dev);
1663 struct adapter *adap = pi->adapter;
1664
1665 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1666 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1667}
1668
1669static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1670{
1671 const struct port_info *pi = netdev_priv(dev);
1672 const struct adapter *adap = pi->adapter;
1673 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1674
1675 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1676 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1677 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1678 return 0;
1679}
1680
1478b3ee
DM		 1681/**
1682 * eeprom_ptov - translate a physical EEPROM address to virtual
1683 * @phys_addr: the physical EEPROM address
1684 * @fn: the PCI function number
1685 * @sz: size of function-specific area
1686 *
1687 * Translate a physical EEPROM address to virtual. The first 1K is
1688 * accessed through virtual addresses starting at 31K, the rest is
1689 * accessed through virtual addresses starting at 0.
1690 *
1691 * The mapping is as follows:
1692 * [0..1K) -> [31K..32K)
1693 * [1K..1K+A) -> [31K-A..31K)
1694 * [1K+A..ES) -> [0..ES-A-1K)
1695 *
1696 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 1697 */
1478b3ee 1698static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 1699{
1478b3ee 1700 fn *= sz;
b8ff05a9
DM		 1701 if (phys_addr < 1024)
1702 return phys_addr + (31 << 10);
1478b3ee
DM		 1703 if (phys_addr < 1024 + fn)
1704 return 31744 - fn + phys_addr - 1024;
b8ff05a9 1705 if (phys_addr < EEPROMSIZE)
1478b3ee 1706 return phys_addr - 1024 - fn;
b8ff05a9
DM		 1707 return -EINVAL;
1708}
1709
1710/*
1711 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 1712 */
1713static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1714{
1478b3ee 1715 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 1716
1717 if (vaddr >= 0)
1718 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1719 return vaddr < 0 ? vaddr : 0;
1720}
1721
1722static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1723{
1478b3ee 1724 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 1725
1726 if (vaddr >= 0)
1727 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1728 return vaddr < 0 ? vaddr : 0;
1729}
1730
1731#define EEPROM_MAGIC 0x38E2F10C
1732
1733static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1734 u8 *data)
1735{
1736 int i, err = 0;
1737 struct adapter *adapter = netdev2adap(dev);
1738
1739 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1740 if (!buf)
1741 return -ENOMEM;
1742
1743 e->magic = EEPROM_MAGIC;
1744 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1745 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1746
1747 if (!err)
1748 memcpy(data, buf + e->offset, e->len);
1749 kfree(buf);
1750 return err;
1751}
1752
1753static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1754 u8 *data)
1755{
1756 u8 *buf;
1757 int err = 0;
1758 u32 aligned_offset, aligned_len, *p;
1759 struct adapter *adapter = netdev2adap(dev);
1760
1761 if (eeprom->magic != EEPROM_MAGIC)
1762 return -EINVAL;
1763
1764 aligned_offset = eeprom->offset & ~3;
1765 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1766
1478b3ee
DM		 1767 if (adapter->fn > 0) {
1768 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1769
1770 if (aligned_offset < start ||
1771 aligned_offset + aligned_len > start + EEPROMPFSIZE)
1772 return -EPERM;
1773 }
1774
b8ff05a9
DM		 1775 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1776 /*
1777 * RMW possibly needed for first or last words.
1778 */
1779 buf = kmalloc(aligned_len, GFP_KERNEL);
1780 if (!buf)
1781 return -ENOMEM;
1782 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1783 if (!err && aligned_len > 4)
1784 err = eeprom_rd_phys(adapter,
1785 aligned_offset + aligned_len - 4,
1786 (u32 *)&buf[aligned_len - 4]);
1787 if (err)
1788 goto out;
1789 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1790 } else
1791 buf = data;
1792
1793 err = t4_seeprom_wp(adapter, false);
1794 if (err)
1795 goto out;
1796
1797 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1798 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1799 aligned_offset += 4;
1800 }
1801
1802 if (!err)
1803 err = t4_seeprom_wp(adapter, true);
1804out:
1805 if (buf != data)
1806 kfree(buf);
1807 return err;
1808}
1809
1810static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1811{
1812 int ret;
1813 const struct firmware *fw;
1814 struct adapter *adap = netdev2adap(netdev);
1815
1816 ef->data[sizeof(ef->data) - 1] = '\0';
1817 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1818 if (ret < 0)
1819 return ret;
1820
1821 ret = t4_load_fw(adap, fw->data, fw->size);
1822 release_firmware(fw);
1823 if (!ret)
1824 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1825 return ret;
1826}
1827
1828#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1829#define BCAST_CRC 0xa0ccc1a6
1830
1831static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1832{
1833 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1834 wol->wolopts = netdev2adap(dev)->wol;
1835 memset(&wol->sopass, 0, sizeof(wol->sopass));
1836}
1837
1838static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1839{
1840 int err = 0;
1841 struct port_info *pi = netdev_priv(dev);
1842
1843 if (wol->wolopts & ~WOL_SUPPORTED)
1844 return -EINVAL;
1845 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1846 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1847 if (wol->wolopts & WAKE_BCAST) {
1848 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1849 ~0ULL, 0, false);
1850 if (!err)
1851 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1852 ~6ULL, ~0ULL, BCAST_CRC, true);
1853 } else
1854 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1855 return err;
1856}
1857
2ed28baa 1858static int cxgb_set_features(struct net_device *dev, u32 features)
87b6cf51 1859{
2ed28baa
MM		 1860 const struct port_info *pi = netdev_priv(dev);
1861 u32 changed = dev->features ^ features;
19ecae2c 1862 int err;
19ecae2c 1863
2ed28baa
MM		 1864 if (!(changed & NETIF_F_HW_VLAN_RX))
1865 return 0;
19ecae2c 1866
2ed28baa
MM		 1867 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1868 -1, -1, -1,
1869 !!(features & NETIF_F_HW_VLAN_RX), true);
1870 if (unlikely(err))
1871 dev->features = features ^ NETIF_F_HW_VLAN_RX;
19ecae2c 1872 return err;
87b6cf51
DM		 1873}
1874
671b0060
DM		 1875static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1876{
1877 const struct port_info *pi = netdev_priv(dev);
1878 unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1879
1880 p->size = pi->rss_size;
1881 while (n--)
1882 p->ring_index[n] = pi->rss[n];
1883 return 0;
1884}
1885
1886static int set_rss_table(struct net_device *dev,
1887 const struct ethtool_rxfh_indir *p)
1888{
1889 unsigned int i;
1890 struct port_info *pi = netdev_priv(dev);
1891
1892 if (p->size != pi->rss_size)
1893 return -EINVAL;
1894 for (i = 0; i < p->size; i++)
1895 if (p->ring_index[i] >= pi->nqsets)
1896 return -EINVAL;
1897 for (i = 0; i < p->size; i++)
1898 pi->rss[i] = p->ring_index[i];
1899 if (pi->adapter->flags & FULL_INIT_DONE)
1900 return write_rss(pi, pi->rss);
1901 return 0;
1902}
1903
1904static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1905 void *rules)
1906{
f796564a
DM		 1907 const struct port_info *pi = netdev_priv(dev);
1908
671b0060 1909 switch (info->cmd) {
f796564a
DM		 1910 case ETHTOOL_GRXFH: {
1911 unsigned int v = pi->rss_mode;
1912
1913 info->data = 0;
1914 switch (info->flow_type) {
1915 case TCP_V4_FLOW:
1916 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1917 info->data = RXH_IP_SRC | RXH_IP_DST |
1918 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1919 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1920 info->data = RXH_IP_SRC | RXH_IP_DST;
1921 break;
1922 case UDP_V4_FLOW:
1923 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1924 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1925 info->data = RXH_IP_SRC | RXH_IP_DST |
1926 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1927 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1928 info->data = RXH_IP_SRC | RXH_IP_DST;
1929 break;
1930 case SCTP_V4_FLOW:
1931 case AH_ESP_V4_FLOW:
1932 case IPV4_FLOW:
1933 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1934 info->data = RXH_IP_SRC | RXH_IP_DST;
1935 break;
1936 case TCP_V6_FLOW:
1937 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1938 info->data = RXH_IP_SRC | RXH_IP_DST |
1939 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1940 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1941 info->data = RXH_IP_SRC | RXH_IP_DST;
1942 break;
1943 case UDP_V6_FLOW:
1944 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1945 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1946 info->data = RXH_IP_SRC | RXH_IP_DST |
1947 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1948 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1949 info->data = RXH_IP_SRC | RXH_IP_DST;
1950 break;
1951 case SCTP_V6_FLOW:
1952 case AH_ESP_V6_FLOW:
1953 case IPV6_FLOW:
1954 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1955 info->data = RXH_IP_SRC | RXH_IP_DST;
1956 break;
1957 }
1958 return 0;
1959 }
671b0060 1960 case ETHTOOL_GRXRINGS:
f796564a 1961 info->data = pi->nqsets;
671b0060
DM		 1962 return 0;
1963 }
1964 return -EOPNOTSUPP;
1965}
1966
b8ff05a9
DM		 1967static struct ethtool_ops cxgb_ethtool_ops = {
1968 .get_settings = get_settings,
1969 .set_settings = set_settings,
1970 .get_drvinfo = get_drvinfo,
1971 .get_msglevel = get_msglevel,
1972 .set_msglevel = set_msglevel,
1973 .get_ringparam = get_sge_param,
1974 .set_ringparam = set_sge_param,
1975 .get_coalesce = get_coalesce,
1976 .set_coalesce = set_coalesce,
1977 .get_eeprom_len = get_eeprom_len,
1978 .get_eeprom = get_eeprom,
1979 .set_eeprom = set_eeprom,
1980 .get_pauseparam = get_pauseparam,
1981 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 1982 .get_link = ethtool_op_get_link,
1983 .get_strings = get_strings,
c5e06360 1984 .set_phys_id = identify_port,
b8ff05a9
DM		 1985 .nway_reset = restart_autoneg,
1986 .get_sset_count = get_sset_count,
1987 .get_ethtool_stats = get_stats,
1988 .get_regs_len = get_regs_len,
1989 .get_regs = get_regs,
1990 .get_wol = get_wol,
1991 .set_wol = set_wol,
671b0060
DM		 1992 .get_rxnfc = get_rxnfc,
1993 .get_rxfh_indir = get_rss_table,
1994 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 1995 .flash_device = set_flash,
1996};
1997
1998/*
1999 * debugfs support
2000 */
2001
2002static int mem_open(struct inode *inode, struct file *file)
2003{
2004 file->private_data = inode->i_private;
2005 return 0;
2006}
2007
2008static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2009 loff_t *ppos)
2010{
2011 loff_t pos = *ppos;
2012 loff_t avail = file->f_path.dentry->d_inode->i_size;
2013 unsigned int mem = (uintptr_t)file->private_data & 3;
2014 struct adapter *adap = file->private_data - mem;
2015
2016 if (pos < 0)
2017 return -EINVAL;
2018 if (pos >= avail)
2019 return 0;
2020 if (count > avail - pos)
2021 count = avail - pos;
2022
2023 while (count) {
2024 size_t len;
2025 int ret, ofst;
2026 __be32 data[16];
2027
2028 if (mem == MEM_MC)
2029 ret = t4_mc_read(adap, pos, data, NULL);
2030 else
2031 ret = t4_edc_read(adap, mem, pos, data, NULL);
2032 if (ret)
2033 return ret;
2034
2035 ofst = pos % sizeof(data);
2036 len = min(count, sizeof(data) - ofst);
2037 if (copy_to_user(buf, (u8 *)data + ofst, len))
2038 return -EFAULT;
2039
2040 buf += len;
2041 pos += len;
2042 count -= len;
2043 }
2044 count = pos - *ppos;
2045 *ppos = pos;
2046 return count;
2047}
2048
2049static const struct file_operations mem_debugfs_fops = {
2050 .owner = THIS_MODULE,
2051 .open = mem_open,
2052 .read = mem_read,
6038f373 2053 .llseek = default_llseek,
b8ff05a9
DM		 2054};
2055
2056static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2057 unsigned int idx, unsigned int size_mb)
2058{
2059 struct dentry *de;
2060
2061 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2062 (void *)adap + idx, &mem_debugfs_fops);
2063 if (de && de->d_inode)
2064 de->d_inode->i_size = size_mb << 20;
2065}
2066
2067static int __devinit setup_debugfs(struct adapter *adap)
2068{
2069 int i;
2070
2071 if (IS_ERR_OR_NULL(adap->debugfs_root))
2072 return -1;
2073
2074 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2075 if (i & EDRAM0_ENABLE)
2076 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2077 if (i & EDRAM1_ENABLE)
2078 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2079 if (i & EXT_MEM_ENABLE)
2080 add_debugfs_mem(adap, "mc", MEM_MC,
2081 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2082 if (adap->l2t)
2083 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2084 &t4_l2t_fops);
2085 return 0;
2086}
2087
2088/*
2089 * upper-layer driver support
2090 */
2091
2092/*
2093 * Allocate an active-open TID and set it to the supplied value.
2094 */
2095int cxgb4_alloc_atid(struct tid_info *t, void *data)
2096{
2097 int atid = -1;
2098
2099 spin_lock_bh(&t->atid_lock);
2100 if (t->afree) {
2101 union aopen_entry *p = t->afree;
2102
2103 atid = p - t->atid_tab;
2104 t->afree = p->next;
2105 p->data = data;
2106 t->atids_in_use++;
2107 }
2108 spin_unlock_bh(&t->atid_lock);
2109 return atid;
2110}
2111EXPORT_SYMBOL(cxgb4_alloc_atid);
2112
2113/*
2114 * Release an active-open TID.
2115 */
2116void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2117{
2118 union aopen_entry *p = &t->atid_tab[atid];
2119
2120 spin_lock_bh(&t->atid_lock);
2121 p->next = t->afree;
2122 t->afree = p;
2123 t->atids_in_use--;
2124 spin_unlock_bh(&t->atid_lock);
2125}
2126EXPORT_SYMBOL(cxgb4_free_atid);
2127
2128/*
2129 * Allocate a server TID and set it to the supplied value.
2130 */
2131int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2132{
2133 int stid;
2134
2135 spin_lock_bh(&t->stid_lock);
2136 if (family == PF_INET) {
2137 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2138 if (stid < t->nstids)
2139 __set_bit(stid, t->stid_bmap);
2140 else
2141 stid = -1;
2142 } else {
2143 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2144 if (stid < 0)
2145 stid = -1;
2146 }
2147 if (stid >= 0) {
2148 t->stid_tab[stid].data = data;
2149 stid += t->stid_base;
2150 t->stids_in_use++;
2151 }
2152 spin_unlock_bh(&t->stid_lock);
2153 return stid;
2154}
2155EXPORT_SYMBOL(cxgb4_alloc_stid);
2156
2157/*
2158 * Release a server TID.
2159 */
2160void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2161{
2162 stid -= t->stid_base;
2163 spin_lock_bh(&t->stid_lock);
2164 if (family == PF_INET)
2165 __clear_bit(stid, t->stid_bmap);
2166 else
2167 bitmap_release_region(t->stid_bmap, stid, 2);
2168 t->stid_tab[stid].data = NULL;
2169 t->stids_in_use--;
2170 spin_unlock_bh(&t->stid_lock);
2171}
2172EXPORT_SYMBOL(cxgb4_free_stid);
2173
2174/*
2175 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2176 */
2177static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2178 unsigned int tid)
2179{
2180 struct cpl_tid_release *req;
2181
2182 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2183 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2184 INIT_TP_WR(req, tid);
2185 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2186}
2187
2188/*
2189 * Queue a TID release request and if necessary schedule a work queue to
2190 * process it.
2191 */
31b9c19b 2192static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2193 unsigned int tid)
b8ff05a9
DM		 2194{
2195 void **p = &t->tid_tab[tid];
2196 struct adapter *adap = container_of(t, struct adapter, tids);
2197
2198 spin_lock_bh(&adap->tid_release_lock);
2199 *p = adap->tid_release_head;
2200 /* Low 2 bits encode the Tx channel number */
2201 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2202 if (!adap->tid_release_task_busy) {
2203 adap->tid_release_task_busy = true;
2204 schedule_work(&adap->tid_release_task);
2205 }
2206 spin_unlock_bh(&adap->tid_release_lock);
2207}
b8ff05a9
DM		 2208
2209/*
2210 * Process the list of pending TID release requests.
2211 */
2212static void process_tid_release_list(struct work_struct *work)
2213{
2214 struct sk_buff *skb;
2215 struct adapter *adap;
2216
2217 adap = container_of(work, struct adapter, tid_release_task);
2218
2219 spin_lock_bh(&adap->tid_release_lock);
2220 while (adap->tid_release_head) {
2221 void **p = adap->tid_release_head;
2222 unsigned int chan = (uintptr_t)p & 3;
2223 p = (void *)p - chan;
2224
2225 adap->tid_release_head = *p;
2226 *p = NULL;
2227 spin_unlock_bh(&adap->tid_release_lock);
2228
2229 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2230 GFP_KERNEL)))
2231 schedule_timeout_uninterruptible(1);
2232
2233 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2234 t4_ofld_send(adap, skb);
2235 spin_lock_bh(&adap->tid_release_lock);
2236 }
2237 adap->tid_release_task_busy = false;
2238 spin_unlock_bh(&adap->tid_release_lock);
2239}
2240
2241/*
2242 * Release a TID and inform HW. If we are unable to allocate the release
2243 * message we defer to a work queue.
2244 */
2245void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2246{
2247 void *old;
2248 struct sk_buff *skb;
2249 struct adapter *adap = container_of(t, struct adapter, tids);
2250
2251 old = t->tid_tab[tid];
2252 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2253 if (likely(skb)) {
2254 t->tid_tab[tid] = NULL;
2255 mk_tid_release(skb, chan, tid);
2256 t4_ofld_send(adap, skb);
2257 } else
2258 cxgb4_queue_tid_release(t, chan, tid);
2259 if (old)
2260 atomic_dec(&t->tids_in_use);
2261}
2262EXPORT_SYMBOL(cxgb4_remove_tid);
2263
2264/*
2265 * Allocate and initialize the TID tables. Returns 0 on success.
2266 */
2267static int tid_init(struct tid_info *t)
2268{
2269 size_t size;
2270 unsigned int natids = t->natids;
2271
2272 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2273 t->nstids * sizeof(*t->stid_tab) +
2274 BITS_TO_LONGS(t->nstids) * sizeof(long);
2275 t->tid_tab = t4_alloc_mem(size);
2276 if (!t->tid_tab)
2277 return -ENOMEM;
2278
2279 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2280 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2281 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2282 spin_lock_init(&t->stid_lock);
2283 spin_lock_init(&t->atid_lock);
2284
2285 t->stids_in_use = 0;
2286 t->afree = NULL;
2287 t->atids_in_use = 0;
2288 atomic_set(&t->tids_in_use, 0);
2289
2290 /* Setup the free list for atid_tab and clear the stid bitmap. */
2291 if (natids) {
2292 while (--natids)
2293 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2294 t->afree = t->atid_tab;
2295 }
2296 bitmap_zero(t->stid_bmap, t->nstids);
2297 return 0;
2298}
2299
2300/**
2301 * cxgb4_create_server - create an IP server
2302 * @dev: the device
2303 * @stid: the server TID
2304 * @sip: local IP address to bind server to
2305 * @sport: the server's TCP port
2306 * @queue: queue to direct messages from this server to
2307 *
2308 * Create an IP server for the given port and address.
2309 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2310 */
2311int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2312 __be32 sip, __be16 sport, unsigned int queue)
2313{
2314 unsigned int chan;
2315 struct sk_buff *skb;
2316 struct adapter *adap;
2317 struct cpl_pass_open_req *req;
2318
2319 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2320 if (!skb)
2321 return -ENOMEM;
2322
2323 adap = netdev2adap(dev);
2324 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2325 INIT_TP_WR(req, 0);
2326 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2327 req->local_port = sport;
2328 req->peer_port = htons(0);
2329 req->local_ip = sip;
2330 req->peer_ip = htonl(0);
e46dab4d 2331 chan = rxq_to_chan(&adap->sge, queue);
b8ff05a9
DM		 2332 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2333 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2334 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2335 return t4_mgmt_tx(adap, skb);
2336}
2337EXPORT_SYMBOL(cxgb4_create_server);
2338
b8ff05a9
DM		 2339/**
2340 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2341 * @mtus: the HW MTU table
2342 * @mtu: the target MTU
2343 * @idx: index of selected entry in the MTU table
2344 *
2345 * Returns the index and the value in the HW MTU table that is closest to
2346 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2347 * table, in which case that smallest available value is selected.
2348 */
2349unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2350 unsigned int *idx)
2351{
2352 unsigned int i = 0;
2353
2354 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2355 ++i;
2356 if (idx)
2357 *idx = i;
2358 return mtus[i];
2359}
2360EXPORT_SYMBOL(cxgb4_best_mtu);
2361
2362/**
2363 * cxgb4_port_chan - get the HW channel of a port
2364 * @dev: the net device for the port
2365 *
2366 * Return the HW Tx channel of the given port.
2367 */
2368unsigned int cxgb4_port_chan(const struct net_device *dev)
2369{
2370 return netdev2pinfo(dev)->tx_chan;
2371}
2372EXPORT_SYMBOL(cxgb4_port_chan);
2373
2374/**
2375 * cxgb4_port_viid - get the VI id of a port
2376 * @dev: the net device for the port
2377 *
2378 * Return the VI id of the given port.
2379 */
2380unsigned int cxgb4_port_viid(const struct net_device *dev)
2381{
2382 return netdev2pinfo(dev)->viid;
2383}
2384EXPORT_SYMBOL(cxgb4_port_viid);
2385
2386/**
2387 * cxgb4_port_idx - get the index of a port
2388 * @dev: the net device for the port
2389 *
2390 * Return the index of the given port.
2391 */
2392unsigned int cxgb4_port_idx(const struct net_device *dev)
2393{
2394 return netdev2pinfo(dev)->port_id;
2395}
2396EXPORT_SYMBOL(cxgb4_port_idx);
2397
b8ff05a9
DM		 2398void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2399 struct tp_tcp_stats *v6)
2400{
2401 struct adapter *adap = pci_get_drvdata(pdev);
2402
2403 spin_lock(&adap->stats_lock);
2404 t4_tp_get_tcp_stats(adap, v4, v6);
2405 spin_unlock(&adap->stats_lock);
2406}
2407EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2408
2409void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2410 const unsigned int *pgsz_order)
2411{
2412 struct adapter *adap = netdev2adap(dev);
2413
2414 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2415 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2416 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2417 HPZ3(pgsz_order[3]));
2418}
2419EXPORT_SYMBOL(cxgb4_iscsi_init);
2420
2421static struct pci_driver cxgb4_driver;
2422
2423static void check_neigh_update(struct neighbour *neigh)
2424{
2425 const struct device *parent;
2426 const struct net_device *netdev = neigh->dev;
2427
2428 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2429 netdev = vlan_dev_real_dev(netdev);
2430 parent = netdev->dev.parent;
2431 if (parent && parent->driver == &cxgb4_driver.driver)
2432 t4_l2t_update(dev_get_drvdata(parent), neigh);
2433}
2434
2435static int netevent_cb(struct notifier_block *nb, unsigned long event,
2436 void *data)
2437{
2438 switch (event) {
2439 case NETEVENT_NEIGH_UPDATE:
2440 check_neigh_update(data);
2441 break;
b8ff05a9
DM		 2442 case NETEVENT_REDIRECT:
2443 default:
2444 break;
2445 }
2446 return 0;
2447}
2448
2449static bool netevent_registered;
2450static struct notifier_block cxgb4_netevent_nb = {
2451 .notifier_call = netevent_cb
2452};
2453
2454static void uld_attach(struct adapter *adap, unsigned int uld)
2455{
2456 void *handle;
2457 struct cxgb4_lld_info lli;
2458
2459 lli.pdev = adap->pdev;
2460 lli.l2t = adap->l2t;
2461 lli.tids = &adap->tids;
2462 lli.ports = adap->port;
2463 lli.vr = &adap->vres;
2464 lli.mtus = adap->params.mtus;
2465 if (uld == CXGB4_ULD_RDMA) {
2466 lli.rxq_ids = adap->sge.rdma_rxq;
2467 lli.nrxq = adap->sge.rdmaqs;
2468 } else if (uld == CXGB4_ULD_ISCSI) {
2469 lli.rxq_ids = adap->sge.ofld_rxq;
2470 lli.nrxq = adap->sge.ofldqsets;
2471 }
2472 lli.ntxq = adap->sge.ofldqsets;
2473 lli.nchan = adap->params.nports;
2474 lli.nports = adap->params.nports;
2475 lli.wr_cred = adap->params.ofldq_wr_cred;
2476 lli.adapter_type = adap->params.rev;
2477 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2478 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 2479 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2480 (adap->fn * 4));
b8ff05a9 2481 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 2482 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2483 (adap->fn * 4));
b8ff05a9
DM		 2484 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2485 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2486 lli.fw_vers = adap->params.fw_vers;
2487
2488 handle = ulds[uld].add(&lli);
2489 if (IS_ERR(handle)) {
2490 dev_warn(adap->pdev_dev,
2491 "could not attach to the %s driver, error %ld\n",
2492 uld_str[uld], PTR_ERR(handle));
2493 return;
2494 }
2495
2496 adap->uld_handle[uld] = handle;
2497
2498 if (!netevent_registered) {
2499 register_netevent_notifier(&cxgb4_netevent_nb);
2500 netevent_registered = true;
2501 }
e29f5dbc
DM		 2502
2503 if (adap->flags & FULL_INIT_DONE)
2504 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 2505}
2506
2507static void attach_ulds(struct adapter *adap)
2508{
2509 unsigned int i;
2510
2511 mutex_lock(&uld_mutex);
2512 list_add_tail(&adap->list_node, &adapter_list);
2513 for (i = 0; i < CXGB4_ULD_MAX; i++)
2514 if (ulds[i].add)
2515 uld_attach(adap, i);
2516 mutex_unlock(&uld_mutex);
2517}
2518
2519static void detach_ulds(struct adapter *adap)
2520{
2521 unsigned int i;
2522
2523 mutex_lock(&uld_mutex);
2524 list_del(&adap->list_node);
2525 for (i = 0; i < CXGB4_ULD_MAX; i++)
2526 if (adap->uld_handle[i]) {
2527 ulds[i].state_change(adap->uld_handle[i],
2528 CXGB4_STATE_DETACH);
2529 adap->uld_handle[i] = NULL;
2530 }
2531 if (netevent_registered && list_empty(&adapter_list)) {
2532 unregister_netevent_notifier(&cxgb4_netevent_nb);
2533 netevent_registered = false;
2534 }
2535 mutex_unlock(&uld_mutex);
2536}
2537
2538static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2539{
2540 unsigned int i;
2541
2542 mutex_lock(&uld_mutex);
2543 for (i = 0; i < CXGB4_ULD_MAX; i++)
2544 if (adap->uld_handle[i])
2545 ulds[i].state_change(adap->uld_handle[i], new_state);
2546 mutex_unlock(&uld_mutex);
2547}
2548
2549/**
2550 * cxgb4_register_uld - register an upper-layer driver
2551 * @type: the ULD type
2552 * @p: the ULD methods
2553 *
2554 * Registers an upper-layer driver with this driver and notifies the ULD
2555 * about any presently available devices that support its type. Returns
2556 * %-EBUSY if a ULD of the same type is already registered.
2557 */
2558int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2559{
2560 int ret = 0;
2561 struct adapter *adap;
2562
2563 if (type >= CXGB4_ULD_MAX)
2564 return -EINVAL;
2565 mutex_lock(&uld_mutex);
2566 if (ulds[type].add) {
2567 ret = -EBUSY;
2568 goto out;
2569 }
2570 ulds[type] = *p;
2571 list_for_each_entry(adap, &adapter_list, list_node)
2572 uld_attach(adap, type);
2573out: mutex_unlock(&uld_mutex);
2574 return ret;
2575}
2576EXPORT_SYMBOL(cxgb4_register_uld);
2577
2578/**
2579 * cxgb4_unregister_uld - unregister an upper-layer driver
2580 * @type: the ULD type
2581 *
2582 * Unregisters an existing upper-layer driver.
2583 */
2584int cxgb4_unregister_uld(enum cxgb4_uld type)
2585{
2586 struct adapter *adap;
2587
2588 if (type >= CXGB4_ULD_MAX)
2589 return -EINVAL;
2590 mutex_lock(&uld_mutex);
2591 list_for_each_entry(adap, &adapter_list, list_node)
2592 adap->uld_handle[type] = NULL;
2593 ulds[type].add = NULL;
2594 mutex_unlock(&uld_mutex);
2595 return 0;
2596}
2597EXPORT_SYMBOL(cxgb4_unregister_uld);
2598
2599/**
2600 * cxgb_up - enable the adapter
2601 * @adap: adapter being enabled
2602 *
2603 * Called when the first port is enabled, this function performs the
2604 * actions necessary to make an adapter operational, such as completing
2605 * the initialization of HW modules, and enabling interrupts.
2606 *
2607 * Must be called with the rtnl lock held.
2608 */
2609static int cxgb_up(struct adapter *adap)
2610{
aaefae9b 2611 int err;
b8ff05a9 2612
aaefae9b
DM		 2613 err = setup_sge_queues(adap);
2614 if (err)
2615 goto out;
2616 err = setup_rss(adap);
2617 if (err)
2618 goto freeq;
b8ff05a9
DM		 2619
2620 if (adap->flags & USING_MSIX) {
aaefae9b 2621 name_msix_vecs(adap);
b8ff05a9
DM		 2622 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2623 adap->msix_info[0].desc, adap);
2624 if (err)
2625 goto irq_err;
2626
2627 err = request_msix_queue_irqs(adap);
2628 if (err) {
2629 free_irq(adap->msix_info[0].vec, adap);
2630 goto irq_err;
2631 }
2632 } else {
2633 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2634 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 2635 adap->port[0]->name, adap);
b8ff05a9
DM		 2636 if (err)
2637 goto irq_err;
2638 }
2639 enable_rx(adap);
2640 t4_sge_start(adap);
2641 t4_intr_enable(adap);
aaefae9b 2642 adap->flags |= FULL_INIT_DONE;
b8ff05a9
DM		 2643 notify_ulds(adap, CXGB4_STATE_UP);
2644 out:
2645 return err;
2646 irq_err:
2647 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 2648 freeq:
2649 t4_free_sge_resources(adap);
b8ff05a9
DM		 2650 goto out;
2651}
2652
2653static void cxgb_down(struct adapter *adapter)
2654{
2655 t4_intr_disable(adapter);
2656 cancel_work_sync(&adapter->tid_release_task);
2657 adapter->tid_release_task_busy = false;
204dc3c0 2658 adapter->tid_release_head = NULL;
b8ff05a9
DM		 2659
2660 if (adapter->flags & USING_MSIX) {
2661 free_msix_queue_irqs(adapter);
2662 free_irq(adapter->msix_info[0].vec, adapter);
2663 } else
2664 free_irq(adapter->pdev->irq, adapter);
2665 quiesce_rx(adapter);
aaefae9b
DM		 2666 t4_sge_stop(adapter);
2667 t4_free_sge_resources(adapter);
2668 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 2669}
2670
2671/*
2672 * net_device operations
2673 */
2674static int cxgb_open(struct net_device *dev)
2675{
2676 int err;
2677 struct port_info *pi = netdev_priv(dev);
2678 struct adapter *adapter = pi->adapter;
2679
6a3c869a
DM		 2680 netif_carrier_off(dev);
2681
aaefae9b
DM		 2682 if (!(adapter->flags & FULL_INIT_DONE)) {
2683 err = cxgb_up(adapter);
2684 if (err < 0)
2685 return err;
2686 }
b8ff05a9 2687
f68707b8
DM		 2688 err = link_start(dev);
2689 if (!err)
2690 netif_tx_start_all_queues(dev);
2691 return err;
b8ff05a9
DM		 2692}
2693
2694static int cxgb_close(struct net_device *dev)
2695{
b8ff05a9
DM		 2696 struct port_info *pi = netdev_priv(dev);
2697 struct adapter *adapter = pi->adapter;
2698
2699 netif_tx_stop_all_queues(dev);
2700 netif_carrier_off(dev);
060e0c75 2701 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 2702}
2703
f5152c90
DM		 2704static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2705 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 2706{
2707 struct port_stats stats;
2708 struct port_info *p = netdev_priv(dev);
2709 struct adapter *adapter = p->adapter;
b8ff05a9
DM		 2710
2711 spin_lock(&adapter->stats_lock);
2712 t4_get_port_stats(adapter, p->tx_chan, &stats);
2713 spin_unlock(&adapter->stats_lock);
2714
2715 ns->tx_bytes = stats.tx_octets;
2716 ns->tx_packets = stats.tx_frames;
2717 ns->rx_bytes = stats.rx_octets;
2718 ns->rx_packets = stats.rx_frames;
2719 ns->multicast = stats.rx_mcast_frames;
2720
2721 /* detailed rx_errors */
2722 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2723 stats.rx_runt;
2724 ns->rx_over_errors = 0;
2725 ns->rx_crc_errors = stats.rx_fcs_err;
2726 ns->rx_frame_errors = stats.rx_symbol_err;
2727 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2728 stats.rx_ovflow2 + stats.rx_ovflow3 +
2729 stats.rx_trunc0 + stats.rx_trunc1 +
2730 stats.rx_trunc2 + stats.rx_trunc3;
2731 ns->rx_missed_errors = 0;
2732
2733 /* detailed tx_errors */
2734 ns->tx_aborted_errors = 0;
2735 ns->tx_carrier_errors = 0;
2736 ns->tx_fifo_errors = 0;
2737 ns->tx_heartbeat_errors = 0;
2738 ns->tx_window_errors = 0;
2739
2740 ns->tx_errors = stats.tx_error_frames;
2741 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2742 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2743 return ns;
2744}
2745
2746static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2747{
060e0c75 2748 unsigned int mbox;
b8ff05a9
DM		 2749 int ret = 0, prtad, devad;
2750 struct port_info *pi = netdev_priv(dev);
2751 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2752
2753 switch (cmd) {
2754 case SIOCGMIIPHY:
2755 if (pi->mdio_addr < 0)
2756 return -EOPNOTSUPP;
2757 data->phy_id = pi->mdio_addr;
2758 break;
2759 case SIOCGMIIREG:
2760 case SIOCSMIIREG:
2761 if (mdio_phy_id_is_c45(data->phy_id)) {
2762 prtad = mdio_phy_id_prtad(data->phy_id);
2763 devad = mdio_phy_id_devad(data->phy_id);
2764 } else if (data->phy_id < 32) {
2765 prtad = data->phy_id;
2766 devad = 0;
2767 data->reg_num &= 0x1f;
2768 } else
2769 return -EINVAL;
2770
060e0c75 2771 mbox = pi->adapter->fn;
b8ff05a9 2772 if (cmd == SIOCGMIIREG)
060e0c75 2773 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 2774 data->reg_num, &data->val_out);
2775 else
060e0c75 2776 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 2777 data->reg_num, data->val_in);
2778 break;
2779 default:
2780 return -EOPNOTSUPP;
2781 }
2782 return ret;
2783}
2784
2785static void cxgb_set_rxmode(struct net_device *dev)
2786{
2787 /* unfortunately we can't return errors to the stack */
2788 set_rxmode(dev, -1, false);
2789}
2790
2791static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2792{
2793 int ret;
2794 struct port_info *pi = netdev_priv(dev);
2795
2796 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2797 return -EINVAL;
060e0c75
DM		 2798 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2799 -1, -1, -1, true);
b8ff05a9
DM		 2800 if (!ret)
2801 dev->mtu = new_mtu;
2802 return ret;
2803}
2804
2805static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2806{
2807 int ret;
2808 struct sockaddr *addr = p;
2809 struct port_info *pi = netdev_priv(dev);
2810
2811 if (!is_valid_ether_addr(addr->sa_data))
2812 return -EINVAL;
2813
060e0c75
DM		 2814 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2815 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 2816 if (ret < 0)
2817 return ret;
2818
2819 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2820 pi->xact_addr_filt = ret;
2821 return 0;
2822}
2823
b8ff05a9
DM		 2824#ifdef CONFIG_NET_POLL_CONTROLLER
2825static void cxgb_netpoll(struct net_device *dev)
2826{
2827 struct port_info *pi = netdev_priv(dev);
2828 struct adapter *adap = pi->adapter;
2829
2830 if (adap->flags & USING_MSIX) {
2831 int i;
2832 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2833
2834 for (i = pi->nqsets; i; i--, rx++)
2835 t4_sge_intr_msix(0, &rx->rspq);
2836 } else
2837 t4_intr_handler(adap)(0, adap);
2838}
2839#endif
2840
2841static const struct net_device_ops cxgb4_netdev_ops = {
2842 .ndo_open = cxgb_open,
2843 .ndo_stop = cxgb_close,
2844 .ndo_start_xmit = t4_eth_xmit,
9be793bf 2845 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 2846 .ndo_set_rx_mode = cxgb_set_rxmode,
2847 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 2848 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 2849 .ndo_validate_addr = eth_validate_addr,
2850 .ndo_do_ioctl = cxgb_ioctl,
2851 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 2852#ifdef CONFIG_NET_POLL_CONTROLLER
2853 .ndo_poll_controller = cxgb_netpoll,
2854#endif
2855};
2856
2857void t4_fatal_err(struct adapter *adap)
2858{
2859 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2860 t4_intr_disable(adap);
2861 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2862}
2863
2864static void setup_memwin(struct adapter *adap)
2865{
2866 u32 bar0;
2867
2868 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2869 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2870 (bar0 + MEMWIN0_BASE) | BIR(0) |
2871 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2872 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2873 (bar0 + MEMWIN1_BASE) | BIR(0) |
2874 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2875 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2876 (bar0 + MEMWIN2_BASE) | BIR(0) |
2877 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1ae970e0
DM		 2878 if (adap->vres.ocq.size) {
2879 unsigned int start, sz_kb;
2880
2881 start = pci_resource_start(adap->pdev, 2) +
2882 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2883 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2884 t4_write_reg(adap,
2885 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2886 start | BIR(1) | WINDOW(ilog2(sz_kb)));
2887 t4_write_reg(adap,
2888 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2889 adap->vres.ocq.start);
2890 t4_read_reg(adap,
2891 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2892 }
b8ff05a9
DM		 2893}
2894
02b5fb8e
DM		 2895static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2896{
2897 u32 v;
2898 int ret;
2899
2900 /* get device capabilities */
2901 memset(c, 0, sizeof(*c));
2902 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2903 FW_CMD_REQUEST | FW_CMD_READ);
2904 c->retval_len16 = htonl(FW_LEN16(*c));
060e0c75 2905 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 2906 if (ret < 0)
2907 return ret;
2908
2909 /* select capabilities we'll be using */
2910 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2911 if (!vf_acls)
2912 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2913 else
2914 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2915 } else if (vf_acls) {
2916 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2917 return ret;
2918 }
2919 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2920 FW_CMD_REQUEST | FW_CMD_WRITE);
060e0c75 2921 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 2922 if (ret < 0)
2923 return ret;
2924
060e0c75 2925 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e
DM		 2926 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2927 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2928 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2929 if (ret < 0)
2930 return ret;
2931
060e0c75
DM		 2932 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
2933 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 2934 if (ret < 0)
2935 return ret;
2936
2937 t4_sge_init(adap);
2938
02b5fb8e
DM		 2939 /* tweak some settings */
2940 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2941 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2942 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2943 v = t4_read_reg(adap, TP_PIO_DATA);
2944 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
060e0c75
DM		 2945
2946 /* get basic stuff going */
2947 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 2948}
2949
b8ff05a9
DM		 2950/*
2951 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2952 */
2953#define MAX_ATIDS 8192U
2954
2955/*
2956 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2957 */
2958static int adap_init0(struct adapter *adap)
2959{
2960 int ret;
2961 u32 v, port_vec;
2962 enum dev_state state;
2963 u32 params[7], val[7];
2964 struct fw_caps_config_cmd c;
2965
2966 ret = t4_check_fw_version(adap);
2967 if (ret == -EINVAL || ret > 0) {
2968 if (upgrade_fw(adap) >= 0) /* recache FW version */
2969 ret = t4_check_fw_version(adap);
2970 }
2971 if (ret < 0)
2972 return ret;
2973
2974 /* contact FW, request master */
060e0c75 2975 ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
b8ff05a9
DM		 2976 if (ret < 0) {
2977 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2978 ret);
2979 return ret;
2980 }
2981
2982 /* reset device */
060e0c75 2983 ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
b8ff05a9
DM		 2984 if (ret < 0)
2985 goto bye;
2986
b8ff05a9
DM		 2987 for (v = 0; v < SGE_NTIMERS - 1; v++)
2988 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
2989 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
2990 adap->sge.counter_val[0] = 1;
2991 for (v = 1; v < SGE_NCOUNTERS; v++)
2992 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
2993 THRESHOLD_3_MASK);
b8ff05a9
DM		 2994#define FW_PARAM_DEV(param) \
2995 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2996 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2997
a0881cab 2998 params[0] = FW_PARAM_DEV(CCLK);
060e0c75 2999 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
a0881cab
DM		 3000 if (ret < 0)
3001 goto bye;
3002 adap->params.vpd.cclk = val[0];
3003
3004 ret = adap_init1(adap, &c);
3005 if (ret < 0)
3006 goto bye;
3007
b8ff05a9
DM		 3008#define FW_PARAM_PFVF(param) \
3009 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
060e0c75
DM		 3010 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3011 FW_PARAMS_PARAM_Y(adap->fn))
b8ff05a9
DM		 3012
3013 params[0] = FW_PARAM_DEV(PORTVEC);
3014 params[1] = FW_PARAM_PFVF(L2T_START);
3015 params[2] = FW_PARAM_PFVF(L2T_END);
3016 params[3] = FW_PARAM_PFVF(FILTER_START);
3017 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d
DM		 3018 params[5] = FW_PARAM_PFVF(IQFLINT_START);
3019 params[6] = FW_PARAM_PFVF(EQ_START);
3020 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
b8ff05a9
DM		 3021 if (ret < 0)
3022 goto bye;
3023 port_vec = val[0];
3024 adap->tids.ftid_base = val[3];
3025 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d
DM		 3026 adap->sge.ingr_start = val[5];
3027 adap->sge.egr_start = val[6];
b8ff05a9
DM		 3028
3029 if (c.ofldcaps) {
3030 /* query offload-related parameters */
3031 params[0] = FW_PARAM_DEV(NTID);
3032 params[1] = FW_PARAM_PFVF(SERVER_START);
3033 params[2] = FW_PARAM_PFVF(SERVER_END);
3034 params[3] = FW_PARAM_PFVF(TDDP_START);
3035 params[4] = FW_PARAM_PFVF(TDDP_END);
3036 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
060e0c75
DM		 3037 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3038 val);
b8ff05a9
DM		 3039 if (ret < 0)
3040 goto bye;
3041 adap->tids.ntids = val[0];
3042 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3043 adap->tids.stid_base = val[1];
3044 adap->tids.nstids = val[2] - val[1] + 1;
3045 adap->vres.ddp.start = val[3];
3046 adap->vres.ddp.size = val[4] - val[3] + 1;
3047 adap->params.ofldq_wr_cred = val[5];
3048 adap->params.offload = 1;
3049 }
3050 if (c.rdmacaps) {
3051 params[0] = FW_PARAM_PFVF(STAG_START);
3052 params[1] = FW_PARAM_PFVF(STAG_END);
3053 params[2] = FW_PARAM_PFVF(RQ_START);
3054 params[3] = FW_PARAM_PFVF(RQ_END);
3055 params[4] = FW_PARAM_PFVF(PBL_START);
3056 params[5] = FW_PARAM_PFVF(PBL_END);
060e0c75
DM		 3057 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3058 val);
b8ff05a9
DM		 3059 if (ret < 0)
3060 goto bye;
3061 adap->vres.stag.start = val[0];
3062 adap->vres.stag.size = val[1] - val[0] + 1;
3063 adap->vres.rq.start = val[2];
3064 adap->vres.rq.size = val[3] - val[2] + 1;
3065 adap->vres.pbl.start = val[4];
3066 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 3067
3068 params[0] = FW_PARAM_PFVF(SQRQ_START);
3069 params[1] = FW_PARAM_PFVF(SQRQ_END);
3070 params[2] = FW_PARAM_PFVF(CQ_START);
3071 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 3072 params[4] = FW_PARAM_PFVF(OCQ_START);
3073 params[5] = FW_PARAM_PFVF(OCQ_END);
060e0c75
DM		 3074 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3075 val);
a0881cab
DM		 3076 if (ret < 0)
3077 goto bye;
3078 adap->vres.qp.start = val[0];
3079 adap->vres.qp.size = val[1] - val[0] + 1;
3080 adap->vres.cq.start = val[2];
3081 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 3082 adap->vres.ocq.start = val[4];
3083 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9
DM		 3084 }
3085 if (c.iscsicaps) {
3086 params[0] = FW_PARAM_PFVF(ISCSI_START);
3087 params[1] = FW_PARAM_PFVF(ISCSI_END);
060e0c75
DM		 3088 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3089 val);
b8ff05a9
DM		 3090 if (ret < 0)
3091 goto bye;
3092 adap->vres.iscsi.start = val[0];
3093 adap->vres.iscsi.size = val[1] - val[0] + 1;
3094 }
3095#undef FW_PARAM_PFVF
3096#undef FW_PARAM_DEV
3097
3098 adap->params.nports = hweight32(port_vec);
3099 adap->params.portvec = port_vec;
3100 adap->flags |= FW_OK;
3101
3102 /* These are finalized by FW initialization, load their values now */
3103 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3104 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3105 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3106 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3107 adap->params.b_wnd);
7ee9ff94
CL		 3108
3109#ifdef CONFIG_PCI_IOV
3110 /*
3111 * Provision resource limits for Virtual Functions. We currently
3112 * grant them all the same static resource limits except for the Port
3113 * Access Rights Mask which we're assigning based on the PF. All of
3114 * the static provisioning stuff for both the PF and VF really needs
3115 * to be managed in a persistent manner for each device which the
3116 * firmware controls.
3117 */
3118 {
3119 int pf, vf;
3120
3121 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3122 if (num_vf[pf] <= 0)
3123 continue;
3124
3125 /* VF numbering starts at 1! */
3126 for (vf = 1; vf <= num_vf[pf]; vf++) {
060e0c75 3127 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
7ee9ff94
CL		 3128 VFRES_NEQ, VFRES_NETHCTRL,
3129 VFRES_NIQFLINT, VFRES_NIQ,
3130 VFRES_TC, VFRES_NVI,
3131 FW_PFVF_CMD_CMASK_MASK,
3132 pfvfres_pmask(adap, pf, vf),
3133 VFRES_NEXACTF,
3134 VFRES_R_CAPS, VFRES_WX_CAPS);
3135 if (ret < 0)
3136 dev_warn(adap->pdev_dev, "failed to "
3137 "provision pf/vf=%d/%d; "
3138 "err=%d\n", pf, vf, ret);
3139 }
3140 }
3141 }
3142#endif
3143
1ae970e0 3144 setup_memwin(adap);
b8ff05a9
DM		 3145 return 0;
3146
3147 /*
3148 * If a command timed out or failed with EIO FW does not operate within
3149 * its spec or something catastrophic happened to HW/FW, stop issuing
3150 * commands.
3151 */
3152bye: if (ret != -ETIMEDOUT && ret != -EIO)
060e0c75 3153 t4_fw_bye(adap, adap->fn);
b8ff05a9
DM		 3154 return ret;
3155}
3156
204dc3c0
DM		 3157/* EEH callbacks */
3158
3159static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3160 pci_channel_state_t state)
3161{
3162 int i;
3163 struct adapter *adap = pci_get_drvdata(pdev);
3164
3165 if (!adap)
3166 goto out;
3167
3168 rtnl_lock();
3169 adap->flags &= ~FW_OK;
3170 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3171 for_each_port(adap, i) {
3172 struct net_device *dev = adap->port[i];
3173
3174 netif_device_detach(dev);
3175 netif_carrier_off(dev);
3176 }
3177 if (adap->flags & FULL_INIT_DONE)
3178 cxgb_down(adap);
3179 rtnl_unlock();
3180 pci_disable_device(pdev);
3181out: return state == pci_channel_io_perm_failure ?
3182 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3183}
3184
3185static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3186{
3187 int i, ret;
3188 struct fw_caps_config_cmd c;
3189 struct adapter *adap = pci_get_drvdata(pdev);
3190
3191 if (!adap) {
3192 pci_restore_state(pdev);
3193 pci_save_state(pdev);
3194 return PCI_ERS_RESULT_RECOVERED;
3195 }
3196
3197 if (pci_enable_device(pdev)) {
3198 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3199 return PCI_ERS_RESULT_DISCONNECT;
3200 }
3201
3202 pci_set_master(pdev);
3203 pci_restore_state(pdev);
3204 pci_save_state(pdev);
3205 pci_cleanup_aer_uncorrect_error_status(pdev);
3206
3207 if (t4_wait_dev_ready(adap) < 0)
3208 return PCI_ERS_RESULT_DISCONNECT;
060e0c75 3209 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
204dc3c0
DM		 3210 return PCI_ERS_RESULT_DISCONNECT;
3211 adap->flags |= FW_OK;
3212 if (adap_init1(adap, &c))
3213 return PCI_ERS_RESULT_DISCONNECT;
3214
3215 for_each_port(adap, i) {
3216 struct port_info *p = adap2pinfo(adap, i);
3217
060e0c75
DM		 3218 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3219 NULL, NULL);
204dc3c0
DM		 3220 if (ret < 0)
3221 return PCI_ERS_RESULT_DISCONNECT;
3222 p->viid = ret;
3223 p->xact_addr_filt = -1;
3224 }
3225
3226 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3227 adap->params.b_wnd);
1ae970e0 3228 setup_memwin(adap);
204dc3c0
DM		 3229 if (cxgb_up(adap))
3230 return PCI_ERS_RESULT_DISCONNECT;
3231 return PCI_ERS_RESULT_RECOVERED;
3232}
3233
3234static void eeh_resume(struct pci_dev *pdev)
3235{
3236 int i;
3237 struct adapter *adap = pci_get_drvdata(pdev);
3238
3239 if (!adap)
3240 return;
3241
3242 rtnl_lock();
3243 for_each_port(adap, i) {
3244 struct net_device *dev = adap->port[i];
3245
3246 if (netif_running(dev)) {
3247 link_start(dev);
3248 cxgb_set_rxmode(dev);
3249 }
3250 netif_device_attach(dev);
3251 }
3252 rtnl_unlock();
3253}
3254
3255static struct pci_error_handlers cxgb4_eeh = {
3256 .error_detected = eeh_err_detected,
3257 .slot_reset = eeh_slot_reset,
3258 .resume = eeh_resume,
3259};
3260
b8ff05a9
DM		 3261static inline bool is_10g_port(const struct link_config *lc)
3262{
3263 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3264}
3265
3266static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3267 unsigned int size, unsigned int iqe_size)
3268{
3269 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3270 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3271 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3272 q->iqe_len = iqe_size;
3273 q->size = size;
3274}
3275
3276/*
3277 * Perform default configuration of DMA queues depending on the number and type
3278 * of ports we found and the number of available CPUs. Most settings can be
3279 * modified by the admin prior to actual use.
3280 */
3281static void __devinit cfg_queues(struct adapter *adap)
3282{
3283 struct sge *s = &adap->sge;
3284 int i, q10g = 0, n10g = 0, qidx = 0;
3285
3286 for_each_port(adap, i)
3287 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3288
3289 /*
3290 * We default to 1 queue per non-10G port and up to # of cores queues
3291 * per 10G port.
3292 */
3293 if (n10g)
3294 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3295 if (q10g > num_online_cpus())
3296 q10g = num_online_cpus();
3297
3298 for_each_port(adap, i) {
3299 struct port_info *pi = adap2pinfo(adap, i);
3300
3301 pi->first_qset = qidx;
3302 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3303 qidx += pi->nqsets;
3304 }
3305
3306 s->ethqsets = qidx;
3307 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3308
3309 if (is_offload(adap)) {
3310 /*
3311 * For offload we use 1 queue/channel if all ports are up to 1G,
3312 * otherwise we divide all available queues amongst the channels
3313 * capped by the number of available cores.
3314 */
3315 if (n10g) {
3316 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3317 num_online_cpus());
3318 s->ofldqsets = roundup(i, adap->params.nports);
3319 } else
3320 s->ofldqsets = adap->params.nports;
3321 /* For RDMA one Rx queue per channel suffices */
3322 s->rdmaqs = adap->params.nports;
3323 }
3324
3325 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3326 struct sge_eth_rxq *r = &s->ethrxq[i];
3327
3328 init_rspq(&r->rspq, 0, 0, 1024, 64);
3329 r->fl.size = 72;
3330 }
3331
3332 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3333 s->ethtxq[i].q.size = 1024;
3334
3335 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3336 s->ctrlq[i].q.size = 512;
3337
3338 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3339 s->ofldtxq[i].q.size = 1024;
3340
3341 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3342 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3343
3344 init_rspq(&r->rspq, 0, 0, 1024, 64);
3345 r->rspq.uld = CXGB4_ULD_ISCSI;
3346 r->fl.size = 72;
3347 }
3348
3349 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3350 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3351
3352 init_rspq(&r->rspq, 0, 0, 511, 64);
3353 r->rspq.uld = CXGB4_ULD_RDMA;
3354 r->fl.size = 72;
3355 }
3356
3357 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3358 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3359}
3360
3361/*
3362 * Reduce the number of Ethernet queues across all ports to at most n.
3363 * n provides at least one queue per port.
3364 */
3365static void __devinit reduce_ethqs(struct adapter *adap, int n)
3366{
3367 int i;
3368 struct port_info *pi;
3369
3370 while (n < adap->sge.ethqsets)
3371 for_each_port(adap, i) {
3372 pi = adap2pinfo(adap, i);
3373 if (pi->nqsets > 1) {
3374 pi->nqsets--;
3375 adap->sge.ethqsets--;
3376 if (adap->sge.ethqsets <= n)
3377 break;
3378 }
3379 }
3380
3381 n = 0;
3382 for_each_port(adap, i) {
3383 pi = adap2pinfo(adap, i);
3384 pi->first_qset = n;
3385 n += pi->nqsets;
3386 }
3387}
3388
3389/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3390#define EXTRA_VECS 2
3391
3392static int __devinit enable_msix(struct adapter *adap)
3393{
3394 int ofld_need = 0;
3395 int i, err, want, need;
3396 struct sge *s = &adap->sge;
3397 unsigned int nchan = adap->params.nports;
3398 struct msix_entry entries[MAX_INGQ + 1];
3399
3400 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3401 entries[i].entry = i;
3402
3403 want = s->max_ethqsets + EXTRA_VECS;
3404 if (is_offload(adap)) {
3405 want += s->rdmaqs + s->ofldqsets;
3406 /* need nchan for each possible ULD */
3407 ofld_need = 2 * nchan;
3408 }
3409 need = adap->params.nports + EXTRA_VECS + ofld_need;
3410
3411 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3412 want = err;
3413
3414 if (!err) {
3415 /*
3416 * Distribute available vectors to the various queue groups.
3417 * Every group gets its minimum requirement and NIC gets top
3418 * priority for leftovers.
3419 */
3420 i = want - EXTRA_VECS - ofld_need;
3421 if (i < s->max_ethqsets) {
3422 s->max_ethqsets = i;
3423 if (i < s->ethqsets)
3424 reduce_ethqs(adap, i);
3425 }
3426 if (is_offload(adap)) {
3427 i = want - EXTRA_VECS - s->max_ethqsets;
3428 i -= ofld_need - nchan;
3429 s->ofldqsets = (i / nchan) * nchan; /* round down */
3430 }
3431 for (i = 0; i < want; ++i)
3432 adap->msix_info[i].vec = entries[i].vector;
3433 } else if (err > 0)
3434 dev_info(adap->pdev_dev,
3435 "only %d MSI-X vectors left, not using MSI-X\n", err);
3436 return err;
3437}
3438
3439#undef EXTRA_VECS
3440
671b0060
DM		 3441static int __devinit init_rss(struct adapter *adap)
3442{
3443 unsigned int i, j;
3444
3445 for_each_port(adap, i) {
3446 struct port_info *pi = adap2pinfo(adap, i);
3447
3448 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3449 if (!pi->rss)
3450 return -ENOMEM;
3451 for (j = 0; j < pi->rss_size; j++)
3452 pi->rss[j] = j % pi->nqsets;
3453 }
3454 return 0;
3455}
3456
118969ed 3457static void __devinit print_port_info(const struct net_device *dev)
b8ff05a9
DM		 3458{
3459 static const char *base[] = {
a0881cab 3460 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
7d5e77aa 3461 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
b8ff05a9
DM		 3462 };
3463
b8ff05a9 3464 char buf[80];
118969ed 3465 char *bufp = buf;
f1a051b9 3466 const char *spd = "";
118969ed
DM		 3467 const struct port_info *pi = netdev_priv(dev);
3468 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 3469
3470 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3471 spd = " 2.5 GT/s";
3472 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3473 spd = " 5 GT/s";
b8ff05a9 3474
118969ed
DM		 3475 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3476 bufp += sprintf(bufp, "100/");
3477 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3478 bufp += sprintf(bufp, "1000/");
3479 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3480 bufp += sprintf(bufp, "10G/");
3481 if (bufp != buf)
3482 --bufp;
3483 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3484
3485 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3486 adap->params.vpd.id, adap->params.rev, buf,
3487 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
3488 (adap->flags & USING_MSIX) ? " MSI-X" :
3489 (adap->flags & USING_MSI) ? " MSI" : "");
3490 netdev_info(dev, "S/N: %s, E/C: %s\n",
3491 adap->params.vpd.sn, adap->params.vpd.ec);
b8ff05a9
DM		 3492}
3493
ef306b50
DM		 3494static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
3495{
3496 u16 v;
3497 int pos;
3498
3499 pos = pci_pcie_cap(dev);
3500 if (pos > 0) {
3501 pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &v);
3502 v |= PCI_EXP_DEVCTL_RELAX_EN;
3503 pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, v);
3504 }
3505}
3506
06546391
DM		 3507/*
3508 * Free the following resources:
3509 * - memory used for tables
3510 * - MSI/MSI-X
3511 * - net devices
3512 * - resources FW is holding for us
3513 */
3514static void free_some_resources(struct adapter *adapter)
3515{
3516 unsigned int i;
3517
3518 t4_free_mem(adapter->l2t);
3519 t4_free_mem(adapter->tids.tid_tab);
3520 disable_msi(adapter);
3521
3522 for_each_port(adapter, i)
671b0060
DM		 3523 if (adapter->port[i]) {
3524 kfree(adap2pinfo(adapter, i)->rss);
06546391 3525 free_netdev(adapter->port[i]);
671b0060 3526 }
06546391 3527 if (adapter->flags & FW_OK)
060e0c75 3528 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 3529}
3530
2ed28baa 3531#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 3532#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9
DM		 3533 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3534
3535static int __devinit init_one(struct pci_dev *pdev,
3536 const struct pci_device_id *ent)
3537{
3538 int func, i, err;
3539 struct port_info *pi;
3540 unsigned int highdma = 0;
3541 struct adapter *adapter = NULL;
3542
3543 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3544
3545 err = pci_request_regions(pdev, KBUILD_MODNAME);
3546 if (err) {
3547 /* Just info, some other driver may have claimed the device. */
3548 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3549 return err;
3550 }
3551
060e0c75 3552 /* We control everything through one PF */
b8ff05a9 3553 func = PCI_FUNC(pdev->devfn);
060e0c75 3554 if (func != ent->driver_data) {
204dc3c0 3555 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 3556 goto sriov;
204dc3c0 3557 }
b8ff05a9
DM		 3558
3559 err = pci_enable_device(pdev);
3560 if (err) {
3561 dev_err(&pdev->dev, "cannot enable PCI device\n");
3562 goto out_release_regions;
3563 }
3564
3565 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3566 highdma = NETIF_F_HIGHDMA;
3567 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3568 if (err) {
3569 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3570 "coherent allocations\n");
3571 goto out_disable_device;
3572 }
3573 } else {
3574 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3575 if (err) {
3576 dev_err(&pdev->dev, "no usable DMA configuration\n");
3577 goto out_disable_device;
3578 }
3579 }
3580
3581 pci_enable_pcie_error_reporting(pdev);
ef306b50 3582 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 3583 pci_set_master(pdev);
3584 pci_save_state(pdev);
3585
3586 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3587 if (!adapter) {
3588 err = -ENOMEM;
3589 goto out_disable_device;
3590 }
3591
3592 adapter->regs = pci_ioremap_bar(pdev, 0);
3593 if (!adapter->regs) {
3594 dev_err(&pdev->dev, "cannot map device registers\n");
3595 err = -ENOMEM;
3596 goto out_free_adapter;
3597 }
3598
3599 adapter->pdev = pdev;
3600 adapter->pdev_dev = &pdev->dev;
060e0c75 3601 adapter->fn = func;
b8ff05a9
DM		 3602 adapter->msg_enable = dflt_msg_enable;
3603 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3604
3605 spin_lock_init(&adapter->stats_lock);
3606 spin_lock_init(&adapter->tid_release_lock);
3607
3608 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3609
3610 err = t4_prep_adapter(adapter);
3611 if (err)
3612 goto out_unmap_bar;
3613 err = adap_init0(adapter);
3614 if (err)
3615 goto out_unmap_bar;
3616
3617 for_each_port(adapter, i) {
3618 struct net_device *netdev;
3619
3620 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3621 MAX_ETH_QSETS);
3622 if (!netdev) {
3623 err = -ENOMEM;
3624 goto out_free_dev;
3625 }
3626
3627 SET_NETDEV_DEV(netdev, &pdev->dev);
3628
3629 adapter->port[i] = netdev;
3630 pi = netdev_priv(netdev);
3631 pi->adapter = adapter;
3632 pi->xact_addr_filt = -1;
b8ff05a9 3633 pi->port_id = i;
b8ff05a9
DM		 3634 netdev->irq = pdev->irq;
3635
2ed28baa
MM		 3636 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
3637 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3638 NETIF_F_RXCSUM | NETIF_F_RXHASH |
3639 NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3640 netdev->features |= netdev->hw_features | highdma;
b8ff05a9
DM		 3641 netdev->vlan_features = netdev->features & VLAN_FEAT;
3642
01789349
JP		 3643 netdev->priv_flags |= IFF_UNICAST_FLT;
3644
b8ff05a9
DM		 3645 netdev->netdev_ops = &cxgb4_netdev_ops;
3646 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3647 }
3648
3649 pci_set_drvdata(pdev, adapter);
3650
3651 if (adapter->flags & FW_OK) {
060e0c75 3652 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 3653 if (err)
3654 goto out_free_dev;
3655 }
3656
3657 /*
3658 * Configure queues and allocate tables now, they can be needed as
3659 * soon as the first register_netdev completes.
3660 */
3661 cfg_queues(adapter);
3662
3663 adapter->l2t = t4_init_l2t();
3664 if (!adapter->l2t) {
3665 /* We tolerate a lack of L2T, giving up some functionality */
3666 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3667 adapter->params.offload = 0;
3668 }
3669
3670 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3671 dev_warn(&pdev->dev, "could not allocate TID table, "
3672 "continuing\n");
3673 adapter->params.offload = 0;
3674 }
3675
f7cabcdd
DM		 3676 /* See what interrupts we'll be using */
3677 if (msi > 1 && enable_msix(adapter) == 0)
3678 adapter->flags |= USING_MSIX;
3679 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3680 adapter->flags |= USING_MSI;
3681
671b0060
DM		 3682 err = init_rss(adapter);
3683 if (err)
3684 goto out_free_dev;
3685
b8ff05a9
DM		 3686 /*
3687 * The card is now ready to go. If any errors occur during device
3688 * registration we do not fail the whole card but rather proceed only
3689 * with the ports we manage to register successfully. However we must
3690 * register at least one net device.
3691 */
3692 for_each_port(adapter, i) {
a57cabe0
DM		 3693 pi = adap2pinfo(adapter, i);
3694 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
3695 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
3696
b8ff05a9
DM		 3697 err = register_netdev(adapter->port[i]);
3698 if (err)
b1a3c2b6 3699 break;
b1a3c2b6
DM		 3700 adapter->chan_map[pi->tx_chan] = i;
3701 print_port_info(adapter->port[i]);
b8ff05a9 3702 }
b1a3c2b6 3703 if (i == 0) {
b8ff05a9
DM		 3704 dev_err(&pdev->dev, "could not register any net devices\n");
3705 goto out_free_dev;
3706 }
b1a3c2b6
DM		 3707 if (err) {
3708 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
3709 err = 0;
6403eab1 3710 }
b8ff05a9
DM		 3711
3712 if (cxgb4_debugfs_root) {
3713 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3714 cxgb4_debugfs_root);
3715 setup_debugfs(adapter);
3716 }
3717
b8ff05a9
DM		 3718 if (is_offload(adapter))
3719 attach_ulds(adapter);
3720
b8ff05a9
DM		 3721sriov:
3722#ifdef CONFIG_PCI_IOV
3723 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3724 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3725 dev_info(&pdev->dev,
3726 "instantiated %u virtual functions\n",
3727 num_vf[func]);
3728#endif
3729 return 0;
3730
3731 out_free_dev:
06546391 3732 free_some_resources(adapter);
b8ff05a9
DM		 3733 out_unmap_bar:
3734 iounmap(adapter->regs);
3735 out_free_adapter:
3736 kfree(adapter);
3737 out_disable_device:
3738 pci_disable_pcie_error_reporting(pdev);
3739 pci_disable_device(pdev);
3740 out_release_regions:
3741 pci_release_regions(pdev);
3742 pci_set_drvdata(pdev, NULL);
3743 return err;
3744}
3745
3746static void __devexit remove_one(struct pci_dev *pdev)
3747{
3748 struct adapter *adapter = pci_get_drvdata(pdev);
3749
3750 pci_disable_sriov(pdev);
3751
3752 if (adapter) {
3753 int i;
3754
3755 if (is_offload(adapter))
3756 detach_ulds(adapter);
3757
3758 for_each_port(adapter, i)
8f3a7676 3759 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 3760 unregister_netdev(adapter->port[i]);
3761
3762 if (adapter->debugfs_root)
3763 debugfs_remove_recursive(adapter->debugfs_root);
3764
aaefae9b
DM		 3765 if (adapter->flags & FULL_INIT_DONE)
3766 cxgb_down(adapter);
b8ff05a9 3767
06546391 3768 free_some_resources(adapter);
b8ff05a9
DM		 3769 iounmap(adapter->regs);
3770 kfree(adapter);
3771 pci_disable_pcie_error_reporting(pdev);
3772 pci_disable_device(pdev);
3773 pci_release_regions(pdev);
3774 pci_set_drvdata(pdev, NULL);
a069ec91 3775 } else
b8ff05a9
DM		 3776 pci_release_regions(pdev);
3777}
3778
3779static struct pci_driver cxgb4_driver = {
3780 .name = KBUILD_MODNAME,
3781 .id_table = cxgb4_pci_tbl,
3782 .probe = init_one,
3783 .remove = __devexit_p(remove_one),
204dc3c0 3784 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 3785};
3786
3787static int __init cxgb4_init_module(void)
3788{
3789 int ret;
3790
3791 /* Debugfs support is optional, just warn if this fails */
3792 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3793 if (!cxgb4_debugfs_root)
3794 pr_warning("could not create debugfs entry, continuing\n");
3795
3796 ret = pci_register_driver(&cxgb4_driver);
3797 if (ret < 0)
3798 debugfs_remove(cxgb4_debugfs_root);
3799 return ret;
3800}
3801
3802static void __exit cxgb4_cleanup_module(void)
3803{
3804 pci_unregister_driver(&cxgb4_driver);
3805 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3806}
3807
3808module_init(cxgb4_init_module);
3809module_exit(cxgb4_cleanup_module);
Ubuntu Artful kernel mirror