]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4: Move offload Rx queue allocation to separate function
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
ce100b8b 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
b8ff05a9
DM		 5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
1ef8019b 64#include <net/bonding.h>
b5a02f50 65#include <net/addrconf.h>
b8ff05a9
DM		 66#include <asm/uaccess.h>
67
68#include "cxgb4.h"
69#include "t4_regs.h"
f612b815 70#include "t4_values.h"
b8ff05a9
DM		 71#include "t4_msg.h"
72#include "t4fw_api.h"
cd6c2f12 73#include "t4fw_version.h"
688848b1 74#include "cxgb4_dcb.h"
fd88b31a 75#include "cxgb4_debugfs.h"
b5a02f50 76#include "clip_tbl.h"
b8ff05a9
DM		 77#include "l2t.h"
78
01bcca68
VP		 79#ifdef DRV_VERSION
80#undef DRV_VERSION
81#endif
3a7f8554
SR		 82#define DRV_VERSION "2.0.0-ko"
83#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9 84
b8ff05a9
DM		 85enum {
86 MAX_TXQ_ENTRIES = 16384,
87 MAX_CTRL_TXQ_ENTRIES = 1024,
88 MAX_RSPQ_ENTRIES = 16384,
89 MAX_RX_BUFFERS = 16384,
90 MIN_TXQ_ENTRIES = 32,
91 MIN_CTRL_TXQ_ENTRIES = 32,
92 MIN_RSPQ_ENTRIES = 128,
93 MIN_FL_ENTRIES = 16
94};
95
f2b7e78d
VP		 96/* Host shadow copy of ingress filter entry. This is in host native format
97 * and doesn't match the ordering or bit order, etc. of the hardware of the
98 * firmware command. The use of bit-field structure elements is purely to
99 * remind ourselves of the field size limitations and save memory in the case
100 * where the filter table is large.
101 */
102struct filter_entry {
103 /* Administrative fields for filter.
104 */
105 u32 valid:1; /* filter allocated and valid */
106 u32 locked:1; /* filter is administratively locked */
107
108 u32 pending:1; /* filter action is pending firmware reply */
109 u32 smtidx:8; /* Source MAC Table index for smac */
110 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
111
112 /* The filter itself. Most of this is a straight copy of information
113 * provided by the extended ioctl(). Some fields are translated to
114 * internal forms -- for instance the Ingress Queue ID passed in from
115 * the ioctl() is translated into the Absolute Ingress Queue ID.
116 */
117 struct ch_filter_specification fs;
118};
119
b8ff05a9
DM		 120#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
121 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
122 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
123
3fedeab1
HS		 124/* Macros needed to support the PCI Device ID Table ...
125 */
126#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
127 static struct pci_device_id cxgb4_pci_tbl[] = {
128#define CH_PCI_DEVICE_ID_FUNCTION 0x4
b8ff05a9 129
3fedeab1
HS		 130/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
131 * called for both.
132 */
133#define CH_PCI_DEVICE_ID_FUNCTION2 0x0
134
135#define CH_PCI_ID_TABLE_ENTRY(devid) \
136 {PCI_VDEVICE(CHELSIO, (devid)), 4}
137
138#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
139 { 0, } \
140 }
141
142#include "t4_pci_id_tbl.h"
b8ff05a9 143
16e47624 144#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 145#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 146#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 147#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 148
149MODULE_DESCRIPTION(DRV_DESC);
150MODULE_AUTHOR("Chelsio Communications");
151MODULE_LICENSE("Dual BSD/GPL");
152MODULE_VERSION(DRV_VERSION);
153MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 154MODULE_FIRMWARE(FW4_FNAME);
0a57a536 155MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 156
636f9d37
VP		 157/*
158 * Normally we're willing to become the firmware's Master PF but will be happy
159 * if another PF has already become the Master and initialized the adapter.
160 * Setting "force_init" will cause this driver to forcibly establish itself as
161 * the Master PF and initialize the adapter.
162 */
163static uint force_init;
164
165module_param(force_init, uint, 0644);
166MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
167
13ee15d3
VP		 168/*
169 * Normally if the firmware we connect to has Configuration File support, we
170 * use that and only fall back to the old Driver-based initialization if the
171 * Configuration File fails for some reason. If force_old_init is set, then
172 * we'll always use the old Driver-based initialization sequence.
173 */
174static uint force_old_init;
175
176module_param(force_old_init, uint, 0644);
06640310
HS		 177MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated"
178 " parameter");
13ee15d3 179
b8ff05a9
DM		 180static int dflt_msg_enable = DFLT_MSG_ENABLE;
181
182module_param(dflt_msg_enable, int, 0644);
183MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
184
185/*
186 * The driver uses the best interrupt scheme available on a platform in the
187 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
188 * of these schemes the driver may consider as follows:
189 *
190 * msi = 2: choose from among all three options
191 * msi = 1: only consider MSI and INTx interrupts
192 * msi = 0: force INTx interrupts
193 */
194static int msi = 2;
195
196module_param(msi, int, 0644);
197MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
198
199/*
200 * Queue interrupt hold-off timer values. Queues default to the first of these
201 * upon creation.
202 */
203static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
204
205module_param_array(intr_holdoff, uint, NULL, 0644);
206MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
06640310 207 "0..4 in microseconds, deprecated parameter");
b8ff05a9
DM		 208
209static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
210
211module_param_array(intr_cnt, uint, NULL, 0644);
212MODULE_PARM_DESC(intr_cnt,
06640310
HS		 213 "thresholds 1..3 for queue interrupt packet counters, "
214 "deprecated parameter");
b8ff05a9 215
636f9d37
VP		 216/*
217 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
218 * offset by 2 bytes in order to have the IP headers line up on 4-byte
219 * boundaries. This is a requirement for many architectures which will throw
220 * a machine check fault if an attempt is made to access one of the 4-byte IP
221 * header fields on a non-4-byte boundary. And it's a major performance issue
222 * even on some architectures which allow it like some implementations of the
223 * x86 ISA. However, some architectures don't mind this and for some very
224 * edge-case performance sensitive applications (like forwarding large volumes
225 * of small packets), setting this DMA offset to 0 will decrease the number of
226 * PCI-E Bus transfers enough to measurably affect performance.
227 */
228static int rx_dma_offset = 2;
229
eb939922 230static bool vf_acls;
b8ff05a9
DM		 231
232#ifdef CONFIG_PCI_IOV
233module_param(vf_acls, bool, 0644);
06640310
HS		 234MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, "
235 "deprecated parameter");
b8ff05a9 236
7d6727cf
SR		 237/* Configure the number of PCI-E Virtual Function which are to be instantiated
238 * on SR-IOV Capable Physical Functions.
0a57a536 239 */
7d6727cf 240static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 241
242module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 243MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 244#endif
245
688848b1
AB		 246/* TX Queue select used to determine what algorithm to use for selecting TX
247 * queue. Select between the kernel provided function (select_queue=0) or user
248 * cxgb_select_queue function (select_queue=1)
249 *
250 * Default: select_queue=0
251 */
252static int select_queue;
253module_param(select_queue, int, 0644);
254MODULE_PARM_DESC(select_queue,
255 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
256
06640310 257static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC;
13ee15d3 258
f2b7e78d 259module_param(tp_vlan_pri_map, uint, 0644);
06640310
HS		 260MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, "
261 "deprecated parameter");
f2b7e78d 262
b8ff05a9
DM		 263static struct dentry *cxgb4_debugfs_root;
264
265static LIST_HEAD(adapter_list);
266static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 267/* Adapter list to be accessed from atomic context */
268static LIST_HEAD(adap_rcu_list);
269static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 270static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
271static const char *uld_str[] = { "RDMA", "iSCSI" };
272
273static void link_report(struct net_device *dev)
274{
275 if (!netif_carrier_ok(dev))
276 netdev_info(dev, "link down\n");
277 else {
278 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
279
280 const char *s = "10Mbps";
281 const struct port_info *p = netdev_priv(dev);
282
283 switch (p->link_cfg.speed) {
e8b39015 284 case 10000:
b8ff05a9
DM		 285 s = "10Gbps";
286 break;
e8b39015 287 case 1000:
b8ff05a9
DM		 288 s = "1000Mbps";
289 break;
e8b39015 290 case 100:
b8ff05a9
DM		 291 s = "100Mbps";
292 break;
e8b39015 293 case 40000:
72aca4bf
KS		 294 s = "40Gbps";
295 break;
b8ff05a9
DM		 296 }
297
298 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
299 fc[p->link_cfg.fc]);
300 }
301}
302
688848b1
AB		 303#ifdef CONFIG_CHELSIO_T4_DCB
304/* Set up/tear down Data Center Bridging Priority mapping for a net device. */
305static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
306{
307 struct port_info *pi = netdev_priv(dev);
308 struct adapter *adap = pi->adapter;
309 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
310 int i;
311
312 /* We use a simple mapping of Port TX Queue Index to DCB
313 * Priority when we're enabling DCB.
314 */
315 for (i = 0; i < pi->nqsets; i++, txq++) {
316 u32 name, value;
317 int err;
318
5167865a
HS		 319 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
320 FW_PARAMS_PARAM_X_V(
321 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
322 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
688848b1
AB		 323 value = enable ? i : 0xffffffff;
324
325 /* Since we can be called while atomic (from "interrupt
326 * level") we need to issue the Set Parameters Commannd
327 * without sleeping (timeout < 0).
328 */
329 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
330 &name, &value);
331
332 if (err)
333 dev_err(adap->pdev_dev,
334 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
335 enable ? "set" : "unset", pi->port_id, i, -err);
10b00466
AB		 336 else
337 txq->dcb_prio = value;
688848b1
AB		 338 }
339}
340#endif /* CONFIG_CHELSIO_T4_DCB */
341
b8ff05a9
DM		 342void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
343{
344 struct net_device *dev = adapter->port[port_id];
345
346 /* Skip changes from disabled ports. */
347 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
348 if (link_stat)
349 netif_carrier_on(dev);
688848b1
AB		 350 else {
351#ifdef CONFIG_CHELSIO_T4_DCB
352 cxgb4_dcb_state_init(dev);
353 dcb_tx_queue_prio_enable(dev, false);
354#endif /* CONFIG_CHELSIO_T4_DCB */
b8ff05a9 355 netif_carrier_off(dev);
688848b1 356 }
b8ff05a9
DM		 357
358 link_report(dev);
359 }
360}
361
362void t4_os_portmod_changed(const struct adapter *adap, int port_id)
363{
364 static const char *mod_str[] = {
a0881cab 365 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 366 };
367
368 const struct net_device *dev = adap->port[port_id];
369 const struct port_info *pi = netdev_priv(dev);
370
371 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
372 netdev_info(dev, "port module unplugged\n");
a0881cab 373 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 374 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
375}
376
377/*
378 * Configure the exact and hash address filters to handle a port's multicast
379 * and secondary unicast MAC addresses.
380 */
381static int set_addr_filters(const struct net_device *dev, bool sleep)
382{
383 u64 mhash = 0;
384 u64 uhash = 0;
385 bool free = true;
386 u16 filt_idx[7];
387 const u8 *addr[7];
388 int ret, naddr = 0;
b8ff05a9
DM		 389 const struct netdev_hw_addr *ha;
390 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 391 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 392 const struct port_info *pi = netdev_priv(dev);
060e0c75 393 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 394
395 /* first do the secondary unicast addresses */
396 netdev_for_each_uc_addr(ha, dev) {
397 addr[naddr++] = ha->addr;
398 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 399 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 400 naddr, addr, filt_idx, &uhash, sleep);
401 if (ret < 0)
402 return ret;
403
404 free = false;
405 naddr = 0;
406 }
407 }
408
409 /* next set up the multicast addresses */
4a35ecf8
DM		 410 netdev_for_each_mc_addr(ha, dev) {
411 addr[naddr++] = ha->addr;
412 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 413 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 414 naddr, addr, filt_idx, &mhash, sleep);
415 if (ret < 0)
416 return ret;
417
418 free = false;
419 naddr = 0;
420 }
421 }
422
060e0c75 423 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 424 uhash | mhash, sleep);
425}
426
3069ee9b
VP		 427int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
428module_param(dbfifo_int_thresh, int, 0644);
429MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
430
404d9e3f
VP		 431/*
432 * usecs to sleep while draining the dbfifo
433 */
434static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 435module_param(dbfifo_drain_delay, int, 0644);
436MODULE_PARM_DESC(dbfifo_drain_delay,
437 "usecs to sleep while draining the dbfifo");
438
b8ff05a9
DM		 439/*
440 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
441 * If @mtu is -1 it is left unchanged.
442 */
443static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
444{
445 int ret;
446 struct port_info *pi = netdev_priv(dev);
447
448 ret = set_addr_filters(dev, sleep_ok);
449 if (ret == 0)
060e0c75 450 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 451 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 452 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 453 sleep_ok);
454 return ret;
455}
456
457/**
458 * link_start - enable a port
459 * @dev: the port to enable
460 *
461 * Performs the MAC and PHY actions needed to enable a port.
462 */
463static int link_start(struct net_device *dev)
464{
465 int ret;
466 struct port_info *pi = netdev_priv(dev);
060e0c75 467 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 468
469 /*
470 * We do not set address filters and promiscuity here, the stack does
471 * that step explicitly.
472 */
060e0c75 473 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 474 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 475 if (ret == 0) {
060e0c75 476 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 477 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 478 true);
b8ff05a9
DM		 479 if (ret >= 0) {
480 pi->xact_addr_filt = ret;
481 ret = 0;
482 }
483 }
484 if (ret == 0)
060e0c75
DM		 485 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
486 &pi->link_cfg);
30f00847
AB		 487 if (ret == 0) {
488 local_bh_disable();
688848b1
AB		 489 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
490 true, CXGB4_DCB_ENABLED);
30f00847
AB		 491 local_bh_enable();
492 }
688848b1 493
b8ff05a9
DM		 494 return ret;
495}
496
688848b1
AB		 497int cxgb4_dcb_enabled(const struct net_device *dev)
498{
499#ifdef CONFIG_CHELSIO_T4_DCB
500 struct port_info *pi = netdev_priv(dev);
501
3bb06261
AB		 502 if (!pi->dcb.enabled)
503 return 0;
504
505 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
506 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
688848b1
AB		 507#else
508 return 0;
509#endif
510}
511EXPORT_SYMBOL(cxgb4_dcb_enabled);
512
513#ifdef CONFIG_CHELSIO_T4_DCB
514/* Handle a Data Center Bridging update message from the firmware. */
515static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
516{
2b5fb1f2 517 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
688848b1
AB		 518 struct net_device *dev = adap->port[port];
519 int old_dcb_enabled = cxgb4_dcb_enabled(dev);
520 int new_dcb_enabled;
521
522 cxgb4_dcb_handle_fw_update(adap, pcmd);
523 new_dcb_enabled = cxgb4_dcb_enabled(dev);
524
525 /* If the DCB has become enabled or disabled on the port then we're
526 * going to need to set up/tear down DCB Priority parameters for the
527 * TX Queues associated with the port.
528 */
529 if (new_dcb_enabled != old_dcb_enabled)
530 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
531}
532#endif /* CONFIG_CHELSIO_T4_DCB */
533
f2b7e78d
VP		 534/* Clear a filter and release any of its resources that we own. This also
535 * clears the filter's "pending" status.
536 */
537static void clear_filter(struct adapter *adap, struct filter_entry *f)
538{
539 /* If the new or old filter have loopback rewriteing rules then we'll
540 * need to free any existing Layer Two Table (L2T) entries of the old
541 * filter rule. The firmware will handle freeing up any Source MAC
542 * Table (SMT) entries used for rewriting Source MAC Addresses in
543 * loopback rules.
544 */
545 if (f->l2t)
546 cxgb4_l2t_release(f->l2t);
547
548 /* The zeroing of the filter rule below clears the filter valid,
549 * pending, locked flags, l2t pointer, etc. so it's all we need for
550 * this operation.
551 */
552 memset(f, 0, sizeof(*f));
553}
554
555/* Handle a filter write/deletion reply.
556 */
557static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
558{
559 unsigned int idx = GET_TID(rpl);
560 unsigned int nidx = idx - adap->tids.ftid_base;
561 unsigned int ret;
562 struct filter_entry *f;
563
564 if (idx >= adap->tids.ftid_base && nidx <
565 (adap->tids.nftids + adap->tids.nsftids)) {
566 idx = nidx;
bdc590b9 567 ret = TCB_COOKIE_G(rpl->cookie);
f2b7e78d
VP		 568 f = &adap->tids.ftid_tab[idx];
569
570 if (ret == FW_FILTER_WR_FLT_DELETED) {
571 /* Clear the filter when we get confirmation from the
572 * hardware that the filter has been deleted.
573 */
574 clear_filter(adap, f);
575 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
576 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
577 idx);
578 clear_filter(adap, f);
579 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
580 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
581 f->pending = 0; /* asynchronous setup completed */
582 f->valid = 1;
583 } else {
584 /* Something went wrong. Issue a warning about the
585 * problem and clear everything out.
586 */
587 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
588 idx, ret);
589 clear_filter(adap, f);
590 }
591 }
592}
593
594/* Response queue handler for the FW event queue.
b8ff05a9
DM		 595 */
596static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
597 const struct pkt_gl *gl)
598{
599 u8 opcode = ((const struct rss_header *)rsp)->opcode;
600
601 rsp++; /* skip RSS header */
b407a4a9
VP		 602
603 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
604 */
605 if (unlikely(opcode == CPL_FW4_MSG &&
606 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
607 rsp++;
608 opcode = ((const struct rss_header *)rsp)->opcode;
609 rsp++;
610 if (opcode != CPL_SGE_EGR_UPDATE) {
611 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
612 , opcode);
613 goto out;
614 }
615 }
616
b8ff05a9
DM		 617 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
618 const struct cpl_sge_egr_update *p = (void *)rsp;
bdc590b9 619 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
e46dab4d 620 struct sge_txq *txq;
b8ff05a9 621
e46dab4d 622 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 623 txq->restarts++;
e46dab4d 624 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 625 struct sge_eth_txq *eq;
626
627 eq = container_of(txq, struct sge_eth_txq, q);
628 netif_tx_wake_queue(eq->txq);
629 } else {
630 struct sge_ofld_txq *oq;
631
632 oq = container_of(txq, struct sge_ofld_txq, q);
633 tasklet_schedule(&oq->qresume_tsk);
634 }
635 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
636 const struct cpl_fw6_msg *p = (void *)rsp;
637
688848b1
AB		 638#ifdef CONFIG_CHELSIO_T4_DCB
639 const struct fw_port_cmd *pcmd = (const void *)p->data;
e2ac9628 640 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
688848b1 641 unsigned int action =
2b5fb1f2 642 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
688848b1
AB		 643
644 if (cmd == FW_PORT_CMD &&
645 action == FW_PORT_ACTION_GET_PORT_INFO) {
2b5fb1f2 646 int port = FW_PORT_CMD_PORTID_G(
688848b1
AB		 647 be32_to_cpu(pcmd->op_to_portid));
648 struct net_device *dev = q->adap->port[port];
649 int state_input = ((pcmd->u.info.dcbxdis_pkd &
2b5fb1f2 650 FW_PORT_CMD_DCBXDIS_F)
688848b1
AB		 651 ? CXGB4_DCB_INPUT_FW_DISABLED
652 : CXGB4_DCB_INPUT_FW_ENABLED);
653
654 cxgb4_dcb_state_fsm(dev, state_input);
655 }
656
657 if (cmd == FW_PORT_CMD &&
658 action == FW_PORT_ACTION_L2_DCB_CFG)
659 dcb_rpl(q->adap, pcmd);
660 else
661#endif
662 if (p->type == 0)
663 t4_handle_fw_rpl(q->adap, p->data);
b8ff05a9
DM		 664 } else if (opcode == CPL_L2T_WRITE_RPL) {
665 const struct cpl_l2t_write_rpl *p = (void *)rsp;
666
667 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 668 } else if (opcode == CPL_SET_TCB_RPL) {
669 const struct cpl_set_tcb_rpl *p = (void *)rsp;
670
671 filter_rpl(q->adap, p);
b8ff05a9
DM		 672 } else
673 dev_err(q->adap->pdev_dev,
674 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 675out:
b8ff05a9
DM		 676 return 0;
677}
678
679/**
680 * uldrx_handler - response queue handler for ULD queues
681 * @q: the response queue that received the packet
682 * @rsp: the response queue descriptor holding the offload message
683 * @gl: the gather list of packet fragments
684 *
685 * Deliver an ingress offload packet to a ULD. All processing is done by
686 * the ULD, we just maintain statistics.
687 */
688static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
689 const struct pkt_gl *gl)
690{
691 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
692
b407a4a9
VP		 693 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
694 */
695 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
696 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
697 rsp += 2;
698
b8ff05a9
DM		 699 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
700 rxq->stats.nomem++;
701 return -1;
702 }
703 if (gl == NULL)
704 rxq->stats.imm++;
705 else if (gl == CXGB4_MSG_AN)
706 rxq->stats.an++;
707 else
708 rxq->stats.pkts++;
709 return 0;
710}
711
712static void disable_msi(struct adapter *adapter)
713{
714 if (adapter->flags & USING_MSIX) {
715 pci_disable_msix(adapter->pdev);
716 adapter->flags &= ~USING_MSIX;
717 } else if (adapter->flags & USING_MSI) {
718 pci_disable_msi(adapter->pdev);
719 adapter->flags &= ~USING_MSI;
720 }
721}
722
723/*
724 * Interrupt handler for non-data events used with MSI-X.
725 */
726static irqreturn_t t4_nondata_intr(int irq, void *cookie)
727{
728 struct adapter *adap = cookie;
0d804338 729 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
b8ff05a9 730
0d804338 731 if (v & PFSW_F) {
b8ff05a9 732 adap->swintr = 1;
0d804338 733 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
b8ff05a9
DM		 734 }
735 t4_slow_intr_handler(adap);
736 return IRQ_HANDLED;
737}
738
739/*
740 * Name the MSI-X interrupts.
741 */
742static void name_msix_vecs(struct adapter *adap)
743{
ba27816c 744 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 745
746 /* non-data interrupts */
b1a3c2b6 747 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 748
749 /* FW events */
b1a3c2b6
DM		 750 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
751 adap->port[0]->name);
b8ff05a9
DM		 752
753 /* Ethernet queues */
754 for_each_port(adap, j) {
755 struct net_device *d = adap->port[j];
756 const struct port_info *pi = netdev_priv(d);
757
ba27816c 758 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 759 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
760 d->name, i);
b8ff05a9
DM		 761 }
762
763 /* offload queues */
ba27816c
DM		 764 for_each_ofldrxq(&adap->sge, i)
765 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 766 adap->port[0]->name, i);
ba27816c
DM		 767
768 for_each_rdmarxq(&adap->sge, i)
769 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 770 adap->port[0]->name, i);
cf38be6d
HS		 771
772 for_each_rdmaciq(&adap->sge, i)
773 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
774 adap->port[0]->name, i);
b8ff05a9
DM		 775}
776
777static int request_msix_queue_irqs(struct adapter *adap)
778{
779 struct sge *s = &adap->sge;
cf38be6d
HS		 780 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
781 int msi_index = 2;
b8ff05a9
DM		 782
783 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
784 adap->msix_info[1].desc, &s->fw_evtq);
785 if (err)
786 return err;
787
788 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 789 err = request_irq(adap->msix_info[msi_index].vec,
790 t4_sge_intr_msix, 0,
791 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 792 &s->ethrxq[ethqidx].rspq);
793 if (err)
794 goto unwind;
404d9e3f 795 msi_index++;
b8ff05a9
DM		 796 }
797 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 798 err = request_irq(adap->msix_info[msi_index].vec,
799 t4_sge_intr_msix, 0,
800 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 801 &s->ofldrxq[ofldqidx].rspq);
802 if (err)
803 goto unwind;
404d9e3f 804 msi_index++;
b8ff05a9
DM		 805 }
806 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 807 err = request_irq(adap->msix_info[msi_index].vec,
808 t4_sge_intr_msix, 0,
809 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 810 &s->rdmarxq[rdmaqidx].rspq);
811 if (err)
812 goto unwind;
404d9e3f 813 msi_index++;
b8ff05a9 814 }
cf38be6d
HS		 815 for_each_rdmaciq(s, rdmaciqqidx) {
816 err = request_irq(adap->msix_info[msi_index].vec,
817 t4_sge_intr_msix, 0,
818 adap->msix_info[msi_index].desc,
819 &s->rdmaciq[rdmaciqqidx].rspq);
820 if (err)
821 goto unwind;
822 msi_index++;
823 }
b8ff05a9
DM		 824 return 0;
825
826unwind:
cf38be6d
HS		 827 while (--rdmaciqqidx >= 0)
828 free_irq(adap->msix_info[--msi_index].vec,
829 &s->rdmaciq[rdmaciqqidx].rspq);
b8ff05a9 830 while (--rdmaqidx >= 0)
404d9e3f 831 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 832 &s->rdmarxq[rdmaqidx].rspq);
833 while (--ofldqidx >= 0)
404d9e3f 834 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 835 &s->ofldrxq[ofldqidx].rspq);
836 while (--ethqidx >= 0)
404d9e3f
VP		 837 free_irq(adap->msix_info[--msi_index].vec,
838 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 839 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
840 return err;
841}
842
843static void free_msix_queue_irqs(struct adapter *adap)
844{
404d9e3f 845 int i, msi_index = 2;
b8ff05a9
DM		 846 struct sge *s = &adap->sge;
847
848 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
849 for_each_ethrxq(s, i)
404d9e3f 850 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 851 for_each_ofldrxq(s, i)
404d9e3f 852 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 853 for_each_rdmarxq(s, i)
404d9e3f 854 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
cf38be6d
HS		 855 for_each_rdmaciq(s, i)
856 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
b8ff05a9
DM		 857}
858
671b0060
DM		 859/**
860 * write_rss - write the RSS table for a given port
861 * @pi: the port
862 * @queues: array of queue indices for RSS
863 *
864 * Sets up the portion of the HW RSS table for the port's VI to distribute
865 * packets to the Rx queues in @queues.
866 */
867static int write_rss(const struct port_info *pi, const u16 *queues)
868{
869 u16 *rss;
870 int i, err;
871 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
872
873 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
874 if (!rss)
875 return -ENOMEM;
876
877 /* map the queue indices to queue ids */
878 for (i = 0; i < pi->rss_size; i++, queues++)
879 rss[i] = q[*queues].rspq.abs_id;
880
060e0c75
DM		 881 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
882 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 883 kfree(rss);
884 return err;
885}
886
b8ff05a9
DM		 887/**
888 * setup_rss - configure RSS
889 * @adap: the adapter
890 *
671b0060 891 * Sets up RSS for each port.
b8ff05a9
DM		 892 */
893static int setup_rss(struct adapter *adap)
894{
671b0060 895 int i, err;
b8ff05a9
DM		 896
897 for_each_port(adap, i) {
898 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 899
671b0060 900 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 901 if (err)
902 return err;
903 }
904 return 0;
905}
906
e46dab4d
DM		 907/*
908 * Return the channel of the ingress queue with the given qid.
909 */
910static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
911{
912 qid -= p->ingr_start;
913 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
914}
915
b8ff05a9
DM		 916/*
917 * Wait until all NAPI handlers are descheduled.
918 */
919static void quiesce_rx(struct adapter *adap)
920{
921 int i;
922
923 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
924 struct sge_rspq *q = adap->sge.ingr_map[i];
925
3a336cb1 926 if (q && q->handler) {
b8ff05a9 927 napi_disable(&q->napi);
3a336cb1
HS		 928 local_bh_disable();
929 while (!cxgb_poll_lock_napi(q))
930 mdelay(1);
931 local_bh_enable();
932 }
933
b8ff05a9
DM		 934 }
935}
936
937/*
938 * Enable NAPI scheduling and interrupt generation for all Rx queues.
939 */
940static void enable_rx(struct adapter *adap)
941{
942 int i;
943
944 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
945 struct sge_rspq *q = adap->sge.ingr_map[i];
946
947 if (!q)
948 continue;
3a336cb1
HS		 949 if (q->handler) {
950 cxgb_busy_poll_init_lock(q);
b8ff05a9 951 napi_enable(&q->napi);
3a336cb1 952 }
b8ff05a9 953 /* 0-increment GTS to start the timer and enable interrupts */
f612b815
HS		 954 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
955 SEINTARM_V(q->intr_params) |
956 INGRESSQID_V(q->cntxt_id));
b8ff05a9
DM		 957 }
958}
959
1c6a5b0e
HS		 960static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
961 unsigned int nq, unsigned int per_chan, int msi_idx,
962 u16 *ids)
963{
964 int i, err;
965
966 for (i = 0; i < nq; i++, q++) {
967 if (msi_idx > 0)
968 msi_idx++;
969 err = t4_sge_alloc_rxq(adap, &q->rspq, false,
970 adap->port[i / per_chan],
971 msi_idx, q->fl.size ? &q->fl : NULL,
972 uldrx_handler);
973 if (err)
974 return err;
975 memset(&q->stats, 0, sizeof(q->stats));
976 if (ids)
977 ids[i] = q->rspq.abs_id;
978 }
979 return 0;
980}
981
b8ff05a9
DM		 982/**
983 * setup_sge_queues - configure SGE Tx/Rx/response queues
984 * @adap: the adapter
985 *
986 * Determines how many sets of SGE queues to use and initializes them.
987 * We support multiple queue sets per port if we have MSI-X, otherwise
988 * just one queue set per port.
989 */
990static int setup_sge_queues(struct adapter *adap)
991{
992 int err, msi_idx, i, j;
993 struct sge *s = &adap->sge;
994
995 bitmap_zero(s->starving_fl, MAX_EGRQ);
996 bitmap_zero(s->txq_maperr, MAX_EGRQ);
997
998 if (adap->flags & USING_MSIX)
999 msi_idx = 1; /* vector 0 is for non-queue interrupts */
1000 else {
1001 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1002 NULL, NULL);
1003 if (err)
1004 return err;
1005 msi_idx = -((int)s->intrq.abs_id + 1);
1006 }
1007
1008 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1009 msi_idx, NULL, fwevtq_handler);
1010 if (err) {
1011freeout: t4_free_sge_resources(adap);
1012 return err;
1013 }
1014
1015 for_each_port(adap, i) {
1016 struct net_device *dev = adap->port[i];
1017 struct port_info *pi = netdev_priv(dev);
1018 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1019 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1020
1021 for (j = 0; j < pi->nqsets; j++, q++) {
1022 if (msi_idx > 0)
1023 msi_idx++;
1024 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1025 msi_idx, &q->fl,
1026 t4_ethrx_handler);
1027 if (err)
1028 goto freeout;
1029 q->rspq.idx = j;
1030 memset(&q->stats, 0, sizeof(q->stats));
1031 }
1032 for (j = 0; j < pi->nqsets; j++, t++) {
1033 err = t4_sge_alloc_eth_txq(adap, t, dev,
1034 netdev_get_tx_queue(dev, j),
1035 s->fw_evtq.cntxt_id);
1036 if (err)
1037 goto freeout;
1038 }
1039 }
1040
1041 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1042 for_each_ofldrxq(s, i) {
1c6a5b0e
HS		 1043 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i],
1044 adap->port[i / j],
b8ff05a9
DM		 1045 s->fw_evtq.cntxt_id);
1046 if (err)
1047 goto freeout;
1048 }
1049
1c6a5b0e
HS		 1050#define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids) do { \
1051 err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids); \
1052 if (err) \
1053 goto freeout; \
1054 if (msi_idx > 0) \
1055 msi_idx += nq; \
1056} while (0)
b8ff05a9 1057
1c6a5b0e
HS		 1058 ALLOC_OFLD_RXQS(s->ofldrxq, s->ofldqsets, j, s->ofld_rxq);
1059 ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq);
1060 ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, 1, s->rdma_ciq);
b8ff05a9 1061
1c6a5b0e 1062#undef ALLOC_OFLD_RXQS
cf38be6d 1063
b8ff05a9
DM		 1064 for_each_port(adap, i) {
1065 /*
1066 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1067 * have RDMA queues, and that's the right value.
1068 */
1069 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1070 s->fw_evtq.cntxt_id,
1071 s->rdmarxq[i].rspq.cntxt_id);
1072 if (err)
1073 goto freeout;
1074 }
1075
9bb59b96 1076 t4_write_reg(adap, is_t4(adap->params.chip) ?
837e4a42
HS		 1077 MPS_TRC_RSS_CONTROL_A :
1078 MPS_T5_TRC_RSS_CONTROL_A,
1079 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1080 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
b8ff05a9
DM		 1081 return 0;
1082}
1083
b8ff05a9
DM		 1084/*
1085 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1086 * The allocated memory is cleared.
1087 */
1088void *t4_alloc_mem(size_t size)
1089{
8be04b93 1090 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1091
1092 if (!p)
89bf67f1 1093 p = vzalloc(size);
b8ff05a9
DM		 1094 return p;
1095}
1096
1097/*
1098 * Free memory allocated through alloc_mem().
1099 */
fd88b31a 1100void t4_free_mem(void *addr)
b8ff05a9
DM		 1101{
1102 if (is_vmalloc_addr(addr))
1103 vfree(addr);
1104 else
1105 kfree(addr);
1106}
1107
f2b7e78d
VP		 1108/* Send a Work Request to write the filter at a specified index. We construct
1109 * a Firmware Filter Work Request to have the work done and put the indicated
1110 * filter into "pending" mode which will prevent any further actions against
1111 * it till we get a reply from the firmware on the completion status of the
1112 * request.
1113 */
1114static int set_filter_wr(struct adapter *adapter, int fidx)
1115{
1116 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1117 struct sk_buff *skb;
1118 struct fw_filter_wr *fwr;
1119 unsigned int ftid;
1120
1121 /* If the new filter requires loopback Destination MAC and/or VLAN
1122 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1123 * the filter.
1124 */
1125 if (f->fs.newdmac || f->fs.newvlan) {
1126 /* allocate L2T entry for new filter */
1127 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1128 if (f->l2t == NULL)
1129 return -EAGAIN;
1130 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1131 f->fs.eport, f->fs.dmac)) {
1132 cxgb4_l2t_release(f->l2t);
1133 f->l2t = NULL;
1134 return -ENOMEM;
1135 }
1136 }
1137
1138 ftid = adapter->tids.ftid_base + fidx;
1139
1140 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1141 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1142 memset(fwr, 0, sizeof(*fwr));
1143
1144 /* It would be nice to put most of the following in t4_hw.c but most
1145 * of the work is translating the cxgbtool ch_filter_specification
1146 * into the Work Request and the definition of that structure is
1147 * currently in cxgbtool.h which isn't appropriate to pull into the
1148 * common code. We may eventually try to come up with a more neutral
1149 * filter specification structure but for now it's easiest to simply
1150 * put this fairly direct code in line ...
1151 */
e2ac9628
HS		 1152 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1153 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
f2b7e78d 1154 fwr->tid_to_iq =
77a80e23
HS		 1155 htonl(FW_FILTER_WR_TID_V(ftid) |
1156 FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1157 FW_FILTER_WR_NOREPLY_V(0) |
1158 FW_FILTER_WR_IQ_V(f->fs.iq));
f2b7e78d 1159 fwr->del_filter_to_l2tix =
77a80e23
HS		 1160 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1161 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1162 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1163 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1164 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1165 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1166 FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1167 FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1168 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
f2b7e78d 1169 f->fs.newvlan == VLAN_REWRITE) |
77a80e23 1170 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
f2b7e78d 1171 f->fs.newvlan == VLAN_REWRITE) |
77a80e23
HS		 1172 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1173 FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1174 FW_FILTER_WR_PRIO_V(f->fs.prio) |
1175 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
f2b7e78d
VP		 1176 fwr->ethtype = htons(f->fs.val.ethtype);
1177 fwr->ethtypem = htons(f->fs.mask.ethtype);
1178 fwr->frag_to_ovlan_vldm =
77a80e23
HS		 1179 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1180 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1181 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1182 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1183 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1184 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
f2b7e78d
VP		 1185 fwr->smac_sel = 0;
1186 fwr->rx_chan_rx_rpl_iq =
77a80e23
HS		 1187 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1188 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
f2b7e78d 1189 fwr->maci_to_matchtypem =
77a80e23
HS		 1190 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1191 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1192 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1193 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1194 FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1195 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1196 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1197 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
f2b7e78d
VP		 1198 fwr->ptcl = f->fs.val.proto;
1199 fwr->ptclm = f->fs.mask.proto;
1200 fwr->ttyp = f->fs.val.tos;
1201 fwr->ttypm = f->fs.mask.tos;
1202 fwr->ivlan = htons(f->fs.val.ivlan);
1203 fwr->ivlanm = htons(f->fs.mask.ivlan);
1204 fwr->ovlan = htons(f->fs.val.ovlan);
1205 fwr->ovlanm = htons(f->fs.mask.ovlan);
1206 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1207 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1208 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1209 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1210 fwr->lp = htons(f->fs.val.lport);
1211 fwr->lpm = htons(f->fs.mask.lport);
1212 fwr->fp = htons(f->fs.val.fport);
1213 fwr->fpm = htons(f->fs.mask.fport);
1214 if (f->fs.newsmac)
1215 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1216
1217 /* Mark the filter as "pending" and ship off the Filter Work Request.
1218 * When we get the Work Request Reply we'll clear the pending status.
1219 */
1220 f->pending = 1;
1221 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1222 t4_ofld_send(adapter, skb);
1223 return 0;
1224}
1225
1226/* Delete the filter at a specified index.
1227 */
1228static int del_filter_wr(struct adapter *adapter, int fidx)
1229{
1230 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1231 struct sk_buff *skb;
1232 struct fw_filter_wr *fwr;
1233 unsigned int len, ftid;
1234
1235 len = sizeof(*fwr);
1236 ftid = adapter->tids.ftid_base + fidx;
1237
1238 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1239 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1240 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1241
1242 /* Mark the filter as "pending" and ship off the Filter Work Request.
1243 * When we get the Work Request Reply we'll clear the pending status.
1244 */
1245 f->pending = 1;
1246 t4_mgmt_tx(adapter, skb);
1247 return 0;
1248}
1249
688848b1
AB		 1250static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1251 void *accel_priv, select_queue_fallback_t fallback)
1252{
1253 int txq;
1254
1255#ifdef CONFIG_CHELSIO_T4_DCB
1256 /* If a Data Center Bridging has been successfully negotiated on this
1257 * link then we'll use the skb's priority to map it to a TX Queue.
1258 * The skb's priority is determined via the VLAN Tag Priority Code
1259 * Point field.
1260 */
1261 if (cxgb4_dcb_enabled(dev)) {
1262 u16 vlan_tci;
1263 int err;
1264
1265 err = vlan_get_tag(skb, &vlan_tci);
1266 if (unlikely(err)) {
1267 if (net_ratelimit())
1268 netdev_warn(dev,
1269 "TX Packet without VLAN Tag on DCB Link\n");
1270 txq = 0;
1271 } else {
1272 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1273 }
1274 return txq;
1275 }
1276#endif /* CONFIG_CHELSIO_T4_DCB */
1277
1278 if (select_queue) {
1279 txq = (skb_rx_queue_recorded(skb)
1280 ? skb_get_rx_queue(skb)
1281 : smp_processor_id());
1282
1283 while (unlikely(txq >= dev->real_num_tx_queues))
1284 txq -= dev->real_num_tx_queues;
1285
1286 return txq;
1287 }
1288
1289 return fallback(dev, skb) % dev->real_num_tx_queues;
1290}
1291
b8ff05a9
DM		 1292static inline int is_offload(const struct adapter *adap)
1293{
1294 return adap->params.offload;
1295}
1296
1297/*
1298 * Implementation of ethtool operations.
1299 */
1300
1301static u32 get_msglevel(struct net_device *dev)
1302{
1303 return netdev2adap(dev)->msg_enable;
1304}
1305
1306static void set_msglevel(struct net_device *dev, u32 val)
1307{
1308 netdev2adap(dev)->msg_enable = val;
1309}
1310
1311static char stats_strings[][ETH_GSTRING_LEN] = {
1312 "TxOctetsOK ",
1313 "TxFramesOK ",
1314 "TxBroadcastFrames ",
1315 "TxMulticastFrames ",
1316 "TxUnicastFrames ",
1317 "TxErrorFrames ",
1318
1319 "TxFrames64 ",
1320 "TxFrames65To127 ",
1321 "TxFrames128To255 ",
1322 "TxFrames256To511 ",
1323 "TxFrames512To1023 ",
1324 "TxFrames1024To1518 ",
1325 "TxFrames1519ToMax ",
1326
1327 "TxFramesDropped ",
1328 "TxPauseFrames ",
1329 "TxPPP0Frames ",
1330 "TxPPP1Frames ",
1331 "TxPPP2Frames ",
1332 "TxPPP3Frames ",
1333 "TxPPP4Frames ",
1334 "TxPPP5Frames ",
1335 "TxPPP6Frames ",
1336 "TxPPP7Frames ",
1337
1338 "RxOctetsOK ",
1339 "RxFramesOK ",
1340 "RxBroadcastFrames ",
1341 "RxMulticastFrames ",
1342 "RxUnicastFrames ",
1343
1344 "RxFramesTooLong ",
1345 "RxJabberErrors ",
1346 "RxFCSErrors ",
1347 "RxLengthErrors ",
1348 "RxSymbolErrors ",
1349 "RxRuntFrames ",
1350
1351 "RxFrames64 ",
1352 "RxFrames65To127 ",
1353 "RxFrames128To255 ",
1354 "RxFrames256To511 ",
1355 "RxFrames512To1023 ",
1356 "RxFrames1024To1518 ",
1357 "RxFrames1519ToMax ",
1358
1359 "RxPauseFrames ",
1360 "RxPPP0Frames ",
1361 "RxPPP1Frames ",
1362 "RxPPP2Frames ",
1363 "RxPPP3Frames ",
1364 "RxPPP4Frames ",
1365 "RxPPP5Frames ",
1366 "RxPPP6Frames ",
1367 "RxPPP7Frames ",
1368
1369 "RxBG0FramesDropped ",
1370 "RxBG1FramesDropped ",
1371 "RxBG2FramesDropped ",
1372 "RxBG3FramesDropped ",
1373 "RxBG0FramesTrunc ",
1374 "RxBG1FramesTrunc ",
1375 "RxBG2FramesTrunc ",
1376 "RxBG3FramesTrunc ",
1377
1378 "TSO ",
1379 "TxCsumOffload ",
1380 "RxCsumGood ",
1381 "VLANextractions ",
1382 "VLANinsertions ",
4a6346d4
DM		 1383 "GROpackets ",
1384 "GROmerged ",
22adfe0a
SR		 1385 "WriteCoalSuccess ",
1386 "WriteCoalFail ",
b8ff05a9
DM		 1387};
1388
1389static int get_sset_count(struct net_device *dev, int sset)
1390{
1391 switch (sset) {
1392 case ETH_SS_STATS:
1393 return ARRAY_SIZE(stats_strings);
1394 default:
1395 return -EOPNOTSUPP;
1396 }
1397}
1398
1399#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1400#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1401
1402static int get_regs_len(struct net_device *dev)
1403{
251f9e88 1404 struct adapter *adap = netdev2adap(dev);
d14807dd 1405 if (is_t4(adap->params.chip))
251f9e88
SR		 1406 return T4_REGMAP_SIZE;
1407 else
1408 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1409}
1410
1411static int get_eeprom_len(struct net_device *dev)
1412{
1413 return EEPROMSIZE;
1414}
1415
1416static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1417{
1418 struct adapter *adapter = netdev2adap(dev);
ba3f8cd5 1419 u32 exprom_vers;
b8ff05a9 1420
23020ab3
RJ		 1421 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1422 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1423 strlcpy(info->bus_info, pci_name(adapter->pdev),
1424 sizeof(info->bus_info));
b8ff05a9 1425
84b40501 1426 if (adapter->params.fw_vers)
b8ff05a9
DM		 1427 snprintf(info->fw_version, sizeof(info->fw_version),
1428 "%u.%u.%u.%u, TP %u.%u.%u.%u",
b2e1a3f0
HS		 1429 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1430 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1431 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1432 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1433 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1434 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1435 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1436 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
ba3f8cd5
HS		 1437
1438 if (!t4_get_exprom_version(adapter, &exprom_vers))
1439 snprintf(info->erom_version, sizeof(info->erom_version),
1440 "%u.%u.%u.%u",
1441 FW_HDR_FW_VER_MAJOR_G(exprom_vers),
1442 FW_HDR_FW_VER_MINOR_G(exprom_vers),
1443 FW_HDR_FW_VER_MICRO_G(exprom_vers),
1444 FW_HDR_FW_VER_BUILD_G(exprom_vers));
b8ff05a9
DM		 1445}
1446
1447static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1448{
1449 if (stringset == ETH_SS_STATS)
1450 memcpy(data, stats_strings, sizeof(stats_strings));
1451}
1452
1453/*
1454 * port stats maintained per queue of the port. They should be in the same
1455 * order as in stats_strings above.
1456 */
1457struct queue_port_stats {
1458 u64 tso;
1459 u64 tx_csum;
1460 u64 rx_csum;
1461 u64 vlan_ex;
1462 u64 vlan_ins;
4a6346d4
DM		 1463 u64 gro_pkts;
1464 u64 gro_merged;
b8ff05a9
DM		 1465};
1466
1467static void collect_sge_port_stats(const struct adapter *adap,
1468 const struct port_info *p, struct queue_port_stats *s)
1469{
1470 int i;
1471 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1472 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1473
1474 memset(s, 0, sizeof(*s));
1475 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1476 s->tso += tx->tso;
1477 s->tx_csum += tx->tx_cso;
1478 s->rx_csum += rx->stats.rx_cso;
1479 s->vlan_ex += rx->stats.vlan_ex;
1480 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1481 s->gro_pkts += rx->stats.lro_pkts;
1482 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1483 }
1484}
1485
1486static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1487 u64 *data)
1488{
1489 struct port_info *pi = netdev_priv(dev);
1490 struct adapter *adapter = pi->adapter;
22adfe0a 1491 u32 val1, val2;
b8ff05a9
DM		 1492
1493 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1494
1495 data += sizeof(struct port_stats) / sizeof(u64);
1496 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1497 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1498 if (!is_t4(adapter->params.chip)) {
f061de42
HS		 1499 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1500 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1501 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
22adfe0a
SR		 1502 *data = val1 - val2;
1503 data++;
1504 *data = val2;
1505 data++;
1506 } else {
1507 memset(data, 0, 2 * sizeof(u64));
1508 *data += 2;
1509 }
b8ff05a9
DM		 1510}
1511
1512/*
1513 * Return a version number to identify the type of adapter. The scheme is:
1514 * - bits 0..9: chip version
1515 * - bits 10..15: chip revision
835bb606 1516 * - bits 16..23: register dump version
b8ff05a9
DM		 1517 */
1518static inline unsigned int mk_adap_vers(const struct adapter *ap)
1519{
d14807dd
HS		 1520 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1521 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1522}
1523
1524static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1525 unsigned int end)
1526{
1527 u32 *p = buf + start;
1528
1529 for ( ; start <= end; start += sizeof(u32))
1530 *p++ = t4_read_reg(ap, start);
1531}
1532
1533static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1534 void *buf)
1535{
251f9e88 1536 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1537 0x1008, 0x1108,
1538 0x1180, 0x11b4,
1539 0x11fc, 0x123c,
1540 0x1300, 0x173c,
1541 0x1800, 0x18fc,
1542 0x3000, 0x30d8,
1543 0x30e0, 0x5924,
1544 0x5960, 0x59d4,
1545 0x5a00, 0x5af8,
1546 0x6000, 0x6098,
1547 0x6100, 0x6150,
1548 0x6200, 0x6208,
1549 0x6240, 0x6248,
1550 0x6280, 0x6338,
1551 0x6370, 0x638c,
1552 0x6400, 0x643c,
1553 0x6500, 0x6524,
1554 0x6a00, 0x6a38,
1555 0x6a60, 0x6a78,
1556 0x6b00, 0x6b84,
1557 0x6bf0, 0x6c84,
1558 0x6cf0, 0x6d84,
1559 0x6df0, 0x6e84,
1560 0x6ef0, 0x6f84,
1561 0x6ff0, 0x7084,
1562 0x70f0, 0x7184,
1563 0x71f0, 0x7284,
1564 0x72f0, 0x7384,
1565 0x73f0, 0x7450,
1566 0x7500, 0x7530,
1567 0x7600, 0x761c,
1568 0x7680, 0x76cc,
1569 0x7700, 0x7798,
1570 0x77c0, 0x77fc,
1571 0x7900, 0x79fc,
1572 0x7b00, 0x7c38,
1573 0x7d00, 0x7efc,
1574 0x8dc0, 0x8e1c,
1575 0x8e30, 0x8e78,
1576 0x8ea0, 0x8f6c,
1577 0x8fc0, 0x9074,
1578 0x90fc, 0x90fc,
1579 0x9400, 0x9458,
1580 0x9600, 0x96bc,
1581 0x9800, 0x9808,
1582 0x9820, 0x983c,
1583 0x9850, 0x9864,
1584 0x9c00, 0x9c6c,
1585 0x9c80, 0x9cec,
1586 0x9d00, 0x9d6c,
1587 0x9d80, 0x9dec,
1588 0x9e00, 0x9e6c,
1589 0x9e80, 0x9eec,
1590 0x9f00, 0x9f6c,
1591 0x9f80, 0x9fec,
1592 0xd004, 0xd03c,
1593 0xdfc0, 0xdfe0,
1594 0xe000, 0xea7c,
3d9103f8
HS		 1595 0xf000, 0x11110,
1596 0x11118, 0x11190,
835bb606
DM		 1597 0x19040, 0x1906c,
1598 0x19078, 0x19080,
1599 0x1908c, 0x19124,
b8ff05a9
DM		 1600 0x19150, 0x191b0,
1601 0x191d0, 0x191e8,
1602 0x19238, 0x1924c,
1603 0x193f8, 0x19474,
1604 0x19490, 0x194f8,
1605 0x19800, 0x19f30,
1606 0x1a000, 0x1a06c,
1607 0x1a0b0, 0x1a120,
1608 0x1a128, 0x1a138,
1609 0x1a190, 0x1a1c4,
1610 0x1a1fc, 0x1a1fc,
1611 0x1e040, 0x1e04c,
835bb606 1612 0x1e284, 0x1e28c,
b8ff05a9
DM		 1613 0x1e2c0, 0x1e2c0,
1614 0x1e2e0, 0x1e2e0,
1615 0x1e300, 0x1e384,
1616 0x1e3c0, 0x1e3c8,
1617 0x1e440, 0x1e44c,
835bb606 1618 0x1e684, 0x1e68c,
b8ff05a9
DM		 1619 0x1e6c0, 0x1e6c0,
1620 0x1e6e0, 0x1e6e0,
1621 0x1e700, 0x1e784,
1622 0x1e7c0, 0x1e7c8,
1623 0x1e840, 0x1e84c,
835bb606 1624 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1625 0x1eac0, 0x1eac0,
1626 0x1eae0, 0x1eae0,
1627 0x1eb00, 0x1eb84,
1628 0x1ebc0, 0x1ebc8,
1629 0x1ec40, 0x1ec4c,
835bb606 1630 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1631 0x1eec0, 0x1eec0,
1632 0x1eee0, 0x1eee0,
1633 0x1ef00, 0x1ef84,
1634 0x1efc0, 0x1efc8,
1635 0x1f040, 0x1f04c,
835bb606 1636 0x1f284, 0x1f28c,
b8ff05a9
DM		 1637 0x1f2c0, 0x1f2c0,
1638 0x1f2e0, 0x1f2e0,
1639 0x1f300, 0x1f384,
1640 0x1f3c0, 0x1f3c8,
1641 0x1f440, 0x1f44c,
835bb606 1642 0x1f684, 0x1f68c,
b8ff05a9
DM		 1643 0x1f6c0, 0x1f6c0,
1644 0x1f6e0, 0x1f6e0,
1645 0x1f700, 0x1f784,
1646 0x1f7c0, 0x1f7c8,
1647 0x1f840, 0x1f84c,
835bb606 1648 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1649 0x1fac0, 0x1fac0,
1650 0x1fae0, 0x1fae0,
1651 0x1fb00, 0x1fb84,
1652 0x1fbc0, 0x1fbc8,
1653 0x1fc40, 0x1fc4c,
835bb606 1654 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1655 0x1fec0, 0x1fec0,
1656 0x1fee0, 0x1fee0,
1657 0x1ff00, 0x1ff84,
1658 0x1ffc0, 0x1ffc8,
1659 0x20000, 0x2002c,
1660 0x20100, 0x2013c,
1661 0x20190, 0x201c8,
1662 0x20200, 0x20318,
1663 0x20400, 0x20528,
1664 0x20540, 0x20614,
1665 0x21000, 0x21040,
1666 0x2104c, 0x21060,
1667 0x210c0, 0x210ec,
1668 0x21200, 0x21268,
1669 0x21270, 0x21284,
1670 0x212fc, 0x21388,
1671 0x21400, 0x21404,
1672 0x21500, 0x21518,
1673 0x2152c, 0x2153c,
1674 0x21550, 0x21554,
1675 0x21600, 0x21600,
1676 0x21608, 0x21628,
1677 0x21630, 0x2163c,
1678 0x21700, 0x2171c,
1679 0x21780, 0x2178c,
1680 0x21800, 0x21c38,
1681 0x21c80, 0x21d7c,
1682 0x21e00, 0x21e04,
1683 0x22000, 0x2202c,
1684 0x22100, 0x2213c,
1685 0x22190, 0x221c8,
1686 0x22200, 0x22318,
1687 0x22400, 0x22528,
1688 0x22540, 0x22614,
1689 0x23000, 0x23040,
1690 0x2304c, 0x23060,
1691 0x230c0, 0x230ec,
1692 0x23200, 0x23268,
1693 0x23270, 0x23284,
1694 0x232fc, 0x23388,
1695 0x23400, 0x23404,
1696 0x23500, 0x23518,
1697 0x2352c, 0x2353c,
1698 0x23550, 0x23554,
1699 0x23600, 0x23600,
1700 0x23608, 0x23628,
1701 0x23630, 0x2363c,
1702 0x23700, 0x2371c,
1703 0x23780, 0x2378c,
1704 0x23800, 0x23c38,
1705 0x23c80, 0x23d7c,
1706 0x23e00, 0x23e04,
1707 0x24000, 0x2402c,
1708 0x24100, 0x2413c,
1709 0x24190, 0x241c8,
1710 0x24200, 0x24318,
1711 0x24400, 0x24528,
1712 0x24540, 0x24614,
1713 0x25000, 0x25040,
1714 0x2504c, 0x25060,
1715 0x250c0, 0x250ec,
1716 0x25200, 0x25268,
1717 0x25270, 0x25284,
1718 0x252fc, 0x25388,
1719 0x25400, 0x25404,
1720 0x25500, 0x25518,
1721 0x2552c, 0x2553c,
1722 0x25550, 0x25554,
1723 0x25600, 0x25600,
1724 0x25608, 0x25628,
1725 0x25630, 0x2563c,
1726 0x25700, 0x2571c,
1727 0x25780, 0x2578c,
1728 0x25800, 0x25c38,
1729 0x25c80, 0x25d7c,
1730 0x25e00, 0x25e04,
1731 0x26000, 0x2602c,
1732 0x26100, 0x2613c,
1733 0x26190, 0x261c8,
1734 0x26200, 0x26318,
1735 0x26400, 0x26528,
1736 0x26540, 0x26614,
1737 0x27000, 0x27040,
1738 0x2704c, 0x27060,
1739 0x270c0, 0x270ec,
1740 0x27200, 0x27268,
1741 0x27270, 0x27284,
1742 0x272fc, 0x27388,
1743 0x27400, 0x27404,
1744 0x27500, 0x27518,
1745 0x2752c, 0x2753c,
1746 0x27550, 0x27554,
1747 0x27600, 0x27600,
1748 0x27608, 0x27628,
1749 0x27630, 0x2763c,
1750 0x27700, 0x2771c,
1751 0x27780, 0x2778c,
1752 0x27800, 0x27c38,
1753 0x27c80, 0x27d7c,
1754 0x27e00, 0x27e04
1755 };
1756
251f9e88
SR		 1757 static const unsigned int t5_reg_ranges[] = {
1758 0x1008, 0x1148,
1759 0x1180, 0x11b4,
1760 0x11fc, 0x123c,
1761 0x1280, 0x173c,
1762 0x1800, 0x18fc,
1763 0x3000, 0x3028,
1764 0x3060, 0x30d8,
1765 0x30e0, 0x30fc,
1766 0x3140, 0x357c,
1767 0x35a8, 0x35cc,
1768 0x35ec, 0x35ec,
1769 0x3600, 0x5624,
1770 0x56cc, 0x575c,
1771 0x580c, 0x5814,
1772 0x5890, 0x58bc,
1773 0x5940, 0x59dc,
1774 0x59fc, 0x5a18,
1775 0x5a60, 0x5a9c,
1776 0x5b9c, 0x5bfc,
1777 0x6000, 0x6040,
1778 0x6058, 0x614c,
1779 0x7700, 0x7798,
1780 0x77c0, 0x78fc,
1781 0x7b00, 0x7c54,
1782 0x7d00, 0x7efc,
1783 0x8dc0, 0x8de0,
1784 0x8df8, 0x8e84,
1785 0x8ea0, 0x8f84,
1786 0x8fc0, 0x90f8,
1787 0x9400, 0x9470,
1788 0x9600, 0x96f4,
1789 0x9800, 0x9808,
1790 0x9820, 0x983c,
1791 0x9850, 0x9864,
1792 0x9c00, 0x9c6c,
1793 0x9c80, 0x9cec,
1794 0x9d00, 0x9d6c,
1795 0x9d80, 0x9dec,
1796 0x9e00, 0x9e6c,
1797 0x9e80, 0x9eec,
1798 0x9f00, 0x9f6c,
1799 0x9f80, 0xa020,
1800 0xd004, 0xd03c,
1801 0xdfc0, 0xdfe0,
1802 0xe000, 0x11088,
3d9103f8
HS		 1803 0x1109c, 0x11110,
1804 0x11118, 0x1117c,
251f9e88
SR		 1805 0x11190, 0x11204,
1806 0x19040, 0x1906c,
1807 0x19078, 0x19080,
1808 0x1908c, 0x19124,
1809 0x19150, 0x191b0,
1810 0x191d0, 0x191e8,
1811 0x19238, 0x19290,
1812 0x193f8, 0x19474,
1813 0x19490, 0x194cc,
1814 0x194f0, 0x194f8,
1815 0x19c00, 0x19c60,
1816 0x19c94, 0x19e10,
1817 0x19e50, 0x19f34,
1818 0x19f40, 0x19f50,
1819 0x19f90, 0x19fe4,
1820 0x1a000, 0x1a06c,
1821 0x1a0b0, 0x1a120,
1822 0x1a128, 0x1a138,
1823 0x1a190, 0x1a1c4,
1824 0x1a1fc, 0x1a1fc,
1825 0x1e008, 0x1e00c,
1826 0x1e040, 0x1e04c,
1827 0x1e284, 0x1e290,
1828 0x1e2c0, 0x1e2c0,
1829 0x1e2e0, 0x1e2e0,
1830 0x1e300, 0x1e384,
1831 0x1e3c0, 0x1e3c8,
1832 0x1e408, 0x1e40c,
1833 0x1e440, 0x1e44c,
1834 0x1e684, 0x1e690,
1835 0x1e6c0, 0x1e6c0,
1836 0x1e6e0, 0x1e6e0,
1837 0x1e700, 0x1e784,
1838 0x1e7c0, 0x1e7c8,
1839 0x1e808, 0x1e80c,
1840 0x1e840, 0x1e84c,
1841 0x1ea84, 0x1ea90,
1842 0x1eac0, 0x1eac0,
1843 0x1eae0, 0x1eae0,
1844 0x1eb00, 0x1eb84,
1845 0x1ebc0, 0x1ebc8,
1846 0x1ec08, 0x1ec0c,
1847 0x1ec40, 0x1ec4c,
1848 0x1ee84, 0x1ee90,
1849 0x1eec0, 0x1eec0,
1850 0x1eee0, 0x1eee0,
1851 0x1ef00, 0x1ef84,
1852 0x1efc0, 0x1efc8,
1853 0x1f008, 0x1f00c,
1854 0x1f040, 0x1f04c,
1855 0x1f284, 0x1f290,
1856 0x1f2c0, 0x1f2c0,
1857 0x1f2e0, 0x1f2e0,
1858 0x1f300, 0x1f384,
1859 0x1f3c0, 0x1f3c8,
1860 0x1f408, 0x1f40c,
1861 0x1f440, 0x1f44c,
1862 0x1f684, 0x1f690,
1863 0x1f6c0, 0x1f6c0,
1864 0x1f6e0, 0x1f6e0,
1865 0x1f700, 0x1f784,
1866 0x1f7c0, 0x1f7c8,
1867 0x1f808, 0x1f80c,
1868 0x1f840, 0x1f84c,
1869 0x1fa84, 0x1fa90,
1870 0x1fac0, 0x1fac0,
1871 0x1fae0, 0x1fae0,
1872 0x1fb00, 0x1fb84,
1873 0x1fbc0, 0x1fbc8,
1874 0x1fc08, 0x1fc0c,
1875 0x1fc40, 0x1fc4c,
1876 0x1fe84, 0x1fe90,
1877 0x1fec0, 0x1fec0,
1878 0x1fee0, 0x1fee0,
1879 0x1ff00, 0x1ff84,
1880 0x1ffc0, 0x1ffc8,
1881 0x30000, 0x30030,
1882 0x30100, 0x30144,
1883 0x30190, 0x301d0,
1884 0x30200, 0x30318,
1885 0x30400, 0x3052c,
1886 0x30540, 0x3061c,
1887 0x30800, 0x30834,
1888 0x308c0, 0x30908,
1889 0x30910, 0x309ac,
1890 0x30a00, 0x30a04,
1891 0x30a0c, 0x30a2c,
1892 0x30a44, 0x30a50,
1893 0x30a74, 0x30c24,
1894 0x30d08, 0x30d14,
1895 0x30d1c, 0x30d20,
1896 0x30d3c, 0x30d50,
1897 0x31200, 0x3120c,
1898 0x31220, 0x31220,
1899 0x31240, 0x31240,
1900 0x31600, 0x31600,
1901 0x31608, 0x3160c,
1902 0x31a00, 0x31a1c,
1903 0x31e04, 0x31e20,
1904 0x31e38, 0x31e3c,
1905 0x31e80, 0x31e80,
1906 0x31e88, 0x31ea8,
1907 0x31eb0, 0x31eb4,
1908 0x31ec8, 0x31ed4,
1909 0x31fb8, 0x32004,
1910 0x32208, 0x3223c,
1911 0x32600, 0x32630,
1912 0x32a00, 0x32abc,
1913 0x32b00, 0x32b70,
1914 0x33000, 0x33048,
1915 0x33060, 0x3309c,
1916 0x330f0, 0x33148,
1917 0x33160, 0x3319c,
1918 0x331f0, 0x332e4,
1919 0x332f8, 0x333e4,
1920 0x333f8, 0x33448,
1921 0x33460, 0x3349c,
1922 0x334f0, 0x33548,
1923 0x33560, 0x3359c,
1924 0x335f0, 0x336e4,
1925 0x336f8, 0x337e4,
1926 0x337f8, 0x337fc,
1927 0x33814, 0x33814,
1928 0x3382c, 0x3382c,
1929 0x33880, 0x3388c,
1930 0x338e8, 0x338ec,
1931 0x33900, 0x33948,
1932 0x33960, 0x3399c,
1933 0x339f0, 0x33ae4,
1934 0x33af8, 0x33b10,
1935 0x33b28, 0x33b28,
1936 0x33b3c, 0x33b50,
1937 0x33bf0, 0x33c10,
1938 0x33c28, 0x33c28,
1939 0x33c3c, 0x33c50,
1940 0x33cf0, 0x33cfc,
1941 0x34000, 0x34030,
1942 0x34100, 0x34144,
1943 0x34190, 0x341d0,
1944 0x34200, 0x34318,
1945 0x34400, 0x3452c,
1946 0x34540, 0x3461c,
1947 0x34800, 0x34834,
1948 0x348c0, 0x34908,
1949 0x34910, 0x349ac,
1950 0x34a00, 0x34a04,
1951 0x34a0c, 0x34a2c,
1952 0x34a44, 0x34a50,
1953 0x34a74, 0x34c24,
1954 0x34d08, 0x34d14,
1955 0x34d1c, 0x34d20,
1956 0x34d3c, 0x34d50,
1957 0x35200, 0x3520c,
1958 0x35220, 0x35220,
1959 0x35240, 0x35240,
1960 0x35600, 0x35600,
1961 0x35608, 0x3560c,
1962 0x35a00, 0x35a1c,
1963 0x35e04, 0x35e20,
1964 0x35e38, 0x35e3c,
1965 0x35e80, 0x35e80,
1966 0x35e88, 0x35ea8,
1967 0x35eb0, 0x35eb4,
1968 0x35ec8, 0x35ed4,
1969 0x35fb8, 0x36004,
1970 0x36208, 0x3623c,
1971 0x36600, 0x36630,
1972 0x36a00, 0x36abc,
1973 0x36b00, 0x36b70,
1974 0x37000, 0x37048,
1975 0x37060, 0x3709c,
1976 0x370f0, 0x37148,
1977 0x37160, 0x3719c,
1978 0x371f0, 0x372e4,
1979 0x372f8, 0x373e4,
1980 0x373f8, 0x37448,
1981 0x37460, 0x3749c,
1982 0x374f0, 0x37548,
1983 0x37560, 0x3759c,
1984 0x375f0, 0x376e4,
1985 0x376f8, 0x377e4,
1986 0x377f8, 0x377fc,
1987 0x37814, 0x37814,
1988 0x3782c, 0x3782c,
1989 0x37880, 0x3788c,
1990 0x378e8, 0x378ec,
1991 0x37900, 0x37948,
1992 0x37960, 0x3799c,
1993 0x379f0, 0x37ae4,
1994 0x37af8, 0x37b10,
1995 0x37b28, 0x37b28,
1996 0x37b3c, 0x37b50,
1997 0x37bf0, 0x37c10,
1998 0x37c28, 0x37c28,
1999 0x37c3c, 0x37c50,
2000 0x37cf0, 0x37cfc,
2001 0x38000, 0x38030,
2002 0x38100, 0x38144,
2003 0x38190, 0x381d0,
2004 0x38200, 0x38318,
2005 0x38400, 0x3852c,
2006 0x38540, 0x3861c,
2007 0x38800, 0x38834,
2008 0x388c0, 0x38908,
2009 0x38910, 0x389ac,
2010 0x38a00, 0x38a04,
2011 0x38a0c, 0x38a2c,
2012 0x38a44, 0x38a50,
2013 0x38a74, 0x38c24,
2014 0x38d08, 0x38d14,
2015 0x38d1c, 0x38d20,
2016 0x38d3c, 0x38d50,
2017 0x39200, 0x3920c,
2018 0x39220, 0x39220,
2019 0x39240, 0x39240,
2020 0x39600, 0x39600,
2021 0x39608, 0x3960c,
2022 0x39a00, 0x39a1c,
2023 0x39e04, 0x39e20,
2024 0x39e38, 0x39e3c,
2025 0x39e80, 0x39e80,
2026 0x39e88, 0x39ea8,
2027 0x39eb0, 0x39eb4,
2028 0x39ec8, 0x39ed4,
2029 0x39fb8, 0x3a004,
2030 0x3a208, 0x3a23c,
2031 0x3a600, 0x3a630,
2032 0x3aa00, 0x3aabc,
2033 0x3ab00, 0x3ab70,
2034 0x3b000, 0x3b048,
2035 0x3b060, 0x3b09c,
2036 0x3b0f0, 0x3b148,
2037 0x3b160, 0x3b19c,
2038 0x3b1f0, 0x3b2e4,
2039 0x3b2f8, 0x3b3e4,
2040 0x3b3f8, 0x3b448,
2041 0x3b460, 0x3b49c,
2042 0x3b4f0, 0x3b548,
2043 0x3b560, 0x3b59c,
2044 0x3b5f0, 0x3b6e4,
2045 0x3b6f8, 0x3b7e4,
2046 0x3b7f8, 0x3b7fc,
2047 0x3b814, 0x3b814,
2048 0x3b82c, 0x3b82c,
2049 0x3b880, 0x3b88c,
2050 0x3b8e8, 0x3b8ec,
2051 0x3b900, 0x3b948,
2052 0x3b960, 0x3b99c,
2053 0x3b9f0, 0x3bae4,
2054 0x3baf8, 0x3bb10,
2055 0x3bb28, 0x3bb28,
2056 0x3bb3c, 0x3bb50,
2057 0x3bbf0, 0x3bc10,
2058 0x3bc28, 0x3bc28,
2059 0x3bc3c, 0x3bc50,
2060 0x3bcf0, 0x3bcfc,
2061 0x3c000, 0x3c030,
2062 0x3c100, 0x3c144,
2063 0x3c190, 0x3c1d0,
2064 0x3c200, 0x3c318,
2065 0x3c400, 0x3c52c,
2066 0x3c540, 0x3c61c,
2067 0x3c800, 0x3c834,
2068 0x3c8c0, 0x3c908,
2069 0x3c910, 0x3c9ac,
2070 0x3ca00, 0x3ca04,
2071 0x3ca0c, 0x3ca2c,
2072 0x3ca44, 0x3ca50,
2073 0x3ca74, 0x3cc24,
2074 0x3cd08, 0x3cd14,
2075 0x3cd1c, 0x3cd20,
2076 0x3cd3c, 0x3cd50,
2077 0x3d200, 0x3d20c,
2078 0x3d220, 0x3d220,
2079 0x3d240, 0x3d240,
2080 0x3d600, 0x3d600,
2081 0x3d608, 0x3d60c,
2082 0x3da00, 0x3da1c,
2083 0x3de04, 0x3de20,
2084 0x3de38, 0x3de3c,
2085 0x3de80, 0x3de80,
2086 0x3de88, 0x3dea8,
2087 0x3deb0, 0x3deb4,
2088 0x3dec8, 0x3ded4,
2089 0x3dfb8, 0x3e004,
2090 0x3e208, 0x3e23c,
2091 0x3e600, 0x3e630,
2092 0x3ea00, 0x3eabc,
2093 0x3eb00, 0x3eb70,
2094 0x3f000, 0x3f048,
2095 0x3f060, 0x3f09c,
2096 0x3f0f0, 0x3f148,
2097 0x3f160, 0x3f19c,
2098 0x3f1f0, 0x3f2e4,
2099 0x3f2f8, 0x3f3e4,
2100 0x3f3f8, 0x3f448,
2101 0x3f460, 0x3f49c,
2102 0x3f4f0, 0x3f548,
2103 0x3f560, 0x3f59c,
2104 0x3f5f0, 0x3f6e4,
2105 0x3f6f8, 0x3f7e4,
2106 0x3f7f8, 0x3f7fc,
2107 0x3f814, 0x3f814,
2108 0x3f82c, 0x3f82c,
2109 0x3f880, 0x3f88c,
2110 0x3f8e8, 0x3f8ec,
2111 0x3f900, 0x3f948,
2112 0x3f960, 0x3f99c,
2113 0x3f9f0, 0x3fae4,
2114 0x3faf8, 0x3fb10,
2115 0x3fb28, 0x3fb28,
2116 0x3fb3c, 0x3fb50,
2117 0x3fbf0, 0x3fc10,
2118 0x3fc28, 0x3fc28,
2119 0x3fc3c, 0x3fc50,
2120 0x3fcf0, 0x3fcfc,
2121 0x40000, 0x4000c,
2122 0x40040, 0x40068,
2123 0x40080, 0x40144,
2124 0x40180, 0x4018c,
2125 0x40200, 0x40298,
2126 0x402ac, 0x4033c,
2127 0x403f8, 0x403fc,
c1f49e3e 2128 0x41304, 0x413c4,
251f9e88
SR		 2129 0x41400, 0x4141c,
2130 0x41480, 0x414d0,
2131 0x44000, 0x44078,
2132 0x440c0, 0x44278,
2133 0x442c0, 0x44478,
2134 0x444c0, 0x44678,
2135 0x446c0, 0x44878,
2136 0x448c0, 0x449fc,
2137 0x45000, 0x45068,
2138 0x45080, 0x45084,
2139 0x450a0, 0x450b0,
2140 0x45200, 0x45268,
2141 0x45280, 0x45284,
2142 0x452a0, 0x452b0,
2143 0x460c0, 0x460e4,
2144 0x47000, 0x4708c,
2145 0x47200, 0x47250,
2146 0x47400, 0x47420,
2147 0x47600, 0x47618,
2148 0x47800, 0x47814,
2149 0x48000, 0x4800c,
2150 0x48040, 0x48068,
2151 0x48080, 0x48144,
2152 0x48180, 0x4818c,
2153 0x48200, 0x48298,
2154 0x482ac, 0x4833c,
2155 0x483f8, 0x483fc,
c1f49e3e 2156 0x49304, 0x493c4,
251f9e88
SR		 2157 0x49400, 0x4941c,
2158 0x49480, 0x494d0,
2159 0x4c000, 0x4c078,
2160 0x4c0c0, 0x4c278,
2161 0x4c2c0, 0x4c478,
2162 0x4c4c0, 0x4c678,
2163 0x4c6c0, 0x4c878,
2164 0x4c8c0, 0x4c9fc,
2165 0x4d000, 0x4d068,
2166 0x4d080, 0x4d084,
2167 0x4d0a0, 0x4d0b0,
2168 0x4d200, 0x4d268,
2169 0x4d280, 0x4d284,
2170 0x4d2a0, 0x4d2b0,
2171 0x4e0c0, 0x4e0e4,
2172 0x4f000, 0x4f08c,
2173 0x4f200, 0x4f250,
2174 0x4f400, 0x4f420,
2175 0x4f600, 0x4f618,
2176 0x4f800, 0x4f814,
2177 0x50000, 0x500cc,
2178 0x50400, 0x50400,
2179 0x50800, 0x508cc,
2180 0x50c00, 0x50c00,
2181 0x51000, 0x5101c,
2182 0x51300, 0x51308,
2183 };
2184
b8ff05a9
DM		 2185 int i;
2186 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2187 static const unsigned int *reg_ranges;
2188 int arr_size = 0, buf_size = 0;
2189
d14807dd 2190 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2191 reg_ranges = &t4_reg_ranges[0];
2192 arr_size = ARRAY_SIZE(t4_reg_ranges);
2193 buf_size = T4_REGMAP_SIZE;
2194 } else {
2195 reg_ranges = &t5_reg_ranges[0];
2196 arr_size = ARRAY_SIZE(t5_reg_ranges);
2197 buf_size = T5_REGMAP_SIZE;
2198 }
b8ff05a9
DM		 2199
2200 regs->version = mk_adap_vers(ap);
2201
251f9e88
SR		 2202 memset(buf, 0, buf_size);
2203 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2204 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2205}
2206
2207static int restart_autoneg(struct net_device *dev)
2208{
2209 struct port_info *p = netdev_priv(dev);
2210
2211 if (!netif_running(dev))
2212 return -EAGAIN;
2213 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2214 return -EINVAL;
060e0c75 2215 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2216 return 0;
2217}
2218
c5e06360
DM		 2219static int identify_port(struct net_device *dev,
2220 enum ethtool_phys_id_state state)
b8ff05a9 2221{
c5e06360 2222 unsigned int val;
060e0c75
DM		 2223 struct adapter *adap = netdev2adap(dev);
2224
c5e06360
DM		 2225 if (state == ETHTOOL_ID_ACTIVE)
2226 val = 0xffff;
2227 else if (state == ETHTOOL_ID_INACTIVE)
2228 val = 0;
2229 else
2230 return -EINVAL;
b8ff05a9 2231
c5e06360 2232 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2233}
2234
40e9de4b 2235static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
b8ff05a9
DM		 2236{
2237 unsigned int v = 0;
2238
a0881cab
DM		 2239 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2240 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2241 v |= SUPPORTED_TP;
2242 if (caps & FW_PORT_CAP_SPEED_100M)
2243 v |= SUPPORTED_100baseT_Full;
2244 if (caps & FW_PORT_CAP_SPEED_1G)
2245 v |= SUPPORTED_1000baseT_Full;
2246 if (caps & FW_PORT_CAP_SPEED_10G)
2247 v |= SUPPORTED_10000baseT_Full;
2248 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2249 v |= SUPPORTED_Backplane;
2250 if (caps & FW_PORT_CAP_SPEED_1G)
2251 v |= SUPPORTED_1000baseKX_Full;
2252 if (caps & FW_PORT_CAP_SPEED_10G)
2253 v |= SUPPORTED_10000baseKX4_Full;
2254 } else if (type == FW_PORT_TYPE_KR)
2255 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2256 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2257 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2258 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2259 else if (type == FW_PORT_TYPE_BP4_AP)
2260 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2261 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2262 SUPPORTED_10000baseKX4_Full;
a0881cab 2263 else if (type == FW_PORT_TYPE_FIBER_XFI ||
40e9de4b
HS		 2264 type == FW_PORT_TYPE_FIBER_XAUI ||
2265 type == FW_PORT_TYPE_SFP ||
2266 type == FW_PORT_TYPE_QSFP_10G ||
2267 type == FW_PORT_TYPE_QSA) {
b8ff05a9 2268 v |= SUPPORTED_FIBRE;
4c2d5186
HS		 2269 if (caps & FW_PORT_CAP_SPEED_1G)
2270 v |= SUPPORTED_1000baseT_Full;
2271 if (caps & FW_PORT_CAP_SPEED_10G)
2272 v |= SUPPORTED_10000baseT_Full;
40e9de4b
HS		 2273 } else if (type == FW_PORT_TYPE_BP40_BA ||
2274 type == FW_PORT_TYPE_QSFP) {
72aca4bf 2275 v |= SUPPORTED_40000baseSR4_Full;
40e9de4b
HS		 2276 v |= SUPPORTED_FIBRE;
2277 }
b8ff05a9
DM		 2278
2279 if (caps & FW_PORT_CAP_ANEG)
2280 v |= SUPPORTED_Autoneg;
2281 return v;
2282}
2283
2284static unsigned int to_fw_linkcaps(unsigned int caps)
2285{
2286 unsigned int v = 0;
2287
2288 if (caps & ADVERTISED_100baseT_Full)
2289 v |= FW_PORT_CAP_SPEED_100M;
2290 if (caps & ADVERTISED_1000baseT_Full)
2291 v |= FW_PORT_CAP_SPEED_1G;
2292 if (caps & ADVERTISED_10000baseT_Full)
2293 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2294 if (caps & ADVERTISED_40000baseSR4_Full)
2295 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2296 return v;
2297}
2298
2299static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2300{
2301 const struct port_info *p = netdev_priv(dev);
2302
2303 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2304 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2305 p->port_type == FW_PORT_TYPE_BT_XAUI)
2306 cmd->port = PORT_TP;
a0881cab
DM		 2307 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2308 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2309 cmd->port = PORT_FIBRE;
3e00a509
HS		 2310 else if (p->port_type == FW_PORT_TYPE_SFP ||
2311 p->port_type == FW_PORT_TYPE_QSFP_10G ||
40e9de4b 2312 p->port_type == FW_PORT_TYPE_QSA ||
3e00a509
HS		 2313 p->port_type == FW_PORT_TYPE_QSFP) {
2314 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2315 p->mod_type == FW_PORT_MOD_TYPE_SR ||
2316 p->mod_type == FW_PORT_MOD_TYPE_ER ||
2317 p->mod_type == FW_PORT_MOD_TYPE_LRM)
2318 cmd->port = PORT_FIBRE;
2319 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2320 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
a0881cab
DM		 2321 cmd->port = PORT_DA;
2322 else
3e00a509 2323 cmd->port = PORT_OTHER;
a0881cab 2324 } else
b8ff05a9
DM		 2325 cmd->port = PORT_OTHER;
2326
2327 if (p->mdio_addr >= 0) {
2328 cmd->phy_address = p->mdio_addr;
2329 cmd->transceiver = XCVR_EXTERNAL;
2330 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2331 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2332 } else {
2333 cmd->phy_address = 0; /* not really, but no better option */
2334 cmd->transceiver = XCVR_INTERNAL;
2335 cmd->mdio_support = 0;
2336 }
2337
2338 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2339 cmd->advertising = from_fw_linkcaps(p->port_type,
2340 p->link_cfg.advertising);
70739497
DD		 2341 ethtool_cmd_speed_set(cmd,
2342 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2343 cmd->duplex = DUPLEX_FULL;
2344 cmd->autoneg = p->link_cfg.autoneg;
2345 cmd->maxtxpkt = 0;
2346 cmd->maxrxpkt = 0;
2347 return 0;
2348}
2349
2350static unsigned int speed_to_caps(int speed)
2351{
e8b39015 2352 if (speed == 100)
b8ff05a9 2353 return FW_PORT_CAP_SPEED_100M;
e8b39015 2354 if (speed == 1000)
b8ff05a9 2355 return FW_PORT_CAP_SPEED_1G;
e8b39015 2356 if (speed == 10000)
b8ff05a9 2357 return FW_PORT_CAP_SPEED_10G;
e8b39015 2358 if (speed == 40000)
72aca4bf 2359 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2360 return 0;
2361}
2362
2363static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2364{
2365 unsigned int cap;
2366 struct port_info *p = netdev_priv(dev);
2367 struct link_config *lc = &p->link_cfg;
25db0338 2368 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2369
2370 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2371 return -EINVAL;
2372
2373 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2374 /*
2375 * PHY offers a single speed. See if that's what's
2376 * being requested.
2377 */
2378 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2379 (lc->supported & speed_to_caps(speed)))
2380 return 0;
b8ff05a9
DM		 2381 return -EINVAL;
2382 }
2383
2384 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2385 cap = speed_to_caps(speed);
b8ff05a9 2386
72aca4bf 2387 if (!(lc->supported & cap) ||
e8b39015
BH		 2388 (speed == 1000) ||
2389 (speed == 10000) ||
72aca4bf 2390 (speed == 40000))
b8ff05a9
DM		 2391 return -EINVAL;
2392 lc->requested_speed = cap;
2393 lc->advertising = 0;
2394 } else {
2395 cap = to_fw_linkcaps(cmd->advertising);
2396 if (!(lc->supported & cap))
2397 return -EINVAL;
2398 lc->requested_speed = 0;
2399 lc->advertising = cap | FW_PORT_CAP_ANEG;
2400 }
2401 lc->autoneg = cmd->autoneg;
2402
2403 if (netif_running(dev))
060e0c75
DM		 2404 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2405 lc);
b8ff05a9
DM		 2406 return 0;
2407}
2408
2409static void get_pauseparam(struct net_device *dev,
2410 struct ethtool_pauseparam *epause)
2411{
2412 struct port_info *p = netdev_priv(dev);
2413
2414 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2415 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2416 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2417}
2418
2419static int set_pauseparam(struct net_device *dev,
2420 struct ethtool_pauseparam *epause)
2421{
2422 struct port_info *p = netdev_priv(dev);
2423 struct link_config *lc = &p->link_cfg;
2424
2425 if (epause->autoneg == AUTONEG_DISABLE)
2426 lc->requested_fc = 0;
2427 else if (lc->supported & FW_PORT_CAP_ANEG)
2428 lc->requested_fc = PAUSE_AUTONEG;
2429 else
2430 return -EINVAL;
2431
2432 if (epause->rx_pause)
2433 lc->requested_fc |= PAUSE_RX;
2434 if (epause->tx_pause)
2435 lc->requested_fc |= PAUSE_TX;
2436 if (netif_running(dev))
060e0c75
DM		 2437 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2438 lc);
b8ff05a9
DM		 2439 return 0;
2440}
2441
b8ff05a9
DM		 2442static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2443{
2444 const struct port_info *pi = netdev_priv(dev);
2445 const struct sge *s = &pi->adapter->sge;
2446
2447 e->rx_max_pending = MAX_RX_BUFFERS;
2448 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2449 e->rx_jumbo_max_pending = 0;
2450 e->tx_max_pending = MAX_TXQ_ENTRIES;
2451
2452 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2453 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2454 e->rx_jumbo_pending = 0;
2455 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2456}
2457
2458static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2459{
2460 int i;
2461 const struct port_info *pi = netdev_priv(dev);
2462 struct adapter *adapter = pi->adapter;
2463 struct sge *s = &adapter->sge;
2464
2465 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2466 e->tx_pending > MAX_TXQ_ENTRIES ||
2467 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2468 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2469 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2470 return -EINVAL;
2471
2472 if (adapter->flags & FULL_INIT_DONE)
2473 return -EBUSY;
2474
2475 for (i = 0; i < pi->nqsets; ++i) {
2476 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2477 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2478 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2479 }
2480 return 0;
2481}
2482
2483static int closest_timer(const struct sge *s, int time)
2484{
2485 int i, delta, match = 0, min_delta = INT_MAX;
2486
2487 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2488 delta = time - s->timer_val[i];
2489 if (delta < 0)
2490 delta = -delta;
2491 if (delta < min_delta) {
2492 min_delta = delta;
2493 match = i;
2494 }
2495 }
2496 return match;
2497}
2498
2499static int closest_thres(const struct sge *s, int thres)
2500{
2501 int i, delta, match = 0, min_delta = INT_MAX;
2502
2503 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2504 delta = thres - s->counter_val[i];
2505 if (delta < 0)
2506 delta = -delta;
2507 if (delta < min_delta) {
2508 min_delta = delta;
2509 match = i;
2510 }
2511 }
2512 return match;
2513}
2514
2515/*
2516 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2517 */
dc9daab2
HS		 2518unsigned int qtimer_val(const struct adapter *adap,
2519 const struct sge_rspq *q)
b8ff05a9
DM		 2520{
2521 unsigned int idx = q->intr_params >> 1;
2522
2523 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2524}
2525
2526/**
c887ad0e 2527 * set_rspq_intr_params - set a queue's interrupt holdoff parameters
b8ff05a9
DM		 2528 * @q: the Rx queue
2529 * @us: the hold-off time in us, or 0 to disable timer
2530 * @cnt: the hold-off packet count, or 0 to disable counter
2531 *
2532 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2533 * one of the two needs to be enabled for the queue to generate interrupts.
2534 */
c887ad0e
HS		 2535static int set_rspq_intr_params(struct sge_rspq *q,
2536 unsigned int us, unsigned int cnt)
b8ff05a9 2537{
c887ad0e
HS		 2538 struct adapter *adap = q->adap;
2539
b8ff05a9
DM		 2540 if ((us | cnt) == 0)
2541 cnt = 1;
2542
2543 if (cnt) {
2544 int err;
2545 u32 v, new_idx;
2546
2547 new_idx = closest_thres(&adap->sge, cnt);
2548 if (q->desc && q->pktcnt_idx != new_idx) {
2549 /* the queue has already been created, update it */
5167865a
HS		 2550 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2551 FW_PARAMS_PARAM_X_V(
2552 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2553 FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
060e0c75
DM		 2554 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2555 &new_idx);
b8ff05a9
DM		 2556 if (err)
2557 return err;
2558 }
2559 q->pktcnt_idx = new_idx;
2560 }
2561
2562 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2563 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2564 return 0;
2565}
2566
c887ad0e
HS		 2567/**
2568 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2569 * @dev: the network device
2570 * @us: the hold-off time in us, or 0 to disable timer
2571 * @cnt: the hold-off packet count, or 0 to disable counter
2572 *
2573 * Set the RX interrupt hold-off parameters for a network device.
2574 */
2575static int set_rx_intr_params(struct net_device *dev,
2576 unsigned int us, unsigned int cnt)
b8ff05a9 2577{
c887ad0e
HS		 2578 int i, err;
2579 struct port_info *pi = netdev_priv(dev);
b8ff05a9 2580 struct adapter *adap = pi->adapter;
c887ad0e
HS		 2581 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2582
2583 for (i = 0; i < pi->nqsets; i++, q++) {
2584 err = set_rspq_intr_params(&q->rspq, us, cnt);
2585 if (err)
2586 return err;
d4fc9dc2 2587 }
c887ad0e
HS		 2588 return 0;
2589}
2590
e553ec3f
HS		 2591static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2592{
2593 int i;
2594 struct port_info *pi = netdev_priv(dev);
2595 struct adapter *adap = pi->adapter;
2596 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2597
2598 for (i = 0; i < pi->nqsets; i++, q++)
2599 q->rspq.adaptive_rx = adaptive_rx;
2600
2601 return 0;
2602}
2603
2604static int get_adaptive_rx_setting(struct net_device *dev)
2605{
2606 struct port_info *pi = netdev_priv(dev);
2607 struct adapter *adap = pi->adapter;
2608 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2609
2610 return q->rspq.adaptive_rx;
2611}
2612
c887ad0e
HS		 2613static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2614{
e553ec3f 2615 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
c887ad0e
HS		 2616 return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2617 c->rx_max_coalesced_frames);
b8ff05a9
DM		 2618}
2619
2620static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2621{
2622 const struct port_info *pi = netdev_priv(dev);
2623 const struct adapter *adap = pi->adapter;
2624 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2625
2626 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2627 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2628 adap->sge.counter_val[rq->pktcnt_idx] : 0;
e553ec3f 2629 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
b8ff05a9
DM		 2630 return 0;
2631}
2632
1478b3ee
DM		 2633/**
2634 * eeprom_ptov - translate a physical EEPROM address to virtual
2635 * @phys_addr: the physical EEPROM address
2636 * @fn: the PCI function number
2637 * @sz: size of function-specific area
2638 *
2639 * Translate a physical EEPROM address to virtual. The first 1K is
2640 * accessed through virtual addresses starting at 31K, the rest is
2641 * accessed through virtual addresses starting at 0.
2642 *
2643 * The mapping is as follows:
2644 * [0..1K) -> [31K..32K)
2645 * [1K..1K+A) -> [31K-A..31K)
2646 * [1K+A..ES) -> [0..ES-A-1K)
2647 *
2648 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2649 */
1478b3ee 2650static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2651{
1478b3ee 2652 fn *= sz;
b8ff05a9
DM		 2653 if (phys_addr < 1024)
2654 return phys_addr + (31 << 10);
1478b3ee
DM		 2655 if (phys_addr < 1024 + fn)
2656 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2657 if (phys_addr < EEPROMSIZE)
1478b3ee 2658 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2659 return -EINVAL;
2660}
2661
2662/*
2663 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2664 */
2665static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2666{
1478b3ee 2667 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2668
2669 if (vaddr >= 0)
2670 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2671 return vaddr < 0 ? vaddr : 0;
2672}
2673
2674static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2675{
1478b3ee 2676 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2677
2678 if (vaddr >= 0)
2679 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2680 return vaddr < 0 ? vaddr : 0;
2681}
2682
2683#define EEPROM_MAGIC 0x38E2F10C
2684
2685static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2686 u8 *data)
2687{
2688 int i, err = 0;
2689 struct adapter *adapter = netdev2adap(dev);
2690
2691 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2692 if (!buf)
2693 return -ENOMEM;
2694
2695 e->magic = EEPROM_MAGIC;
2696 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2697 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2698
2699 if (!err)
2700 memcpy(data, buf + e->offset, e->len);
2701 kfree(buf);
2702 return err;
2703}
2704
2705static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2706 u8 *data)
2707{
2708 u8 *buf;
2709 int err = 0;
2710 u32 aligned_offset, aligned_len, *p;
2711 struct adapter *adapter = netdev2adap(dev);
2712
2713 if (eeprom->magic != EEPROM_MAGIC)
2714 return -EINVAL;
2715
2716 aligned_offset = eeprom->offset & ~3;
2717 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2718
1478b3ee
DM		 2719 if (adapter->fn > 0) {
2720 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2721
2722 if (aligned_offset < start ||
2723 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2724 return -EPERM;
2725 }
2726
b8ff05a9
DM		 2727 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2728 /*
2729 * RMW possibly needed for first or last words.
2730 */
2731 buf = kmalloc(aligned_len, GFP_KERNEL);
2732 if (!buf)
2733 return -ENOMEM;
2734 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2735 if (!err && aligned_len > 4)
2736 err = eeprom_rd_phys(adapter,
2737 aligned_offset + aligned_len - 4,
2738 (u32 *)&buf[aligned_len - 4]);
2739 if (err)
2740 goto out;
2741 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2742 } else
2743 buf = data;
2744
2745 err = t4_seeprom_wp(adapter, false);
2746 if (err)
2747 goto out;
2748
2749 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2750 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2751 aligned_offset += 4;
2752 }
2753
2754 if (!err)
2755 err = t4_seeprom_wp(adapter, true);
2756out:
2757 if (buf != data)
2758 kfree(buf);
2759 return err;
2760}
2761
2762static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2763{
2764 int ret;
2765 const struct firmware *fw;
2766 struct adapter *adap = netdev2adap(netdev);
b2e1a3f0 2767 unsigned int mbox = PCIE_FW_MASTER_M + 1;
b8ff05a9
DM		 2768
2769 ef->data[sizeof(ef->data) - 1] = '\0';
2770 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2771 if (ret < 0)
2772 return ret;
2773
22c0b963
HS		 2774 /* If the adapter has been fully initialized then we'll go ahead and
2775 * try to get the firmware's cooperation in upgrading to the new
2776 * firmware image otherwise we'll try to do the entire job from the
2777 * host ... and we always "force" the operation in this path.
2778 */
2779 if (adap->flags & FULL_INIT_DONE)
2780 mbox = adap->mbox;
2781
2782 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
b8ff05a9
DM		 2783 release_firmware(fw);
2784 if (!ret)
22c0b963
HS		 2785 dev_info(adap->pdev_dev, "loaded firmware %s,"
2786 " reload cxgb4 driver\n", ef->data);
b8ff05a9
DM		 2787 return ret;
2788}
2789
2790#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2791#define BCAST_CRC 0xa0ccc1a6
2792
2793static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2794{
2795 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2796 wol->wolopts = netdev2adap(dev)->wol;
2797 memset(&wol->sopass, 0, sizeof(wol->sopass));
2798}
2799
2800static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2801{
2802 int err = 0;
2803 struct port_info *pi = netdev_priv(dev);
2804
2805 if (wol->wolopts & ~WOL_SUPPORTED)
2806 return -EINVAL;
2807 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2808 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2809 if (wol->wolopts & WAKE_BCAST) {
2810 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2811 ~0ULL, 0, false);
2812 if (!err)
2813 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2814 ~6ULL, ~0ULL, BCAST_CRC, true);
2815 } else
2816 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2817 return err;
2818}
2819
c8f44aff 2820static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2821{
2ed28baa 2822 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2823 netdev_features_t changed = dev->features ^ features;
19ecae2c 2824 int err;
19ecae2c 2825
f646968f 2826 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2827 return 0;
19ecae2c 2828
2ed28baa
MM		 2829 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2830 -1, -1, -1,
f646968f 2831 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2832 if (unlikely(err))
f646968f 2833 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2834 return err;
87b6cf51
DM		 2835}
2836
7850f63f 2837static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2838{
2839 const struct port_info *pi = netdev_priv(dev);
671b0060 2840
7850f63f
BH		 2841 return pi->rss_size;
2842}
2843
892311f6 2844static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
7850f63f
BH		 2845{
2846 const struct port_info *pi = netdev_priv(dev);
2847 unsigned int n = pi->rss_size;
2848
892311f6
EP		 2849 if (hfunc)
2850 *hfunc = ETH_RSS_HASH_TOP;
2851 if (!p)
2852 return 0;
671b0060 2853 while (n--)
7850f63f 2854 p[n] = pi->rss[n];
671b0060
DM		 2855 return 0;
2856}
2857
892311f6
EP		 2858static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2859 const u8 hfunc)
671b0060
DM		 2860{
2861 unsigned int i;
2862 struct port_info *pi = netdev_priv(dev);
2863
892311f6
EP		 2864 /* We require at least one supported parameter to be changed and no
2865 * change in any of the unsupported parameters
2866 */
2867 if (key ||
2868 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2869 return -EOPNOTSUPP;
2870 if (!p)
2871 return 0;
2872
7850f63f
BH		 2873 for (i = 0; i < pi->rss_size; i++)
2874 pi->rss[i] = p[i];
671b0060
DM		 2875 if (pi->adapter->flags & FULL_INIT_DONE)
2876 return write_rss(pi, pi->rss);
2877 return 0;
2878}
2879
2880static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2881 u32 *rules)
671b0060 2882{
f796564a
DM		 2883 const struct port_info *pi = netdev_priv(dev);
2884
671b0060 2885 switch (info->cmd) {
f796564a
DM		 2886 case ETHTOOL_GRXFH: {
2887 unsigned int v = pi->rss_mode;
2888
2889 info->data = 0;
2890 switch (info->flow_type) {
2891 case TCP_V4_FLOW:
b2e1a3f0 2892 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
f796564a
DM		 2893 info->data = RXH_IP_SRC | RXH_IP_DST |
2894 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2895 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2896 info->data = RXH_IP_SRC | RXH_IP_DST;
2897 break;
2898 case UDP_V4_FLOW:
b2e1a3f0
HS		 2899 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2900 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2901 info->data = RXH_IP_SRC | RXH_IP_DST |
2902 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2903 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2904 info->data = RXH_IP_SRC | RXH_IP_DST;
2905 break;
2906 case SCTP_V4_FLOW:
2907 case AH_ESP_V4_FLOW:
2908 case IPV4_FLOW:
b2e1a3f0 2909 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2910 info->data = RXH_IP_SRC | RXH_IP_DST;
2911 break;
2912 case TCP_V6_FLOW:
b2e1a3f0 2913 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
f796564a
DM		 2914 info->data = RXH_IP_SRC | RXH_IP_DST |
2915 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2916 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2917 info->data = RXH_IP_SRC | RXH_IP_DST;
2918 break;
2919 case UDP_V6_FLOW:
b2e1a3f0
HS		 2920 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
2921 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2922 info->data = RXH_IP_SRC | RXH_IP_DST |
2923 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2924 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2925 info->data = RXH_IP_SRC | RXH_IP_DST;
2926 break;
2927 case SCTP_V6_FLOW:
2928 case AH_ESP_V6_FLOW:
2929 case IPV6_FLOW:
b2e1a3f0 2930 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2931 info->data = RXH_IP_SRC | RXH_IP_DST;
2932 break;
2933 }
2934 return 0;
2935 }
671b0060 2936 case ETHTOOL_GRXRINGS:
f796564a 2937 info->data = pi->nqsets;
671b0060
DM		 2938 return 0;
2939 }
2940 return -EOPNOTSUPP;
2941}
2942
9b07be4b 2943static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 2944 .get_settings = get_settings,
2945 .set_settings = set_settings,
2946 .get_drvinfo = get_drvinfo,
2947 .get_msglevel = get_msglevel,
2948 .set_msglevel = set_msglevel,
2949 .get_ringparam = get_sge_param,
2950 .set_ringparam = set_sge_param,
2951 .get_coalesce = get_coalesce,
2952 .set_coalesce = set_coalesce,
2953 .get_eeprom_len = get_eeprom_len,
2954 .get_eeprom = get_eeprom,
2955 .set_eeprom = set_eeprom,
2956 .get_pauseparam = get_pauseparam,
2957 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2958 .get_link = ethtool_op_get_link,
2959 .get_strings = get_strings,
c5e06360 2960 .set_phys_id = identify_port,
b8ff05a9
DM		 2961 .nway_reset = restart_autoneg,
2962 .get_sset_count = get_sset_count,
2963 .get_ethtool_stats = get_stats,
2964 .get_regs_len = get_regs_len,
2965 .get_regs = get_regs,
2966 .get_wol = get_wol,
2967 .set_wol = set_wol,
671b0060 2968 .get_rxnfc = get_rxnfc,
7850f63f 2969 .get_rxfh_indir_size = get_rss_table_size,
fe62d001
BH		 2970 .get_rxfh = get_rss_table,
2971 .set_rxfh = set_rss_table,
b8ff05a9
DM		 2972 .flash_device = set_flash,
2973};
2974
91744948 2975static int setup_debugfs(struct adapter *adap)
b8ff05a9 2976{
b8ff05a9
DM		 2977 if (IS_ERR_OR_NULL(adap->debugfs_root))
2978 return -1;
2979
fd88b31a
HS		 2980#ifdef CONFIG_DEBUG_FS
2981 t4_setup_debugfs(adap);
2982#endif
b8ff05a9
DM		 2983 return 0;
2984}
2985
2986/*
2987 * upper-layer driver support
2988 */
2989
2990/*
2991 * Allocate an active-open TID and set it to the supplied value.
2992 */
2993int cxgb4_alloc_atid(struct tid_info *t, void *data)
2994{
2995 int atid = -1;
2996
2997 spin_lock_bh(&t->atid_lock);
2998 if (t->afree) {
2999 union aopen_entry *p = t->afree;
3000
f2b7e78d 3001 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 3002 t->afree = p->next;
3003 p->data = data;
3004 t->atids_in_use++;
3005 }
3006 spin_unlock_bh(&t->atid_lock);
3007 return atid;
3008}
3009EXPORT_SYMBOL(cxgb4_alloc_atid);
3010
3011/*
3012 * Release an active-open TID.
3013 */
3014void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3015{
f2b7e78d 3016 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 3017
3018 spin_lock_bh(&t->atid_lock);
3019 p->next = t->afree;
3020 t->afree = p;
3021 t->atids_in_use--;
3022 spin_unlock_bh(&t->atid_lock);
3023}
3024EXPORT_SYMBOL(cxgb4_free_atid);
3025
3026/*
3027 * Allocate a server TID and set it to the supplied value.
3028 */
3029int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3030{
3031 int stid;
3032
3033 spin_lock_bh(&t->stid_lock);
3034 if (family == PF_INET) {
3035 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3036 if (stid < t->nstids)
3037 __set_bit(stid, t->stid_bmap);
3038 else
3039 stid = -1;
3040 } else {
3041 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3042 if (stid < 0)
3043 stid = -1;
3044 }
3045 if (stid >= 0) {
3046 t->stid_tab[stid].data = data;
3047 stid += t->stid_base;
15f63b74
KS		 3048 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3049 * This is equivalent to 4 TIDs. With CLIP enabled it
3050 * needs 2 TIDs.
3051 */
3052 if (family == PF_INET)
3053 t->stids_in_use++;
3054 else
3055 t->stids_in_use += 4;
b8ff05a9
DM		 3056 }
3057 spin_unlock_bh(&t->stid_lock);
3058 return stid;
3059}
3060EXPORT_SYMBOL(cxgb4_alloc_stid);
3061
dca4faeb
VP		 3062/* Allocate a server filter TID and set it to the supplied value.
3063 */
3064int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3065{
3066 int stid;
3067
3068 spin_lock_bh(&t->stid_lock);
3069 if (family == PF_INET) {
3070 stid = find_next_zero_bit(t->stid_bmap,
3071 t->nstids + t->nsftids, t->nstids);
3072 if (stid < (t->nstids + t->nsftids))
3073 __set_bit(stid, t->stid_bmap);
3074 else
3075 stid = -1;
3076 } else {
3077 stid = -1;
3078 }
3079 if (stid >= 0) {
3080 t->stid_tab[stid].data = data;
470c60c4
KS		 3081 stid -= t->nstids;
3082 stid += t->sftid_base;
dca4faeb
VP		 3083 t->stids_in_use++;
3084 }
3085 spin_unlock_bh(&t->stid_lock);
3086 return stid;
3087}
3088EXPORT_SYMBOL(cxgb4_alloc_sftid);
3089
3090/* Release a server TID.
b8ff05a9
DM		 3091 */
3092void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3093{
470c60c4
KS		 3094 /* Is it a server filter TID? */
3095 if (t->nsftids && (stid >= t->sftid_base)) {
3096 stid -= t->sftid_base;
3097 stid += t->nstids;
3098 } else {
3099 stid -= t->stid_base;
3100 }
3101
b8ff05a9
DM		 3102 spin_lock_bh(&t->stid_lock);
3103 if (family == PF_INET)
3104 __clear_bit(stid, t->stid_bmap);
3105 else
3106 bitmap_release_region(t->stid_bmap, stid, 2);
3107 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3108 if (family == PF_INET)
3109 t->stids_in_use--;
3110 else
3111 t->stids_in_use -= 4;
b8ff05a9
DM		 3112 spin_unlock_bh(&t->stid_lock);
3113}
3114EXPORT_SYMBOL(cxgb4_free_stid);
3115
3116/*
3117 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3118 */
3119static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3120 unsigned int tid)
3121{
3122 struct cpl_tid_release *req;
3123
3124 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3125 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3126 INIT_TP_WR(req, tid);
3127 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3128}
3129
3130/*
3131 * Queue a TID release request and if necessary schedule a work queue to
3132 * process it.
3133 */
31b9c19b 3134static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3135 unsigned int tid)
b8ff05a9
DM		 3136{
3137 void **p = &t->tid_tab[tid];
3138 struct adapter *adap = container_of(t, struct adapter, tids);
3139
3140 spin_lock_bh(&adap->tid_release_lock);
3141 *p = adap->tid_release_head;
3142 /* Low 2 bits encode the Tx channel number */
3143 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3144 if (!adap->tid_release_task_busy) {
3145 adap->tid_release_task_busy = true;
29aaee65 3146 queue_work(adap->workq, &adap->tid_release_task);
b8ff05a9
DM		 3147 }
3148 spin_unlock_bh(&adap->tid_release_lock);
3149}
b8ff05a9
DM		 3150
3151/*
3152 * Process the list of pending TID release requests.
3153 */
3154static void process_tid_release_list(struct work_struct *work)
3155{
3156 struct sk_buff *skb;
3157 struct adapter *adap;
3158
3159 adap = container_of(work, struct adapter, tid_release_task);
3160
3161 spin_lock_bh(&adap->tid_release_lock);
3162 while (adap->tid_release_head) {
3163 void **p = adap->tid_release_head;
3164 unsigned int chan = (uintptr_t)p & 3;
3165 p = (void *)p - chan;
3166
3167 adap->tid_release_head = *p;
3168 *p = NULL;
3169 spin_unlock_bh(&adap->tid_release_lock);
3170
3171 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3172 GFP_KERNEL)))
3173 schedule_timeout_uninterruptible(1);
3174
3175 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3176 t4_ofld_send(adap, skb);
3177 spin_lock_bh(&adap->tid_release_lock);
3178 }
3179 adap->tid_release_task_busy = false;
3180 spin_unlock_bh(&adap->tid_release_lock);
3181}
3182
3183/*
3184 * Release a TID and inform HW. If we are unable to allocate the release
3185 * message we defer to a work queue.
3186 */
3187void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3188{
3189 void *old;
3190 struct sk_buff *skb;
3191 struct adapter *adap = container_of(t, struct adapter, tids);
3192
3193 old = t->tid_tab[tid];
3194 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3195 if (likely(skb)) {
3196 t->tid_tab[tid] = NULL;
3197 mk_tid_release(skb, chan, tid);
3198 t4_ofld_send(adap, skb);
3199 } else
3200 cxgb4_queue_tid_release(t, chan, tid);
3201 if (old)
3202 atomic_dec(&t->tids_in_use);
3203}
3204EXPORT_SYMBOL(cxgb4_remove_tid);
3205
3206/*
3207 * Allocate and initialize the TID tables. Returns 0 on success.
3208 */
3209static int tid_init(struct tid_info *t)
3210{
3211 size_t size;
f2b7e78d 3212 unsigned int stid_bmap_size;
b8ff05a9 3213 unsigned int natids = t->natids;
b6f8eaec 3214 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3215
dca4faeb 3216 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3217 size = t->ntids * sizeof(*t->tid_tab) +
3218 natids * sizeof(*t->atid_tab) +
b8ff05a9 3219 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3220 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3221 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3222 t->nftids * sizeof(*t->ftid_tab) +
3223 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3224
b8ff05a9
DM		 3225 t->tid_tab = t4_alloc_mem(size);
3226 if (!t->tid_tab)
3227 return -ENOMEM;
3228
3229 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3230 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3231 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3232 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3233 spin_lock_init(&t->stid_lock);
3234 spin_lock_init(&t->atid_lock);
3235
3236 t->stids_in_use = 0;
3237 t->afree = NULL;
3238 t->atids_in_use = 0;
3239 atomic_set(&t->tids_in_use, 0);
3240
3241 /* Setup the free list for atid_tab and clear the stid bitmap. */
3242 if (natids) {
3243 while (--natids)
3244 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3245 t->afree = t->atid_tab;
3246 }
dca4faeb 3247 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3248 /* Reserve stid 0 for T4/T5 adapters */
3249 if (!t->stid_base &&
3250 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3251 __set_bit(0, t->stid_bmap);
3252
b8ff05a9
DM		 3253 return 0;
3254}
3255
3256/**
3257 * cxgb4_create_server - create an IP server
3258 * @dev: the device
3259 * @stid: the server TID
3260 * @sip: local IP address to bind server to
3261 * @sport: the server's TCP port
3262 * @queue: queue to direct messages from this server to
3263 *
3264 * Create an IP server for the given port and address.
3265 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3266 */
3267int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3268 __be32 sip, __be16 sport, __be16 vlan,
3269 unsigned int queue)
b8ff05a9
DM		 3270{
3271 unsigned int chan;
3272 struct sk_buff *skb;
3273 struct adapter *adap;
3274 struct cpl_pass_open_req *req;
80f40c1f 3275 int ret;
b8ff05a9
DM		 3276
3277 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3278 if (!skb)
3279 return -ENOMEM;
3280
3281 adap = netdev2adap(dev);
3282 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3283 INIT_TP_WR(req, 0);
3284 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3285 req->local_port = sport;
3286 req->peer_port = htons(0);
3287 req->local_ip = sip;
3288 req->peer_ip = htonl(0);
e46dab4d 3289 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3290 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
6c53e938
HS		 3291 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3292 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
80f40c1f
VP		 3293 ret = t4_mgmt_tx(adap, skb);
3294 return net_xmit_eval(ret);
b8ff05a9
DM		 3295}
3296EXPORT_SYMBOL(cxgb4_create_server);
3297
80f40c1f
VP		 3298/* cxgb4_create_server6 - create an IPv6 server
3299 * @dev: the device
3300 * @stid: the server TID
3301 * @sip: local IPv6 address to bind server to
3302 * @sport: the server's TCP port
3303 * @queue: queue to direct messages from this server to
3304 *
3305 * Create an IPv6 server for the given port and address.
3306 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3307 */
3308int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3309 const struct in6_addr *sip, __be16 sport,
3310 unsigned int queue)
3311{
3312 unsigned int chan;
3313 struct sk_buff *skb;
3314 struct adapter *adap;
3315 struct cpl_pass_open_req6 *req;
3316 int ret;
3317
3318 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3319 if (!skb)
3320 return -ENOMEM;
3321
3322 adap = netdev2adap(dev);
3323 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3324 INIT_TP_WR(req, 0);
3325 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3326 req->local_port = sport;
3327 req->peer_port = htons(0);
3328 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3329 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3330 req->peer_ip_hi = cpu_to_be64(0);
3331 req->peer_ip_lo = cpu_to_be64(0);
3332 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3333 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
6c53e938
HS		 3334 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3335 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
80f40c1f
VP		 3336 ret = t4_mgmt_tx(adap, skb);
3337 return net_xmit_eval(ret);
3338}
3339EXPORT_SYMBOL(cxgb4_create_server6);
3340
3341int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3342 unsigned int queue, bool ipv6)
3343{
3344 struct sk_buff *skb;
3345 struct adapter *adap;
3346 struct cpl_close_listsvr_req *req;
3347 int ret;
3348
3349 adap = netdev2adap(dev);
3350
3351 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3352 if (!skb)
3353 return -ENOMEM;
3354
3355 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3356 INIT_TP_WR(req, 0);
3357 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
bdc590b9
HS		 3358 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
3359 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
80f40c1f
VP		 3360 ret = t4_mgmt_tx(adap, skb);
3361 return net_xmit_eval(ret);
3362}
3363EXPORT_SYMBOL(cxgb4_remove_server);
3364
b8ff05a9
DM		 3365/**
3366 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3367 * @mtus: the HW MTU table
3368 * @mtu: the target MTU
3369 * @idx: index of selected entry in the MTU table
3370 *
3371 * Returns the index and the value in the HW MTU table that is closest to
3372 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3373 * table, in which case that smallest available value is selected.
3374 */
3375unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3376 unsigned int *idx)
3377{
3378 unsigned int i = 0;
3379
3380 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3381 ++i;
3382 if (idx)
3383 *idx = i;
3384 return mtus[i];
3385}
3386EXPORT_SYMBOL(cxgb4_best_mtu);
3387
92e7ae71
HS		 3388/**
3389 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3390 * @mtus: the HW MTU table
3391 * @header_size: Header Size
3392 * @data_size_max: maximum Data Segment Size
3393 * @data_size_align: desired Data Segment Size Alignment (2^N)
3394 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3395 *
3396 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3397 * MTU Table based solely on a Maximum MTU parameter, we break that
3398 * parameter up into a Header Size and Maximum Data Segment Size, and
3399 * provide a desired Data Segment Size Alignment. If we find an MTU in
3400 * the Hardware MTU Table which will result in a Data Segment Size with
3401 * the requested alignment _and_ that MTU isn't "too far" from the
3402 * closest MTU, then we'll return that rather than the closest MTU.
3403 */
3404unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3405 unsigned short header_size,
3406 unsigned short data_size_max,
3407 unsigned short data_size_align,
3408 unsigned int *mtu_idxp)
3409{
3410 unsigned short max_mtu = header_size + data_size_max;
3411 unsigned short data_size_align_mask = data_size_align - 1;
3412 int mtu_idx, aligned_mtu_idx;
3413
3414 /* Scan the MTU Table till we find an MTU which is larger than our
3415 * Maximum MTU or we reach the end of the table. Along the way,
3416 * record the last MTU found, if any, which will result in a Data
3417 * Segment Length matching the requested alignment.
3418 */
3419 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3420 unsigned short data_size = mtus[mtu_idx] - header_size;
3421
3422 /* If this MTU minus the Header Size would result in a
3423 * Data Segment Size of the desired alignment, remember it.
3424 */
3425 if ((data_size & data_size_align_mask) == 0)
3426 aligned_mtu_idx = mtu_idx;
3427
3428 /* If we're not at the end of the Hardware MTU Table and the
3429 * next element is larger than our Maximum MTU, drop out of
3430 * the loop.
3431 */
3432 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3433 break;
3434 }
3435
3436 /* If we fell out of the loop because we ran to the end of the table,
3437 * then we just have to use the last [largest] entry.
3438 */
3439 if (mtu_idx == NMTUS)
3440 mtu_idx--;
3441
3442 /* If we found an MTU which resulted in the requested Data Segment
3443 * Length alignment and that's "not far" from the largest MTU which is
3444 * less than or equal to the maximum MTU, then use that.
3445 */
3446 if (aligned_mtu_idx >= 0 &&
3447 mtu_idx - aligned_mtu_idx <= 1)
3448 mtu_idx = aligned_mtu_idx;
3449
3450 /* If the caller has passed in an MTU Index pointer, pass the
3451 * MTU Index back. Return the MTU value.
3452 */
3453 if (mtu_idxp)
3454 *mtu_idxp = mtu_idx;
3455 return mtus[mtu_idx];
3456}
3457EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3458
b8ff05a9
DM		 3459/**
3460 * cxgb4_port_chan - get the HW channel of a port
3461 * @dev: the net device for the port
3462 *
3463 * Return the HW Tx channel of the given port.
3464 */
3465unsigned int cxgb4_port_chan(const struct net_device *dev)
3466{
3467 return netdev2pinfo(dev)->tx_chan;
3468}
3469EXPORT_SYMBOL(cxgb4_port_chan);
3470
881806bc
VP		 3471unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3472{
3473 struct adapter *adap = netdev2adap(dev);
2cc301d2 3474 u32 v1, v2, lp_count, hp_count;
881806bc 3475
f061de42
HS		 3476 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3477 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3478 if (is_t4(adap->params.chip)) {
f061de42
HS		 3479 lp_count = LP_COUNT_G(v1);
3480 hp_count = HP_COUNT_G(v1);
2cc301d2 3481 } else {
f061de42
HS		 3482 lp_count = LP_COUNT_T5_G(v1);
3483 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3484 }
3485 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3486}
3487EXPORT_SYMBOL(cxgb4_dbfifo_count);
3488
b8ff05a9
DM		 3489/**
3490 * cxgb4_port_viid - get the VI id of a port
3491 * @dev: the net device for the port
3492 *
3493 * Return the VI id of the given port.
3494 */
3495unsigned int cxgb4_port_viid(const struct net_device *dev)
3496{
3497 return netdev2pinfo(dev)->viid;
3498}
3499EXPORT_SYMBOL(cxgb4_port_viid);
3500
3501/**
3502 * cxgb4_port_idx - get the index of a port
3503 * @dev: the net device for the port
3504 *
3505 * Return the index of the given port.
3506 */
3507unsigned int cxgb4_port_idx(const struct net_device *dev)
3508{
3509 return netdev2pinfo(dev)->port_id;
3510}
3511EXPORT_SYMBOL(cxgb4_port_idx);
3512
b8ff05a9
DM		 3513void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3514 struct tp_tcp_stats *v6)
3515{
3516 struct adapter *adap = pci_get_drvdata(pdev);
3517
3518 spin_lock(&adap->stats_lock);
3519 t4_tp_get_tcp_stats(adap, v4, v6);
3520 spin_unlock(&adap->stats_lock);
3521}
3522EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3523
3524void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3525 const unsigned int *pgsz_order)
3526{
3527 struct adapter *adap = netdev2adap(dev);
3528
0d804338
HS		 3529 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3530 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3531 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3532 HPZ3_V(pgsz_order[3]));
b8ff05a9
DM		 3533}
3534EXPORT_SYMBOL(cxgb4_iscsi_init);
3535
3069ee9b
VP		 3536int cxgb4_flush_eq_cache(struct net_device *dev)
3537{
3538 struct adapter *adap = netdev2adap(dev);
3539 int ret;
3540
3541 ret = t4_fwaddrspace_write(adap, adap->mbox,
f061de42 3542 0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3069ee9b
VP		 3543 return ret;
3544}
3545EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3546
3547static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3548{
f061de42 3549 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3069ee9b
VP		 3550 __be64 indices;
3551 int ret;
3552
fc5ab020
HS		 3553 spin_lock(&adap->win0_lock);
3554 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3555 sizeof(indices), (__be32 *)&indices,
3556 T4_MEMORY_READ);
3557 spin_unlock(&adap->win0_lock);
3069ee9b 3558 if (!ret) {
404d9e3f
VP		 3559 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3560 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3561 }
3562 return ret;
3563}
3564
3565int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3566 u16 size)
3567{
3568 struct adapter *adap = netdev2adap(dev);
3569 u16 hw_pidx, hw_cidx;
3570 int ret;
3571
3572 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3573 if (ret)
3574 goto out;
3575
3576 if (pidx != hw_pidx) {
3577 u16 delta;
f612b815 3578 u32 val;
3069ee9b
VP		 3579
3580 if (pidx >= hw_pidx)
3581 delta = pidx - hw_pidx;
3582 else
3583 delta = size - hw_pidx + pidx;
f612b815
HS		 3584
3585 if (is_t4(adap->params.chip))
3586 val = PIDX_V(delta);
3587 else
3588 val = PIDX_T5_V(delta);
3069ee9b 3589 wmb();
f612b815
HS		 3590 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3591 QID_V(qid) | val);
3069ee9b
VP		 3592 }
3593out:
3594 return ret;
3595}
3596EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3597
3cbdb928
VP		 3598void cxgb4_disable_db_coalescing(struct net_device *dev)
3599{
3600 struct adapter *adap;
3601
3602 adap = netdev2adap(dev);
f061de42 3603 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
f612b815 3604 NOCOALESCE_F);
3cbdb928
VP		 3605}
3606EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3607
3608void cxgb4_enable_db_coalescing(struct net_device *dev)
3609{
3610 struct adapter *adap;
3611
3612 adap = netdev2adap(dev);
f061de42 3613 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3cbdb928
VP		 3614}
3615EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3616
031cf476
HS		 3617int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3618{
3619 struct adapter *adap;
3620 u32 offset, memtype, memaddr;
6559a7e8 3621 u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
031cf476
HS		 3622 u32 edc0_end, edc1_end, mc0_end, mc1_end;
3623 int ret;
3624
3625 adap = netdev2adap(dev);
3626
3627 offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3628
3629 /* Figure out where the offset lands in the Memory Type/Address scheme.
3630 * This code assumes that the memory is laid out starting at offset 0
3631 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3632 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have
3633 * MC0, and some have both MC0 and MC1.
3634 */
6559a7e8
HS		 3635 size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3636 edc0_size = EDRAM0_SIZE_G(size) << 20;
3637 size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3638 edc1_size = EDRAM1_SIZE_G(size) << 20;
3639 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3640 mc0_size = EXT_MEM0_SIZE_G(size) << 20;
031cf476
HS		 3641
3642 edc0_end = edc0_size;
3643 edc1_end = edc0_end + edc1_size;
3644 mc0_end = edc1_end + mc0_size;
3645
3646 if (offset < edc0_end) {
3647 memtype = MEM_EDC0;
3648 memaddr = offset;
3649 } else if (offset < edc1_end) {
3650 memtype = MEM_EDC1;
3651 memaddr = offset - edc0_end;
3652 } else {
3653 if (offset < mc0_end) {
3654 memtype = MEM_MC0;
3655 memaddr = offset - edc1_end;
3656 } else if (is_t4(adap->params.chip)) {
3657 /* T4 only has a single memory channel */
3658 goto err;
3659 } else {
6559a7e8
HS		 3660 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3661 mc1_size = EXT_MEM1_SIZE_G(size) << 20;
031cf476
HS		 3662 mc1_end = mc0_end + mc1_size;
3663 if (offset < mc1_end) {
3664 memtype = MEM_MC1;
3665 memaddr = offset - mc0_end;
3666 } else {
3667 /* offset beyond the end of any memory */
3668 goto err;
3669 }
3670 }
3671 }
3672
3673 spin_lock(&adap->win0_lock);
3674 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3675 spin_unlock(&adap->win0_lock);
3676 return ret;
3677
3678err:
3679 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3680 stag, offset);
3681 return -EINVAL;
3682}
3683EXPORT_SYMBOL(cxgb4_read_tpte);
3684
7730b4c7
HS		 3685u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3686{
3687 u32 hi, lo;
3688 struct adapter *adap;
3689
3690 adap = netdev2adap(dev);
f612b815
HS		 3691 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3692 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
7730b4c7
HS		 3693
3694 return ((u64)hi << 32) | (u64)lo;
3695}
3696EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3697
df64e4d3
HS		 3698int cxgb4_bar2_sge_qregs(struct net_device *dev,
3699 unsigned int qid,
3700 enum cxgb4_bar2_qtype qtype,
3701 u64 *pbar2_qoffset,
3702 unsigned int *pbar2_qid)
3703{
dd0bcc0b 3704 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
df64e4d3
HS		 3705 qid,
3706 (qtype == CXGB4_BAR2_QTYPE_EGRESS
3707 ? T4_BAR2_QTYPE_EGRESS
3708 : T4_BAR2_QTYPE_INGRESS),
3709 pbar2_qoffset,
3710 pbar2_qid);
3711}
3712EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3713
b8ff05a9
DM		 3714static struct pci_driver cxgb4_driver;
3715
3716static void check_neigh_update(struct neighbour *neigh)
3717{
3718 const struct device *parent;
3719 const struct net_device *netdev = neigh->dev;
3720
3721 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3722 netdev = vlan_dev_real_dev(netdev);
3723 parent = netdev->dev.parent;
3724 if (parent && parent->driver == &cxgb4_driver.driver)
3725 t4_l2t_update(dev_get_drvdata(parent), neigh);
3726}
3727
3728static int netevent_cb(struct notifier_block *nb, unsigned long event,
3729 void *data)
3730{
3731 switch (event) {
3732 case NETEVENT_NEIGH_UPDATE:
3733 check_neigh_update(data);
3734 break;
b8ff05a9
DM		 3735 case NETEVENT_REDIRECT:
3736 default:
3737 break;
3738 }
3739 return 0;
3740}
3741
3742static bool netevent_registered;
3743static struct notifier_block cxgb4_netevent_nb = {
3744 .notifier_call = netevent_cb
3745};
3746
3069ee9b
VP		 3747static void drain_db_fifo(struct adapter *adap, int usecs)
3748{
2cc301d2 3749 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3750
3751 do {
f061de42
HS		 3752 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3753 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3754 if (is_t4(adap->params.chip)) {
f061de42
HS		 3755 lp_count = LP_COUNT_G(v1);
3756 hp_count = HP_COUNT_G(v1);
2cc301d2 3757 } else {
f061de42
HS		 3758 lp_count = LP_COUNT_T5_G(v1);
3759 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3760 }
3761
3762 if (lp_count == 0 && hp_count == 0)
3763 break;
3069ee9b
VP		 3764 set_current_state(TASK_UNINTERRUPTIBLE);
3765 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3766 } while (1);
3767}
3768
3769static void disable_txq_db(struct sge_txq *q)
3770{
05eb2389
SW		 3771 unsigned long flags;
3772
3773 spin_lock_irqsave(&q->db_lock, flags);
3069ee9b 3774 q->db_disabled = 1;
05eb2389 3775 spin_unlock_irqrestore(&q->db_lock, flags);
3069ee9b
VP		 3776}
3777
05eb2389 3778static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3069ee9b
VP		 3779{
3780 spin_lock_irq(&q->db_lock);
05eb2389
SW		 3781 if (q->db_pidx_inc) {
3782 /* Make sure that all writes to the TX descriptors
3783 * are committed before we tell HW about them.
3784 */
3785 wmb();
f612b815
HS		 3786 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3787 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
05eb2389
SW		 3788 q->db_pidx_inc = 0;
3789 }
3069ee9b
VP		 3790 q->db_disabled = 0;
3791 spin_unlock_irq(&q->db_lock);
3792}
3793
3794static void disable_dbs(struct adapter *adap)
3795{
3796 int i;
3797
3798 for_each_ethrxq(&adap->sge, i)
3799 disable_txq_db(&adap->sge.ethtxq[i].q);
3800 for_each_ofldrxq(&adap->sge, i)
3801 disable_txq_db(&adap->sge.ofldtxq[i].q);
3802 for_each_port(adap, i)
3803 disable_txq_db(&adap->sge.ctrlq[i].q);
3804}
3805
3806static void enable_dbs(struct adapter *adap)
3807{
3808 int i;
3809
3810 for_each_ethrxq(&adap->sge, i)
05eb2389 3811 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3069ee9b 3812 for_each_ofldrxq(&adap->sge, i)
05eb2389 3813 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3069ee9b 3814 for_each_port(adap, i)
05eb2389
SW		 3815 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3816}
3817
3818static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3819{
3820 if (adap->uld_handle[CXGB4_ULD_RDMA])
3821 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3822 cmd);
3823}
3824
3825static void process_db_full(struct work_struct *work)
3826{
3827 struct adapter *adap;
3828
3829 adap = container_of(work, struct adapter, db_full_task);
3830
3831 drain_db_fifo(adap, dbfifo_drain_delay);
3832 enable_dbs(adap);
3833 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
f612b815
HS		 3834 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3835 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3836 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3069ee9b
VP		 3837}
3838
3839static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3840{
3841 u16 hw_pidx, hw_cidx;
3842 int ret;
3843
05eb2389 3844 spin_lock_irq(&q->db_lock);
3069ee9b
VP		 3845 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3846 if (ret)
3847 goto out;
3848 if (q->db_pidx != hw_pidx) {
3849 u16 delta;
f612b815 3850 u32 val;
3069ee9b
VP		 3851
3852 if (q->db_pidx >= hw_pidx)
3853 delta = q->db_pidx - hw_pidx;
3854 else
3855 delta = q->size - hw_pidx + q->db_pidx;
f612b815
HS		 3856
3857 if (is_t4(adap->params.chip))
3858 val = PIDX_V(delta);
3859 else
3860 val = PIDX_T5_V(delta);
3069ee9b 3861 wmb();
f612b815
HS		 3862 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3863 QID_V(q->cntxt_id) | val);
3069ee9b
VP		 3864 }
3865out:
3866 q->db_disabled = 0;
05eb2389
SW		 3867 q->db_pidx_inc = 0;
3868 spin_unlock_irq(&q->db_lock);
3069ee9b
VP		 3869 if (ret)
3870 CH_WARN(adap, "DB drop recovery failed.\n");
3871}
3872static void recover_all_queues(struct adapter *adap)
3873{
3874 int i;
3875
3876 for_each_ethrxq(&adap->sge, i)
3877 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3878 for_each_ofldrxq(&adap->sge, i)
3879 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3880 for_each_port(adap, i)
3881 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3882}
3883
881806bc
VP		 3884static void process_db_drop(struct work_struct *work)
3885{
3886 struct adapter *adap;
881806bc 3887
3069ee9b 3888 adap = container_of(work, struct adapter, db_drop_task);
881806bc 3889
d14807dd 3890 if (is_t4(adap->params.chip)) {
05eb2389 3891 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3892 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
05eb2389 3893 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3894 recover_all_queues(adap);
05eb2389 3895 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3896 enable_dbs(adap);
05eb2389 3897 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2cc301d2
SR		 3898 } else {
3899 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3900 u16 qid = (dropped_db >> 15) & 0x1ffff;
3901 u16 pidx_inc = dropped_db & 0x1fff;
df64e4d3
HS		 3902 u64 bar2_qoffset;
3903 unsigned int bar2_qid;
3904 int ret;
2cc301d2 3905
dd0bcc0b 3906 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
df64e4d3
HS		 3907 &bar2_qoffset, &bar2_qid);
3908 if (ret)
3909 dev_err(adap->pdev_dev, "doorbell drop recovery: "
3910 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
3911 else
f612b815 3912 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
df64e4d3 3913 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
2cc301d2
SR		 3914
3915 /* Re-enable BAR2 WC */
3916 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3917 }
3918
f061de42 3919 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
881806bc
VP		 3920}
3921
3922void t4_db_full(struct adapter *adap)
3923{
d14807dd 3924 if (is_t4(adap->params.chip)) {
05eb2389
SW		 3925 disable_dbs(adap);
3926 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
f612b815
HS		 3927 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3928 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
29aaee65 3929 queue_work(adap->workq, &adap->db_full_task);
2cc301d2 3930 }
881806bc
VP		 3931}
3932
3933void t4_db_dropped(struct adapter *adap)
3934{
05eb2389
SW		 3935 if (is_t4(adap->params.chip)) {
3936 disable_dbs(adap);
3937 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3938 }
29aaee65 3939 queue_work(adap->workq, &adap->db_drop_task);
881806bc
VP		 3940}
3941
b8ff05a9
DM		 3942static void uld_attach(struct adapter *adap, unsigned int uld)
3943{
3944 void *handle;
3945 struct cxgb4_lld_info lli;
dca4faeb 3946 unsigned short i;
b8ff05a9
DM		 3947
3948 lli.pdev = adap->pdev;
35b1de55 3949 lli.pf = adap->fn;
b8ff05a9
DM		 3950 lli.l2t = adap->l2t;
3951 lli.tids = &adap->tids;
3952 lli.ports = adap->port;
3953 lli.vr = &adap->vres;
3954 lli.mtus = adap->params.mtus;
3955 if (uld == CXGB4_ULD_RDMA) {
3956 lli.rxq_ids = adap->sge.rdma_rxq;
cf38be6d 3957 lli.ciq_ids = adap->sge.rdma_ciq;
b8ff05a9 3958 lli.nrxq = adap->sge.rdmaqs;
cf38be6d 3959 lli.nciq = adap->sge.rdmaciqs;
b8ff05a9
DM		 3960 } else if (uld == CXGB4_ULD_ISCSI) {
3961 lli.rxq_ids = adap->sge.ofld_rxq;
3962 lli.nrxq = adap->sge.ofldqsets;
3963 }
3964 lli.ntxq = adap->sge.ofldqsets;
3965 lli.nchan = adap->params.nports;
3966 lli.nports = adap->params.nports;
3967 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 3968 lli.adapter_type = adap->params.chip;
837e4a42 3969 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
7730b4c7 3970 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
df64e4d3
HS		 3971 lli.udb_density = 1 << adap->params.sge.eq_qpp;
3972 lli.ucq_density = 1 << adap->params.sge.iq_qpp;
dcf7b6f5 3973 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 3974 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3975 for (i = 0; i < NCHAN; i++)
3976 lli.tx_modq[i] = i;
f612b815
HS		 3977 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
3978 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
b8ff05a9 3979 lli.fw_vers = adap->params.fw_vers;
3069ee9b 3980 lli.dbfifo_int_thresh = dbfifo_int_thresh;
04e10e21
HS		 3981 lli.sge_ingpadboundary = adap->sge.fl_align;
3982 lli.sge_egrstatuspagesize = adap->sge.stat_len;
dca4faeb
VP		 3983 lli.sge_pktshift = adap->sge.pktshift;
3984 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
4c2c5763
HS		 3985 lli.max_ordird_qp = adap->params.max_ordird_qp;
3986 lli.max_ird_adapter = adap->params.max_ird_adapter;
1ac0f095 3987 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 3988
3989 handle = ulds[uld].add(&lli);
3990 if (IS_ERR(handle)) {
3991 dev_warn(adap->pdev_dev,
3992 "could not attach to the %s driver, error %ld\n",
3993 uld_str[uld], PTR_ERR(handle));
3994 return;
3995 }
3996
3997 adap->uld_handle[uld] = handle;
3998
3999 if (!netevent_registered) {
4000 register_netevent_notifier(&cxgb4_netevent_nb);
4001 netevent_registered = true;
4002 }
e29f5dbc
DM		 4003
4004 if (adap->flags & FULL_INIT_DONE)
4005 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 4006}
4007
4008static void attach_ulds(struct adapter *adap)
4009{
4010 unsigned int i;
4011
01bcca68
VP		 4012 spin_lock(&adap_rcu_lock);
4013 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4014 spin_unlock(&adap_rcu_lock);
4015
b8ff05a9
DM		 4016 mutex_lock(&uld_mutex);
4017 list_add_tail(&adap->list_node, &adapter_list);
4018 for (i = 0; i < CXGB4_ULD_MAX; i++)
4019 if (ulds[i].add)
4020 uld_attach(adap, i);
4021 mutex_unlock(&uld_mutex);
4022}
4023
4024static void detach_ulds(struct adapter *adap)
4025{
4026 unsigned int i;
4027
4028 mutex_lock(&uld_mutex);
4029 list_del(&adap->list_node);
4030 for (i = 0; i < CXGB4_ULD_MAX; i++)
4031 if (adap->uld_handle[i]) {
4032 ulds[i].state_change(adap->uld_handle[i],
4033 CXGB4_STATE_DETACH);
4034 adap->uld_handle[i] = NULL;
4035 }
4036 if (netevent_registered && list_empty(&adapter_list)) {
4037 unregister_netevent_notifier(&cxgb4_netevent_nb);
4038 netevent_registered = false;
4039 }
4040 mutex_unlock(&uld_mutex);
01bcca68
VP		 4041
4042 spin_lock(&adap_rcu_lock);
4043 list_del_rcu(&adap->rcu_node);
4044 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 4045}
4046
4047static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4048{
4049 unsigned int i;
4050
4051 mutex_lock(&uld_mutex);
4052 for (i = 0; i < CXGB4_ULD_MAX; i++)
4053 if (adap->uld_handle[i])
4054 ulds[i].state_change(adap->uld_handle[i], new_state);
4055 mutex_unlock(&uld_mutex);
4056}
4057
4058/**
4059 * cxgb4_register_uld - register an upper-layer driver
4060 * @type: the ULD type
4061 * @p: the ULD methods
4062 *
4063 * Registers an upper-layer driver with this driver and notifies the ULD
4064 * about any presently available devices that support its type. Returns
4065 * %-EBUSY if a ULD of the same type is already registered.
4066 */
4067int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4068{
4069 int ret = 0;
4070 struct adapter *adap;
4071
4072 if (type >= CXGB4_ULD_MAX)
4073 return -EINVAL;
4074 mutex_lock(&uld_mutex);
4075 if (ulds[type].add) {
4076 ret = -EBUSY;
4077 goto out;
4078 }
4079 ulds[type] = *p;
4080 list_for_each_entry(adap, &adapter_list, list_node)
4081 uld_attach(adap, type);
4082out: mutex_unlock(&uld_mutex);
4083 return ret;
4084}
4085EXPORT_SYMBOL(cxgb4_register_uld);
4086
4087/**
4088 * cxgb4_unregister_uld - unregister an upper-layer driver
4089 * @type: the ULD type
4090 *
4091 * Unregisters an existing upper-layer driver.
4092 */
4093int cxgb4_unregister_uld(enum cxgb4_uld type)
4094{
4095 struct adapter *adap;
4096
4097 if (type >= CXGB4_ULD_MAX)
4098 return -EINVAL;
4099 mutex_lock(&uld_mutex);
4100 list_for_each_entry(adap, &adapter_list, list_node)
4101 adap->uld_handle[type] = NULL;
4102 ulds[type].add = NULL;
4103 mutex_unlock(&uld_mutex);
4104 return 0;
4105}
4106EXPORT_SYMBOL(cxgb4_unregister_uld);
4107
1bb60376 4108#if IS_ENABLED(CONFIG_IPV6)
b5a02f50
AB		 4109static int cxgb4_inet6addr_handler(struct notifier_block *this,
4110 unsigned long event, void *data)
01bcca68 4111{
b5a02f50
AB		 4112 struct inet6_ifaddr *ifa = data;
4113 struct net_device *event_dev = ifa->idev->dev;
4114 const struct device *parent = NULL;
4115#if IS_ENABLED(CONFIG_BONDING)
01bcca68 4116 struct adapter *adap;
b5a02f50
AB		 4117#endif
4118 if (event_dev->priv_flags & IFF_802_1Q_VLAN)
4119 event_dev = vlan_dev_real_dev(event_dev);
4120#if IS_ENABLED(CONFIG_BONDING)
4121 if (event_dev->flags & IFF_MASTER) {
4122 list_for_each_entry(adap, &adapter_list, list_node) {
4123 switch (event) {
4124 case NETDEV_UP:
4125 cxgb4_clip_get(adap->port[0],
4126 (const u32 *)ifa, 1);
4127 break;
4128 case NETDEV_DOWN:
4129 cxgb4_clip_release(adap->port[0],
4130 (const u32 *)ifa, 1);
4131 break;
4132 default:
4133 break;
4134 }
4135 }
4136 return NOTIFY_OK;
4137 }
4138#endif
01bcca68 4139
b5a02f50
AB		 4140 if (event_dev)
4141 parent = event_dev->dev.parent;
01bcca68 4142
b5a02f50 4143 if (parent && parent->driver == &cxgb4_driver.driver) {
01bcca68
VP		 4144 switch (event) {
4145 case NETDEV_UP:
b5a02f50 4146 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
01bcca68
VP		 4147 break;
4148 case NETDEV_DOWN:
b5a02f50 4149 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
01bcca68
VP		 4150 break;
4151 default:
4152 break;
4153 }
4154 }
b5a02f50 4155 return NOTIFY_OK;
01bcca68
VP		 4156}
4157
b5a02f50 4158static bool inet6addr_registered;
01bcca68
VP		 4159static struct notifier_block cxgb4_inet6addr_notifier = {
4160 .notifier_call = cxgb4_inet6addr_handler
4161};
4162
01bcca68
VP		 4163static void update_clip(const struct adapter *adap)
4164{
4165 int i;
4166 struct net_device *dev;
4167 int ret;
4168
4169 rcu_read_lock();
4170
4171 for (i = 0; i < MAX_NPORTS; i++) {
4172 dev = adap->port[i];
4173 ret = 0;
4174
4175 if (dev)
b5a02f50 4176 ret = cxgb4_update_root_dev_clip(dev);
01bcca68
VP		 4177
4178 if (ret < 0)
4179 break;
4180 }
4181 rcu_read_unlock();
4182}
1bb60376 4183#endif /* IS_ENABLED(CONFIG_IPV6) */
01bcca68 4184
b8ff05a9
DM		 4185/**
4186 * cxgb_up - enable the adapter
4187 * @adap: adapter being enabled
4188 *
4189 * Called when the first port is enabled, this function performs the
4190 * actions necessary to make an adapter operational, such as completing
4191 * the initialization of HW modules, and enabling interrupts.
4192 *
4193 * Must be called with the rtnl lock held.
4194 */
4195static int cxgb_up(struct adapter *adap)
4196{
aaefae9b 4197 int err;
b8ff05a9 4198
aaefae9b
DM		 4199 err = setup_sge_queues(adap);
4200 if (err)
4201 goto out;
4202 err = setup_rss(adap);
4203 if (err)
4204 goto freeq;
b8ff05a9
DM		 4205
4206 if (adap->flags & USING_MSIX) {
aaefae9b 4207 name_msix_vecs(adap);
b8ff05a9
DM		 4208 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4209 adap->msix_info[0].desc, adap);
4210 if (err)
4211 goto irq_err;
4212
4213 err = request_msix_queue_irqs(adap);
4214 if (err) {
4215 free_irq(adap->msix_info[0].vec, adap);
4216 goto irq_err;
4217 }
4218 } else {
4219 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4220 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4221 adap->port[0]->name, adap);
b8ff05a9
DM		 4222 if (err)
4223 goto irq_err;
4224 }
4225 enable_rx(adap);
4226 t4_sge_start(adap);
4227 t4_intr_enable(adap);
aaefae9b 4228 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4229 notify_ulds(adap, CXGB4_STATE_UP);
1bb60376 4230#if IS_ENABLED(CONFIG_IPV6)
01bcca68 4231 update_clip(adap);
1bb60376 4232#endif
b8ff05a9
DM		 4233 out:
4234 return err;
4235 irq_err:
4236 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4237 freeq:
4238 t4_free_sge_resources(adap);
b8ff05a9
DM		 4239 goto out;
4240}
4241
4242static void cxgb_down(struct adapter *adapter)
4243{
4244 t4_intr_disable(adapter);
4245 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4246 cancel_work_sync(&adapter->db_full_task);
4247 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4248 adapter->tid_release_task_busy = false;
204dc3c0 4249 adapter->tid_release_head = NULL;
b8ff05a9
DM		 4250
4251 if (adapter->flags & USING_MSIX) {
4252 free_msix_queue_irqs(adapter);
4253 free_irq(adapter->msix_info[0].vec, adapter);
4254 } else
4255 free_irq(adapter->pdev->irq, adapter);
4256 quiesce_rx(adapter);
aaefae9b
DM		 4257 t4_sge_stop(adapter);
4258 t4_free_sge_resources(adapter);
4259 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4260}
4261
4262/*
4263 * net_device operations
4264 */
4265static int cxgb_open(struct net_device *dev)
4266{
4267 int err;
4268 struct port_info *pi = netdev_priv(dev);
4269 struct adapter *adapter = pi->adapter;
4270
6a3c869a
DM		 4271 netif_carrier_off(dev);
4272
aaefae9b
DM		 4273 if (!(adapter->flags & FULL_INIT_DONE)) {
4274 err = cxgb_up(adapter);
4275 if (err < 0)
4276 return err;
4277 }
b8ff05a9 4278
f68707b8
DM		 4279 err = link_start(dev);
4280 if (!err)
4281 netif_tx_start_all_queues(dev);
4282 return err;
b8ff05a9
DM		 4283}
4284
4285static int cxgb_close(struct net_device *dev)
4286{
b8ff05a9
DM		 4287 struct port_info *pi = netdev_priv(dev);
4288 struct adapter *adapter = pi->adapter;
4289
4290 netif_tx_stop_all_queues(dev);
4291 netif_carrier_off(dev);
060e0c75 4292 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4293}
4294
f2b7e78d
VP		 4295/* Return an error number if the indicated filter isn't writable ...
4296 */
4297static int writable_filter(struct filter_entry *f)
4298{
4299 if (f->locked)
4300 return -EPERM;
4301 if (f->pending)
4302 return -EBUSY;
4303
4304 return 0;
4305}
4306
4307/* Delete the filter at the specified index (if valid). The checks for all
4308 * the common problems with doing this like the filter being locked, currently
4309 * pending in another operation, etc.
4310 */
4311static int delete_filter(struct adapter *adapter, unsigned int fidx)
4312{
4313 struct filter_entry *f;
4314 int ret;
4315
dca4faeb 4316 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4317 return -EINVAL;
4318
4319 f = &adapter->tids.ftid_tab[fidx];
4320 ret = writable_filter(f);
4321 if (ret)
4322 return ret;
4323 if (f->valid)
4324 return del_filter_wr(adapter, fidx);
4325
4326 return 0;
4327}
4328
dca4faeb 4329int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4330 __be32 sip, __be16 sport, __be16 vlan,
4331 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4332{
4333 int ret;
4334 struct filter_entry *f;
4335 struct adapter *adap;
4336 int i;
4337 u8 *val;
4338
4339 adap = netdev2adap(dev);
4340
1cab775c 4341 /* Adjust stid to correct filter index */
470c60c4 4342 stid -= adap->tids.sftid_base;
1cab775c
VP		 4343 stid += adap->tids.nftids;
4344
dca4faeb
VP		 4345 /* Check to make sure the filter requested is writable ...
4346 */
4347 f = &adap->tids.ftid_tab[stid];
4348 ret = writable_filter(f);
4349 if (ret)
4350 return ret;
4351
4352 /* Clear out any old resources being used by the filter before
4353 * we start constructing the new filter.
4354 */
4355 if (f->valid)
4356 clear_filter(adap, f);
4357
4358 /* Clear out filter specifications */
4359 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4360 f->fs.val.lport = cpu_to_be16(sport);
4361 f->fs.mask.lport = ~0;
4362 val = (u8 *)&sip;
793dad94 4363 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4364 for (i = 0; i < 4; i++) {
4365 f->fs.val.lip[i] = val[i];
4366 f->fs.mask.lip[i] = ~0;
4367 }
0d804338 4368 if (adap->params.tp.vlan_pri_map & PORT_F) {
793dad94
VP		 4369 f->fs.val.iport = port;
4370 f->fs.mask.iport = mask;
4371 }
4372 }
dca4faeb 4373
0d804338 4374 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
7c89e555
KS		 4375 f->fs.val.proto = IPPROTO_TCP;
4376 f->fs.mask.proto = ~0;
4377 }
4378
dca4faeb
VP		 4379 f->fs.dirsteer = 1;
4380 f->fs.iq = queue;
4381 /* Mark filter as locked */
4382 f->locked = 1;
4383 f->fs.rpttid = 1;
4384
4385 ret = set_filter_wr(adap, stid);
4386 if (ret) {
4387 clear_filter(adap, f);
4388 return ret;
4389 }
4390
4391 return 0;
4392}
4393EXPORT_SYMBOL(cxgb4_create_server_filter);
4394
4395int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4396 unsigned int queue, bool ipv6)
4397{
4398 int ret;
4399 struct filter_entry *f;
4400 struct adapter *adap;
4401
4402 adap = netdev2adap(dev);
1cab775c
VP		 4403
4404 /* Adjust stid to correct filter index */
470c60c4 4405 stid -= adap->tids.sftid_base;
1cab775c
VP		 4406 stid += adap->tids.nftids;
4407
dca4faeb
VP		 4408 f = &adap->tids.ftid_tab[stid];
4409 /* Unlock the filter */
4410 f->locked = 0;
4411
4412 ret = delete_filter(adap, stid);
4413 if (ret)
4414 return ret;
4415
4416 return 0;
4417}
4418EXPORT_SYMBOL(cxgb4_remove_server_filter);
4419
f5152c90
DM		 4420static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4421 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4422{
4423 struct port_stats stats;
4424 struct port_info *p = netdev_priv(dev);
4425 struct adapter *adapter = p->adapter;
b8ff05a9 4426
9fe6cb58
GS		 4427 /* Block retrieving statistics during EEH error
4428 * recovery. Otherwise, the recovery might fail
4429 * and the PCI device will be removed permanently
4430 */
b8ff05a9 4431 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4432 if (!netif_device_present(dev)) {
4433 spin_unlock(&adapter->stats_lock);
4434 return ns;
4435 }
b8ff05a9
DM		 4436 t4_get_port_stats(adapter, p->tx_chan, &stats);
4437 spin_unlock(&adapter->stats_lock);
4438
4439 ns->tx_bytes = stats.tx_octets;
4440 ns->tx_packets = stats.tx_frames;
4441 ns->rx_bytes = stats.rx_octets;
4442 ns->rx_packets = stats.rx_frames;
4443 ns->multicast = stats.rx_mcast_frames;
4444
4445 /* detailed rx_errors */
4446 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4447 stats.rx_runt;
4448 ns->rx_over_errors = 0;
4449 ns->rx_crc_errors = stats.rx_fcs_err;
4450 ns->rx_frame_errors = stats.rx_symbol_err;
4451 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4452 stats.rx_ovflow2 + stats.rx_ovflow3 +
4453 stats.rx_trunc0 + stats.rx_trunc1 +
4454 stats.rx_trunc2 + stats.rx_trunc3;
4455 ns->rx_missed_errors = 0;
4456
4457 /* detailed tx_errors */
4458 ns->tx_aborted_errors = 0;
4459 ns->tx_carrier_errors = 0;
4460 ns->tx_fifo_errors = 0;
4461 ns->tx_heartbeat_errors = 0;
4462 ns->tx_window_errors = 0;
4463
4464 ns->tx_errors = stats.tx_error_frames;
4465 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4466 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4467 return ns;
4468}
4469
4470static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4471{
060e0c75 4472 unsigned int mbox;
b8ff05a9
DM		 4473 int ret = 0, prtad, devad;
4474 struct port_info *pi = netdev_priv(dev);
4475 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4476
4477 switch (cmd) {
4478 case SIOCGMIIPHY:
4479 if (pi->mdio_addr < 0)
4480 return -EOPNOTSUPP;
4481 data->phy_id = pi->mdio_addr;
4482 break;
4483 case SIOCGMIIREG:
4484 case SIOCSMIIREG:
4485 if (mdio_phy_id_is_c45(data->phy_id)) {
4486 prtad = mdio_phy_id_prtad(data->phy_id);
4487 devad = mdio_phy_id_devad(data->phy_id);
4488 } else if (data->phy_id < 32) {
4489 prtad = data->phy_id;
4490 devad = 0;
4491 data->reg_num &= 0x1f;
4492 } else
4493 return -EINVAL;
4494
060e0c75 4495 mbox = pi->adapter->fn;
b8ff05a9 4496 if (cmd == SIOCGMIIREG)
060e0c75 4497 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4498 data->reg_num, &data->val_out);
4499 else
060e0c75 4500 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4501 data->reg_num, data->val_in);
4502 break;
4503 default:
4504 return -EOPNOTSUPP;
4505 }
4506 return ret;
4507}
4508
4509static void cxgb_set_rxmode(struct net_device *dev)
4510{
4511 /* unfortunately we can't return errors to the stack */
4512 set_rxmode(dev, -1, false);
4513}
4514
4515static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4516{
4517 int ret;
4518 struct port_info *pi = netdev_priv(dev);
4519
4520 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4521 return -EINVAL;
060e0c75
DM		 4522 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4523 -1, -1, -1, true);
b8ff05a9
DM		 4524 if (!ret)
4525 dev->mtu = new_mtu;
4526 return ret;
4527}
4528
4529static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4530{
4531 int ret;
4532 struct sockaddr *addr = p;
4533 struct port_info *pi = netdev_priv(dev);
4534
4535 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4536 return -EADDRNOTAVAIL;
b8ff05a9 4537
060e0c75
DM		 4538 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4539 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4540 if (ret < 0)
4541 return ret;
4542
4543 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4544 pi->xact_addr_filt = ret;
4545 return 0;
4546}
4547
b8ff05a9
DM		 4548#ifdef CONFIG_NET_POLL_CONTROLLER
4549static void cxgb_netpoll(struct net_device *dev)
4550{
4551 struct port_info *pi = netdev_priv(dev);
4552 struct adapter *adap = pi->adapter;
4553
4554 if (adap->flags & USING_MSIX) {
4555 int i;
4556 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4557
4558 for (i = pi->nqsets; i; i--, rx++)
4559 t4_sge_intr_msix(0, &rx->rspq);
4560 } else
4561 t4_intr_handler(adap)(0, adap);
4562}
4563#endif
4564
4565static const struct net_device_ops cxgb4_netdev_ops = {
4566 .ndo_open = cxgb_open,
4567 .ndo_stop = cxgb_close,
4568 .ndo_start_xmit = t4_eth_xmit,
688848b1 4569 .ndo_select_queue = cxgb_select_queue,
9be793bf 4570 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4571 .ndo_set_rx_mode = cxgb_set_rxmode,
4572 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4573 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4574 .ndo_validate_addr = eth_validate_addr,
4575 .ndo_do_ioctl = cxgb_ioctl,
4576 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4577#ifdef CONFIG_NET_POLL_CONTROLLER
4578 .ndo_poll_controller = cxgb_netpoll,
4579#endif
3a336cb1
HS		 4580#ifdef CONFIG_NET_RX_BUSY_POLL
4581 .ndo_busy_poll = cxgb_busy_poll,
4582#endif
4583
b8ff05a9
DM		 4584};
4585
4586void t4_fatal_err(struct adapter *adap)
4587{
f612b815 4588 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
b8ff05a9
DM		 4589 t4_intr_disable(adap);
4590 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4591}
4592
0abfd152
HS		 4593/* Return the specified PCI-E Configuration Space register from our Physical
4594 * Function. We try first via a Firmware LDST Command since we prefer to let
4595 * the firmware own all of these registers, but if that fails we go for it
4596 * directly ourselves.
4597 */
4598static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4599{
4600 struct fw_ldst_cmd ldst_cmd;
4601 u32 val;
4602 int ret;
4603
4604 /* Construct and send the Firmware LDST Command to retrieve the
4605 * specified PCI-E Configuration Space register.
4606 */
4607 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4608 ldst_cmd.op_to_addrspace =
e2ac9628
HS		 4609 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4610 FW_CMD_REQUEST_F |
4611 FW_CMD_READ_F |
5167865a 4612 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
0abfd152 4613 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
5167865a 4614 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
0abfd152 4615 ldst_cmd.u.pcie.ctrl_to_fn =
5167865a 4616 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
0abfd152
HS		 4617 ldst_cmd.u.pcie.r = reg;
4618 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4619 &ldst_cmd);
4620
4621 /* If the LDST Command suucceeded, exctract the returned register
4622 * value. Otherwise read it directly ourself.
4623 */
4624 if (ret == 0)
4625 val = ntohl(ldst_cmd.u.pcie.data[0]);
4626 else
4627 t4_hw_pci_read_cfg4(adap, reg, &val);
4628
4629 return val;
4630}
4631
b8ff05a9
DM		 4632static void setup_memwin(struct adapter *adap)
4633{
0abfd152 4634 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
b8ff05a9 4635
d14807dd 4636 if (is_t4(adap->params.chip)) {
0abfd152
HS		 4637 u32 bar0;
4638
4639 /* Truncation intentional: we only read the bottom 32-bits of
4640 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor
4641 * mechanism to read BAR0 instead of using
4642 * pci_resource_start() because we could be operating from
4643 * within a Virtual Machine which is trapping our accesses to
4644 * our Configuration Space and we need to set up the PCI-E
4645 * Memory Window decoders with the actual addresses which will
4646 * be coming across the PCI-E link.
4647 */
4648 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4649 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4650 adap->t4_bar0 = bar0;
4651
19dd37ba
SR		 4652 mem_win0_base = bar0 + MEMWIN0_BASE;
4653 mem_win1_base = bar0 + MEMWIN1_BASE;
4654 mem_win2_base = bar0 + MEMWIN2_BASE;
0abfd152 4655 mem_win2_aperture = MEMWIN2_APERTURE;
19dd37ba
SR		 4656 } else {
4657 /* For T5, only relative offset inside the PCIe BAR is passed */
4658 mem_win0_base = MEMWIN0_BASE;
0abfd152 4659 mem_win1_base = MEMWIN1_BASE;
19dd37ba 4660 mem_win2_base = MEMWIN2_BASE_T5;
0abfd152 4661 mem_win2_aperture = MEMWIN2_APERTURE_T5;
19dd37ba 4662 }
f061de42
HS		 4663 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4664 mem_win0_base | BIR_V(0) |
4665 WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4666 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4667 mem_win1_base | BIR_V(0) |
4668 WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4669 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4670 mem_win2_base | BIR_V(0) |
4671 WINDOW_V(ilog2(mem_win2_aperture) - 10));
4672 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
636f9d37
VP		 4673}
4674
4675static void setup_memwin_rdma(struct adapter *adap)
4676{
1ae970e0 4677 if (adap->vres.ocq.size) {
0abfd152
HS		 4678 u32 start;
4679 unsigned int sz_kb;
1ae970e0 4680
0abfd152
HS		 4681 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4682 start &= PCI_BASE_ADDRESS_MEM_MASK;
4683 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
1ae970e0
DM		 4684 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4685 t4_write_reg(adap,
f061de42
HS		 4686 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4687 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
1ae970e0 4688 t4_write_reg(adap,
f061de42 4689 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
1ae970e0
DM		 4690 adap->vres.ocq.start);
4691 t4_read_reg(adap,
f061de42 4692 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
1ae970e0 4693 }
b8ff05a9
DM		 4694}
4695
02b5fb8e
DM		 4696static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4697{
4698 u32 v;
4699 int ret;
4700
4701 /* get device capabilities */
4702 memset(c, 0, sizeof(*c));
e2ac9628
HS		 4703 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4704 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 4705 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4706 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4707 if (ret < 0)
4708 return ret;
4709
4710 /* select capabilities we'll be using */
4711 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4712 if (!vf_acls)
4713 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4714 else
4715 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4716 } else if (vf_acls) {
4717 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4718 return ret;
4719 }
e2ac9628
HS		 4720 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4721 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
060e0c75 4722 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4723 if (ret < 0)
4724 return ret;
4725
060e0c75 4726 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e 4727 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
b2e1a3f0
HS		 4728 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4729 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
02b5fb8e
DM		 4730 if (ret < 0)
4731 return ret;
4732
060e0c75
DM		 4733 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4734 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 4735 if (ret < 0)
4736 return ret;
4737
4738 t4_sge_init(adap);
4739
02b5fb8e 4740 /* tweak some settings */
837e4a42 4741 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
0d804338 4742 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
837e4a42
HS		 4743 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4744 v = t4_read_reg(adap, TP_PIO_DATA_A);
4745 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
060e0c75 4746
dca4faeb
VP		 4747 /* first 4 Tx modulation queues point to consecutive Tx channels */
4748 adap->params.tp.tx_modq_map = 0xE4;
0d804338
HS		 4749 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4750 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
dca4faeb
VP		 4751
4752 /* associate each Tx modulation queue with consecutive Tx channels */
4753 v = 0x84218421;
837e4a42 4754 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4755 &v, 1, TP_TX_SCHED_HDR_A);
837e4a42 4756 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4757 &v, 1, TP_TX_SCHED_FIFO_A);
837e4a42 4758 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4759 &v, 1, TP_TX_SCHED_PCMD_A);
dca4faeb
VP		 4760
4761#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4762 if (is_offload(adap)) {
0d804338
HS		 4763 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4764 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4765 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4766 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4767 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4768 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4769 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4770 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4771 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4772 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
dca4faeb
VP		 4773 }
4774
060e0c75
DM		 4775 /* get basic stuff going */
4776 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4777}
4778
b8ff05a9
DM		 4779/*
4780 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4781 */
4782#define MAX_ATIDS 8192U
4783
636f9d37
VP		 4784/*
4785 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4786 *
4787 * If the firmware we're dealing with has Configuration File support, then
4788 * we use that to perform all configuration
4789 */
4790
4791/*
4792 * Tweak configuration based on module parameters, etc. Most of these have
4793 * defaults assigned to them by Firmware Configuration Files (if we're using
4794 * them) but need to be explicitly set if we're using hard-coded
4795 * initialization. But even in the case of using Firmware Configuration
4796 * Files, we'd like to expose the ability to change these via module
4797 * parameters so these are essentially common tweaks/settings for
4798 * Configuration Files and hard-coded initialization ...
4799 */
4800static int adap_init0_tweaks(struct adapter *adapter)
4801{
4802 /*
4803 * Fix up various Host-Dependent Parameters like Page Size, Cache
4804 * Line Size, etc. The firmware default is for a 4KB Page Size and
4805 * 64B Cache Line Size ...
4806 */
4807 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4808
4809 /*
4810 * Process module parameters which affect early initialization.
4811 */
4812 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4813 dev_err(&adapter->pdev->dev,
4814 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4815 rx_dma_offset);
4816 rx_dma_offset = 2;
4817 }
f612b815
HS		 4818 t4_set_reg_field(adapter, SGE_CONTROL_A,
4819 PKTSHIFT_V(PKTSHIFT_M),
4820 PKTSHIFT_V(rx_dma_offset));
636f9d37
VP		 4821
4822 /*
4823 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4824 * adds the pseudo header itself.
4825 */
837e4a42
HS		 4826 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
4827 CSUM_HAS_PSEUDO_HDR_F, 0);
636f9d37
VP		 4828
4829 return 0;
4830}
4831
4832/*
4833 * Attempt to initialize the adapter via a Firmware Configuration File.
4834 */
4835static int adap_init0_config(struct adapter *adapter, int reset)
4836{
4837 struct fw_caps_config_cmd caps_cmd;
4838 const struct firmware *cf;
4839 unsigned long mtype = 0, maddr = 0;
4840 u32 finiver, finicsum, cfcsum;
16e47624
HS		 4841 int ret;
4842 int config_issued = 0;
0a57a536 4843 char *fw_config_file, fw_config_file_path[256];
16e47624 4844 char *config_name = NULL;
636f9d37
VP		 4845
4846 /*
4847 * Reset device if necessary.
4848 */
4849 if (reset) {
4850 ret = t4_fw_reset(adapter, adapter->mbox,
0d804338 4851 PIORSTMODE_F | PIORST_F);
636f9d37
VP		 4852 if (ret < 0)
4853 goto bye;
4854 }
4855
4856 /*
4857 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4858 * then use that. Otherwise, use the configuration file stored
4859 * in the adapter flash ...
4860 */
d14807dd 4861 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 4862 case CHELSIO_T4:
16e47624 4863 fw_config_file = FW4_CFNAME;
0a57a536
SR		 4864 break;
4865 case CHELSIO_T5:
4866 fw_config_file = FW5_CFNAME;
4867 break;
4868 default:
4869 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4870 adapter->pdev->device);
4871 ret = -EINVAL;
4872 goto bye;
4873 }
4874
4875 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 4876 if (ret < 0) {
16e47624 4877 config_name = "On FLASH";
636f9d37
VP		 4878 mtype = FW_MEMTYPE_CF_FLASH;
4879 maddr = t4_flash_cfg_addr(adapter);
4880 } else {
4881 u32 params[7], val[7];
4882
16e47624
HS		 4883 sprintf(fw_config_file_path,
4884 "/lib/firmware/%s", fw_config_file);
4885 config_name = fw_config_file_path;
4886
636f9d37
VP		 4887 if (cf->size >= FLASH_CFG_MAX_SIZE)
4888 ret = -ENOMEM;
4889 else {
5167865a
HS		 4890 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
4891 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
636f9d37
VP		 4892 ret = t4_query_params(adapter, adapter->mbox,
4893 adapter->fn, 0, 1, params, val);
4894 if (ret == 0) {
4895 /*
fc5ab020 4896 * For t4_memory_rw() below addresses and
636f9d37
VP		 4897 * sizes have to be in terms of multiples of 4
4898 * bytes. So, if the Configuration File isn't
4899 * a multiple of 4 bytes in length we'll have
4900 * to write that out separately since we can't
4901 * guarantee that the bytes following the
4902 * residual byte in the buffer returned by
4903 * request_firmware() are zeroed out ...
4904 */
4905 size_t resid = cf->size & 0x3;
4906 size_t size = cf->size & ~0x3;
4907 __be32 *data = (__be32 *)cf->data;
4908
5167865a
HS		 4909 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
4910 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
636f9d37 4911
fc5ab020
HS		 4912 spin_lock(&adapter->win0_lock);
4913 ret = t4_memory_rw(adapter, 0, mtype, maddr,
4914 size, data, T4_MEMORY_WRITE);
636f9d37
VP		 4915 if (ret == 0 && resid != 0) {
4916 union {
4917 __be32 word;
4918 char buf[4];
4919 } last;
4920 int i;
4921
4922 last.word = data[size >> 2];
4923 for (i = resid; i < 4; i++)
4924 last.buf[i] = 0;
fc5ab020
HS		 4925 ret = t4_memory_rw(adapter, 0, mtype,
4926 maddr + size,
4927 4, &last.word,
4928 T4_MEMORY_WRITE);
636f9d37 4929 }
fc5ab020 4930 spin_unlock(&adapter->win0_lock);
636f9d37
VP		 4931 }
4932 }
4933
4934 release_firmware(cf);
4935 if (ret)
4936 goto bye;
4937 }
4938
4939 /*
4940 * Issue a Capability Configuration command to the firmware to get it
4941 * to parse the Configuration File. We don't use t4_fw_config_file()
4942 * because we want the ability to modify various features after we've
4943 * processed the configuration file ...
4944 */
4945 memset(&caps_cmd, 0, sizeof(caps_cmd));
4946 caps_cmd.op_to_write =
e2ac9628
HS		 4947 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4948 FW_CMD_REQUEST_F |
4949 FW_CMD_READ_F);
ce91a923 4950 caps_cmd.cfvalid_to_len16 =
5167865a
HS		 4951 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
4952 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
4953 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
636f9d37
VP		 4954 FW_LEN16(caps_cmd));
4955 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4956 &caps_cmd);
16e47624
HS		 4957
4958 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4959 * Configuration File in FLASH), our last gasp effort is to use the
4960 * Firmware Configuration File which is embedded in the firmware. A
4961 * very few early versions of the firmware didn't have one embedded
4962 * but we can ignore those.
4963 */
4964 if (ret == -ENOENT) {
4965 memset(&caps_cmd, 0, sizeof(caps_cmd));
4966 caps_cmd.op_to_write =
e2ac9628
HS		 4967 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4968 FW_CMD_REQUEST_F |
4969 FW_CMD_READ_F);
16e47624
HS		 4970 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4971 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4972 sizeof(caps_cmd), &caps_cmd);
4973 config_name = "Firmware Default";
4974 }
4975
4976 config_issued = 1;
636f9d37
VP		 4977 if (ret < 0)
4978 goto bye;
4979
4980 finiver = ntohl(caps_cmd.finiver);
4981 finicsum = ntohl(caps_cmd.finicsum);
4982 cfcsum = ntohl(caps_cmd.cfcsum);
4983 if (finicsum != cfcsum)
4984 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4985 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4986 finicsum, cfcsum);
4987
636f9d37
VP		 4988 /*
4989 * And now tell the firmware to use the configuration we just loaded.
4990 */
4991 caps_cmd.op_to_write =
e2ac9628
HS		 4992 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4993 FW_CMD_REQUEST_F |
4994 FW_CMD_WRITE_F);
ce91a923 4995 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 4996 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4997 NULL);
4998 if (ret < 0)
4999 goto bye;
5000
5001 /*
5002 * Tweak configuration based on system architecture, module
5003 * parameters, etc.
5004 */
5005 ret = adap_init0_tweaks(adapter);
5006 if (ret < 0)
5007 goto bye;
5008
5009 /*
5010 * And finally tell the firmware to initialize itself using the
5011 * parameters from the Configuration File.
5012 */
5013 ret = t4_fw_initialize(adapter, adapter->mbox);
5014 if (ret < 0)
5015 goto bye;
5016
06640310
HS		 5017 /* Emit Firmware Configuration File information and return
5018 * successfully.
636f9d37 5019 */
636f9d37 5020 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 5021 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5022 config_name, finiver, cfcsum);
636f9d37
VP		 5023 return 0;
5024
5025 /*
5026 * Something bad happened. Return the error ... (If the "error"
5027 * is that there's no Configuration File on the adapter we don't
5028 * want to issue a warning since this is fairly common.)
5029 */
5030bye:
16e47624
HS		 5031 if (config_issued && ret != -ENOENT)
5032 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5033 config_name, -ret);
636f9d37
VP		 5034 return ret;
5035}
5036
16e47624
HS		 5037static struct fw_info fw_info_array[] = {
5038 {
5039 .chip = CHELSIO_T4,
5040 .fs_name = FW4_CFNAME,
5041 .fw_mod_name = FW4_FNAME,
5042 .fw_hdr = {
5043 .chip = FW_HDR_CHIP_T4,
5044 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5045 .intfver_nic = FW_INTFVER(T4, NIC),
5046 .intfver_vnic = FW_INTFVER(T4, VNIC),
5047 .intfver_ri = FW_INTFVER(T4, RI),
5048 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5049 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5050 },
5051 }, {
5052 .chip = CHELSIO_T5,
5053 .fs_name = FW5_CFNAME,
5054 .fw_mod_name = FW5_FNAME,
5055 .fw_hdr = {
5056 .chip = FW_HDR_CHIP_T5,
5057 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5058 .intfver_nic = FW_INTFVER(T5, NIC),
5059 .intfver_vnic = FW_INTFVER(T5, VNIC),
5060 .intfver_ri = FW_INTFVER(T5, RI),
5061 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5062 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5063 },
5064 }
5065};
5066
5067static struct fw_info *find_fw_info(int chip)
5068{
5069 int i;
5070
5071 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5072 if (fw_info_array[i].chip == chip)
5073 return &fw_info_array[i];
5074 }
5075 return NULL;
5076}
5077
b8ff05a9
DM		 5078/*
5079 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5080 */
5081static int adap_init0(struct adapter *adap)
5082{
5083 int ret;
5084 u32 v, port_vec;
5085 enum dev_state state;
5086 u32 params[7], val[7];
9a4da2cd 5087 struct fw_caps_config_cmd caps_cmd;
49aa284f
HS		 5088 struct fw_devlog_cmd devlog_cmd;
5089 u32 devlog_meminfo;
dcf7b6f5 5090 int reset = 1;
b8ff05a9 5091
666224d4
HS		 5092 /* Contact FW, advertising Master capability */
5093 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
b8ff05a9
DM		 5094 if (ret < 0) {
5095 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5096 ret);
5097 return ret;
5098 }
636f9d37
VP		 5099 if (ret == adap->mbox)
5100 adap->flags |= MASTER_PF;
b8ff05a9 5101
636f9d37
VP		 5102 /*
5103 * If we're the Master PF Driver and the device is uninitialized,
5104 * then let's consider upgrading the firmware ... (We always want
5105 * to check the firmware version number in order to A. get it for
5106 * later reporting and B. to warn if the currently loaded firmware
5107 * is excessively mismatched relative to the driver.)
5108 */
16e47624
HS		 5109 t4_get_fw_version(adap, &adap->params.fw_vers);
5110 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5111 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5112 struct fw_info *fw_info;
5113 struct fw_hdr *card_fw;
5114 const struct firmware *fw;
5115 const u8 *fw_data = NULL;
5116 unsigned int fw_size = 0;
5117
5118 /* This is the firmware whose headers the driver was compiled
5119 * against
5120 */
5121 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5122 if (fw_info == NULL) {
5123 dev_err(adap->pdev_dev,
5124 "unable to get firmware info for chip %d.\n",
5125 CHELSIO_CHIP_VERSION(adap->params.chip));
5126 return -EINVAL;
636f9d37 5127 }
16e47624
HS		 5128
5129 /* allocate memory to read the header of the firmware on the
5130 * card
5131 */
5132 card_fw = t4_alloc_mem(sizeof(*card_fw));
5133
5134 /* Get FW from from /lib/firmware/ */
5135 ret = request_firmware(&fw, fw_info->fw_mod_name,
5136 adap->pdev_dev);
5137 if (ret < 0) {
5138 dev_err(adap->pdev_dev,
5139 "unable to load firmware image %s, error %d\n",
5140 fw_info->fw_mod_name, ret);
5141 } else {
5142 fw_data = fw->data;
5143 fw_size = fw->size;
5144 }
5145
5146 /* upgrade FW logic */
5147 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5148 state, &reset);
5149
5150 /* Cleaning up */
0b5b6bee 5151 release_firmware(fw);
16e47624
HS		 5152 t4_free_mem(card_fw);
5153
636f9d37 5154 if (ret < 0)
16e47624 5155 goto bye;
636f9d37 5156 }
b8ff05a9 5157
636f9d37
VP		 5158 /*
5159 * Grab VPD parameters. This should be done after we establish a
5160 * connection to the firmware since some of the VPD parameters
5161 * (notably the Core Clock frequency) are retrieved via requests to
5162 * the firmware. On the other hand, we need these fairly early on
5163 * so we do this right after getting ahold of the firmware.
5164 */
5165 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5166 if (ret < 0)
5167 goto bye;
a0881cab 5168
49aa284f
HS		 5169 /* Read firmware device log parameters. We really need to find a way
5170 * to get these parameters initialized with some default values (which
5171 * are likely to be correct) for the case where we either don't
5172 * attache to the firmware or it's crashed when we probe the adapter.
5173 * That way we'll still be able to perform early firmware startup
5174 * debugging ... If the request to get the Firmware's Device Log
5175 * parameters fails, we'll live so we don't make that a fatal error.
5176 */
5177 memset(&devlog_cmd, 0, sizeof(devlog_cmd));
5178 devlog_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_DEVLOG_CMD) |
5179 FW_CMD_REQUEST_F | FW_CMD_READ_F);
5180 devlog_cmd.retval_len16 = htonl(FW_LEN16(devlog_cmd));
5181 ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
5182 &devlog_cmd);
5183 if (ret == 0) {
5184 devlog_meminfo =
5185 ntohl(devlog_cmd.memtype_devlog_memaddr16_devlog);
5186 adap->params.devlog.memtype =
5187 FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
5188 adap->params.devlog.start =
5189 FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
5190 adap->params.devlog.size = ntohl(devlog_cmd.memsize_devlog);
5191 }
5192
636f9d37 5193 /*
13ee15d3
VP		 5194 * Find out what ports are available to us. Note that we need to do
5195 * this before calling adap_init0_no_config() since it needs nports
5196 * and portvec ...
636f9d37
VP		 5197 */
5198 v =
5167865a
HS		 5199 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5200 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
636f9d37 5201 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5202 if (ret < 0)
5203 goto bye;
5204
636f9d37
VP		 5205 adap->params.nports = hweight32(port_vec);
5206 adap->params.portvec = port_vec;
5207
06640310
HS		 5208 /* If the firmware is initialized already, emit a simply note to that
5209 * effect. Otherwise, it's time to try initializing the adapter.
636f9d37
VP		 5210 */
5211 if (state == DEV_STATE_INIT) {
5212 dev_info(adap->pdev_dev, "Coming up as %s: "\
5213 "Adapter already initialized\n",
5214 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
636f9d37
VP		 5215 } else {
5216 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5217 "Initializing adapter\n");
06640310
HS		 5218
5219 /* Find out whether we're dealing with a version of the
5220 * firmware which has configuration file support.
636f9d37 5221 */
06640310
HS		 5222 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5223 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5224 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5225 params, val);
13ee15d3 5226
06640310
HS		 5227 /* If the firmware doesn't support Configuration Files,
5228 * return an error.
5229 */
5230 if (ret < 0) {
5231 dev_err(adap->pdev_dev, "firmware doesn't support "
5232 "Firmware Configuration Files\n");
5233 goto bye;
5234 }
5235
5236 /* The firmware provides us with a memory buffer where we can
5237 * load a Configuration File from the host if we want to
5238 * override the Configuration File in flash.
5239 */
5240 ret = adap_init0_config(adap, reset);
5241 if (ret == -ENOENT) {
5242 dev_err(adap->pdev_dev, "no Configuration File "
5243 "present on adapter.\n");
5244 goto bye;
636f9d37
VP		 5245 }
5246 if (ret < 0) {
06640310
HS		 5247 dev_err(adap->pdev_dev, "could not initialize "
5248 "adapter, error %d\n", -ret);
636f9d37
VP		 5249 goto bye;
5250 }
5251 }
5252
06640310
HS		 5253 /* Give the SGE code a chance to pull in anything that it needs ...
5254 * Note that this must be called after we retrieve our VPD parameters
5255 * in order to know how to convert core ticks to seconds, etc.
636f9d37 5256 */
06640310
HS		 5257 ret = t4_sge_init(adap);
5258 if (ret < 0)
5259 goto bye;
636f9d37 5260
9a4da2cd
VP		 5261 if (is_bypass_device(adap->pdev->device))
5262 adap->params.bypass = 1;
5263
636f9d37
VP		 5264 /*
5265 * Grab some of our basic fundamental operating parameters.
5266 */
5267#define FW_PARAM_DEV(param) \
5167865a
HS		 5268 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5269 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
636f9d37 5270
b8ff05a9 5271#define FW_PARAM_PFVF(param) \
5167865a
HS		 5272 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5273 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
5274 FW_PARAMS_PARAM_Y_V(0) | \
5275 FW_PARAMS_PARAM_Z_V(0)
b8ff05a9 5276
636f9d37 5277 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5278 params[1] = FW_PARAM_PFVF(L2T_START);
5279 params[2] = FW_PARAM_PFVF(L2T_END);
5280 params[3] = FW_PARAM_PFVF(FILTER_START);
5281 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5282 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5283 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5284 if (ret < 0)
5285 goto bye;
636f9d37
VP		 5286 adap->sge.egr_start = val[0];
5287 adap->l2t_start = val[1];
5288 adap->l2t_end = val[2];
b8ff05a9
DM		 5289 adap->tids.ftid_base = val[3];
5290 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5291 adap->sge.ingr_start = val[5];
b8ff05a9 5292
b5a02f50
AB		 5293 params[0] = FW_PARAM_PFVF(CLIP_START);
5294 params[1] = FW_PARAM_PFVF(CLIP_END);
5295 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5296 if (ret < 0)
5297 goto bye;
5298 adap->clipt_start = val[0];
5299 adap->clipt_end = val[1];
5300
636f9d37
VP		 5301 /* query params related to active filter region */
5302 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5303 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5304 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5305 /* If Active filter size is set we enable establishing
5306 * offload connection through firmware work request
5307 */
5308 if ((val[0] != val[1]) && (ret >= 0)) {
5309 adap->flags |= FW_OFLD_CONN;
5310 adap->tids.aftid_base = val[0];
5311 adap->tids.aftid_end = val[1];
5312 }
5313
b407a4a9
VP		 5314 /* If we're running on newer firmware, let it know that we're
5315 * prepared to deal with encapsulated CPL messages. Older
5316 * firmware won't understand this and we'll just get
5317 * unencapsulated messages ...
5318 */
5319 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5320 val[0] = 1;
5321 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5322
1ac0f095
KS		 5323 /*
5324 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5325 * capability. Earlier versions of the firmware didn't have the
5326 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5327 * permission to use ULPTX MEMWRITE DSGL.
5328 */
5329 if (is_t4(adap->params.chip)) {
5330 adap->params.ulptx_memwrite_dsgl = false;
5331 } else {
5332 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5333 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5334 1, params, val);
5335 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5336 }
5337
636f9d37
VP		 5338 /*
5339 * Get device capabilities so we can determine what resources we need
5340 * to manage.
5341 */
5342 memset(&caps_cmd, 0, sizeof(caps_cmd));
e2ac9628
HS		 5343 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5344 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 5345 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5346 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5347 &caps_cmd);
5348 if (ret < 0)
5349 goto bye;
5350
13ee15d3 5351 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5352 /* query offload-related parameters */
5353 params[0] = FW_PARAM_DEV(NTID);
5354 params[1] = FW_PARAM_PFVF(SERVER_START);
5355 params[2] = FW_PARAM_PFVF(SERVER_END);
5356 params[3] = FW_PARAM_PFVF(TDDP_START);
5357 params[4] = FW_PARAM_PFVF(TDDP_END);
5358 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5359 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5360 params, val);
b8ff05a9
DM		 5361 if (ret < 0)
5362 goto bye;
5363 adap->tids.ntids = val[0];
5364 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5365 adap->tids.stid_base = val[1];
5366 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37
VP		 5367 /*
5368 * Setup server filter region. Divide the availble filter
5369 * region into two parts. Regular filters get 1/3rd and server
5370 * filters get 2/3rd part. This is only enabled if workarond
5371 * path is enabled.
5372 * 1. For regular filters.
5373 * 2. Server filter: This are special filters which are used
5374 * to redirect SYN packets to offload queue.
5375 */
5376 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5377 adap->tids.sftid_base = adap->tids.ftid_base +
5378 DIV_ROUND_UP(adap->tids.nftids, 3);
5379 adap->tids.nsftids = adap->tids.nftids -
5380 DIV_ROUND_UP(adap->tids.nftids, 3);
5381 adap->tids.nftids = adap->tids.sftid_base -
5382 adap->tids.ftid_base;
5383 }
b8ff05a9
DM		 5384 adap->vres.ddp.start = val[3];
5385 adap->vres.ddp.size = val[4] - val[3] + 1;
5386 adap->params.ofldq_wr_cred = val[5];
636f9d37 5387
b8ff05a9
DM		 5388 adap->params.offload = 1;
5389 }
636f9d37 5390 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5391 params[0] = FW_PARAM_PFVF(STAG_START);
5392 params[1] = FW_PARAM_PFVF(STAG_END);
5393 params[2] = FW_PARAM_PFVF(RQ_START);
5394 params[3] = FW_PARAM_PFVF(RQ_END);
5395 params[4] = FW_PARAM_PFVF(PBL_START);
5396 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5397 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5398 params, val);
b8ff05a9
DM		 5399 if (ret < 0)
5400 goto bye;
5401 adap->vres.stag.start = val[0];
5402 adap->vres.stag.size = val[1] - val[0] + 1;
5403 adap->vres.rq.start = val[2];
5404 adap->vres.rq.size = val[3] - val[2] + 1;
5405 adap->vres.pbl.start = val[4];
5406 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5407
5408 params[0] = FW_PARAM_PFVF(SQRQ_START);
5409 params[1] = FW_PARAM_PFVF(SQRQ_END);
5410 params[2] = FW_PARAM_PFVF(CQ_START);
5411 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5412 params[4] = FW_PARAM_PFVF(OCQ_START);
5413 params[5] = FW_PARAM_PFVF(OCQ_END);
5c937dd3
HS		 5414 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5415 val);
a0881cab
DM		 5416 if (ret < 0)
5417 goto bye;
5418 adap->vres.qp.start = val[0];
5419 adap->vres.qp.size = val[1] - val[0] + 1;
5420 adap->vres.cq.start = val[2];
5421 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5422 adap->vres.ocq.start = val[4];
5423 adap->vres.ocq.size = val[5] - val[4] + 1;
4c2c5763
HS		 5424
5425 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5426 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5c937dd3
HS		 5427 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5428 val);
4c2c5763
HS		 5429 if (ret < 0) {
5430 adap->params.max_ordird_qp = 8;
5431 adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5432 ret = 0;
5433 } else {
5434 adap->params.max_ordird_qp = val[0];
5435 adap->params.max_ird_adapter = val[1];
5436 }
5437 dev_info(adap->pdev_dev,
5438 "max_ordird_qp %d max_ird_adapter %d\n",
5439 adap->params.max_ordird_qp,
5440 adap->params.max_ird_adapter);
b8ff05a9 5441 }
636f9d37 5442 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5443 params[0] = FW_PARAM_PFVF(ISCSI_START);
5444 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5445 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5446 params, val);
b8ff05a9
DM		 5447 if (ret < 0)
5448 goto bye;
5449 adap->vres.iscsi.start = val[0];
5450 adap->vres.iscsi.size = val[1] - val[0] + 1;
5451 }
5452#undef FW_PARAM_PFVF
5453#undef FW_PARAM_DEV
5454
92e7ae71
HS		 5455 /* The MTU/MSS Table is initialized by now, so load their values. If
5456 * we're initializing the adapter, then we'll make any modifications
5457 * we want to the MTU/MSS Table and also initialize the congestion
5458 * parameters.
636f9d37 5459 */
b8ff05a9 5460 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
92e7ae71
HS		 5461 if (state != DEV_STATE_INIT) {
5462 int i;
5463
5464 /* The default MTU Table contains values 1492 and 1500.
5465 * However, for TCP, it's better to have two values which are
5466 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5467 * This allows us to have a TCP Data Payload which is a
5468 * multiple of 8 regardless of what combination of TCP Options
5469 * are in use (always a multiple of 4 bytes) which is
5470 * important for performance reasons. For instance, if no
5471 * options are in use, then we have a 20-byte IP header and a
5472 * 20-byte TCP header. In this case, a 1500-byte MSS would
5473 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5474 * which is not a multiple of 8. So using an MSS of 1488 in
5475 * this case results in a TCP Data Payload of 1448 bytes which
5476 * is a multiple of 8. On the other hand, if 12-byte TCP Time
5477 * Stamps have been negotiated, then an MTU of 1500 bytes
5478 * results in a TCP Data Payload of 1448 bytes which, as
5479 * above, is a multiple of 8 bytes ...
5480 */
5481 for (i = 0; i < NMTUS; i++)
5482 if (adap->params.mtus[i] == 1492) {
5483 adap->params.mtus[i] = 1488;
5484 break;
5485 }
7ee9ff94 5486
92e7ae71
HS		 5487 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5488 adap->params.b_wnd);
5489 }
df64e4d3 5490 t4_init_sge_params(adap);
dcf7b6f5 5491 t4_init_tp_params(adap);
636f9d37 5492 adap->flags |= FW_OK;
b8ff05a9
DM		 5493 return 0;
5494
5495 /*
636f9d37
VP		 5496 * Something bad happened. If a command timed out or failed with EIO
5497 * FW does not operate within its spec or something catastrophic
5498 * happened to HW/FW, stop issuing commands.
b8ff05a9 5499 */
636f9d37
VP		 5500bye:
5501 if (ret != -ETIMEDOUT && ret != -EIO)
5502 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5503 return ret;
5504}
5505
204dc3c0
DM		 5506/* EEH callbacks */
5507
5508static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5509 pci_channel_state_t state)
5510{
5511 int i;
5512 struct adapter *adap = pci_get_drvdata(pdev);
5513
5514 if (!adap)
5515 goto out;
5516
5517 rtnl_lock();
5518 adap->flags &= ~FW_OK;
5519 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5520 spin_lock(&adap->stats_lock);
204dc3c0
DM		 5521 for_each_port(adap, i) {
5522 struct net_device *dev = adap->port[i];
5523
5524 netif_device_detach(dev);
5525 netif_carrier_off(dev);
5526 }
9fe6cb58 5527 spin_unlock(&adap->stats_lock);
204dc3c0
DM		 5528 if (adap->flags & FULL_INIT_DONE)
5529 cxgb_down(adap);
5530 rtnl_unlock();
144be3d9
GS		 5531 if ((adap->flags & DEV_ENABLED)) {
5532 pci_disable_device(pdev);
5533 adap->flags &= ~DEV_ENABLED;
5534 }
204dc3c0
DM		 5535out: return state == pci_channel_io_perm_failure ?
5536 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5537}
5538
5539static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5540{
5541 int i, ret;
5542 struct fw_caps_config_cmd c;
5543 struct adapter *adap = pci_get_drvdata(pdev);
5544
5545 if (!adap) {
5546 pci_restore_state(pdev);
5547 pci_save_state(pdev);
5548 return PCI_ERS_RESULT_RECOVERED;
5549 }
5550
144be3d9
GS		 5551 if (!(adap->flags & DEV_ENABLED)) {
5552 if (pci_enable_device(pdev)) {
5553 dev_err(&pdev->dev, "Cannot reenable PCI "
5554 "device after reset\n");
5555 return PCI_ERS_RESULT_DISCONNECT;
5556 }
5557 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 5558 }
5559
5560 pci_set_master(pdev);
5561 pci_restore_state(pdev);
5562 pci_save_state(pdev);
5563 pci_cleanup_aer_uncorrect_error_status(pdev);
5564
8203b509 5565 if (t4_wait_dev_ready(adap->regs) < 0)
204dc3c0 5566 return PCI_ERS_RESULT_DISCONNECT;
777c2300 5567 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 5568 return PCI_ERS_RESULT_DISCONNECT;
5569 adap->flags |= FW_OK;
5570 if (adap_init1(adap, &c))
5571 return PCI_ERS_RESULT_DISCONNECT;
5572
5573 for_each_port(adap, i) {
5574 struct port_info *p = adap2pinfo(adap, i);
5575
060e0c75
DM		 5576 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5577 NULL, NULL);
204dc3c0
DM		 5578 if (ret < 0)
5579 return PCI_ERS_RESULT_DISCONNECT;
5580 p->viid = ret;
5581 p->xact_addr_filt = -1;
5582 }
5583
5584 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5585 adap->params.b_wnd);
1ae970e0 5586 setup_memwin(adap);
204dc3c0
DM		 5587 if (cxgb_up(adap))
5588 return PCI_ERS_RESULT_DISCONNECT;
5589 return PCI_ERS_RESULT_RECOVERED;
5590}
5591
5592static void eeh_resume(struct pci_dev *pdev)
5593{
5594 int i;
5595 struct adapter *adap = pci_get_drvdata(pdev);
5596
5597 if (!adap)
5598 return;
5599
5600 rtnl_lock();
5601 for_each_port(adap, i) {
5602 struct net_device *dev = adap->port[i];
5603
5604 if (netif_running(dev)) {
5605 link_start(dev);
5606 cxgb_set_rxmode(dev);
5607 }
5608 netif_device_attach(dev);
5609 }
5610 rtnl_unlock();
5611}
5612
3646f0e5 5613static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 5614 .error_detected = eeh_err_detected,
5615 .slot_reset = eeh_slot_reset,
5616 .resume = eeh_resume,
5617};
5618
57d8b764 5619static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 5620{
57d8b764
KS		 5621 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5622 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 5623}
5624
c887ad0e
HS		 5625static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
5626 unsigned int us, unsigned int cnt,
b8ff05a9
DM		 5627 unsigned int size, unsigned int iqe_size)
5628{
c887ad0e
HS		 5629 q->adap = adap;
5630 set_rspq_intr_params(q, us, cnt);
b8ff05a9
DM		 5631 q->iqe_len = iqe_size;
5632 q->size = size;
5633}
5634
5635/*
5636 * Perform default configuration of DMA queues depending on the number and type
5637 * of ports we found and the number of available CPUs. Most settings can be
5638 * modified by the admin prior to actual use.
5639 */
91744948 5640static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 5641{
5642 struct sge *s = &adap->sge;
688848b1
AB		 5643 int i, n10g = 0, qidx = 0;
5644#ifndef CONFIG_CHELSIO_T4_DCB
5645 int q10g = 0;
5646#endif
cf38be6d 5647 int ciq_size;
b8ff05a9
DM		 5648
5649 for_each_port(adap, i)
57d8b764 5650 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
688848b1
AB		 5651#ifdef CONFIG_CHELSIO_T4_DCB
5652 /* For Data Center Bridging support we need to be able to support up
5653 * to 8 Traffic Priorities; each of which will be assigned to its
5654 * own TX Queue in order to prevent Head-Of-Line Blocking.
5655 */
5656 if (adap->params.nports * 8 > MAX_ETH_QSETS) {
5657 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
5658 MAX_ETH_QSETS, adap->params.nports * 8);
5659 BUG_ON(1);
5660 }
b8ff05a9 5661
688848b1
AB		 5662 for_each_port(adap, i) {
5663 struct port_info *pi = adap2pinfo(adap, i);
5664
5665 pi->first_qset = qidx;
5666 pi->nqsets = 8;
5667 qidx += pi->nqsets;
5668 }
5669#else /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 5670 /*
5671 * We default to 1 queue per non-10G port and up to # of cores queues
5672 * per 10G port.
5673 */
5674 if (n10g)
5675 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 5676 if (q10g > netif_get_num_default_rss_queues())
5677 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 5678
5679 for_each_port(adap, i) {
5680 struct port_info *pi = adap2pinfo(adap, i);
5681
5682 pi->first_qset = qidx;
57d8b764 5683 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 5684 qidx += pi->nqsets;
5685 }
688848b1 5686#endif /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 5687
5688 s->ethqsets = qidx;
5689 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5690
5691 if (is_offload(adap)) {
5692 /*
5693 * For offload we use 1 queue/channel if all ports are up to 1G,
5694 * otherwise we divide all available queues amongst the channels
5695 * capped by the number of available cores.
5696 */
5697 if (n10g) {
5698 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5699 num_online_cpus());
5700 s->ofldqsets = roundup(i, adap->params.nports);
5701 } else
5702 s->ofldqsets = adap->params.nports;
5703 /* For RDMA one Rx queue per channel suffices */
5704 s->rdmaqs = adap->params.nports;
cf38be6d 5705 s->rdmaciqs = adap->params.nports;
b8ff05a9
DM		 5706 }
5707
5708 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5709 struct sge_eth_rxq *r = &s->ethrxq[i];
5710
c887ad0e 5711 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
b8ff05a9
DM		 5712 r->fl.size = 72;
5713 }
5714
5715 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5716 s->ethtxq[i].q.size = 1024;
5717
5718 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5719 s->ctrlq[i].q.size = 512;
5720
5721 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5722 s->ofldtxq[i].q.size = 1024;
5723
5724 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5725 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5726
c887ad0e 5727 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
b8ff05a9
DM		 5728 r->rspq.uld = CXGB4_ULD_ISCSI;
5729 r->fl.size = 72;
5730 }
5731
5732 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5733 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5734
c887ad0e 5735 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
b8ff05a9
DM		 5736 r->rspq.uld = CXGB4_ULD_RDMA;
5737 r->fl.size = 72;
5738 }
5739
cf38be6d
HS		 5740 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5741 if (ciq_size > SGE_MAX_IQ_SIZE) {
5742 CH_WARN(adap, "CIQ size too small for available IQs\n");
5743 ciq_size = SGE_MAX_IQ_SIZE;
5744 }
5745
5746 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5747 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5748
c887ad0e 5749 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
cf38be6d
HS		 5750 r->rspq.uld = CXGB4_ULD_RDMA;
5751 }
5752
c887ad0e
HS		 5753 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5754 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
b8ff05a9
DM		 5755}
5756
5757/*
5758 * Reduce the number of Ethernet queues across all ports to at most n.
5759 * n provides at least one queue per port.
5760 */
91744948 5761static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 5762{
5763 int i;
5764 struct port_info *pi;
5765
5766 while (n < adap->sge.ethqsets)
5767 for_each_port(adap, i) {
5768 pi = adap2pinfo(adap, i);
5769 if (pi->nqsets > 1) {
5770 pi->nqsets--;
5771 adap->sge.ethqsets--;
5772 if (adap->sge.ethqsets <= n)
5773 break;
5774 }
5775 }
5776
5777 n = 0;
5778 for_each_port(adap, i) {
5779 pi = adap2pinfo(adap, i);
5780 pi->first_qset = n;
5781 n += pi->nqsets;
5782 }
5783}
5784
5785/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5786#define EXTRA_VECS 2
5787
91744948 5788static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 5789{
5790 int ofld_need = 0;
c32ad224 5791 int i, want, need;
b8ff05a9
DM		 5792 struct sge *s = &adap->sge;
5793 unsigned int nchan = adap->params.nports;
5794 struct msix_entry entries[MAX_INGQ + 1];
5795
5796 for (i = 0; i < ARRAY_SIZE(entries); ++i)
5797 entries[i].entry = i;
5798
5799 want = s->max_ethqsets + EXTRA_VECS;
5800 if (is_offload(adap)) {
cf38be6d 5801 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
b8ff05a9 5802 /* need nchan for each possible ULD */
cf38be6d 5803 ofld_need = 3 * nchan;
b8ff05a9 5804 }
688848b1
AB		 5805#ifdef CONFIG_CHELSIO_T4_DCB
5806 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
5807 * each port.
5808 */
5809 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
5810#else
b8ff05a9 5811 need = adap->params.nports + EXTRA_VECS + ofld_need;
688848b1 5812#endif
c32ad224
AG		 5813 want = pci_enable_msix_range(adap->pdev, entries, need, want);
5814 if (want < 0)
5815 return want;
b8ff05a9 5816
c32ad224
AG		 5817 /*
5818 * Distribute available vectors to the various queue groups.
5819 * Every group gets its minimum requirement and NIC gets top
5820 * priority for leftovers.
5821 */
5822 i = want - EXTRA_VECS - ofld_need;
5823 if (i < s->max_ethqsets) {
5824 s->max_ethqsets = i;
5825 if (i < s->ethqsets)
5826 reduce_ethqs(adap, i);
5827 }
5828 if (is_offload(adap)) {
5829 i = want - EXTRA_VECS - s->max_ethqsets;
5830 i -= ofld_need - nchan;
5831 s->ofldqsets = (i / nchan) * nchan; /* round down */
5832 }
5833 for (i = 0; i < want; ++i)
5834 adap->msix_info[i].vec = entries[i].vector;
5835
5836 return 0;
b8ff05a9
DM		 5837}
5838
5839#undef EXTRA_VECS
5840
91744948 5841static int init_rss(struct adapter *adap)
671b0060
DM		 5842{
5843 unsigned int i, j;
5844
5845 for_each_port(adap, i) {
5846 struct port_info *pi = adap2pinfo(adap, i);
5847
5848 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5849 if (!pi->rss)
5850 return -ENOMEM;
5851 for (j = 0; j < pi->rss_size; j++)
278bc429 5852 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 5853 }
5854 return 0;
5855}
5856
91744948 5857static void print_port_info(const struct net_device *dev)
b8ff05a9 5858{
b8ff05a9 5859 char buf[80];
118969ed 5860 char *bufp = buf;
f1a051b9 5861 const char *spd = "";
118969ed
DM		 5862 const struct port_info *pi = netdev_priv(dev);
5863 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 5864
5865 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5866 spd = " 2.5 GT/s";
5867 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5868 spd = " 5 GT/s";
d2e752db
RD		 5869 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5870 spd = " 8 GT/s";
b8ff05a9 5871
118969ed
DM		 5872 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5873 bufp += sprintf(bufp, "100/");
5874 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5875 bufp += sprintf(bufp, "1000/");
5876 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5877 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 5878 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5879 bufp += sprintf(bufp, "40G/");
118969ed
DM		 5880 if (bufp != buf)
5881 --bufp;
72aca4bf 5882 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 5883
5884 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 5885 adap->params.vpd.id,
d14807dd 5886 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 5887 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5888 (adap->flags & USING_MSIX) ? " MSI-X" :
5889 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 5890 netdev_info(dev, "S/N: %s, P/N: %s\n",
5891 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 5892}
5893
91744948 5894static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 5895{
e5c8ae5f 5896 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 5897}
5898
06546391
DM		 5899/*
5900 * Free the following resources:
5901 * - memory used for tables
5902 * - MSI/MSI-X
5903 * - net devices
5904 * - resources FW is holding for us
5905 */
5906static void free_some_resources(struct adapter *adapter)
5907{
5908 unsigned int i;
5909
5910 t4_free_mem(adapter->l2t);
5911 t4_free_mem(adapter->tids.tid_tab);
5912 disable_msi(adapter);
5913
5914 for_each_port(adapter, i)
671b0060
DM		 5915 if (adapter->port[i]) {
5916 kfree(adap2pinfo(adapter, i)->rss);
06546391 5917 free_netdev(adapter->port[i]);
671b0060 5918 }
06546391 5919 if (adapter->flags & FW_OK)
060e0c75 5920 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 5921}
5922
2ed28baa 5923#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 5924#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 5925 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 5926#define SEGMENT_SIZE 128
b8ff05a9 5927
1dd06ae8 5928static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 5929{
22adfe0a 5930 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 5931 struct port_info *pi;
c8f44aff 5932 bool highdma = false;
b8ff05a9 5933 struct adapter *adapter = NULL;
d6ce2628 5934 void __iomem *regs;
b8ff05a9
DM		 5935
5936 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5937
5938 err = pci_request_regions(pdev, KBUILD_MODNAME);
5939 if (err) {
5940 /* Just info, some other driver may have claimed the device. */
5941 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5942 return err;
5943 }
5944
b8ff05a9
DM		 5945 err = pci_enable_device(pdev);
5946 if (err) {
5947 dev_err(&pdev->dev, "cannot enable PCI device\n");
5948 goto out_release_regions;
5949 }
5950
d6ce2628
HS		 5951 regs = pci_ioremap_bar(pdev, 0);
5952 if (!regs) {
5953 dev_err(&pdev->dev, "cannot map device registers\n");
5954 err = -ENOMEM;
5955 goto out_disable_device;
5956 }
5957
8203b509
HS		 5958 err = t4_wait_dev_ready(regs);
5959 if (err < 0)
5960 goto out_unmap_bar0;
5961
d6ce2628 5962 /* We control everything through one PF */
0d804338 5963 func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
d6ce2628
HS		 5964 if (func != ent->driver_data) {
5965 iounmap(regs);
5966 pci_disable_device(pdev);
5967 pci_save_state(pdev); /* to restore SR-IOV later */
5968 goto sriov;
5969 }
5970
b8ff05a9 5971 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 5972 highdma = true;
b8ff05a9
DM		 5973 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5974 if (err) {
5975 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5976 "coherent allocations\n");
d6ce2628 5977 goto out_unmap_bar0;
b8ff05a9
DM		 5978 }
5979 } else {
5980 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5981 if (err) {
5982 dev_err(&pdev->dev, "no usable DMA configuration\n");
d6ce2628 5983 goto out_unmap_bar0;
b8ff05a9
DM		 5984 }
5985 }
5986
5987 pci_enable_pcie_error_reporting(pdev);
ef306b50 5988 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 5989 pci_set_master(pdev);
5990 pci_save_state(pdev);
5991
5992 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5993 if (!adapter) {
5994 err = -ENOMEM;
d6ce2628 5995 goto out_unmap_bar0;
b8ff05a9
DM		 5996 }
5997
29aaee65
AB		 5998 adapter->workq = create_singlethread_workqueue("cxgb4");
5999 if (!adapter->workq) {
6000 err = -ENOMEM;
6001 goto out_free_adapter;
6002 }
6003
144be3d9
GS		 6004 /* PCI device has been enabled */
6005 adapter->flags |= DEV_ENABLED;
6006
d6ce2628 6007 adapter->regs = regs;
b8ff05a9
DM		 6008 adapter->pdev = pdev;
6009 adapter->pdev_dev = &pdev->dev;
3069ee9b 6010 adapter->mbox = func;
060e0c75 6011 adapter->fn = func;
b8ff05a9
DM		 6012 adapter->msg_enable = dflt_msg_enable;
6013 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6014
6015 spin_lock_init(&adapter->stats_lock);
6016 spin_lock_init(&adapter->tid_release_lock);
e327c225 6017 spin_lock_init(&adapter->win0_lock);
b8ff05a9
DM		 6018
6019 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 6020 INIT_WORK(&adapter->db_full_task, process_db_full);
6021 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 6022
6023 err = t4_prep_adapter(adapter);
6024 if (err)
d6ce2628
HS		 6025 goto out_free_adapter;
6026
22adfe0a 6027
d14807dd 6028 if (!is_t4(adapter->params.chip)) {
f612b815
HS		 6029 s_qpp = (QUEUESPERPAGEPF0_S +
6030 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6031 adapter->fn);
6032 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6033 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
22adfe0a
SR		 6034 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6035
6036 /* Each segment size is 128B. Write coalescing is enabled only
6037 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6038 * queue is less no of segments that can be accommodated in
6039 * a page size.
6040 */
6041 if (qpp > num_seg) {
6042 dev_err(&pdev->dev,
6043 "Incorrect number of egress queues per page\n");
6044 err = -EINVAL;
d6ce2628 6045 goto out_free_adapter;
22adfe0a
SR		 6046 }
6047 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6048 pci_resource_len(pdev, 2));
6049 if (!adapter->bar2) {
6050 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6051 err = -ENOMEM;
d6ce2628 6052 goto out_free_adapter;
22adfe0a
SR		 6053 }
6054 }
6055
636f9d37 6056 setup_memwin(adapter);
b8ff05a9 6057 err = adap_init0(adapter);
636f9d37 6058 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6059 if (err)
6060 goto out_unmap_bar;
6061
6062 for_each_port(adapter, i) {
6063 struct net_device *netdev;
6064
6065 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6066 MAX_ETH_QSETS);
6067 if (!netdev) {
6068 err = -ENOMEM;
6069 goto out_free_dev;
6070 }
6071
6072 SET_NETDEV_DEV(netdev, &pdev->dev);
6073
6074 adapter->port[i] = netdev;
6075 pi = netdev_priv(netdev);
6076 pi->adapter = adapter;
6077 pi->xact_addr_filt = -1;
b8ff05a9 6078 pi->port_id = i;
b8ff05a9
DM		 6079 netdev->irq = pdev->irq;
6080
2ed28baa
MM		 6081 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6082 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6083 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6084 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6085 if (highdma)
6086 netdev->hw_features |= NETIF_F_HIGHDMA;
6087 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6088 netdev->vlan_features = netdev->features & VLAN_FEAT;
6089
01789349
JP		 6090 netdev->priv_flags |= IFF_UNICAST_FLT;
6091
b8ff05a9 6092 netdev->netdev_ops = &cxgb4_netdev_ops;
688848b1
AB		 6093#ifdef CONFIG_CHELSIO_T4_DCB
6094 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6095 cxgb4_dcb_state_init(netdev);
6096#endif
7ad24ea4 6097 netdev->ethtool_ops = &cxgb_ethtool_ops;
b8ff05a9
DM		 6098 }
6099
6100 pci_set_drvdata(pdev, adapter);
6101
6102 if (adapter->flags & FW_OK) {
060e0c75 6103 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6104 if (err)
6105 goto out_free_dev;
6106 }
6107
6108 /*
6109 * Configure queues and allocate tables now, they can be needed as
6110 * soon as the first register_netdev completes.
6111 */
6112 cfg_queues(adapter);
6113
6114 adapter->l2t = t4_init_l2t();
6115 if (!adapter->l2t) {
6116 /* We tolerate a lack of L2T, giving up some functionality */
6117 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6118 adapter->params.offload = 0;
6119 }
6120
b5a02f50
AB		 6121#if IS_ENABLED(CONFIG_IPV6)
6122 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
6123 adapter->clipt_end);
6124 if (!adapter->clipt) {
6125 /* We tolerate a lack of clip_table, giving up
6126 * some functionality
6127 */
6128 dev_warn(&pdev->dev,
6129 "could not allocate Clip table, continuing\n");
6130 adapter->params.offload = 0;
6131 }
6132#endif
b8ff05a9
DM		 6133 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6134 dev_warn(&pdev->dev, "could not allocate TID table, "
6135 "continuing\n");
6136 adapter->params.offload = 0;
6137 }
6138
f7cabcdd
DM		 6139 /* See what interrupts we'll be using */
6140 if (msi > 1 && enable_msix(adapter) == 0)
6141 adapter->flags |= USING_MSIX;
6142 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6143 adapter->flags |= USING_MSI;
6144
671b0060
DM		 6145 err = init_rss(adapter);
6146 if (err)
6147 goto out_free_dev;
6148
b8ff05a9
DM		 6149 /*
6150 * The card is now ready to go. If any errors occur during device
6151 * registration we do not fail the whole card but rather proceed only
6152 * with the ports we manage to register successfully. However we must
6153 * register at least one net device.
6154 */
6155 for_each_port(adapter, i) {
a57cabe0
DM		 6156 pi = adap2pinfo(adapter, i);
6157 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6158 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6159
b8ff05a9
DM		 6160 err = register_netdev(adapter->port[i]);
6161 if (err)
b1a3c2b6 6162 break;
b1a3c2b6
DM		 6163 adapter->chan_map[pi->tx_chan] = i;
6164 print_port_info(adapter->port[i]);
b8ff05a9 6165 }
b1a3c2b6 6166 if (i == 0) {
b8ff05a9
DM		 6167 dev_err(&pdev->dev, "could not register any net devices\n");
6168 goto out_free_dev;
6169 }
b1a3c2b6
DM		 6170 if (err) {
6171 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6172 err = 0;
6403eab1 6173 }
b8ff05a9
DM		 6174
6175 if (cxgb4_debugfs_root) {
6176 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6177 cxgb4_debugfs_root);
6178 setup_debugfs(adapter);
6179 }
6180
6482aa7c
DLR		 6181 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6182 pdev->needs_freset = 1;
6183
b8ff05a9
DM		 6184 if (is_offload(adapter))
6185 attach_ulds(adapter);
6186
8e1e6059 6187sriov:
b8ff05a9 6188#ifdef CONFIG_PCI_IOV
7d6727cf 6189 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6190 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6191 dev_info(&pdev->dev,
6192 "instantiated %u virtual functions\n",
6193 num_vf[func]);
6194#endif
6195 return 0;
6196
6197 out_free_dev:
06546391 6198 free_some_resources(adapter);
b8ff05a9 6199 out_unmap_bar:
d14807dd 6200 if (!is_t4(adapter->params.chip))
22adfe0a 6201 iounmap(adapter->bar2);
b8ff05a9 6202 out_free_adapter:
29aaee65
AB		 6203 if (adapter->workq)
6204 destroy_workqueue(adapter->workq);
6205
b8ff05a9 6206 kfree(adapter);
d6ce2628
HS		 6207 out_unmap_bar0:
6208 iounmap(regs);
b8ff05a9
DM		 6209 out_disable_device:
6210 pci_disable_pcie_error_reporting(pdev);
6211 pci_disable_device(pdev);
6212 out_release_regions:
6213 pci_release_regions(pdev);
b8ff05a9
DM		 6214 return err;
6215}
6216
91744948 6217static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6218{
6219 struct adapter *adapter = pci_get_drvdata(pdev);
6220
636f9d37 6221#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6222 pci_disable_sriov(pdev);
6223
636f9d37
VP		 6224#endif
6225
b8ff05a9
DM		 6226 if (adapter) {
6227 int i;
6228
29aaee65
AB		 6229 /* Tear down per-adapter Work Queue first since it can contain
6230 * references to our adapter data structure.
6231 */
6232 destroy_workqueue(adapter->workq);
6233
b8ff05a9
DM		 6234 if (is_offload(adapter))
6235 detach_ulds(adapter);
6236
6237 for_each_port(adapter, i)
8f3a7676 6238 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6239 unregister_netdev(adapter->port[i]);
6240
9f16dc2e 6241 debugfs_remove_recursive(adapter->debugfs_root);
b8ff05a9 6242
f2b7e78d
VP		 6243 /* If we allocated filters, free up state associated with any
6244 * valid filters ...
6245 */
6246 if (adapter->tids.ftid_tab) {
6247 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6248 for (i = 0; i < (adapter->tids.nftids +
6249 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6250 if (f->valid)
6251 clear_filter(adapter, f);
6252 }
6253
aaefae9b
DM		 6254 if (adapter->flags & FULL_INIT_DONE)
6255 cxgb_down(adapter);
b8ff05a9 6256
06546391 6257 free_some_resources(adapter);
b5a02f50
AB		 6258#if IS_ENABLED(CONFIG_IPV6)
6259 t4_cleanup_clip_tbl(adapter);
6260#endif
b8ff05a9 6261 iounmap(adapter->regs);
d14807dd 6262 if (!is_t4(adapter->params.chip))
22adfe0a 6263 iounmap(adapter->bar2);
b8ff05a9 6264 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6265 if ((adapter->flags & DEV_ENABLED)) {
6266 pci_disable_device(pdev);
6267 adapter->flags &= ~DEV_ENABLED;
6268 }
b8ff05a9 6269 pci_release_regions(pdev);
ee9a33b2 6270 synchronize_rcu();
8b662fe7 6271 kfree(adapter);
a069ec91 6272 } else
b8ff05a9
DM		 6273 pci_release_regions(pdev);
6274}
6275
6276static struct pci_driver cxgb4_driver = {
6277 .name = KBUILD_MODNAME,
6278 .id_table = cxgb4_pci_tbl,
6279 .probe = init_one,
91744948 6280 .remove = remove_one,
687d705c 6281 .shutdown = remove_one,
204dc3c0 6282 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6283};
6284
6285static int __init cxgb4_init_module(void)
6286{
6287 int ret;
6288
6289 /* Debugfs support is optional, just warn if this fails */
6290 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6291 if (!cxgb4_debugfs_root)
428ac43f 6292 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6293
6294 ret = pci_register_driver(&cxgb4_driver);
29aaee65 6295 if (ret < 0)
b8ff05a9 6296 debugfs_remove(cxgb4_debugfs_root);
01bcca68 6297
1bb60376 6298#if IS_ENABLED(CONFIG_IPV6)
b5a02f50
AB		 6299 if (!inet6addr_registered) {
6300 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6301 inet6addr_registered = true;
6302 }
1bb60376 6303#endif
01bcca68 6304
b8ff05a9
DM		 6305 return ret;
6306}
6307
6308static void __exit cxgb4_cleanup_module(void)
6309{
1bb60376 6310#if IS_ENABLED(CONFIG_IPV6)
1793c798 6311 if (inet6addr_registered) {
b5a02f50
AB		 6312 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6313 inet6addr_registered = false;
6314 }
1bb60376 6315#endif
b8ff05a9
DM		 6316 pci_unregister_driver(&cxgb4_driver);
6317 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
6318}
6319
6320module_init(cxgb4_init_module);
6321module_exit(cxgb4_cleanup_module);
Ubuntu Artful kernel mirror