]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
netdevice.h: remove iflink description
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
ce100b8b 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
b8ff05a9
DM		 5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
1ef8019b 64#include <net/bonding.h>
b5a02f50 65#include <net/addrconf.h>
b8ff05a9
DM		 66#include <asm/uaccess.h>
67
68#include "cxgb4.h"
69#include "t4_regs.h"
f612b815 70#include "t4_values.h"
b8ff05a9
DM		 71#include "t4_msg.h"
72#include "t4fw_api.h"
cd6c2f12 73#include "t4fw_version.h"
688848b1 74#include "cxgb4_dcb.h"
fd88b31a 75#include "cxgb4_debugfs.h"
b5a02f50 76#include "clip_tbl.h"
b8ff05a9
DM		 77#include "l2t.h"
78
01bcca68
VP		 79#ifdef DRV_VERSION
80#undef DRV_VERSION
81#endif
3a7f8554
SR		 82#define DRV_VERSION "2.0.0-ko"
83#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9 84
b8ff05a9
DM		 85enum {
86 MAX_TXQ_ENTRIES = 16384,
87 MAX_CTRL_TXQ_ENTRIES = 1024,
88 MAX_RSPQ_ENTRIES = 16384,
89 MAX_RX_BUFFERS = 16384,
90 MIN_TXQ_ENTRIES = 32,
91 MIN_CTRL_TXQ_ENTRIES = 32,
92 MIN_RSPQ_ENTRIES = 128,
93 MIN_FL_ENTRIES = 16
94};
95
f2b7e78d
VP		 96/* Host shadow copy of ingress filter entry. This is in host native format
97 * and doesn't match the ordering or bit order, etc. of the hardware of the
98 * firmware command. The use of bit-field structure elements is purely to
99 * remind ourselves of the field size limitations and save memory in the case
100 * where the filter table is large.
101 */
102struct filter_entry {
103 /* Administrative fields for filter.
104 */
105 u32 valid:1; /* filter allocated and valid */
106 u32 locked:1; /* filter is administratively locked */
107
108 u32 pending:1; /* filter action is pending firmware reply */
109 u32 smtidx:8; /* Source MAC Table index for smac */
110 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
111
112 /* The filter itself. Most of this is a straight copy of information
113 * provided by the extended ioctl(). Some fields are translated to
114 * internal forms -- for instance the Ingress Queue ID passed in from
115 * the ioctl() is translated into the Absolute Ingress Queue ID.
116 */
117 struct ch_filter_specification fs;
118};
119
b8ff05a9
DM		 120#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
121 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
122 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
123
3fedeab1
HS		 124/* Macros needed to support the PCI Device ID Table ...
125 */
126#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
768ffc66 127 static const struct pci_device_id cxgb4_pci_tbl[] = {
3fedeab1 128#define CH_PCI_DEVICE_ID_FUNCTION 0x4
b8ff05a9 129
3fedeab1
HS		 130/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
131 * called for both.
132 */
133#define CH_PCI_DEVICE_ID_FUNCTION2 0x0
134
135#define CH_PCI_ID_TABLE_ENTRY(devid) \
136 {PCI_VDEVICE(CHELSIO, (devid)), 4}
137
138#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
139 { 0, } \
140 }
141
142#include "t4_pci_id_tbl.h"
b8ff05a9 143
16e47624 144#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 145#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 146#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 147#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 148
149MODULE_DESCRIPTION(DRV_DESC);
150MODULE_AUTHOR("Chelsio Communications");
151MODULE_LICENSE("Dual BSD/GPL");
152MODULE_VERSION(DRV_VERSION);
153MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 154MODULE_FIRMWARE(FW4_FNAME);
0a57a536 155MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 156
636f9d37
VP		 157/*
158 * Normally we're willing to become the firmware's Master PF but will be happy
159 * if another PF has already become the Master and initialized the adapter.
160 * Setting "force_init" will cause this driver to forcibly establish itself as
161 * the Master PF and initialize the adapter.
162 */
163static uint force_init;
164
165module_param(force_init, uint, 0644);
166MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
167
13ee15d3
VP		 168/*
169 * Normally if the firmware we connect to has Configuration File support, we
170 * use that and only fall back to the old Driver-based initialization if the
171 * Configuration File fails for some reason. If force_old_init is set, then
172 * we'll always use the old Driver-based initialization sequence.
173 */
174static uint force_old_init;
175
176module_param(force_old_init, uint, 0644);
06640310
HS		 177MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated"
178 " parameter");
13ee15d3 179
b8ff05a9
DM		 180static int dflt_msg_enable = DFLT_MSG_ENABLE;
181
182module_param(dflt_msg_enable, int, 0644);
183MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
184
185/*
186 * The driver uses the best interrupt scheme available on a platform in the
187 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
188 * of these schemes the driver may consider as follows:
189 *
190 * msi = 2: choose from among all three options
191 * msi = 1: only consider MSI and INTx interrupts
192 * msi = 0: force INTx interrupts
193 */
194static int msi = 2;
195
196module_param(msi, int, 0644);
197MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
198
199/*
200 * Queue interrupt hold-off timer values. Queues default to the first of these
201 * upon creation.
202 */
203static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
204
205module_param_array(intr_holdoff, uint, NULL, 0644);
206MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
06640310 207 "0..4 in microseconds, deprecated parameter");
b8ff05a9
DM		 208
209static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
210
211module_param_array(intr_cnt, uint, NULL, 0644);
212MODULE_PARM_DESC(intr_cnt,
06640310
HS		 213 "thresholds 1..3 for queue interrupt packet counters, "
214 "deprecated parameter");
b8ff05a9 215
636f9d37
VP		 216/*
217 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
218 * offset by 2 bytes in order to have the IP headers line up on 4-byte
219 * boundaries. This is a requirement for many architectures which will throw
220 * a machine check fault if an attempt is made to access one of the 4-byte IP
221 * header fields on a non-4-byte boundary. And it's a major performance issue
222 * even on some architectures which allow it like some implementations of the
223 * x86 ISA. However, some architectures don't mind this and for some very
224 * edge-case performance sensitive applications (like forwarding large volumes
225 * of small packets), setting this DMA offset to 0 will decrease the number of
226 * PCI-E Bus transfers enough to measurably affect performance.
227 */
228static int rx_dma_offset = 2;
229
eb939922 230static bool vf_acls;
b8ff05a9
DM		 231
232#ifdef CONFIG_PCI_IOV
233module_param(vf_acls, bool, 0644);
06640310
HS		 234MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, "
235 "deprecated parameter");
b8ff05a9 236
7d6727cf
SR		 237/* Configure the number of PCI-E Virtual Function which are to be instantiated
238 * on SR-IOV Capable Physical Functions.
0a57a536 239 */
7d6727cf 240static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 241
242module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 243MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 244#endif
245
688848b1
AB		 246/* TX Queue select used to determine what algorithm to use for selecting TX
247 * queue. Select between the kernel provided function (select_queue=0) or user
248 * cxgb_select_queue function (select_queue=1)
249 *
250 * Default: select_queue=0
251 */
252static int select_queue;
253module_param(select_queue, int, 0644);
254MODULE_PARM_DESC(select_queue,
255 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
256
06640310 257static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC;
13ee15d3 258
f2b7e78d 259module_param(tp_vlan_pri_map, uint, 0644);
06640310
HS		 260MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, "
261 "deprecated parameter");
f2b7e78d 262
b8ff05a9
DM		 263static struct dentry *cxgb4_debugfs_root;
264
265static LIST_HEAD(adapter_list);
266static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 267/* Adapter list to be accessed from atomic context */
268static LIST_HEAD(adap_rcu_list);
269static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 270static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
271static const char *uld_str[] = { "RDMA", "iSCSI" };
272
273static void link_report(struct net_device *dev)
274{
275 if (!netif_carrier_ok(dev))
276 netdev_info(dev, "link down\n");
277 else {
278 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
279
280 const char *s = "10Mbps";
281 const struct port_info *p = netdev_priv(dev);
282
283 switch (p->link_cfg.speed) {
e8b39015 284 case 10000:
b8ff05a9
DM		 285 s = "10Gbps";
286 break;
e8b39015 287 case 1000:
b8ff05a9
DM		 288 s = "1000Mbps";
289 break;
e8b39015 290 case 100:
b8ff05a9
DM		 291 s = "100Mbps";
292 break;
e8b39015 293 case 40000:
72aca4bf
KS		 294 s = "40Gbps";
295 break;
b8ff05a9
DM		 296 }
297
298 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
299 fc[p->link_cfg.fc]);
300 }
301}
302
688848b1
AB		 303#ifdef CONFIG_CHELSIO_T4_DCB
304/* Set up/tear down Data Center Bridging Priority mapping for a net device. */
305static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
306{
307 struct port_info *pi = netdev_priv(dev);
308 struct adapter *adap = pi->adapter;
309 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
310 int i;
311
312 /* We use a simple mapping of Port TX Queue Index to DCB
313 * Priority when we're enabling DCB.
314 */
315 for (i = 0; i < pi->nqsets; i++, txq++) {
316 u32 name, value;
317 int err;
318
5167865a
HS		 319 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
320 FW_PARAMS_PARAM_X_V(
321 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
322 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
688848b1
AB		 323 value = enable ? i : 0xffffffff;
324
325 /* Since we can be called while atomic (from "interrupt
326 * level") we need to issue the Set Parameters Commannd
327 * without sleeping (timeout < 0).
328 */
329 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
330 &name, &value);
331
332 if (err)
333 dev_err(adap->pdev_dev,
334 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
335 enable ? "set" : "unset", pi->port_id, i, -err);
10b00466
AB		 336 else
337 txq->dcb_prio = value;
688848b1
AB		 338 }
339}
340#endif /* CONFIG_CHELSIO_T4_DCB */
341
b8ff05a9
DM		 342void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
343{
344 struct net_device *dev = adapter->port[port_id];
345
346 /* Skip changes from disabled ports. */
347 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
348 if (link_stat)
349 netif_carrier_on(dev);
688848b1
AB		 350 else {
351#ifdef CONFIG_CHELSIO_T4_DCB
352 cxgb4_dcb_state_init(dev);
353 dcb_tx_queue_prio_enable(dev, false);
354#endif /* CONFIG_CHELSIO_T4_DCB */
b8ff05a9 355 netif_carrier_off(dev);
688848b1 356 }
b8ff05a9
DM		 357
358 link_report(dev);
359 }
360}
361
362void t4_os_portmod_changed(const struct adapter *adap, int port_id)
363{
364 static const char *mod_str[] = {
a0881cab 365 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 366 };
367
368 const struct net_device *dev = adap->port[port_id];
369 const struct port_info *pi = netdev_priv(dev);
370
371 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
372 netdev_info(dev, "port module unplugged\n");
a0881cab 373 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 374 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
375}
376
377/*
378 * Configure the exact and hash address filters to handle a port's multicast
379 * and secondary unicast MAC addresses.
380 */
381static int set_addr_filters(const struct net_device *dev, bool sleep)
382{
383 u64 mhash = 0;
384 u64 uhash = 0;
385 bool free = true;
386 u16 filt_idx[7];
387 const u8 *addr[7];
388 int ret, naddr = 0;
b8ff05a9
DM		 389 const struct netdev_hw_addr *ha;
390 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 391 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 392 const struct port_info *pi = netdev_priv(dev);
060e0c75 393 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 394
395 /* first do the secondary unicast addresses */
396 netdev_for_each_uc_addr(ha, dev) {
397 addr[naddr++] = ha->addr;
398 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 399 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 400 naddr, addr, filt_idx, &uhash, sleep);
401 if (ret < 0)
402 return ret;
403
404 free = false;
405 naddr = 0;
406 }
407 }
408
409 /* next set up the multicast addresses */
4a35ecf8
DM		 410 netdev_for_each_mc_addr(ha, dev) {
411 addr[naddr++] = ha->addr;
412 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 413 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 414 naddr, addr, filt_idx, &mhash, sleep);
415 if (ret < 0)
416 return ret;
417
418 free = false;
419 naddr = 0;
420 }
421 }
422
060e0c75 423 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 424 uhash | mhash, sleep);
425}
426
3069ee9b
VP		 427int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
428module_param(dbfifo_int_thresh, int, 0644);
429MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
430
404d9e3f
VP		 431/*
432 * usecs to sleep while draining the dbfifo
433 */
434static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 435module_param(dbfifo_drain_delay, int, 0644);
436MODULE_PARM_DESC(dbfifo_drain_delay,
437 "usecs to sleep while draining the dbfifo");
438
b8ff05a9
DM		 439/*
440 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
441 * If @mtu is -1 it is left unchanged.
442 */
443static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
444{
445 int ret;
446 struct port_info *pi = netdev_priv(dev);
447
448 ret = set_addr_filters(dev, sleep_ok);
449 if (ret == 0)
060e0c75 450 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 451 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 452 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 453 sleep_ok);
454 return ret;
455}
456
457/**
458 * link_start - enable a port
459 * @dev: the port to enable
460 *
461 * Performs the MAC and PHY actions needed to enable a port.
462 */
463static int link_start(struct net_device *dev)
464{
465 int ret;
466 struct port_info *pi = netdev_priv(dev);
060e0c75 467 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 468
469 /*
470 * We do not set address filters and promiscuity here, the stack does
471 * that step explicitly.
472 */
060e0c75 473 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 474 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 475 if (ret == 0) {
060e0c75 476 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 477 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 478 true);
b8ff05a9
DM		 479 if (ret >= 0) {
480 pi->xact_addr_filt = ret;
481 ret = 0;
482 }
483 }
484 if (ret == 0)
060e0c75
DM		 485 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
486 &pi->link_cfg);
30f00847
AB		 487 if (ret == 0) {
488 local_bh_disable();
688848b1
AB		 489 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
490 true, CXGB4_DCB_ENABLED);
30f00847
AB		 491 local_bh_enable();
492 }
688848b1 493
b8ff05a9
DM		 494 return ret;
495}
496
688848b1
AB		 497int cxgb4_dcb_enabled(const struct net_device *dev)
498{
499#ifdef CONFIG_CHELSIO_T4_DCB
500 struct port_info *pi = netdev_priv(dev);
501
3bb06261
AB		 502 if (!pi->dcb.enabled)
503 return 0;
504
505 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
506 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
688848b1
AB		 507#else
508 return 0;
509#endif
510}
511EXPORT_SYMBOL(cxgb4_dcb_enabled);
512
513#ifdef CONFIG_CHELSIO_T4_DCB
514/* Handle a Data Center Bridging update message from the firmware. */
515static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
516{
2b5fb1f2 517 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
688848b1
AB		 518 struct net_device *dev = adap->port[port];
519 int old_dcb_enabled = cxgb4_dcb_enabled(dev);
520 int new_dcb_enabled;
521
522 cxgb4_dcb_handle_fw_update(adap, pcmd);
523 new_dcb_enabled = cxgb4_dcb_enabled(dev);
524
525 /* If the DCB has become enabled or disabled on the port then we're
526 * going to need to set up/tear down DCB Priority parameters for the
527 * TX Queues associated with the port.
528 */
529 if (new_dcb_enabled != old_dcb_enabled)
530 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
531}
532#endif /* CONFIG_CHELSIO_T4_DCB */
533
f2b7e78d
VP		 534/* Clear a filter and release any of its resources that we own. This also
535 * clears the filter's "pending" status.
536 */
537static void clear_filter(struct adapter *adap, struct filter_entry *f)
538{
539 /* If the new or old filter have loopback rewriteing rules then we'll
540 * need to free any existing Layer Two Table (L2T) entries of the old
541 * filter rule. The firmware will handle freeing up any Source MAC
542 * Table (SMT) entries used for rewriting Source MAC Addresses in
543 * loopback rules.
544 */
545 if (f->l2t)
546 cxgb4_l2t_release(f->l2t);
547
548 /* The zeroing of the filter rule below clears the filter valid,
549 * pending, locked flags, l2t pointer, etc. so it's all we need for
550 * this operation.
551 */
552 memset(f, 0, sizeof(*f));
553}
554
555/* Handle a filter write/deletion reply.
556 */
557static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
558{
559 unsigned int idx = GET_TID(rpl);
560 unsigned int nidx = idx - adap->tids.ftid_base;
561 unsigned int ret;
562 struct filter_entry *f;
563
564 if (idx >= adap->tids.ftid_base && nidx <
565 (adap->tids.nftids + adap->tids.nsftids)) {
566 idx = nidx;
bdc590b9 567 ret = TCB_COOKIE_G(rpl->cookie);
f2b7e78d
VP		 568 f = &adap->tids.ftid_tab[idx];
569
570 if (ret == FW_FILTER_WR_FLT_DELETED) {
571 /* Clear the filter when we get confirmation from the
572 * hardware that the filter has been deleted.
573 */
574 clear_filter(adap, f);
575 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
576 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
577 idx);
578 clear_filter(adap, f);
579 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
580 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
581 f->pending = 0; /* asynchronous setup completed */
582 f->valid = 1;
583 } else {
584 /* Something went wrong. Issue a warning about the
585 * problem and clear everything out.
586 */
587 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
588 idx, ret);
589 clear_filter(adap, f);
590 }
591 }
592}
593
594/* Response queue handler for the FW event queue.
b8ff05a9
DM		 595 */
596static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
597 const struct pkt_gl *gl)
598{
599 u8 opcode = ((const struct rss_header *)rsp)->opcode;
600
601 rsp++; /* skip RSS header */
b407a4a9
VP		 602
603 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
604 */
605 if (unlikely(opcode == CPL_FW4_MSG &&
606 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
607 rsp++;
608 opcode = ((const struct rss_header *)rsp)->opcode;
609 rsp++;
610 if (opcode != CPL_SGE_EGR_UPDATE) {
611 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
612 , opcode);
613 goto out;
614 }
615 }
616
b8ff05a9
DM		 617 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
618 const struct cpl_sge_egr_update *p = (void *)rsp;
bdc590b9 619 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
e46dab4d 620 struct sge_txq *txq;
b8ff05a9 621
e46dab4d 622 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 623 txq->restarts++;
e46dab4d 624 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 625 struct sge_eth_txq *eq;
626
627 eq = container_of(txq, struct sge_eth_txq, q);
628 netif_tx_wake_queue(eq->txq);
629 } else {
630 struct sge_ofld_txq *oq;
631
632 oq = container_of(txq, struct sge_ofld_txq, q);
633 tasklet_schedule(&oq->qresume_tsk);
634 }
635 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
636 const struct cpl_fw6_msg *p = (void *)rsp;
637
688848b1
AB		 638#ifdef CONFIG_CHELSIO_T4_DCB
639 const struct fw_port_cmd *pcmd = (const void *)p->data;
e2ac9628 640 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
688848b1 641 unsigned int action =
2b5fb1f2 642 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
688848b1
AB		 643
644 if (cmd == FW_PORT_CMD &&
645 action == FW_PORT_ACTION_GET_PORT_INFO) {
2b5fb1f2 646 int port = FW_PORT_CMD_PORTID_G(
688848b1
AB		 647 be32_to_cpu(pcmd->op_to_portid));
648 struct net_device *dev = q->adap->port[port];
649 int state_input = ((pcmd->u.info.dcbxdis_pkd &
2b5fb1f2 650 FW_PORT_CMD_DCBXDIS_F)
688848b1
AB		 651 ? CXGB4_DCB_INPUT_FW_DISABLED
652 : CXGB4_DCB_INPUT_FW_ENABLED);
653
654 cxgb4_dcb_state_fsm(dev, state_input);
655 }
656
657 if (cmd == FW_PORT_CMD &&
658 action == FW_PORT_ACTION_L2_DCB_CFG)
659 dcb_rpl(q->adap, pcmd);
660 else
661#endif
662 if (p->type == 0)
663 t4_handle_fw_rpl(q->adap, p->data);
b8ff05a9
DM		 664 } else if (opcode == CPL_L2T_WRITE_RPL) {
665 const struct cpl_l2t_write_rpl *p = (void *)rsp;
666
667 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 668 } else if (opcode == CPL_SET_TCB_RPL) {
669 const struct cpl_set_tcb_rpl *p = (void *)rsp;
670
671 filter_rpl(q->adap, p);
b8ff05a9
DM		 672 } else
673 dev_err(q->adap->pdev_dev,
674 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 675out:
b8ff05a9
DM		 676 return 0;
677}
678
679/**
680 * uldrx_handler - response queue handler for ULD queues
681 * @q: the response queue that received the packet
682 * @rsp: the response queue descriptor holding the offload message
683 * @gl: the gather list of packet fragments
684 *
685 * Deliver an ingress offload packet to a ULD. All processing is done by
686 * the ULD, we just maintain statistics.
687 */
688static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
689 const struct pkt_gl *gl)
690{
691 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
692
b407a4a9
VP		 693 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
694 */
695 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
696 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
697 rsp += 2;
698
b8ff05a9
DM		 699 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
700 rxq->stats.nomem++;
701 return -1;
702 }
703 if (gl == NULL)
704 rxq->stats.imm++;
705 else if (gl == CXGB4_MSG_AN)
706 rxq->stats.an++;
707 else
708 rxq->stats.pkts++;
709 return 0;
710}
711
712static void disable_msi(struct adapter *adapter)
713{
714 if (adapter->flags & USING_MSIX) {
715 pci_disable_msix(adapter->pdev);
716 adapter->flags &= ~USING_MSIX;
717 } else if (adapter->flags & USING_MSI) {
718 pci_disable_msi(adapter->pdev);
719 adapter->flags &= ~USING_MSI;
720 }
721}
722
723/*
724 * Interrupt handler for non-data events used with MSI-X.
725 */
726static irqreturn_t t4_nondata_intr(int irq, void *cookie)
727{
728 struct adapter *adap = cookie;
0d804338 729 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
b8ff05a9 730
0d804338 731 if (v & PFSW_F) {
b8ff05a9 732 adap->swintr = 1;
0d804338 733 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
b8ff05a9
DM		 734 }
735 t4_slow_intr_handler(adap);
736 return IRQ_HANDLED;
737}
738
739/*
740 * Name the MSI-X interrupts.
741 */
742static void name_msix_vecs(struct adapter *adap)
743{
ba27816c 744 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 745
746 /* non-data interrupts */
b1a3c2b6 747 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 748
749 /* FW events */
b1a3c2b6
DM		 750 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
751 adap->port[0]->name);
b8ff05a9
DM		 752
753 /* Ethernet queues */
754 for_each_port(adap, j) {
755 struct net_device *d = adap->port[j];
756 const struct port_info *pi = netdev_priv(d);
757
ba27816c 758 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 759 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
760 d->name, i);
b8ff05a9
DM		 761 }
762
763 /* offload queues */
ba27816c
DM		 764 for_each_ofldrxq(&adap->sge, i)
765 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 766 adap->port[0]->name, i);
ba27816c
DM		 767
768 for_each_rdmarxq(&adap->sge, i)
769 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 770 adap->port[0]->name, i);
cf38be6d
HS		 771
772 for_each_rdmaciq(&adap->sge, i)
773 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
774 adap->port[0]->name, i);
b8ff05a9
DM		 775}
776
777static int request_msix_queue_irqs(struct adapter *adap)
778{
779 struct sge *s = &adap->sge;
cf38be6d
HS		 780 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
781 int msi_index = 2;
b8ff05a9
DM		 782
783 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
784 adap->msix_info[1].desc, &s->fw_evtq);
785 if (err)
786 return err;
787
788 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 789 err = request_irq(adap->msix_info[msi_index].vec,
790 t4_sge_intr_msix, 0,
791 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 792 &s->ethrxq[ethqidx].rspq);
793 if (err)
794 goto unwind;
404d9e3f 795 msi_index++;
b8ff05a9
DM		 796 }
797 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 798 err = request_irq(adap->msix_info[msi_index].vec,
799 t4_sge_intr_msix, 0,
800 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 801 &s->ofldrxq[ofldqidx].rspq);
802 if (err)
803 goto unwind;
404d9e3f 804 msi_index++;
b8ff05a9
DM		 805 }
806 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 807 err = request_irq(adap->msix_info[msi_index].vec,
808 t4_sge_intr_msix, 0,
809 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 810 &s->rdmarxq[rdmaqidx].rspq);
811 if (err)
812 goto unwind;
404d9e3f 813 msi_index++;
b8ff05a9 814 }
cf38be6d
HS		 815 for_each_rdmaciq(s, rdmaciqqidx) {
816 err = request_irq(adap->msix_info[msi_index].vec,
817 t4_sge_intr_msix, 0,
818 adap->msix_info[msi_index].desc,
819 &s->rdmaciq[rdmaciqqidx].rspq);
820 if (err)
821 goto unwind;
822 msi_index++;
823 }
b8ff05a9
DM		 824 return 0;
825
826unwind:
cf38be6d
HS		 827 while (--rdmaciqqidx >= 0)
828 free_irq(adap->msix_info[--msi_index].vec,
829 &s->rdmaciq[rdmaciqqidx].rspq);
b8ff05a9 830 while (--rdmaqidx >= 0)
404d9e3f 831 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 832 &s->rdmarxq[rdmaqidx].rspq);
833 while (--ofldqidx >= 0)
404d9e3f 834 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 835 &s->ofldrxq[ofldqidx].rspq);
836 while (--ethqidx >= 0)
404d9e3f
VP		 837 free_irq(adap->msix_info[--msi_index].vec,
838 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 839 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
840 return err;
841}
842
843static void free_msix_queue_irqs(struct adapter *adap)
844{
404d9e3f 845 int i, msi_index = 2;
b8ff05a9
DM		 846 struct sge *s = &adap->sge;
847
848 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
849 for_each_ethrxq(s, i)
404d9e3f 850 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 851 for_each_ofldrxq(s, i)
404d9e3f 852 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 853 for_each_rdmarxq(s, i)
404d9e3f 854 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
cf38be6d
HS		 855 for_each_rdmaciq(s, i)
856 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
b8ff05a9
DM		 857}
858
671b0060
DM		 859/**
860 * write_rss - write the RSS table for a given port
861 * @pi: the port
862 * @queues: array of queue indices for RSS
863 *
864 * Sets up the portion of the HW RSS table for the port's VI to distribute
865 * packets to the Rx queues in @queues.
866 */
867static int write_rss(const struct port_info *pi, const u16 *queues)
868{
869 u16 *rss;
870 int i, err;
871 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
872
873 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
874 if (!rss)
875 return -ENOMEM;
876
877 /* map the queue indices to queue ids */
878 for (i = 0; i < pi->rss_size; i++, queues++)
879 rss[i] = q[*queues].rspq.abs_id;
880
060e0c75
DM		 881 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
882 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 883 kfree(rss);
884 return err;
885}
886
b8ff05a9
DM		 887/**
888 * setup_rss - configure RSS
889 * @adap: the adapter
890 *
671b0060 891 * Sets up RSS for each port.
b8ff05a9
DM		 892 */
893static int setup_rss(struct adapter *adap)
894{
671b0060 895 int i, err;
b8ff05a9
DM		 896
897 for_each_port(adap, i) {
898 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 899
671b0060 900 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 901 if (err)
902 return err;
903 }
904 return 0;
905}
906
e46dab4d
DM		 907/*
908 * Return the channel of the ingress queue with the given qid.
909 */
910static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
911{
912 qid -= p->ingr_start;
913 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
914}
915
b8ff05a9
DM		 916/*
917 * Wait until all NAPI handlers are descheduled.
918 */
919static void quiesce_rx(struct adapter *adap)
920{
921 int i;
922
4b8e27a8 923 for (i = 0; i < adap->sge.ingr_sz; i++) {
b8ff05a9
DM		 924 struct sge_rspq *q = adap->sge.ingr_map[i];
925
3a336cb1 926 if (q && q->handler) {
b8ff05a9 927 napi_disable(&q->napi);
3a336cb1
HS		 928 local_bh_disable();
929 while (!cxgb_poll_lock_napi(q))
930 mdelay(1);
931 local_bh_enable();
932 }
933
b8ff05a9
DM		 934 }
935}
936
b37987e8
HS		 937/* Disable interrupt and napi handler */
938static void disable_interrupts(struct adapter *adap)
939{
940 if (adap->flags & FULL_INIT_DONE) {
941 t4_intr_disable(adap);
942 if (adap->flags & USING_MSIX) {
943 free_msix_queue_irqs(adap);
944 free_irq(adap->msix_info[0].vec, adap);
945 } else {
946 free_irq(adap->pdev->irq, adap);
947 }
948 quiesce_rx(adap);
949 }
950}
951
b8ff05a9
DM		 952/*
953 * Enable NAPI scheduling and interrupt generation for all Rx queues.
954 */
955static void enable_rx(struct adapter *adap)
956{
957 int i;
958
4b8e27a8 959 for (i = 0; i < adap->sge.ingr_sz; i++) {
b8ff05a9
DM		 960 struct sge_rspq *q = adap->sge.ingr_map[i];
961
962 if (!q)
963 continue;
3a336cb1
HS		 964 if (q->handler) {
965 cxgb_busy_poll_init_lock(q);
b8ff05a9 966 napi_enable(&q->napi);
3a336cb1 967 }
b8ff05a9 968 /* 0-increment GTS to start the timer and enable interrupts */
f612b815
HS		 969 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
970 SEINTARM_V(q->intr_params) |
971 INGRESSQID_V(q->cntxt_id));
b8ff05a9
DM		 972 }
973}
974
1c6a5b0e
HS		 975static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
976 unsigned int nq, unsigned int per_chan, int msi_idx,
977 u16 *ids)
978{
979 int i, err;
980
981 for (i = 0; i < nq; i++, q++) {
982 if (msi_idx > 0)
983 msi_idx++;
984 err = t4_sge_alloc_rxq(adap, &q->rspq, false,
985 adap->port[i / per_chan],
986 msi_idx, q->fl.size ? &q->fl : NULL,
987 uldrx_handler);
988 if (err)
989 return err;
990 memset(&q->stats, 0, sizeof(q->stats));
991 if (ids)
992 ids[i] = q->rspq.abs_id;
993 }
994 return 0;
995}
996
b8ff05a9
DM		 997/**
998 * setup_sge_queues - configure SGE Tx/Rx/response queues
999 * @adap: the adapter
1000 *
1001 * Determines how many sets of SGE queues to use and initializes them.
1002 * We support multiple queue sets per port if we have MSI-X, otherwise
1003 * just one queue set per port.
1004 */
1005static int setup_sge_queues(struct adapter *adap)
1006{
1007 int err, msi_idx, i, j;
1008 struct sge *s = &adap->sge;
1009
4b8e27a8
HS		 1010 bitmap_zero(s->starving_fl, s->egr_sz);
1011 bitmap_zero(s->txq_maperr, s->egr_sz);
b8ff05a9
DM		 1012
1013 if (adap->flags & USING_MSIX)
1014 msi_idx = 1; /* vector 0 is for non-queue interrupts */
1015 else {
1016 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1017 NULL, NULL);
1018 if (err)
1019 return err;
1020 msi_idx = -((int)s->intrq.abs_id + 1);
1021 }
1022
4b8e27a8
HS		 1023 /* NOTE: If you add/delete any Ingress/Egress Queue allocations in here,
1024 * don't forget to update the following which need to be
1025 * synchronized to and changes here.
1026 *
1027 * 1. The calculations of MAX_INGQ in cxgb4.h.
1028 *
1029 * 2. Update enable_msix/name_msix_vecs/request_msix_queue_irqs
1030 * to accommodate any new/deleted Ingress Queues
1031 * which need MSI-X Vectors.
1032 *
1033 * 3. Update sge_qinfo_show() to include information on the
1034 * new/deleted queues.
1035 */
b8ff05a9
DM		 1036 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1037 msi_idx, NULL, fwevtq_handler);
1038 if (err) {
1039freeout: t4_free_sge_resources(adap);
1040 return err;
1041 }
1042
1043 for_each_port(adap, i) {
1044 struct net_device *dev = adap->port[i];
1045 struct port_info *pi = netdev_priv(dev);
1046 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1047 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1048
1049 for (j = 0; j < pi->nqsets; j++, q++) {
1050 if (msi_idx > 0)
1051 msi_idx++;
1052 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1053 msi_idx, &q->fl,
1054 t4_ethrx_handler);
1055 if (err)
1056 goto freeout;
1057 q->rspq.idx = j;
1058 memset(&q->stats, 0, sizeof(q->stats));
1059 }
1060 for (j = 0; j < pi->nqsets; j++, t++) {
1061 err = t4_sge_alloc_eth_txq(adap, t, dev,
1062 netdev_get_tx_queue(dev, j),
1063 s->fw_evtq.cntxt_id);
1064 if (err)
1065 goto freeout;
1066 }
1067 }
1068
1069 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1070 for_each_ofldrxq(s, i) {
1c6a5b0e
HS		 1071 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i],
1072 adap->port[i / j],
b8ff05a9
DM		 1073 s->fw_evtq.cntxt_id);
1074 if (err)
1075 goto freeout;
1076 }
1077
1c6a5b0e
HS		 1078#define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids) do { \
1079 err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids); \
1080 if (err) \
1081 goto freeout; \
1082 if (msi_idx > 0) \
1083 msi_idx += nq; \
1084} while (0)
b8ff05a9 1085
1c6a5b0e
HS		 1086 ALLOC_OFLD_RXQS(s->ofldrxq, s->ofldqsets, j, s->ofld_rxq);
1087 ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq);
f36e58e5
HS		 1088 j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */
1089 ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq);
b8ff05a9 1090
1c6a5b0e 1091#undef ALLOC_OFLD_RXQS
cf38be6d 1092
b8ff05a9
DM		 1093 for_each_port(adap, i) {
1094 /*
1095 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1096 * have RDMA queues, and that's the right value.
1097 */
1098 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1099 s->fw_evtq.cntxt_id,
1100 s->rdmarxq[i].rspq.cntxt_id);
1101 if (err)
1102 goto freeout;
1103 }
1104
9bb59b96 1105 t4_write_reg(adap, is_t4(adap->params.chip) ?
837e4a42
HS		 1106 MPS_TRC_RSS_CONTROL_A :
1107 MPS_T5_TRC_RSS_CONTROL_A,
1108 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1109 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
b8ff05a9
DM		 1110 return 0;
1111}
1112
b8ff05a9
DM		 1113/*
1114 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1115 * The allocated memory is cleared.
1116 */
1117void *t4_alloc_mem(size_t size)
1118{
8be04b93 1119 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1120
1121 if (!p)
89bf67f1 1122 p = vzalloc(size);
b8ff05a9
DM		 1123 return p;
1124}
1125
1126/*
1127 * Free memory allocated through alloc_mem().
1128 */
fd88b31a 1129void t4_free_mem(void *addr)
b8ff05a9
DM		 1130{
1131 if (is_vmalloc_addr(addr))
1132 vfree(addr);
1133 else
1134 kfree(addr);
1135}
1136
f2b7e78d
VP		 1137/* Send a Work Request to write the filter at a specified index. We construct
1138 * a Firmware Filter Work Request to have the work done and put the indicated
1139 * filter into "pending" mode which will prevent any further actions against
1140 * it till we get a reply from the firmware on the completion status of the
1141 * request.
1142 */
1143static int set_filter_wr(struct adapter *adapter, int fidx)
1144{
1145 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1146 struct sk_buff *skb;
1147 struct fw_filter_wr *fwr;
1148 unsigned int ftid;
1149
1150 /* If the new filter requires loopback Destination MAC and/or VLAN
1151 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1152 * the filter.
1153 */
1154 if (f->fs.newdmac || f->fs.newvlan) {
1155 /* allocate L2T entry for new filter */
1156 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1157 if (f->l2t == NULL)
1158 return -EAGAIN;
1159 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1160 f->fs.eport, f->fs.dmac)) {
1161 cxgb4_l2t_release(f->l2t);
1162 f->l2t = NULL;
1163 return -ENOMEM;
1164 }
1165 }
1166
1167 ftid = adapter->tids.ftid_base + fidx;
1168
1169 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1170 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1171 memset(fwr, 0, sizeof(*fwr));
1172
1173 /* It would be nice to put most of the following in t4_hw.c but most
1174 * of the work is translating the cxgbtool ch_filter_specification
1175 * into the Work Request and the definition of that structure is
1176 * currently in cxgbtool.h which isn't appropriate to pull into the
1177 * common code. We may eventually try to come up with a more neutral
1178 * filter specification structure but for now it's easiest to simply
1179 * put this fairly direct code in line ...
1180 */
e2ac9628
HS		 1181 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1182 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
f2b7e78d 1183 fwr->tid_to_iq =
77a80e23
HS		 1184 htonl(FW_FILTER_WR_TID_V(ftid) |
1185 FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1186 FW_FILTER_WR_NOREPLY_V(0) |
1187 FW_FILTER_WR_IQ_V(f->fs.iq));
f2b7e78d 1188 fwr->del_filter_to_l2tix =
77a80e23
HS		 1189 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1190 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1191 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1192 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1193 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1194 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1195 FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1196 FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1197 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
f2b7e78d 1198 f->fs.newvlan == VLAN_REWRITE) |
77a80e23 1199 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
f2b7e78d 1200 f->fs.newvlan == VLAN_REWRITE) |
77a80e23
HS		 1201 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1202 FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1203 FW_FILTER_WR_PRIO_V(f->fs.prio) |
1204 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
f2b7e78d
VP		 1205 fwr->ethtype = htons(f->fs.val.ethtype);
1206 fwr->ethtypem = htons(f->fs.mask.ethtype);
1207 fwr->frag_to_ovlan_vldm =
77a80e23
HS		 1208 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1209 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1210 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1211 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1212 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1213 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
f2b7e78d
VP		 1214 fwr->smac_sel = 0;
1215 fwr->rx_chan_rx_rpl_iq =
77a80e23
HS		 1216 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1217 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
f2b7e78d 1218 fwr->maci_to_matchtypem =
77a80e23
HS		 1219 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1220 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1221 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1222 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1223 FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1224 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1225 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1226 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
f2b7e78d
VP		 1227 fwr->ptcl = f->fs.val.proto;
1228 fwr->ptclm = f->fs.mask.proto;
1229 fwr->ttyp = f->fs.val.tos;
1230 fwr->ttypm = f->fs.mask.tos;
1231 fwr->ivlan = htons(f->fs.val.ivlan);
1232 fwr->ivlanm = htons(f->fs.mask.ivlan);
1233 fwr->ovlan = htons(f->fs.val.ovlan);
1234 fwr->ovlanm = htons(f->fs.mask.ovlan);
1235 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1236 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1237 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1238 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1239 fwr->lp = htons(f->fs.val.lport);
1240 fwr->lpm = htons(f->fs.mask.lport);
1241 fwr->fp = htons(f->fs.val.fport);
1242 fwr->fpm = htons(f->fs.mask.fport);
1243 if (f->fs.newsmac)
1244 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1245
1246 /* Mark the filter as "pending" and ship off the Filter Work Request.
1247 * When we get the Work Request Reply we'll clear the pending status.
1248 */
1249 f->pending = 1;
1250 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1251 t4_ofld_send(adapter, skb);
1252 return 0;
1253}
1254
1255/* Delete the filter at a specified index.
1256 */
1257static int del_filter_wr(struct adapter *adapter, int fidx)
1258{
1259 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1260 struct sk_buff *skb;
1261 struct fw_filter_wr *fwr;
1262 unsigned int len, ftid;
1263
1264 len = sizeof(*fwr);
1265 ftid = adapter->tids.ftid_base + fidx;
1266
1267 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1268 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1269 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1270
1271 /* Mark the filter as "pending" and ship off the Filter Work Request.
1272 * When we get the Work Request Reply we'll clear the pending status.
1273 */
1274 f->pending = 1;
1275 t4_mgmt_tx(adapter, skb);
1276 return 0;
1277}
1278
688848b1
AB		 1279static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1280 void *accel_priv, select_queue_fallback_t fallback)
1281{
1282 int txq;
1283
1284#ifdef CONFIG_CHELSIO_T4_DCB
1285 /* If a Data Center Bridging has been successfully negotiated on this
1286 * link then we'll use the skb's priority to map it to a TX Queue.
1287 * The skb's priority is determined via the VLAN Tag Priority Code
1288 * Point field.
1289 */
1290 if (cxgb4_dcb_enabled(dev)) {
1291 u16 vlan_tci;
1292 int err;
1293
1294 err = vlan_get_tag(skb, &vlan_tci);
1295 if (unlikely(err)) {
1296 if (net_ratelimit())
1297 netdev_warn(dev,
1298 "TX Packet without VLAN Tag on DCB Link\n");
1299 txq = 0;
1300 } else {
1301 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
84a200b3
VP		 1302#ifdef CONFIG_CHELSIO_T4_FCOE
1303 if (skb->protocol == htons(ETH_P_FCOE))
1304 txq = skb->priority & 0x7;
1305#endif /* CONFIG_CHELSIO_T4_FCOE */
688848b1
AB		 1306 }
1307 return txq;
1308 }
1309#endif /* CONFIG_CHELSIO_T4_DCB */
1310
1311 if (select_queue) {
1312 txq = (skb_rx_queue_recorded(skb)
1313 ? skb_get_rx_queue(skb)
1314 : smp_processor_id());
1315
1316 while (unlikely(txq >= dev->real_num_tx_queues))
1317 txq -= dev->real_num_tx_queues;
1318
1319 return txq;
1320 }
1321
1322 return fallback(dev, skb) % dev->real_num_tx_queues;
1323}
1324
b8ff05a9
DM		 1325static inline int is_offload(const struct adapter *adap)
1326{
1327 return adap->params.offload;
1328}
1329
1330/*
1331 * Implementation of ethtool operations.
1332 */
1333
1334static u32 get_msglevel(struct net_device *dev)
1335{
1336 return netdev2adap(dev)->msg_enable;
1337}
1338
1339static void set_msglevel(struct net_device *dev, u32 val)
1340{
1341 netdev2adap(dev)->msg_enable = val;
1342}
1343
1344static char stats_strings[][ETH_GSTRING_LEN] = {
1345 "TxOctetsOK ",
1346 "TxFramesOK ",
1347 "TxBroadcastFrames ",
1348 "TxMulticastFrames ",
1349 "TxUnicastFrames ",
1350 "TxErrorFrames ",
1351
1352 "TxFrames64 ",
1353 "TxFrames65To127 ",
1354 "TxFrames128To255 ",
1355 "TxFrames256To511 ",
1356 "TxFrames512To1023 ",
1357 "TxFrames1024To1518 ",
1358 "TxFrames1519ToMax ",
1359
1360 "TxFramesDropped ",
1361 "TxPauseFrames ",
1362 "TxPPP0Frames ",
1363 "TxPPP1Frames ",
1364 "TxPPP2Frames ",
1365 "TxPPP3Frames ",
1366 "TxPPP4Frames ",
1367 "TxPPP5Frames ",
1368 "TxPPP6Frames ",
1369 "TxPPP7Frames ",
1370
1371 "RxOctetsOK ",
1372 "RxFramesOK ",
1373 "RxBroadcastFrames ",
1374 "RxMulticastFrames ",
1375 "RxUnicastFrames ",
1376
1377 "RxFramesTooLong ",
1378 "RxJabberErrors ",
1379 "RxFCSErrors ",
1380 "RxLengthErrors ",
1381 "RxSymbolErrors ",
1382 "RxRuntFrames ",
1383
1384 "RxFrames64 ",
1385 "RxFrames65To127 ",
1386 "RxFrames128To255 ",
1387 "RxFrames256To511 ",
1388 "RxFrames512To1023 ",
1389 "RxFrames1024To1518 ",
1390 "RxFrames1519ToMax ",
1391
1392 "RxPauseFrames ",
1393 "RxPPP0Frames ",
1394 "RxPPP1Frames ",
1395 "RxPPP2Frames ",
1396 "RxPPP3Frames ",
1397 "RxPPP4Frames ",
1398 "RxPPP5Frames ",
1399 "RxPPP6Frames ",
1400 "RxPPP7Frames ",
1401
1402 "RxBG0FramesDropped ",
1403 "RxBG1FramesDropped ",
1404 "RxBG2FramesDropped ",
1405 "RxBG3FramesDropped ",
1406 "RxBG0FramesTrunc ",
1407 "RxBG1FramesTrunc ",
1408 "RxBG2FramesTrunc ",
1409 "RxBG3FramesTrunc ",
1410
1411 "TSO ",
1412 "TxCsumOffload ",
1413 "RxCsumGood ",
1414 "VLANextractions ",
1415 "VLANinsertions ",
4a6346d4
DM		 1416 "GROpackets ",
1417 "GROmerged ",
22adfe0a
SR		 1418 "WriteCoalSuccess ",
1419 "WriteCoalFail ",
b8ff05a9
DM		 1420};
1421
1422static int get_sset_count(struct net_device *dev, int sset)
1423{
1424 switch (sset) {
1425 case ETH_SS_STATS:
1426 return ARRAY_SIZE(stats_strings);
1427 default:
1428 return -EOPNOTSUPP;
1429 }
1430}
1431
1432#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1433#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1434
1435static int get_regs_len(struct net_device *dev)
1436{
251f9e88 1437 struct adapter *adap = netdev2adap(dev);
d14807dd 1438 if (is_t4(adap->params.chip))
251f9e88
SR		 1439 return T4_REGMAP_SIZE;
1440 else
1441 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1442}
1443
1444static int get_eeprom_len(struct net_device *dev)
1445{
1446 return EEPROMSIZE;
1447}
1448
1449static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1450{
1451 struct adapter *adapter = netdev2adap(dev);
ba3f8cd5 1452 u32 exprom_vers;
b8ff05a9 1453
23020ab3
RJ		 1454 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1455 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1456 strlcpy(info->bus_info, pci_name(adapter->pdev),
1457 sizeof(info->bus_info));
b8ff05a9 1458
84b40501 1459 if (adapter->params.fw_vers)
b8ff05a9
DM		 1460 snprintf(info->fw_version, sizeof(info->fw_version),
1461 "%u.%u.%u.%u, TP %u.%u.%u.%u",
b2e1a3f0
HS		 1462 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1463 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1464 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1465 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1466 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1467 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1468 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1469 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
ba3f8cd5
HS		 1470
1471 if (!t4_get_exprom_version(adapter, &exprom_vers))
1472 snprintf(info->erom_version, sizeof(info->erom_version),
1473 "%u.%u.%u.%u",
1474 FW_HDR_FW_VER_MAJOR_G(exprom_vers),
1475 FW_HDR_FW_VER_MINOR_G(exprom_vers),
1476 FW_HDR_FW_VER_MICRO_G(exprom_vers),
1477 FW_HDR_FW_VER_BUILD_G(exprom_vers));
b8ff05a9
DM		 1478}
1479
1480static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1481{
1482 if (stringset == ETH_SS_STATS)
1483 memcpy(data, stats_strings, sizeof(stats_strings));
1484}
1485
1486/*
1487 * port stats maintained per queue of the port. They should be in the same
1488 * order as in stats_strings above.
1489 */
1490struct queue_port_stats {
1491 u64 tso;
1492 u64 tx_csum;
1493 u64 rx_csum;
1494 u64 vlan_ex;
1495 u64 vlan_ins;
4a6346d4
DM		 1496 u64 gro_pkts;
1497 u64 gro_merged;
b8ff05a9
DM		 1498};
1499
1500static void collect_sge_port_stats(const struct adapter *adap,
1501 const struct port_info *p, struct queue_port_stats *s)
1502{
1503 int i;
1504 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1505 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1506
1507 memset(s, 0, sizeof(*s));
1508 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1509 s->tso += tx->tso;
1510 s->tx_csum += tx->tx_cso;
1511 s->rx_csum += rx->stats.rx_cso;
1512 s->vlan_ex += rx->stats.vlan_ex;
1513 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1514 s->gro_pkts += rx->stats.lro_pkts;
1515 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1516 }
1517}
1518
1519static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1520 u64 *data)
1521{
1522 struct port_info *pi = netdev_priv(dev);
1523 struct adapter *adapter = pi->adapter;
22adfe0a 1524 u32 val1, val2;
b8ff05a9
DM		 1525
1526 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1527
1528 data += sizeof(struct port_stats) / sizeof(u64);
1529 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1530 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1531 if (!is_t4(adapter->params.chip)) {
f061de42
HS		 1532 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1533 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1534 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
22adfe0a
SR		 1535 *data = val1 - val2;
1536 data++;
1537 *data = val2;
1538 data++;
1539 } else {
1540 memset(data, 0, 2 * sizeof(u64));
1541 *data += 2;
1542 }
b8ff05a9
DM		 1543}
1544
1545/*
1546 * Return a version number to identify the type of adapter. The scheme is:
1547 * - bits 0..9: chip version
1548 * - bits 10..15: chip revision
835bb606 1549 * - bits 16..23: register dump version
b8ff05a9
DM		 1550 */
1551static inline unsigned int mk_adap_vers(const struct adapter *ap)
1552{
d14807dd
HS		 1553 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1554 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1555}
1556
1557static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1558 unsigned int end)
1559{
1560 u32 *p = buf + start;
1561
1562 for ( ; start <= end; start += sizeof(u32))
1563 *p++ = t4_read_reg(ap, start);
1564}
1565
1566static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1567 void *buf)
1568{
251f9e88 1569 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1570 0x1008, 0x1108,
1571 0x1180, 0x11b4,
1572 0x11fc, 0x123c,
1573 0x1300, 0x173c,
1574 0x1800, 0x18fc,
1575 0x3000, 0x30d8,
1576 0x30e0, 0x5924,
1577 0x5960, 0x59d4,
1578 0x5a00, 0x5af8,
1579 0x6000, 0x6098,
1580 0x6100, 0x6150,
1581 0x6200, 0x6208,
1582 0x6240, 0x6248,
1583 0x6280, 0x6338,
1584 0x6370, 0x638c,
1585 0x6400, 0x643c,
1586 0x6500, 0x6524,
1587 0x6a00, 0x6a38,
1588 0x6a60, 0x6a78,
1589 0x6b00, 0x6b84,
1590 0x6bf0, 0x6c84,
1591 0x6cf0, 0x6d84,
1592 0x6df0, 0x6e84,
1593 0x6ef0, 0x6f84,
1594 0x6ff0, 0x7084,
1595 0x70f0, 0x7184,
1596 0x71f0, 0x7284,
1597 0x72f0, 0x7384,
1598 0x73f0, 0x7450,
1599 0x7500, 0x7530,
1600 0x7600, 0x761c,
1601 0x7680, 0x76cc,
1602 0x7700, 0x7798,
1603 0x77c0, 0x77fc,
1604 0x7900, 0x79fc,
1605 0x7b00, 0x7c38,
1606 0x7d00, 0x7efc,
1607 0x8dc0, 0x8e1c,
1608 0x8e30, 0x8e78,
1609 0x8ea0, 0x8f6c,
1610 0x8fc0, 0x9074,
1611 0x90fc, 0x90fc,
1612 0x9400, 0x9458,
1613 0x9600, 0x96bc,
1614 0x9800, 0x9808,
1615 0x9820, 0x983c,
1616 0x9850, 0x9864,
1617 0x9c00, 0x9c6c,
1618 0x9c80, 0x9cec,
1619 0x9d00, 0x9d6c,
1620 0x9d80, 0x9dec,
1621 0x9e00, 0x9e6c,
1622 0x9e80, 0x9eec,
1623 0x9f00, 0x9f6c,
1624 0x9f80, 0x9fec,
1625 0xd004, 0xd03c,
1626 0xdfc0, 0xdfe0,
1627 0xe000, 0xea7c,
3d9103f8
HS		 1628 0xf000, 0x11110,
1629 0x11118, 0x11190,
835bb606
DM		 1630 0x19040, 0x1906c,
1631 0x19078, 0x19080,
1632 0x1908c, 0x19124,
b8ff05a9
DM		 1633 0x19150, 0x191b0,
1634 0x191d0, 0x191e8,
1635 0x19238, 0x1924c,
1636 0x193f8, 0x19474,
1637 0x19490, 0x194f8,
1638 0x19800, 0x19f30,
1639 0x1a000, 0x1a06c,
1640 0x1a0b0, 0x1a120,
1641 0x1a128, 0x1a138,
1642 0x1a190, 0x1a1c4,
1643 0x1a1fc, 0x1a1fc,
1644 0x1e040, 0x1e04c,
835bb606 1645 0x1e284, 0x1e28c,
b8ff05a9
DM		 1646 0x1e2c0, 0x1e2c0,
1647 0x1e2e0, 0x1e2e0,
1648 0x1e300, 0x1e384,
1649 0x1e3c0, 0x1e3c8,
1650 0x1e440, 0x1e44c,
835bb606 1651 0x1e684, 0x1e68c,
b8ff05a9
DM		 1652 0x1e6c0, 0x1e6c0,
1653 0x1e6e0, 0x1e6e0,
1654 0x1e700, 0x1e784,
1655 0x1e7c0, 0x1e7c8,
1656 0x1e840, 0x1e84c,
835bb606 1657 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1658 0x1eac0, 0x1eac0,
1659 0x1eae0, 0x1eae0,
1660 0x1eb00, 0x1eb84,
1661 0x1ebc0, 0x1ebc8,
1662 0x1ec40, 0x1ec4c,
835bb606 1663 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1664 0x1eec0, 0x1eec0,
1665 0x1eee0, 0x1eee0,
1666 0x1ef00, 0x1ef84,
1667 0x1efc0, 0x1efc8,
1668 0x1f040, 0x1f04c,
835bb606 1669 0x1f284, 0x1f28c,
b8ff05a9
DM		 1670 0x1f2c0, 0x1f2c0,
1671 0x1f2e0, 0x1f2e0,
1672 0x1f300, 0x1f384,
1673 0x1f3c0, 0x1f3c8,
1674 0x1f440, 0x1f44c,
835bb606 1675 0x1f684, 0x1f68c,
b8ff05a9
DM		 1676 0x1f6c0, 0x1f6c0,
1677 0x1f6e0, 0x1f6e0,
1678 0x1f700, 0x1f784,
1679 0x1f7c0, 0x1f7c8,
1680 0x1f840, 0x1f84c,
835bb606 1681 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1682 0x1fac0, 0x1fac0,
1683 0x1fae0, 0x1fae0,
1684 0x1fb00, 0x1fb84,
1685 0x1fbc0, 0x1fbc8,
1686 0x1fc40, 0x1fc4c,
835bb606 1687 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1688 0x1fec0, 0x1fec0,
1689 0x1fee0, 0x1fee0,
1690 0x1ff00, 0x1ff84,
1691 0x1ffc0, 0x1ffc8,
1692 0x20000, 0x2002c,
1693 0x20100, 0x2013c,
1694 0x20190, 0x201c8,
1695 0x20200, 0x20318,
1696 0x20400, 0x20528,
1697 0x20540, 0x20614,
1698 0x21000, 0x21040,
1699 0x2104c, 0x21060,
1700 0x210c0, 0x210ec,
1701 0x21200, 0x21268,
1702 0x21270, 0x21284,
1703 0x212fc, 0x21388,
1704 0x21400, 0x21404,
1705 0x21500, 0x21518,
1706 0x2152c, 0x2153c,
1707 0x21550, 0x21554,
1708 0x21600, 0x21600,
1709 0x21608, 0x21628,
1710 0x21630, 0x2163c,
1711 0x21700, 0x2171c,
1712 0x21780, 0x2178c,
1713 0x21800, 0x21c38,
1714 0x21c80, 0x21d7c,
1715 0x21e00, 0x21e04,
1716 0x22000, 0x2202c,
1717 0x22100, 0x2213c,
1718 0x22190, 0x221c8,
1719 0x22200, 0x22318,
1720 0x22400, 0x22528,
1721 0x22540, 0x22614,
1722 0x23000, 0x23040,
1723 0x2304c, 0x23060,
1724 0x230c0, 0x230ec,
1725 0x23200, 0x23268,
1726 0x23270, 0x23284,
1727 0x232fc, 0x23388,
1728 0x23400, 0x23404,
1729 0x23500, 0x23518,
1730 0x2352c, 0x2353c,
1731 0x23550, 0x23554,
1732 0x23600, 0x23600,
1733 0x23608, 0x23628,
1734 0x23630, 0x2363c,
1735 0x23700, 0x2371c,
1736 0x23780, 0x2378c,
1737 0x23800, 0x23c38,
1738 0x23c80, 0x23d7c,
1739 0x23e00, 0x23e04,
1740 0x24000, 0x2402c,
1741 0x24100, 0x2413c,
1742 0x24190, 0x241c8,
1743 0x24200, 0x24318,
1744 0x24400, 0x24528,
1745 0x24540, 0x24614,
1746 0x25000, 0x25040,
1747 0x2504c, 0x25060,
1748 0x250c0, 0x250ec,
1749 0x25200, 0x25268,
1750 0x25270, 0x25284,
1751 0x252fc, 0x25388,
1752 0x25400, 0x25404,
1753 0x25500, 0x25518,
1754 0x2552c, 0x2553c,
1755 0x25550, 0x25554,
1756 0x25600, 0x25600,
1757 0x25608, 0x25628,
1758 0x25630, 0x2563c,
1759 0x25700, 0x2571c,
1760 0x25780, 0x2578c,
1761 0x25800, 0x25c38,
1762 0x25c80, 0x25d7c,
1763 0x25e00, 0x25e04,
1764 0x26000, 0x2602c,
1765 0x26100, 0x2613c,
1766 0x26190, 0x261c8,
1767 0x26200, 0x26318,
1768 0x26400, 0x26528,
1769 0x26540, 0x26614,
1770 0x27000, 0x27040,
1771 0x2704c, 0x27060,
1772 0x270c0, 0x270ec,
1773 0x27200, 0x27268,
1774 0x27270, 0x27284,
1775 0x272fc, 0x27388,
1776 0x27400, 0x27404,
1777 0x27500, 0x27518,
1778 0x2752c, 0x2753c,
1779 0x27550, 0x27554,
1780 0x27600, 0x27600,
1781 0x27608, 0x27628,
1782 0x27630, 0x2763c,
1783 0x27700, 0x2771c,
1784 0x27780, 0x2778c,
1785 0x27800, 0x27c38,
1786 0x27c80, 0x27d7c,
1787 0x27e00, 0x27e04
1788 };
1789
251f9e88
SR		 1790 static const unsigned int t5_reg_ranges[] = {
1791 0x1008, 0x1148,
1792 0x1180, 0x11b4,
1793 0x11fc, 0x123c,
1794 0x1280, 0x173c,
1795 0x1800, 0x18fc,
1796 0x3000, 0x3028,
1797 0x3060, 0x30d8,
1798 0x30e0, 0x30fc,
1799 0x3140, 0x357c,
1800 0x35a8, 0x35cc,
1801 0x35ec, 0x35ec,
1802 0x3600, 0x5624,
1803 0x56cc, 0x575c,
1804 0x580c, 0x5814,
1805 0x5890, 0x58bc,
1806 0x5940, 0x59dc,
1807 0x59fc, 0x5a18,
1808 0x5a60, 0x5a9c,
1809 0x5b9c, 0x5bfc,
1810 0x6000, 0x6040,
1811 0x6058, 0x614c,
1812 0x7700, 0x7798,
1813 0x77c0, 0x78fc,
1814 0x7b00, 0x7c54,
1815 0x7d00, 0x7efc,
1816 0x8dc0, 0x8de0,
1817 0x8df8, 0x8e84,
1818 0x8ea0, 0x8f84,
1819 0x8fc0, 0x90f8,
1820 0x9400, 0x9470,
1821 0x9600, 0x96f4,
1822 0x9800, 0x9808,
1823 0x9820, 0x983c,
1824 0x9850, 0x9864,
1825 0x9c00, 0x9c6c,
1826 0x9c80, 0x9cec,
1827 0x9d00, 0x9d6c,
1828 0x9d80, 0x9dec,
1829 0x9e00, 0x9e6c,
1830 0x9e80, 0x9eec,
1831 0x9f00, 0x9f6c,
1832 0x9f80, 0xa020,
1833 0xd004, 0xd03c,
1834 0xdfc0, 0xdfe0,
1835 0xe000, 0x11088,
3d9103f8
HS		 1836 0x1109c, 0x11110,
1837 0x11118, 0x1117c,
251f9e88
SR		 1838 0x11190, 0x11204,
1839 0x19040, 0x1906c,
1840 0x19078, 0x19080,
1841 0x1908c, 0x19124,
1842 0x19150, 0x191b0,
1843 0x191d0, 0x191e8,
1844 0x19238, 0x19290,
1845 0x193f8, 0x19474,
1846 0x19490, 0x194cc,
1847 0x194f0, 0x194f8,
1848 0x19c00, 0x19c60,
1849 0x19c94, 0x19e10,
1850 0x19e50, 0x19f34,
1851 0x19f40, 0x19f50,
1852 0x19f90, 0x19fe4,
1853 0x1a000, 0x1a06c,
1854 0x1a0b0, 0x1a120,
1855 0x1a128, 0x1a138,
1856 0x1a190, 0x1a1c4,
1857 0x1a1fc, 0x1a1fc,
1858 0x1e008, 0x1e00c,
1859 0x1e040, 0x1e04c,
1860 0x1e284, 0x1e290,
1861 0x1e2c0, 0x1e2c0,
1862 0x1e2e0, 0x1e2e0,
1863 0x1e300, 0x1e384,
1864 0x1e3c0, 0x1e3c8,
1865 0x1e408, 0x1e40c,
1866 0x1e440, 0x1e44c,
1867 0x1e684, 0x1e690,
1868 0x1e6c0, 0x1e6c0,
1869 0x1e6e0, 0x1e6e0,
1870 0x1e700, 0x1e784,
1871 0x1e7c0, 0x1e7c8,
1872 0x1e808, 0x1e80c,
1873 0x1e840, 0x1e84c,
1874 0x1ea84, 0x1ea90,
1875 0x1eac0, 0x1eac0,
1876 0x1eae0, 0x1eae0,
1877 0x1eb00, 0x1eb84,
1878 0x1ebc0, 0x1ebc8,
1879 0x1ec08, 0x1ec0c,
1880 0x1ec40, 0x1ec4c,
1881 0x1ee84, 0x1ee90,
1882 0x1eec0, 0x1eec0,
1883 0x1eee0, 0x1eee0,
1884 0x1ef00, 0x1ef84,
1885 0x1efc0, 0x1efc8,
1886 0x1f008, 0x1f00c,
1887 0x1f040, 0x1f04c,
1888 0x1f284, 0x1f290,
1889 0x1f2c0, 0x1f2c0,
1890 0x1f2e0, 0x1f2e0,
1891 0x1f300, 0x1f384,
1892 0x1f3c0, 0x1f3c8,
1893 0x1f408, 0x1f40c,
1894 0x1f440, 0x1f44c,
1895 0x1f684, 0x1f690,
1896 0x1f6c0, 0x1f6c0,
1897 0x1f6e0, 0x1f6e0,
1898 0x1f700, 0x1f784,
1899 0x1f7c0, 0x1f7c8,
1900 0x1f808, 0x1f80c,
1901 0x1f840, 0x1f84c,
1902 0x1fa84, 0x1fa90,
1903 0x1fac0, 0x1fac0,
1904 0x1fae0, 0x1fae0,
1905 0x1fb00, 0x1fb84,
1906 0x1fbc0, 0x1fbc8,
1907 0x1fc08, 0x1fc0c,
1908 0x1fc40, 0x1fc4c,
1909 0x1fe84, 0x1fe90,
1910 0x1fec0, 0x1fec0,
1911 0x1fee0, 0x1fee0,
1912 0x1ff00, 0x1ff84,
1913 0x1ffc0, 0x1ffc8,
1914 0x30000, 0x30030,
1915 0x30100, 0x30144,
1916 0x30190, 0x301d0,
1917 0x30200, 0x30318,
1918 0x30400, 0x3052c,
1919 0x30540, 0x3061c,
1920 0x30800, 0x30834,
1921 0x308c0, 0x30908,
1922 0x30910, 0x309ac,
1923 0x30a00, 0x30a04,
1924 0x30a0c, 0x30a2c,
1925 0x30a44, 0x30a50,
1926 0x30a74, 0x30c24,
1927 0x30d08, 0x30d14,
1928 0x30d1c, 0x30d20,
1929 0x30d3c, 0x30d50,
1930 0x31200, 0x3120c,
1931 0x31220, 0x31220,
1932 0x31240, 0x31240,
1933 0x31600, 0x31600,
1934 0x31608, 0x3160c,
1935 0x31a00, 0x31a1c,
1936 0x31e04, 0x31e20,
1937 0x31e38, 0x31e3c,
1938 0x31e80, 0x31e80,
1939 0x31e88, 0x31ea8,
1940 0x31eb0, 0x31eb4,
1941 0x31ec8, 0x31ed4,
1942 0x31fb8, 0x32004,
1943 0x32208, 0x3223c,
1944 0x32600, 0x32630,
1945 0x32a00, 0x32abc,
1946 0x32b00, 0x32b70,
1947 0x33000, 0x33048,
1948 0x33060, 0x3309c,
1949 0x330f0, 0x33148,
1950 0x33160, 0x3319c,
1951 0x331f0, 0x332e4,
1952 0x332f8, 0x333e4,
1953 0x333f8, 0x33448,
1954 0x33460, 0x3349c,
1955 0x334f0, 0x33548,
1956 0x33560, 0x3359c,
1957 0x335f0, 0x336e4,
1958 0x336f8, 0x337e4,
1959 0x337f8, 0x337fc,
1960 0x33814, 0x33814,
1961 0x3382c, 0x3382c,
1962 0x33880, 0x3388c,
1963 0x338e8, 0x338ec,
1964 0x33900, 0x33948,
1965 0x33960, 0x3399c,
1966 0x339f0, 0x33ae4,
1967 0x33af8, 0x33b10,
1968 0x33b28, 0x33b28,
1969 0x33b3c, 0x33b50,
1970 0x33bf0, 0x33c10,
1971 0x33c28, 0x33c28,
1972 0x33c3c, 0x33c50,
1973 0x33cf0, 0x33cfc,
1974 0x34000, 0x34030,
1975 0x34100, 0x34144,
1976 0x34190, 0x341d0,
1977 0x34200, 0x34318,
1978 0x34400, 0x3452c,
1979 0x34540, 0x3461c,
1980 0x34800, 0x34834,
1981 0x348c0, 0x34908,
1982 0x34910, 0x349ac,
1983 0x34a00, 0x34a04,
1984 0x34a0c, 0x34a2c,
1985 0x34a44, 0x34a50,
1986 0x34a74, 0x34c24,
1987 0x34d08, 0x34d14,
1988 0x34d1c, 0x34d20,
1989 0x34d3c, 0x34d50,
1990 0x35200, 0x3520c,
1991 0x35220, 0x35220,
1992 0x35240, 0x35240,
1993 0x35600, 0x35600,
1994 0x35608, 0x3560c,
1995 0x35a00, 0x35a1c,
1996 0x35e04, 0x35e20,
1997 0x35e38, 0x35e3c,
1998 0x35e80, 0x35e80,
1999 0x35e88, 0x35ea8,
2000 0x35eb0, 0x35eb4,
2001 0x35ec8, 0x35ed4,
2002 0x35fb8, 0x36004,
2003 0x36208, 0x3623c,
2004 0x36600, 0x36630,
2005 0x36a00, 0x36abc,
2006 0x36b00, 0x36b70,
2007 0x37000, 0x37048,
2008 0x37060, 0x3709c,
2009 0x370f0, 0x37148,
2010 0x37160, 0x3719c,
2011 0x371f0, 0x372e4,
2012 0x372f8, 0x373e4,
2013 0x373f8, 0x37448,
2014 0x37460, 0x3749c,
2015 0x374f0, 0x37548,
2016 0x37560, 0x3759c,
2017 0x375f0, 0x376e4,
2018 0x376f8, 0x377e4,
2019 0x377f8, 0x377fc,
2020 0x37814, 0x37814,
2021 0x3782c, 0x3782c,
2022 0x37880, 0x3788c,
2023 0x378e8, 0x378ec,
2024 0x37900, 0x37948,
2025 0x37960, 0x3799c,
2026 0x379f0, 0x37ae4,
2027 0x37af8, 0x37b10,
2028 0x37b28, 0x37b28,
2029 0x37b3c, 0x37b50,
2030 0x37bf0, 0x37c10,
2031 0x37c28, 0x37c28,
2032 0x37c3c, 0x37c50,
2033 0x37cf0, 0x37cfc,
2034 0x38000, 0x38030,
2035 0x38100, 0x38144,
2036 0x38190, 0x381d0,
2037 0x38200, 0x38318,
2038 0x38400, 0x3852c,
2039 0x38540, 0x3861c,
2040 0x38800, 0x38834,
2041 0x388c0, 0x38908,
2042 0x38910, 0x389ac,
2043 0x38a00, 0x38a04,
2044 0x38a0c, 0x38a2c,
2045 0x38a44, 0x38a50,
2046 0x38a74, 0x38c24,
2047 0x38d08, 0x38d14,
2048 0x38d1c, 0x38d20,
2049 0x38d3c, 0x38d50,
2050 0x39200, 0x3920c,
2051 0x39220, 0x39220,
2052 0x39240, 0x39240,
2053 0x39600, 0x39600,
2054 0x39608, 0x3960c,
2055 0x39a00, 0x39a1c,
2056 0x39e04, 0x39e20,
2057 0x39e38, 0x39e3c,
2058 0x39e80, 0x39e80,
2059 0x39e88, 0x39ea8,
2060 0x39eb0, 0x39eb4,
2061 0x39ec8, 0x39ed4,
2062 0x39fb8, 0x3a004,
2063 0x3a208, 0x3a23c,
2064 0x3a600, 0x3a630,
2065 0x3aa00, 0x3aabc,
2066 0x3ab00, 0x3ab70,
2067 0x3b000, 0x3b048,
2068 0x3b060, 0x3b09c,
2069 0x3b0f0, 0x3b148,
2070 0x3b160, 0x3b19c,
2071 0x3b1f0, 0x3b2e4,
2072 0x3b2f8, 0x3b3e4,
2073 0x3b3f8, 0x3b448,
2074 0x3b460, 0x3b49c,
2075 0x3b4f0, 0x3b548,
2076 0x3b560, 0x3b59c,
2077 0x3b5f0, 0x3b6e4,
2078 0x3b6f8, 0x3b7e4,
2079 0x3b7f8, 0x3b7fc,
2080 0x3b814, 0x3b814,
2081 0x3b82c, 0x3b82c,
2082 0x3b880, 0x3b88c,
2083 0x3b8e8, 0x3b8ec,
2084 0x3b900, 0x3b948,
2085 0x3b960, 0x3b99c,
2086 0x3b9f0, 0x3bae4,
2087 0x3baf8, 0x3bb10,
2088 0x3bb28, 0x3bb28,
2089 0x3bb3c, 0x3bb50,
2090 0x3bbf0, 0x3bc10,
2091 0x3bc28, 0x3bc28,
2092 0x3bc3c, 0x3bc50,
2093 0x3bcf0, 0x3bcfc,
2094 0x3c000, 0x3c030,
2095 0x3c100, 0x3c144,
2096 0x3c190, 0x3c1d0,
2097 0x3c200, 0x3c318,
2098 0x3c400, 0x3c52c,
2099 0x3c540, 0x3c61c,
2100 0x3c800, 0x3c834,
2101 0x3c8c0, 0x3c908,
2102 0x3c910, 0x3c9ac,
2103 0x3ca00, 0x3ca04,
2104 0x3ca0c, 0x3ca2c,
2105 0x3ca44, 0x3ca50,
2106 0x3ca74, 0x3cc24,
2107 0x3cd08, 0x3cd14,
2108 0x3cd1c, 0x3cd20,
2109 0x3cd3c, 0x3cd50,
2110 0x3d200, 0x3d20c,
2111 0x3d220, 0x3d220,
2112 0x3d240, 0x3d240,
2113 0x3d600, 0x3d600,
2114 0x3d608, 0x3d60c,
2115 0x3da00, 0x3da1c,
2116 0x3de04, 0x3de20,
2117 0x3de38, 0x3de3c,
2118 0x3de80, 0x3de80,
2119 0x3de88, 0x3dea8,
2120 0x3deb0, 0x3deb4,
2121 0x3dec8, 0x3ded4,
2122 0x3dfb8, 0x3e004,
2123 0x3e208, 0x3e23c,
2124 0x3e600, 0x3e630,
2125 0x3ea00, 0x3eabc,
2126 0x3eb00, 0x3eb70,
2127 0x3f000, 0x3f048,
2128 0x3f060, 0x3f09c,
2129 0x3f0f0, 0x3f148,
2130 0x3f160, 0x3f19c,
2131 0x3f1f0, 0x3f2e4,
2132 0x3f2f8, 0x3f3e4,
2133 0x3f3f8, 0x3f448,
2134 0x3f460, 0x3f49c,
2135 0x3f4f0, 0x3f548,
2136 0x3f560, 0x3f59c,
2137 0x3f5f0, 0x3f6e4,
2138 0x3f6f8, 0x3f7e4,
2139 0x3f7f8, 0x3f7fc,
2140 0x3f814, 0x3f814,
2141 0x3f82c, 0x3f82c,
2142 0x3f880, 0x3f88c,
2143 0x3f8e8, 0x3f8ec,
2144 0x3f900, 0x3f948,
2145 0x3f960, 0x3f99c,
2146 0x3f9f0, 0x3fae4,
2147 0x3faf8, 0x3fb10,
2148 0x3fb28, 0x3fb28,
2149 0x3fb3c, 0x3fb50,
2150 0x3fbf0, 0x3fc10,
2151 0x3fc28, 0x3fc28,
2152 0x3fc3c, 0x3fc50,
2153 0x3fcf0, 0x3fcfc,
2154 0x40000, 0x4000c,
2155 0x40040, 0x40068,
2156 0x40080, 0x40144,
2157 0x40180, 0x4018c,
2158 0x40200, 0x40298,
2159 0x402ac, 0x4033c,
2160 0x403f8, 0x403fc,
c1f49e3e 2161 0x41304, 0x413c4,
251f9e88
SR		 2162 0x41400, 0x4141c,
2163 0x41480, 0x414d0,
2164 0x44000, 0x44078,
2165 0x440c0, 0x44278,
2166 0x442c0, 0x44478,
2167 0x444c0, 0x44678,
2168 0x446c0, 0x44878,
2169 0x448c0, 0x449fc,
2170 0x45000, 0x45068,
2171 0x45080, 0x45084,
2172 0x450a0, 0x450b0,
2173 0x45200, 0x45268,
2174 0x45280, 0x45284,
2175 0x452a0, 0x452b0,
2176 0x460c0, 0x460e4,
2177 0x47000, 0x4708c,
2178 0x47200, 0x47250,
2179 0x47400, 0x47420,
2180 0x47600, 0x47618,
2181 0x47800, 0x47814,
2182 0x48000, 0x4800c,
2183 0x48040, 0x48068,
2184 0x48080, 0x48144,
2185 0x48180, 0x4818c,
2186 0x48200, 0x48298,
2187 0x482ac, 0x4833c,
2188 0x483f8, 0x483fc,
c1f49e3e 2189 0x49304, 0x493c4,
251f9e88
SR		 2190 0x49400, 0x4941c,
2191 0x49480, 0x494d0,
2192 0x4c000, 0x4c078,
2193 0x4c0c0, 0x4c278,
2194 0x4c2c0, 0x4c478,
2195 0x4c4c0, 0x4c678,
2196 0x4c6c0, 0x4c878,
2197 0x4c8c0, 0x4c9fc,
2198 0x4d000, 0x4d068,
2199 0x4d080, 0x4d084,
2200 0x4d0a0, 0x4d0b0,
2201 0x4d200, 0x4d268,
2202 0x4d280, 0x4d284,
2203 0x4d2a0, 0x4d2b0,
2204 0x4e0c0, 0x4e0e4,
2205 0x4f000, 0x4f08c,
2206 0x4f200, 0x4f250,
2207 0x4f400, 0x4f420,
2208 0x4f600, 0x4f618,
2209 0x4f800, 0x4f814,
2210 0x50000, 0x500cc,
2211 0x50400, 0x50400,
2212 0x50800, 0x508cc,
2213 0x50c00, 0x50c00,
2214 0x51000, 0x5101c,
2215 0x51300, 0x51308,
2216 };
2217
b8ff05a9
DM		 2218 int i;
2219 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2220 static const unsigned int *reg_ranges;
2221 int arr_size = 0, buf_size = 0;
2222
d14807dd 2223 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2224 reg_ranges = &t4_reg_ranges[0];
2225 arr_size = ARRAY_SIZE(t4_reg_ranges);
2226 buf_size = T4_REGMAP_SIZE;
2227 } else {
2228 reg_ranges = &t5_reg_ranges[0];
2229 arr_size = ARRAY_SIZE(t5_reg_ranges);
2230 buf_size = T5_REGMAP_SIZE;
2231 }
b8ff05a9
DM		 2232
2233 regs->version = mk_adap_vers(ap);
2234
251f9e88
SR		 2235 memset(buf, 0, buf_size);
2236 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2237 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2238}
2239
2240static int restart_autoneg(struct net_device *dev)
2241{
2242 struct port_info *p = netdev_priv(dev);
2243
2244 if (!netif_running(dev))
2245 return -EAGAIN;
2246 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2247 return -EINVAL;
060e0c75 2248 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2249 return 0;
2250}
2251
c5e06360
DM		 2252static int identify_port(struct net_device *dev,
2253 enum ethtool_phys_id_state state)
b8ff05a9 2254{
c5e06360 2255 unsigned int val;
060e0c75
DM		 2256 struct adapter *adap = netdev2adap(dev);
2257
c5e06360
DM		 2258 if (state == ETHTOOL_ID_ACTIVE)
2259 val = 0xffff;
2260 else if (state == ETHTOOL_ID_INACTIVE)
2261 val = 0;
2262 else
2263 return -EINVAL;
b8ff05a9 2264
c5e06360 2265 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2266}
2267
40e9de4b 2268static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
b8ff05a9
DM		 2269{
2270 unsigned int v = 0;
2271
a0881cab
DM		 2272 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2273 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2274 v |= SUPPORTED_TP;
2275 if (caps & FW_PORT_CAP_SPEED_100M)
2276 v |= SUPPORTED_100baseT_Full;
2277 if (caps & FW_PORT_CAP_SPEED_1G)
2278 v |= SUPPORTED_1000baseT_Full;
2279 if (caps & FW_PORT_CAP_SPEED_10G)
2280 v |= SUPPORTED_10000baseT_Full;
2281 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2282 v |= SUPPORTED_Backplane;
2283 if (caps & FW_PORT_CAP_SPEED_1G)
2284 v |= SUPPORTED_1000baseKX_Full;
2285 if (caps & FW_PORT_CAP_SPEED_10G)
2286 v |= SUPPORTED_10000baseKX4_Full;
2287 } else if (type == FW_PORT_TYPE_KR)
2288 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2289 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2290 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2291 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2292 else if (type == FW_PORT_TYPE_BP4_AP)
2293 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2294 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2295 SUPPORTED_10000baseKX4_Full;
a0881cab 2296 else if (type == FW_PORT_TYPE_FIBER_XFI ||
40e9de4b
HS		 2297 type == FW_PORT_TYPE_FIBER_XAUI ||
2298 type == FW_PORT_TYPE_SFP ||
2299 type == FW_PORT_TYPE_QSFP_10G ||
2300 type == FW_PORT_TYPE_QSA) {
b8ff05a9 2301 v |= SUPPORTED_FIBRE;
4c2d5186
HS		 2302 if (caps & FW_PORT_CAP_SPEED_1G)
2303 v |= SUPPORTED_1000baseT_Full;
2304 if (caps & FW_PORT_CAP_SPEED_10G)
2305 v |= SUPPORTED_10000baseT_Full;
40e9de4b
HS		 2306 } else if (type == FW_PORT_TYPE_BP40_BA ||
2307 type == FW_PORT_TYPE_QSFP) {
72aca4bf 2308 v |= SUPPORTED_40000baseSR4_Full;
40e9de4b
HS		 2309 v |= SUPPORTED_FIBRE;
2310 }
b8ff05a9
DM		 2311
2312 if (caps & FW_PORT_CAP_ANEG)
2313 v |= SUPPORTED_Autoneg;
2314 return v;
2315}
2316
2317static unsigned int to_fw_linkcaps(unsigned int caps)
2318{
2319 unsigned int v = 0;
2320
2321 if (caps & ADVERTISED_100baseT_Full)
2322 v |= FW_PORT_CAP_SPEED_100M;
2323 if (caps & ADVERTISED_1000baseT_Full)
2324 v |= FW_PORT_CAP_SPEED_1G;
2325 if (caps & ADVERTISED_10000baseT_Full)
2326 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2327 if (caps & ADVERTISED_40000baseSR4_Full)
2328 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2329 return v;
2330}
2331
2332static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2333{
2334 const struct port_info *p = netdev_priv(dev);
2335
2336 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2337 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2338 p->port_type == FW_PORT_TYPE_BT_XAUI)
2339 cmd->port = PORT_TP;
a0881cab
DM		 2340 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2341 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2342 cmd->port = PORT_FIBRE;
3e00a509
HS		 2343 else if (p->port_type == FW_PORT_TYPE_SFP ||
2344 p->port_type == FW_PORT_TYPE_QSFP_10G ||
40e9de4b 2345 p->port_type == FW_PORT_TYPE_QSA ||
3e00a509
HS		 2346 p->port_type == FW_PORT_TYPE_QSFP) {
2347 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2348 p->mod_type == FW_PORT_MOD_TYPE_SR ||
2349 p->mod_type == FW_PORT_MOD_TYPE_ER ||
2350 p->mod_type == FW_PORT_MOD_TYPE_LRM)
2351 cmd->port = PORT_FIBRE;
2352 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2353 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
a0881cab
DM		 2354 cmd->port = PORT_DA;
2355 else
3e00a509 2356 cmd->port = PORT_OTHER;
a0881cab 2357 } else
b8ff05a9
DM		 2358 cmd->port = PORT_OTHER;
2359
2360 if (p->mdio_addr >= 0) {
2361 cmd->phy_address = p->mdio_addr;
2362 cmd->transceiver = XCVR_EXTERNAL;
2363 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2364 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2365 } else {
2366 cmd->phy_address = 0; /* not really, but no better option */
2367 cmd->transceiver = XCVR_INTERNAL;
2368 cmd->mdio_support = 0;
2369 }
2370
2371 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2372 cmd->advertising = from_fw_linkcaps(p->port_type,
2373 p->link_cfg.advertising);
70739497
DD		 2374 ethtool_cmd_speed_set(cmd,
2375 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2376 cmd->duplex = DUPLEX_FULL;
2377 cmd->autoneg = p->link_cfg.autoneg;
2378 cmd->maxtxpkt = 0;
2379 cmd->maxrxpkt = 0;
2380 return 0;
2381}
2382
2383static unsigned int speed_to_caps(int speed)
2384{
e8b39015 2385 if (speed == 100)
b8ff05a9 2386 return FW_PORT_CAP_SPEED_100M;
e8b39015 2387 if (speed == 1000)
b8ff05a9 2388 return FW_PORT_CAP_SPEED_1G;
e8b39015 2389 if (speed == 10000)
b8ff05a9 2390 return FW_PORT_CAP_SPEED_10G;
e8b39015 2391 if (speed == 40000)
72aca4bf 2392 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2393 return 0;
2394}
2395
2396static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2397{
2398 unsigned int cap;
2399 struct port_info *p = netdev_priv(dev);
2400 struct link_config *lc = &p->link_cfg;
25db0338 2401 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2402
2403 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2404 return -EINVAL;
2405
2406 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2407 /*
2408 * PHY offers a single speed. See if that's what's
2409 * being requested.
2410 */
2411 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2412 (lc->supported & speed_to_caps(speed)))
2413 return 0;
b8ff05a9
DM		 2414 return -EINVAL;
2415 }
2416
2417 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2418 cap = speed_to_caps(speed);
b8ff05a9 2419
72aca4bf 2420 if (!(lc->supported & cap) ||
e8b39015
BH		 2421 (speed == 1000) ||
2422 (speed == 10000) ||
72aca4bf 2423 (speed == 40000))
b8ff05a9
DM		 2424 return -EINVAL;
2425 lc->requested_speed = cap;
2426 lc->advertising = 0;
2427 } else {
2428 cap = to_fw_linkcaps(cmd->advertising);
2429 if (!(lc->supported & cap))
2430 return -EINVAL;
2431 lc->requested_speed = 0;
2432 lc->advertising = cap | FW_PORT_CAP_ANEG;
2433 }
2434 lc->autoneg = cmd->autoneg;
2435
2436 if (netif_running(dev))
060e0c75
DM		 2437 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2438 lc);
b8ff05a9
DM		 2439 return 0;
2440}
2441
2442static void get_pauseparam(struct net_device *dev,
2443 struct ethtool_pauseparam *epause)
2444{
2445 struct port_info *p = netdev_priv(dev);
2446
2447 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2448 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2449 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2450}
2451
2452static int set_pauseparam(struct net_device *dev,
2453 struct ethtool_pauseparam *epause)
2454{
2455 struct port_info *p = netdev_priv(dev);
2456 struct link_config *lc = &p->link_cfg;
2457
2458 if (epause->autoneg == AUTONEG_DISABLE)
2459 lc->requested_fc = 0;
2460 else if (lc->supported & FW_PORT_CAP_ANEG)
2461 lc->requested_fc = PAUSE_AUTONEG;
2462 else
2463 return -EINVAL;
2464
2465 if (epause->rx_pause)
2466 lc->requested_fc |= PAUSE_RX;
2467 if (epause->tx_pause)
2468 lc->requested_fc |= PAUSE_TX;
2469 if (netif_running(dev))
060e0c75
DM		 2470 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2471 lc);
b8ff05a9
DM		 2472 return 0;
2473}
2474
b8ff05a9
DM		 2475static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2476{
2477 const struct port_info *pi = netdev_priv(dev);
2478 const struct sge *s = &pi->adapter->sge;
2479
2480 e->rx_max_pending = MAX_RX_BUFFERS;
2481 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2482 e->rx_jumbo_max_pending = 0;
2483 e->tx_max_pending = MAX_TXQ_ENTRIES;
2484
2485 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2486 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2487 e->rx_jumbo_pending = 0;
2488 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2489}
2490
2491static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2492{
2493 int i;
2494 const struct port_info *pi = netdev_priv(dev);
2495 struct adapter *adapter = pi->adapter;
2496 struct sge *s = &adapter->sge;
2497
2498 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2499 e->tx_pending > MAX_TXQ_ENTRIES ||
2500 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2501 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2502 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2503 return -EINVAL;
2504
2505 if (adapter->flags & FULL_INIT_DONE)
2506 return -EBUSY;
2507
2508 for (i = 0; i < pi->nqsets; ++i) {
2509 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2510 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2511 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2512 }
2513 return 0;
2514}
2515
2516static int closest_timer(const struct sge *s, int time)
2517{
2518 int i, delta, match = 0, min_delta = INT_MAX;
2519
2520 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2521 delta = time - s->timer_val[i];
2522 if (delta < 0)
2523 delta = -delta;
2524 if (delta < min_delta) {
2525 min_delta = delta;
2526 match = i;
2527 }
2528 }
2529 return match;
2530}
2531
2532static int closest_thres(const struct sge *s, int thres)
2533{
2534 int i, delta, match = 0, min_delta = INT_MAX;
2535
2536 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2537 delta = thres - s->counter_val[i];
2538 if (delta < 0)
2539 delta = -delta;
2540 if (delta < min_delta) {
2541 min_delta = delta;
2542 match = i;
2543 }
2544 }
2545 return match;
2546}
2547
2548/*
2549 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2550 */
dc9daab2
HS		 2551unsigned int qtimer_val(const struct adapter *adap,
2552 const struct sge_rspq *q)
b8ff05a9
DM		 2553{
2554 unsigned int idx = q->intr_params >> 1;
2555
2556 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2557}
2558
2559/**
c887ad0e 2560 * set_rspq_intr_params - set a queue's interrupt holdoff parameters
b8ff05a9
DM		 2561 * @q: the Rx queue
2562 * @us: the hold-off time in us, or 0 to disable timer
2563 * @cnt: the hold-off packet count, or 0 to disable counter
2564 *
2565 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2566 * one of the two needs to be enabled for the queue to generate interrupts.
2567 */
c887ad0e
HS		 2568static int set_rspq_intr_params(struct sge_rspq *q,
2569 unsigned int us, unsigned int cnt)
b8ff05a9 2570{
c887ad0e
HS		 2571 struct adapter *adap = q->adap;
2572
b8ff05a9
DM		 2573 if ((us | cnt) == 0)
2574 cnt = 1;
2575
2576 if (cnt) {
2577 int err;
2578 u32 v, new_idx;
2579
2580 new_idx = closest_thres(&adap->sge, cnt);
2581 if (q->desc && q->pktcnt_idx != new_idx) {
2582 /* the queue has already been created, update it */
5167865a
HS		 2583 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2584 FW_PARAMS_PARAM_X_V(
2585 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2586 FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
060e0c75
DM		 2587 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2588 &new_idx);
b8ff05a9
DM		 2589 if (err)
2590 return err;
2591 }
2592 q->pktcnt_idx = new_idx;
2593 }
2594
2595 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2596 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2597 return 0;
2598}
2599
c887ad0e
HS		 2600/**
2601 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2602 * @dev: the network device
2603 * @us: the hold-off time in us, or 0 to disable timer
2604 * @cnt: the hold-off packet count, or 0 to disable counter
2605 *
2606 * Set the RX interrupt hold-off parameters for a network device.
2607 */
2608static int set_rx_intr_params(struct net_device *dev,
2609 unsigned int us, unsigned int cnt)
b8ff05a9 2610{
c887ad0e
HS		 2611 int i, err;
2612 struct port_info *pi = netdev_priv(dev);
b8ff05a9 2613 struct adapter *adap = pi->adapter;
c887ad0e
HS		 2614 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2615
2616 for (i = 0; i < pi->nqsets; i++, q++) {
2617 err = set_rspq_intr_params(&q->rspq, us, cnt);
2618 if (err)
2619 return err;
d4fc9dc2 2620 }
c887ad0e
HS		 2621 return 0;
2622}
2623
e553ec3f
HS		 2624static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2625{
2626 int i;
2627 struct port_info *pi = netdev_priv(dev);
2628 struct adapter *adap = pi->adapter;
2629 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2630
2631 for (i = 0; i < pi->nqsets; i++, q++)
2632 q->rspq.adaptive_rx = adaptive_rx;
2633
2634 return 0;
2635}
2636
2637static int get_adaptive_rx_setting(struct net_device *dev)
2638{
2639 struct port_info *pi = netdev_priv(dev);
2640 struct adapter *adap = pi->adapter;
2641 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2642
2643 return q->rspq.adaptive_rx;
2644}
2645
c887ad0e
HS		 2646static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2647{
e553ec3f 2648 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
c887ad0e
HS		 2649 return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2650 c->rx_max_coalesced_frames);
b8ff05a9
DM		 2651}
2652
2653static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2654{
2655 const struct port_info *pi = netdev_priv(dev);
2656 const struct adapter *adap = pi->adapter;
2657 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2658
2659 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2660 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2661 adap->sge.counter_val[rq->pktcnt_idx] : 0;
e553ec3f 2662 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
b8ff05a9
DM		 2663 return 0;
2664}
2665
1478b3ee
DM		 2666/**
2667 * eeprom_ptov - translate a physical EEPROM address to virtual
2668 * @phys_addr: the physical EEPROM address
2669 * @fn: the PCI function number
2670 * @sz: size of function-specific area
2671 *
2672 * Translate a physical EEPROM address to virtual. The first 1K is
2673 * accessed through virtual addresses starting at 31K, the rest is
2674 * accessed through virtual addresses starting at 0.
2675 *
2676 * The mapping is as follows:
2677 * [0..1K) -> [31K..32K)
2678 * [1K..1K+A) -> [31K-A..31K)
2679 * [1K+A..ES) -> [0..ES-A-1K)
2680 *
2681 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2682 */
1478b3ee 2683static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2684{
1478b3ee 2685 fn *= sz;
b8ff05a9
DM		 2686 if (phys_addr < 1024)
2687 return phys_addr + (31 << 10);
1478b3ee
DM		 2688 if (phys_addr < 1024 + fn)
2689 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2690 if (phys_addr < EEPROMSIZE)
1478b3ee 2691 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2692 return -EINVAL;
2693}
2694
2695/*
2696 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2697 */
2698static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2699{
1478b3ee 2700 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2701
2702 if (vaddr >= 0)
2703 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2704 return vaddr < 0 ? vaddr : 0;
2705}
2706
2707static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2708{
1478b3ee 2709 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2710
2711 if (vaddr >= 0)
2712 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2713 return vaddr < 0 ? vaddr : 0;
2714}
2715
2716#define EEPROM_MAGIC 0x38E2F10C
2717
2718static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2719 u8 *data)
2720{
2721 int i, err = 0;
2722 struct adapter *adapter = netdev2adap(dev);
2723
2724 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2725 if (!buf)
2726 return -ENOMEM;
2727
2728 e->magic = EEPROM_MAGIC;
2729 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2730 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2731
2732 if (!err)
2733 memcpy(data, buf + e->offset, e->len);
2734 kfree(buf);
2735 return err;
2736}
2737
2738static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2739 u8 *data)
2740{
2741 u8 *buf;
2742 int err = 0;
2743 u32 aligned_offset, aligned_len, *p;
2744 struct adapter *adapter = netdev2adap(dev);
2745
2746 if (eeprom->magic != EEPROM_MAGIC)
2747 return -EINVAL;
2748
2749 aligned_offset = eeprom->offset & ~3;
2750 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2751
1478b3ee
DM		 2752 if (adapter->fn > 0) {
2753 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2754
2755 if (aligned_offset < start ||
2756 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2757 return -EPERM;
2758 }
2759
b8ff05a9
DM		 2760 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2761 /*
2762 * RMW possibly needed for first or last words.
2763 */
2764 buf = kmalloc(aligned_len, GFP_KERNEL);
2765 if (!buf)
2766 return -ENOMEM;
2767 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2768 if (!err && aligned_len > 4)
2769 err = eeprom_rd_phys(adapter,
2770 aligned_offset + aligned_len - 4,
2771 (u32 *)&buf[aligned_len - 4]);
2772 if (err)
2773 goto out;
2774 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2775 } else
2776 buf = data;
2777
2778 err = t4_seeprom_wp(adapter, false);
2779 if (err)
2780 goto out;
2781
2782 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2783 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2784 aligned_offset += 4;
2785 }
2786
2787 if (!err)
2788 err = t4_seeprom_wp(adapter, true);
2789out:
2790 if (buf != data)
2791 kfree(buf);
2792 return err;
2793}
2794
2795static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2796{
2797 int ret;
2798 const struct firmware *fw;
2799 struct adapter *adap = netdev2adap(netdev);
b2e1a3f0 2800 unsigned int mbox = PCIE_FW_MASTER_M + 1;
b8ff05a9
DM		 2801
2802 ef->data[sizeof(ef->data) - 1] = '\0';
2803 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2804 if (ret < 0)
2805 return ret;
2806
22c0b963
HS		 2807 /* If the adapter has been fully initialized then we'll go ahead and
2808 * try to get the firmware's cooperation in upgrading to the new
2809 * firmware image otherwise we'll try to do the entire job from the
2810 * host ... and we always "force" the operation in this path.
2811 */
2812 if (adap->flags & FULL_INIT_DONE)
2813 mbox = adap->mbox;
2814
2815 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
b8ff05a9
DM		 2816 release_firmware(fw);
2817 if (!ret)
22c0b963
HS		 2818 dev_info(adap->pdev_dev, "loaded firmware %s,"
2819 " reload cxgb4 driver\n", ef->data);
b8ff05a9
DM		 2820 return ret;
2821}
2822
2823#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2824#define BCAST_CRC 0xa0ccc1a6
2825
2826static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2827{
2828 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2829 wol->wolopts = netdev2adap(dev)->wol;
2830 memset(&wol->sopass, 0, sizeof(wol->sopass));
2831}
2832
2833static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2834{
2835 int err = 0;
2836 struct port_info *pi = netdev_priv(dev);
2837
2838 if (wol->wolopts & ~WOL_SUPPORTED)
2839 return -EINVAL;
2840 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2841 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2842 if (wol->wolopts & WAKE_BCAST) {
2843 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2844 ~0ULL, 0, false);
2845 if (!err)
2846 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2847 ~6ULL, ~0ULL, BCAST_CRC, true);
2848 } else
2849 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2850 return err;
2851}
2852
c8f44aff 2853static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2854{
2ed28baa 2855 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2856 netdev_features_t changed = dev->features ^ features;
19ecae2c 2857 int err;
19ecae2c 2858
f646968f 2859 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2860 return 0;
19ecae2c 2861
2ed28baa
MM		 2862 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2863 -1, -1, -1,
f646968f 2864 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2865 if (unlikely(err))
f646968f 2866 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2867 return err;
87b6cf51
DM		 2868}
2869
7850f63f 2870static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2871{
2872 const struct port_info *pi = netdev_priv(dev);
671b0060 2873
7850f63f
BH		 2874 return pi->rss_size;
2875}
2876
892311f6 2877static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
7850f63f
BH		 2878{
2879 const struct port_info *pi = netdev_priv(dev);
2880 unsigned int n = pi->rss_size;
2881
892311f6
EP		 2882 if (hfunc)
2883 *hfunc = ETH_RSS_HASH_TOP;
2884 if (!p)
2885 return 0;
671b0060 2886 while (n--)
7850f63f 2887 p[n] = pi->rss[n];
671b0060
DM		 2888 return 0;
2889}
2890
892311f6
EP		 2891static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2892 const u8 hfunc)
671b0060
DM		 2893{
2894 unsigned int i;
2895 struct port_info *pi = netdev_priv(dev);
2896
892311f6
EP		 2897 /* We require at least one supported parameter to be changed and no
2898 * change in any of the unsupported parameters
2899 */
2900 if (key ||
2901 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2902 return -EOPNOTSUPP;
2903 if (!p)
2904 return 0;
2905
7850f63f
BH		 2906 for (i = 0; i < pi->rss_size; i++)
2907 pi->rss[i] = p[i];
671b0060
DM		 2908 if (pi->adapter->flags & FULL_INIT_DONE)
2909 return write_rss(pi, pi->rss);
2910 return 0;
2911}
2912
2913static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2914 u32 *rules)
671b0060 2915{
f796564a
DM		 2916 const struct port_info *pi = netdev_priv(dev);
2917
671b0060 2918 switch (info->cmd) {
f796564a
DM		 2919 case ETHTOOL_GRXFH: {
2920 unsigned int v = pi->rss_mode;
2921
2922 info->data = 0;
2923 switch (info->flow_type) {
2924 case TCP_V4_FLOW:
b2e1a3f0 2925 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
f796564a
DM		 2926 info->data = RXH_IP_SRC | RXH_IP_DST |
2927 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2928 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2929 info->data = RXH_IP_SRC | RXH_IP_DST;
2930 break;
2931 case UDP_V4_FLOW:
b2e1a3f0
HS		 2932 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2933 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2934 info->data = RXH_IP_SRC | RXH_IP_DST |
2935 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2936 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2937 info->data = RXH_IP_SRC | RXH_IP_DST;
2938 break;
2939 case SCTP_V4_FLOW:
2940 case AH_ESP_V4_FLOW:
2941 case IPV4_FLOW:
b2e1a3f0 2942 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2943 info->data = RXH_IP_SRC | RXH_IP_DST;
2944 break;
2945 case TCP_V6_FLOW:
b2e1a3f0 2946 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
f796564a
DM		 2947 info->data = RXH_IP_SRC | RXH_IP_DST |
2948 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2949 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2950 info->data = RXH_IP_SRC | RXH_IP_DST;
2951 break;
2952 case UDP_V6_FLOW:
b2e1a3f0
HS		 2953 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
2954 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2955 info->data = RXH_IP_SRC | RXH_IP_DST |
2956 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2957 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2958 info->data = RXH_IP_SRC | RXH_IP_DST;
2959 break;
2960 case SCTP_V6_FLOW:
2961 case AH_ESP_V6_FLOW:
2962 case IPV6_FLOW:
b2e1a3f0 2963 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2964 info->data = RXH_IP_SRC | RXH_IP_DST;
2965 break;
2966 }
2967 return 0;
2968 }
671b0060 2969 case ETHTOOL_GRXRINGS:
f796564a 2970 info->data = pi->nqsets;
671b0060
DM		 2971 return 0;
2972 }
2973 return -EOPNOTSUPP;
2974}
2975
9b07be4b 2976static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 2977 .get_settings = get_settings,
2978 .set_settings = set_settings,
2979 .get_drvinfo = get_drvinfo,
2980 .get_msglevel = get_msglevel,
2981 .set_msglevel = set_msglevel,
2982 .get_ringparam = get_sge_param,
2983 .set_ringparam = set_sge_param,
2984 .get_coalesce = get_coalesce,
2985 .set_coalesce = set_coalesce,
2986 .get_eeprom_len = get_eeprom_len,
2987 .get_eeprom = get_eeprom,
2988 .set_eeprom = set_eeprom,
2989 .get_pauseparam = get_pauseparam,
2990 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2991 .get_link = ethtool_op_get_link,
2992 .get_strings = get_strings,
c5e06360 2993 .set_phys_id = identify_port,
b8ff05a9
DM		 2994 .nway_reset = restart_autoneg,
2995 .get_sset_count = get_sset_count,
2996 .get_ethtool_stats = get_stats,
2997 .get_regs_len = get_regs_len,
2998 .get_regs = get_regs,
2999 .get_wol = get_wol,
3000 .set_wol = set_wol,
671b0060 3001 .get_rxnfc = get_rxnfc,
7850f63f 3002 .get_rxfh_indir_size = get_rss_table_size,
fe62d001
BH		 3003 .get_rxfh = get_rss_table,
3004 .set_rxfh = set_rss_table,
b8ff05a9
DM		 3005 .flash_device = set_flash,
3006};
3007
91744948 3008static int setup_debugfs(struct adapter *adap)
b8ff05a9 3009{
b8ff05a9
DM		 3010 if (IS_ERR_OR_NULL(adap->debugfs_root))
3011 return -1;
3012
fd88b31a
HS		 3013#ifdef CONFIG_DEBUG_FS
3014 t4_setup_debugfs(adap);
3015#endif
b8ff05a9
DM		 3016 return 0;
3017}
3018
3019/*
3020 * upper-layer driver support
3021 */
3022
3023/*
3024 * Allocate an active-open TID and set it to the supplied value.
3025 */
3026int cxgb4_alloc_atid(struct tid_info *t, void *data)
3027{
3028 int atid = -1;
3029
3030 spin_lock_bh(&t->atid_lock);
3031 if (t->afree) {
3032 union aopen_entry *p = t->afree;
3033
f2b7e78d 3034 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 3035 t->afree = p->next;
3036 p->data = data;
3037 t->atids_in_use++;
3038 }
3039 spin_unlock_bh(&t->atid_lock);
3040 return atid;
3041}
3042EXPORT_SYMBOL(cxgb4_alloc_atid);
3043
3044/*
3045 * Release an active-open TID.
3046 */
3047void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3048{
f2b7e78d 3049 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 3050
3051 spin_lock_bh(&t->atid_lock);
3052 p->next = t->afree;
3053 t->afree = p;
3054 t->atids_in_use--;
3055 spin_unlock_bh(&t->atid_lock);
3056}
3057EXPORT_SYMBOL(cxgb4_free_atid);
3058
3059/*
3060 * Allocate a server TID and set it to the supplied value.
3061 */
3062int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3063{
3064 int stid;
3065
3066 spin_lock_bh(&t->stid_lock);
3067 if (family == PF_INET) {
3068 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3069 if (stid < t->nstids)
3070 __set_bit(stid, t->stid_bmap);
3071 else
3072 stid = -1;
3073 } else {
3074 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3075 if (stid < 0)
3076 stid = -1;
3077 }
3078 if (stid >= 0) {
3079 t->stid_tab[stid].data = data;
3080 stid += t->stid_base;
15f63b74
KS		 3081 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3082 * This is equivalent to 4 TIDs. With CLIP enabled it
3083 * needs 2 TIDs.
3084 */
3085 if (family == PF_INET)
3086 t->stids_in_use++;
3087 else
3088 t->stids_in_use += 4;
b8ff05a9
DM		 3089 }
3090 spin_unlock_bh(&t->stid_lock);
3091 return stid;
3092}
3093EXPORT_SYMBOL(cxgb4_alloc_stid);
3094
dca4faeb
VP		 3095/* Allocate a server filter TID and set it to the supplied value.
3096 */
3097int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3098{
3099 int stid;
3100
3101 spin_lock_bh(&t->stid_lock);
3102 if (family == PF_INET) {
3103 stid = find_next_zero_bit(t->stid_bmap,
3104 t->nstids + t->nsftids, t->nstids);
3105 if (stid < (t->nstids + t->nsftids))
3106 __set_bit(stid, t->stid_bmap);
3107 else
3108 stid = -1;
3109 } else {
3110 stid = -1;
3111 }
3112 if (stid >= 0) {
3113 t->stid_tab[stid].data = data;
470c60c4
KS		 3114 stid -= t->nstids;
3115 stid += t->sftid_base;
dca4faeb
VP		 3116 t->stids_in_use++;
3117 }
3118 spin_unlock_bh(&t->stid_lock);
3119 return stid;
3120}
3121EXPORT_SYMBOL(cxgb4_alloc_sftid);
3122
3123/* Release a server TID.
b8ff05a9
DM		 3124 */
3125void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3126{
470c60c4
KS		 3127 /* Is it a server filter TID? */
3128 if (t->nsftids && (stid >= t->sftid_base)) {
3129 stid -= t->sftid_base;
3130 stid += t->nstids;
3131 } else {
3132 stid -= t->stid_base;
3133 }
3134
b8ff05a9
DM		 3135 spin_lock_bh(&t->stid_lock);
3136 if (family == PF_INET)
3137 __clear_bit(stid, t->stid_bmap);
3138 else
3139 bitmap_release_region(t->stid_bmap, stid, 2);
3140 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3141 if (family == PF_INET)
3142 t->stids_in_use--;
3143 else
3144 t->stids_in_use -= 4;
b8ff05a9
DM		 3145 spin_unlock_bh(&t->stid_lock);
3146}
3147EXPORT_SYMBOL(cxgb4_free_stid);
3148
3149/*
3150 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3151 */
3152static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3153 unsigned int tid)
3154{
3155 struct cpl_tid_release *req;
3156
3157 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3158 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3159 INIT_TP_WR(req, tid);
3160 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3161}
3162
3163/*
3164 * Queue a TID release request and if necessary schedule a work queue to
3165 * process it.
3166 */
31b9c19b 3167static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3168 unsigned int tid)
b8ff05a9
DM		 3169{
3170 void **p = &t->tid_tab[tid];
3171 struct adapter *adap = container_of(t, struct adapter, tids);
3172
3173 spin_lock_bh(&adap->tid_release_lock);
3174 *p = adap->tid_release_head;
3175 /* Low 2 bits encode the Tx channel number */
3176 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3177 if (!adap->tid_release_task_busy) {
3178 adap->tid_release_task_busy = true;
29aaee65 3179 queue_work(adap->workq, &adap->tid_release_task);
b8ff05a9
DM		 3180 }
3181 spin_unlock_bh(&adap->tid_release_lock);
3182}
b8ff05a9
DM		 3183
3184/*
3185 * Process the list of pending TID release requests.
3186 */
3187static void process_tid_release_list(struct work_struct *work)
3188{
3189 struct sk_buff *skb;
3190 struct adapter *adap;
3191
3192 adap = container_of(work, struct adapter, tid_release_task);
3193
3194 spin_lock_bh(&adap->tid_release_lock);
3195 while (adap->tid_release_head) {
3196 void **p = adap->tid_release_head;
3197 unsigned int chan = (uintptr_t)p & 3;
3198 p = (void *)p - chan;
3199
3200 adap->tid_release_head = *p;
3201 *p = NULL;
3202 spin_unlock_bh(&adap->tid_release_lock);
3203
3204 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3205 GFP_KERNEL)))
3206 schedule_timeout_uninterruptible(1);
3207
3208 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3209 t4_ofld_send(adap, skb);
3210 spin_lock_bh(&adap->tid_release_lock);
3211 }
3212 adap->tid_release_task_busy = false;
3213 spin_unlock_bh(&adap->tid_release_lock);
3214}
3215
3216/*
3217 * Release a TID and inform HW. If we are unable to allocate the release
3218 * message we defer to a work queue.
3219 */
3220void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3221{
3222 void *old;
3223 struct sk_buff *skb;
3224 struct adapter *adap = container_of(t, struct adapter, tids);
3225
3226 old = t->tid_tab[tid];
3227 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3228 if (likely(skb)) {
3229 t->tid_tab[tid] = NULL;
3230 mk_tid_release(skb, chan, tid);
3231 t4_ofld_send(adap, skb);
3232 } else
3233 cxgb4_queue_tid_release(t, chan, tid);
3234 if (old)
3235 atomic_dec(&t->tids_in_use);
3236}
3237EXPORT_SYMBOL(cxgb4_remove_tid);
3238
3239/*
3240 * Allocate and initialize the TID tables. Returns 0 on success.
3241 */
3242static int tid_init(struct tid_info *t)
3243{
3244 size_t size;
f2b7e78d 3245 unsigned int stid_bmap_size;
b8ff05a9 3246 unsigned int natids = t->natids;
b6f8eaec 3247 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3248
dca4faeb 3249 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3250 size = t->ntids * sizeof(*t->tid_tab) +
3251 natids * sizeof(*t->atid_tab) +
b8ff05a9 3252 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3253 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3254 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3255 t->nftids * sizeof(*t->ftid_tab) +
3256 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3257
b8ff05a9
DM		 3258 t->tid_tab = t4_alloc_mem(size);
3259 if (!t->tid_tab)
3260 return -ENOMEM;
3261
3262 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3263 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3264 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3265 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3266 spin_lock_init(&t->stid_lock);
3267 spin_lock_init(&t->atid_lock);
3268
3269 t->stids_in_use = 0;
3270 t->afree = NULL;
3271 t->atids_in_use = 0;
3272 atomic_set(&t->tids_in_use, 0);
3273
3274 /* Setup the free list for atid_tab and clear the stid bitmap. */
3275 if (natids) {
3276 while (--natids)
3277 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3278 t->afree = t->atid_tab;
3279 }
dca4faeb 3280 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3281 /* Reserve stid 0 for T4/T5 adapters */
3282 if (!t->stid_base &&
3283 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3284 __set_bit(0, t->stid_bmap);
3285
b8ff05a9
DM		 3286 return 0;
3287}
3288
3289/**
3290 * cxgb4_create_server - create an IP server
3291 * @dev: the device
3292 * @stid: the server TID
3293 * @sip: local IP address to bind server to
3294 * @sport: the server's TCP port
3295 * @queue: queue to direct messages from this server to
3296 *
3297 * Create an IP server for the given port and address.
3298 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3299 */
3300int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3301 __be32 sip, __be16 sport, __be16 vlan,
3302 unsigned int queue)
b8ff05a9
DM		 3303{
3304 unsigned int chan;
3305 struct sk_buff *skb;
3306 struct adapter *adap;
3307 struct cpl_pass_open_req *req;
80f40c1f 3308 int ret;
b8ff05a9
DM		 3309
3310 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3311 if (!skb)
3312 return -ENOMEM;
3313
3314 adap = netdev2adap(dev);
3315 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3316 INIT_TP_WR(req, 0);
3317 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3318 req->local_port = sport;
3319 req->peer_port = htons(0);
3320 req->local_ip = sip;
3321 req->peer_ip = htonl(0);
e46dab4d 3322 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3323 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
6c53e938
HS		 3324 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3325 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
80f40c1f
VP		 3326 ret = t4_mgmt_tx(adap, skb);
3327 return net_xmit_eval(ret);
b8ff05a9
DM		 3328}
3329EXPORT_SYMBOL(cxgb4_create_server);
3330
80f40c1f
VP		 3331/* cxgb4_create_server6 - create an IPv6 server
3332 * @dev: the device
3333 * @stid: the server TID
3334 * @sip: local IPv6 address to bind server to
3335 * @sport: the server's TCP port
3336 * @queue: queue to direct messages from this server to
3337 *
3338 * Create an IPv6 server for the given port and address.
3339 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3340 */
3341int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3342 const struct in6_addr *sip, __be16 sport,
3343 unsigned int queue)
3344{
3345 unsigned int chan;
3346 struct sk_buff *skb;
3347 struct adapter *adap;
3348 struct cpl_pass_open_req6 *req;
3349 int ret;
3350
3351 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3352 if (!skb)
3353 return -ENOMEM;
3354
3355 adap = netdev2adap(dev);
3356 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3357 INIT_TP_WR(req, 0);
3358 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3359 req->local_port = sport;
3360 req->peer_port = htons(0);
3361 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3362 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3363 req->peer_ip_hi = cpu_to_be64(0);
3364 req->peer_ip_lo = cpu_to_be64(0);
3365 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3366 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
6c53e938
HS		 3367 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3368 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
80f40c1f
VP		 3369 ret = t4_mgmt_tx(adap, skb);
3370 return net_xmit_eval(ret);
3371}
3372EXPORT_SYMBOL(cxgb4_create_server6);
3373
3374int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3375 unsigned int queue, bool ipv6)
3376{
3377 struct sk_buff *skb;
3378 struct adapter *adap;
3379 struct cpl_close_listsvr_req *req;
3380 int ret;
3381
3382 adap = netdev2adap(dev);
3383
3384 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3385 if (!skb)
3386 return -ENOMEM;
3387
3388 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3389 INIT_TP_WR(req, 0);
3390 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
bdc590b9
HS		 3391 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
3392 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
80f40c1f
VP		 3393 ret = t4_mgmt_tx(adap, skb);
3394 return net_xmit_eval(ret);
3395}
3396EXPORT_SYMBOL(cxgb4_remove_server);
3397
b8ff05a9
DM		 3398/**
3399 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3400 * @mtus: the HW MTU table
3401 * @mtu: the target MTU
3402 * @idx: index of selected entry in the MTU table
3403 *
3404 * Returns the index and the value in the HW MTU table that is closest to
3405 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3406 * table, in which case that smallest available value is selected.
3407 */
3408unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3409 unsigned int *idx)
3410{
3411 unsigned int i = 0;
3412
3413 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3414 ++i;
3415 if (idx)
3416 *idx = i;
3417 return mtus[i];
3418}
3419EXPORT_SYMBOL(cxgb4_best_mtu);
3420
92e7ae71
HS		 3421/**
3422 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3423 * @mtus: the HW MTU table
3424 * @header_size: Header Size
3425 * @data_size_max: maximum Data Segment Size
3426 * @data_size_align: desired Data Segment Size Alignment (2^N)
3427 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3428 *
3429 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3430 * MTU Table based solely on a Maximum MTU parameter, we break that
3431 * parameter up into a Header Size and Maximum Data Segment Size, and
3432 * provide a desired Data Segment Size Alignment. If we find an MTU in
3433 * the Hardware MTU Table which will result in a Data Segment Size with
3434 * the requested alignment _and_ that MTU isn't "too far" from the
3435 * closest MTU, then we'll return that rather than the closest MTU.
3436 */
3437unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3438 unsigned short header_size,
3439 unsigned short data_size_max,
3440 unsigned short data_size_align,
3441 unsigned int *mtu_idxp)
3442{
3443 unsigned short max_mtu = header_size + data_size_max;
3444 unsigned short data_size_align_mask = data_size_align - 1;
3445 int mtu_idx, aligned_mtu_idx;
3446
3447 /* Scan the MTU Table till we find an MTU which is larger than our
3448 * Maximum MTU or we reach the end of the table. Along the way,
3449 * record the last MTU found, if any, which will result in a Data
3450 * Segment Length matching the requested alignment.
3451 */
3452 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3453 unsigned short data_size = mtus[mtu_idx] - header_size;
3454
3455 /* If this MTU minus the Header Size would result in a
3456 * Data Segment Size of the desired alignment, remember it.
3457 */
3458 if ((data_size & data_size_align_mask) == 0)
3459 aligned_mtu_idx = mtu_idx;
3460
3461 /* If we're not at the end of the Hardware MTU Table and the
3462 * next element is larger than our Maximum MTU, drop out of
3463 * the loop.
3464 */
3465 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3466 break;
3467 }
3468
3469 /* If we fell out of the loop because we ran to the end of the table,
3470 * then we just have to use the last [largest] entry.
3471 */
3472 if (mtu_idx == NMTUS)
3473 mtu_idx--;
3474
3475 /* If we found an MTU which resulted in the requested Data Segment
3476 * Length alignment and that's "not far" from the largest MTU which is
3477 * less than or equal to the maximum MTU, then use that.
3478 */
3479 if (aligned_mtu_idx >= 0 &&
3480 mtu_idx - aligned_mtu_idx <= 1)
3481 mtu_idx = aligned_mtu_idx;
3482
3483 /* If the caller has passed in an MTU Index pointer, pass the
3484 * MTU Index back. Return the MTU value.
3485 */
3486 if (mtu_idxp)
3487 *mtu_idxp = mtu_idx;
3488 return mtus[mtu_idx];
3489}
3490EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3491
b8ff05a9
DM		 3492/**
3493 * cxgb4_port_chan - get the HW channel of a port
3494 * @dev: the net device for the port
3495 *
3496 * Return the HW Tx channel of the given port.
3497 */
3498unsigned int cxgb4_port_chan(const struct net_device *dev)
3499{
3500 return netdev2pinfo(dev)->tx_chan;
3501}
3502EXPORT_SYMBOL(cxgb4_port_chan);
3503
881806bc
VP		 3504unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3505{
3506 struct adapter *adap = netdev2adap(dev);
2cc301d2 3507 u32 v1, v2, lp_count, hp_count;
881806bc 3508
f061de42
HS		 3509 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3510 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3511 if (is_t4(adap->params.chip)) {
f061de42
HS		 3512 lp_count = LP_COUNT_G(v1);
3513 hp_count = HP_COUNT_G(v1);
2cc301d2 3514 } else {
f061de42
HS		 3515 lp_count = LP_COUNT_T5_G(v1);
3516 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3517 }
3518 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3519}
3520EXPORT_SYMBOL(cxgb4_dbfifo_count);
3521
b8ff05a9
DM		 3522/**
3523 * cxgb4_port_viid - get the VI id of a port
3524 * @dev: the net device for the port
3525 *
3526 * Return the VI id of the given port.
3527 */
3528unsigned int cxgb4_port_viid(const struct net_device *dev)
3529{
3530 return netdev2pinfo(dev)->viid;
3531}
3532EXPORT_SYMBOL(cxgb4_port_viid);
3533
3534/**
3535 * cxgb4_port_idx - get the index of a port
3536 * @dev: the net device for the port
3537 *
3538 * Return the index of the given port.
3539 */
3540unsigned int cxgb4_port_idx(const struct net_device *dev)
3541{
3542 return netdev2pinfo(dev)->port_id;
3543}
3544EXPORT_SYMBOL(cxgb4_port_idx);
3545
b8ff05a9
DM		 3546void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3547 struct tp_tcp_stats *v6)
3548{
3549 struct adapter *adap = pci_get_drvdata(pdev);
3550
3551 spin_lock(&adap->stats_lock);
3552 t4_tp_get_tcp_stats(adap, v4, v6);
3553 spin_unlock(&adap->stats_lock);
3554}
3555EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3556
3557void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3558 const unsigned int *pgsz_order)
3559{
3560 struct adapter *adap = netdev2adap(dev);
3561
0d804338
HS		 3562 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3563 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3564 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3565 HPZ3_V(pgsz_order[3]));
b8ff05a9
DM		 3566}
3567EXPORT_SYMBOL(cxgb4_iscsi_init);
3568
3069ee9b
VP		 3569int cxgb4_flush_eq_cache(struct net_device *dev)
3570{
3571 struct adapter *adap = netdev2adap(dev);
3572 int ret;
3573
3574 ret = t4_fwaddrspace_write(adap, adap->mbox,
f061de42 3575 0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3069ee9b
VP		 3576 return ret;
3577}
3578EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3579
3580static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3581{
f061de42 3582 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3069ee9b
VP		 3583 __be64 indices;
3584 int ret;
3585
fc5ab020
HS		 3586 spin_lock(&adap->win0_lock);
3587 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3588 sizeof(indices), (__be32 *)&indices,
3589 T4_MEMORY_READ);
3590 spin_unlock(&adap->win0_lock);
3069ee9b 3591 if (!ret) {
404d9e3f
VP		 3592 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3593 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3594 }
3595 return ret;
3596}
3597
3598int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3599 u16 size)
3600{
3601 struct adapter *adap = netdev2adap(dev);
3602 u16 hw_pidx, hw_cidx;
3603 int ret;
3604
3605 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3606 if (ret)
3607 goto out;
3608
3609 if (pidx != hw_pidx) {
3610 u16 delta;
f612b815 3611 u32 val;
3069ee9b
VP		 3612
3613 if (pidx >= hw_pidx)
3614 delta = pidx - hw_pidx;
3615 else
3616 delta = size - hw_pidx + pidx;
f612b815
HS		 3617
3618 if (is_t4(adap->params.chip))
3619 val = PIDX_V(delta);
3620 else
3621 val = PIDX_T5_V(delta);
3069ee9b 3622 wmb();
f612b815
HS		 3623 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3624 QID_V(qid) | val);
3069ee9b
VP		 3625 }
3626out:
3627 return ret;
3628}
3629EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3630
3cbdb928
VP		 3631void cxgb4_disable_db_coalescing(struct net_device *dev)
3632{
3633 struct adapter *adap;
3634
3635 adap = netdev2adap(dev);
f061de42 3636 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
f612b815 3637 NOCOALESCE_F);
3cbdb928
VP		 3638}
3639EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3640
3641void cxgb4_enable_db_coalescing(struct net_device *dev)
3642{
3643 struct adapter *adap;
3644
3645 adap = netdev2adap(dev);
f061de42 3646 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3cbdb928
VP		 3647}
3648EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3649
031cf476
HS		 3650int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3651{
3652 struct adapter *adap;
3653 u32 offset, memtype, memaddr;
6559a7e8 3654 u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
031cf476
HS		 3655 u32 edc0_end, edc1_end, mc0_end, mc1_end;
3656 int ret;
3657
3658 adap = netdev2adap(dev);
3659
3660 offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3661
3662 /* Figure out where the offset lands in the Memory Type/Address scheme.
3663 * This code assumes that the memory is laid out starting at offset 0
3664 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3665 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have
3666 * MC0, and some have both MC0 and MC1.
3667 */
6559a7e8
HS		 3668 size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3669 edc0_size = EDRAM0_SIZE_G(size) << 20;
3670 size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3671 edc1_size = EDRAM1_SIZE_G(size) << 20;
3672 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3673 mc0_size = EXT_MEM0_SIZE_G(size) << 20;
031cf476
HS		 3674
3675 edc0_end = edc0_size;
3676 edc1_end = edc0_end + edc1_size;
3677 mc0_end = edc1_end + mc0_size;
3678
3679 if (offset < edc0_end) {
3680 memtype = MEM_EDC0;
3681 memaddr = offset;
3682 } else if (offset < edc1_end) {
3683 memtype = MEM_EDC1;
3684 memaddr = offset - edc0_end;
3685 } else {
3686 if (offset < mc0_end) {
3687 memtype = MEM_MC0;
3688 memaddr = offset - edc1_end;
3689 } else if (is_t4(adap->params.chip)) {
3690 /* T4 only has a single memory channel */
3691 goto err;
3692 } else {
6559a7e8
HS		 3693 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3694 mc1_size = EXT_MEM1_SIZE_G(size) << 20;
031cf476
HS		 3695 mc1_end = mc0_end + mc1_size;
3696 if (offset < mc1_end) {
3697 memtype = MEM_MC1;
3698 memaddr = offset - mc0_end;
3699 } else {
3700 /* offset beyond the end of any memory */
3701 goto err;
3702 }
3703 }
3704 }
3705
3706 spin_lock(&adap->win0_lock);
3707 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3708 spin_unlock(&adap->win0_lock);
3709 return ret;
3710
3711err:
3712 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3713 stag, offset);
3714 return -EINVAL;
3715}
3716EXPORT_SYMBOL(cxgb4_read_tpte);
3717
7730b4c7
HS		 3718u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3719{
3720 u32 hi, lo;
3721 struct adapter *adap;
3722
3723 adap = netdev2adap(dev);
f612b815
HS		 3724 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3725 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
7730b4c7
HS		 3726
3727 return ((u64)hi << 32) | (u64)lo;
3728}
3729EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3730
df64e4d3
HS		 3731int cxgb4_bar2_sge_qregs(struct net_device *dev,
3732 unsigned int qid,
3733 enum cxgb4_bar2_qtype qtype,
3734 u64 *pbar2_qoffset,
3735 unsigned int *pbar2_qid)
3736{
dd0bcc0b 3737 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
df64e4d3
HS		 3738 qid,
3739 (qtype == CXGB4_BAR2_QTYPE_EGRESS
3740 ? T4_BAR2_QTYPE_EGRESS
3741 : T4_BAR2_QTYPE_INGRESS),
3742 pbar2_qoffset,
3743 pbar2_qid);
3744}
3745EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3746
b8ff05a9
DM		 3747static struct pci_driver cxgb4_driver;
3748
3749static void check_neigh_update(struct neighbour *neigh)
3750{
3751 const struct device *parent;
3752 const struct net_device *netdev = neigh->dev;
3753
3754 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3755 netdev = vlan_dev_real_dev(netdev);
3756 parent = netdev->dev.parent;
3757 if (parent && parent->driver == &cxgb4_driver.driver)
3758 t4_l2t_update(dev_get_drvdata(parent), neigh);
3759}
3760
3761static int netevent_cb(struct notifier_block *nb, unsigned long event,
3762 void *data)
3763{
3764 switch (event) {
3765 case NETEVENT_NEIGH_UPDATE:
3766 check_neigh_update(data);
3767 break;
b8ff05a9
DM		 3768 case NETEVENT_REDIRECT:
3769 default:
3770 break;
3771 }
3772 return 0;
3773}
3774
3775static bool netevent_registered;
3776static struct notifier_block cxgb4_netevent_nb = {
3777 .notifier_call = netevent_cb
3778};
3779
3069ee9b
VP		 3780static void drain_db_fifo(struct adapter *adap, int usecs)
3781{
2cc301d2 3782 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3783
3784 do {
f061de42
HS		 3785 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3786 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3787 if (is_t4(adap->params.chip)) {
f061de42
HS		 3788 lp_count = LP_COUNT_G(v1);
3789 hp_count = HP_COUNT_G(v1);
2cc301d2 3790 } else {
f061de42
HS		 3791 lp_count = LP_COUNT_T5_G(v1);
3792 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3793 }
3794
3795 if (lp_count == 0 && hp_count == 0)
3796 break;
3069ee9b
VP		 3797 set_current_state(TASK_UNINTERRUPTIBLE);
3798 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3799 } while (1);
3800}
3801
3802static void disable_txq_db(struct sge_txq *q)
3803{
05eb2389
SW		 3804 unsigned long flags;
3805
3806 spin_lock_irqsave(&q->db_lock, flags);
3069ee9b 3807 q->db_disabled = 1;
05eb2389 3808 spin_unlock_irqrestore(&q->db_lock, flags);
3069ee9b
VP		 3809}
3810
05eb2389 3811static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3069ee9b
VP		 3812{
3813 spin_lock_irq(&q->db_lock);
05eb2389
SW		 3814 if (q->db_pidx_inc) {
3815 /* Make sure that all writes to the TX descriptors
3816 * are committed before we tell HW about them.
3817 */
3818 wmb();
f612b815
HS		 3819 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3820 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
05eb2389
SW		 3821 q->db_pidx_inc = 0;
3822 }
3069ee9b
VP		 3823 q->db_disabled = 0;
3824 spin_unlock_irq(&q->db_lock);
3825}
3826
3827static void disable_dbs(struct adapter *adap)
3828{
3829 int i;
3830
3831 for_each_ethrxq(&adap->sge, i)
3832 disable_txq_db(&adap->sge.ethtxq[i].q);
3833 for_each_ofldrxq(&adap->sge, i)
3834 disable_txq_db(&adap->sge.ofldtxq[i].q);
3835 for_each_port(adap, i)
3836 disable_txq_db(&adap->sge.ctrlq[i].q);
3837}
3838
3839static void enable_dbs(struct adapter *adap)
3840{
3841 int i;
3842
3843 for_each_ethrxq(&adap->sge, i)
05eb2389 3844 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3069ee9b 3845 for_each_ofldrxq(&adap->sge, i)
05eb2389 3846 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3069ee9b 3847 for_each_port(adap, i)
05eb2389
SW		 3848 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3849}
3850
3851static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3852{
3853 if (adap->uld_handle[CXGB4_ULD_RDMA])
3854 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3855 cmd);
3856}
3857
3858static void process_db_full(struct work_struct *work)
3859{
3860 struct adapter *adap;
3861
3862 adap = container_of(work, struct adapter, db_full_task);
3863
3864 drain_db_fifo(adap, dbfifo_drain_delay);
3865 enable_dbs(adap);
3866 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
f612b815
HS		 3867 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3868 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3869 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3069ee9b
VP		 3870}
3871
3872static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3873{
3874 u16 hw_pidx, hw_cidx;
3875 int ret;
3876
05eb2389 3877 spin_lock_irq(&q->db_lock);
3069ee9b
VP		 3878 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3879 if (ret)
3880 goto out;
3881 if (q->db_pidx != hw_pidx) {
3882 u16 delta;
f612b815 3883 u32 val;
3069ee9b
VP		 3884
3885 if (q->db_pidx >= hw_pidx)
3886 delta = q->db_pidx - hw_pidx;
3887 else
3888 delta = q->size - hw_pidx + q->db_pidx;
f612b815
HS		 3889
3890 if (is_t4(adap->params.chip))
3891 val = PIDX_V(delta);
3892 else
3893 val = PIDX_T5_V(delta);
3069ee9b 3894 wmb();
f612b815
HS		 3895 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3896 QID_V(q->cntxt_id) | val);
3069ee9b
VP		 3897 }
3898out:
3899 q->db_disabled = 0;
05eb2389
SW		 3900 q->db_pidx_inc = 0;
3901 spin_unlock_irq(&q->db_lock);
3069ee9b
VP		 3902 if (ret)
3903 CH_WARN(adap, "DB drop recovery failed.\n");
3904}
3905static void recover_all_queues(struct adapter *adap)
3906{
3907 int i;
3908
3909 for_each_ethrxq(&adap->sge, i)
3910 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3911 for_each_ofldrxq(&adap->sge, i)
3912 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3913 for_each_port(adap, i)
3914 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3915}
3916
881806bc
VP		 3917static void process_db_drop(struct work_struct *work)
3918{
3919 struct adapter *adap;
881806bc 3920
3069ee9b 3921 adap = container_of(work, struct adapter, db_drop_task);
881806bc 3922
d14807dd 3923 if (is_t4(adap->params.chip)) {
05eb2389 3924 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3925 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
05eb2389 3926 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3927 recover_all_queues(adap);
05eb2389 3928 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3929 enable_dbs(adap);
05eb2389 3930 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2cc301d2
SR		 3931 } else {
3932 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3933 u16 qid = (dropped_db >> 15) & 0x1ffff;
3934 u16 pidx_inc = dropped_db & 0x1fff;
df64e4d3
HS		 3935 u64 bar2_qoffset;
3936 unsigned int bar2_qid;
3937 int ret;
2cc301d2 3938
dd0bcc0b 3939 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
df64e4d3
HS		 3940 &bar2_qoffset, &bar2_qid);
3941 if (ret)
3942 dev_err(adap->pdev_dev, "doorbell drop recovery: "
3943 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
3944 else
f612b815 3945 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
df64e4d3 3946 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
2cc301d2
SR		 3947
3948 /* Re-enable BAR2 WC */
3949 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3950 }
3951
f061de42 3952 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
881806bc
VP		 3953}
3954
3955void t4_db_full(struct adapter *adap)
3956{
d14807dd 3957 if (is_t4(adap->params.chip)) {
05eb2389
SW		 3958 disable_dbs(adap);
3959 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
f612b815
HS		 3960 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3961 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
29aaee65 3962 queue_work(adap->workq, &adap->db_full_task);
2cc301d2 3963 }
881806bc
VP		 3964}
3965
3966void t4_db_dropped(struct adapter *adap)
3967{
05eb2389
SW		 3968 if (is_t4(adap->params.chip)) {
3969 disable_dbs(adap);
3970 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3971 }
29aaee65 3972 queue_work(adap->workq, &adap->db_drop_task);
881806bc
VP		 3973}
3974
b8ff05a9
DM		 3975static void uld_attach(struct adapter *adap, unsigned int uld)
3976{
3977 void *handle;
3978 struct cxgb4_lld_info lli;
dca4faeb 3979 unsigned short i;
b8ff05a9
DM		 3980
3981 lli.pdev = adap->pdev;
35b1de55 3982 lli.pf = adap->fn;
b8ff05a9
DM		 3983 lli.l2t = adap->l2t;
3984 lli.tids = &adap->tids;
3985 lli.ports = adap->port;
3986 lli.vr = &adap->vres;
3987 lli.mtus = adap->params.mtus;
3988 if (uld == CXGB4_ULD_RDMA) {
3989 lli.rxq_ids = adap->sge.rdma_rxq;
cf38be6d 3990 lli.ciq_ids = adap->sge.rdma_ciq;
b8ff05a9 3991 lli.nrxq = adap->sge.rdmaqs;
cf38be6d 3992 lli.nciq = adap->sge.rdmaciqs;
b8ff05a9
DM		 3993 } else if (uld == CXGB4_ULD_ISCSI) {
3994 lli.rxq_ids = adap->sge.ofld_rxq;
3995 lli.nrxq = adap->sge.ofldqsets;
3996 }
3997 lli.ntxq = adap->sge.ofldqsets;
3998 lli.nchan = adap->params.nports;
3999 lli.nports = adap->params.nports;
4000 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 4001 lli.adapter_type = adap->params.chip;
837e4a42 4002 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
7730b4c7 4003 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
df64e4d3
HS		 4004 lli.udb_density = 1 << adap->params.sge.eq_qpp;
4005 lli.ucq_density = 1 << adap->params.sge.iq_qpp;
dcf7b6f5 4006 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 4007 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
4008 for (i = 0; i < NCHAN; i++)
4009 lli.tx_modq[i] = i;
f612b815
HS		 4010 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
4011 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
b8ff05a9 4012 lli.fw_vers = adap->params.fw_vers;
3069ee9b 4013 lli.dbfifo_int_thresh = dbfifo_int_thresh;
04e10e21
HS		 4014 lli.sge_ingpadboundary = adap->sge.fl_align;
4015 lli.sge_egrstatuspagesize = adap->sge.stat_len;
dca4faeb
VP		 4016 lli.sge_pktshift = adap->sge.pktshift;
4017 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
4c2c5763
HS		 4018 lli.max_ordird_qp = adap->params.max_ordird_qp;
4019 lli.max_ird_adapter = adap->params.max_ird_adapter;
1ac0f095 4020 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 4021
4022 handle = ulds[uld].add(&lli);
4023 if (IS_ERR(handle)) {
4024 dev_warn(adap->pdev_dev,
4025 "could not attach to the %s driver, error %ld\n",
4026 uld_str[uld], PTR_ERR(handle));
4027 return;
4028 }
4029
4030 adap->uld_handle[uld] = handle;
4031
4032 if (!netevent_registered) {
4033 register_netevent_notifier(&cxgb4_netevent_nb);
4034 netevent_registered = true;
4035 }
e29f5dbc
DM		 4036
4037 if (adap->flags & FULL_INIT_DONE)
4038 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 4039}
4040
4041static void attach_ulds(struct adapter *adap)
4042{
4043 unsigned int i;
4044
01bcca68
VP		 4045 spin_lock(&adap_rcu_lock);
4046 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4047 spin_unlock(&adap_rcu_lock);
4048
b8ff05a9
DM		 4049 mutex_lock(&uld_mutex);
4050 list_add_tail(&adap->list_node, &adapter_list);
4051 for (i = 0; i < CXGB4_ULD_MAX; i++)
4052 if (ulds[i].add)
4053 uld_attach(adap, i);
4054 mutex_unlock(&uld_mutex);
4055}
4056
4057static void detach_ulds(struct adapter *adap)
4058{
4059 unsigned int i;
4060
4061 mutex_lock(&uld_mutex);
4062 list_del(&adap->list_node);
4063 for (i = 0; i < CXGB4_ULD_MAX; i++)
4064 if (adap->uld_handle[i]) {
4065 ulds[i].state_change(adap->uld_handle[i],
4066 CXGB4_STATE_DETACH);
4067 adap->uld_handle[i] = NULL;
4068 }
4069 if (netevent_registered && list_empty(&adapter_list)) {
4070 unregister_netevent_notifier(&cxgb4_netevent_nb);
4071 netevent_registered = false;
4072 }
4073 mutex_unlock(&uld_mutex);
01bcca68
VP		 4074
4075 spin_lock(&adap_rcu_lock);
4076 list_del_rcu(&adap->rcu_node);
4077 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 4078}
4079
4080static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4081{
4082 unsigned int i;
4083
4084 mutex_lock(&uld_mutex);
4085 for (i = 0; i < CXGB4_ULD_MAX; i++)
4086 if (adap->uld_handle[i])
4087 ulds[i].state_change(adap->uld_handle[i], new_state);
4088 mutex_unlock(&uld_mutex);
4089}
4090
4091/**
4092 * cxgb4_register_uld - register an upper-layer driver
4093 * @type: the ULD type
4094 * @p: the ULD methods
4095 *
4096 * Registers an upper-layer driver with this driver and notifies the ULD
4097 * about any presently available devices that support its type. Returns
4098 * %-EBUSY if a ULD of the same type is already registered.
4099 */
4100int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4101{
4102 int ret = 0;
4103 struct adapter *adap;
4104
4105 if (type >= CXGB4_ULD_MAX)
4106 return -EINVAL;
4107 mutex_lock(&uld_mutex);
4108 if (ulds[type].add) {
4109 ret = -EBUSY;
4110 goto out;
4111 }
4112 ulds[type] = *p;
4113 list_for_each_entry(adap, &adapter_list, list_node)
4114 uld_attach(adap, type);
4115out: mutex_unlock(&uld_mutex);
4116 return ret;
4117}
4118EXPORT_SYMBOL(cxgb4_register_uld);
4119
4120/**
4121 * cxgb4_unregister_uld - unregister an upper-layer driver
4122 * @type: the ULD type
4123 *
4124 * Unregisters an existing upper-layer driver.
4125 */
4126int cxgb4_unregister_uld(enum cxgb4_uld type)
4127{
4128 struct adapter *adap;
4129
4130 if (type >= CXGB4_ULD_MAX)
4131 return -EINVAL;
4132 mutex_lock(&uld_mutex);
4133 list_for_each_entry(adap, &adapter_list, list_node)
4134 adap->uld_handle[type] = NULL;
4135 ulds[type].add = NULL;
4136 mutex_unlock(&uld_mutex);
4137 return 0;
4138}
4139EXPORT_SYMBOL(cxgb4_unregister_uld);
4140
1bb60376 4141#if IS_ENABLED(CONFIG_IPV6)
b5a02f50
AB		 4142static int cxgb4_inet6addr_handler(struct notifier_block *this,
4143 unsigned long event, void *data)
01bcca68 4144{
b5a02f50
AB		 4145 struct inet6_ifaddr *ifa = data;
4146 struct net_device *event_dev = ifa->idev->dev;
4147 const struct device *parent = NULL;
4148#if IS_ENABLED(CONFIG_BONDING)
01bcca68 4149 struct adapter *adap;
b5a02f50
AB		 4150#endif
4151 if (event_dev->priv_flags & IFF_802_1Q_VLAN)
4152 event_dev = vlan_dev_real_dev(event_dev);
4153#if IS_ENABLED(CONFIG_BONDING)
4154 if (event_dev->flags & IFF_MASTER) {
4155 list_for_each_entry(adap, &adapter_list, list_node) {
4156 switch (event) {
4157 case NETDEV_UP:
4158 cxgb4_clip_get(adap->port[0],
4159 (const u32 *)ifa, 1);
4160 break;
4161 case NETDEV_DOWN:
4162 cxgb4_clip_release(adap->port[0],
4163 (const u32 *)ifa, 1);
4164 break;
4165 default:
4166 break;
4167 }
4168 }
4169 return NOTIFY_OK;
4170 }
4171#endif
01bcca68 4172
b5a02f50
AB		 4173 if (event_dev)
4174 parent = event_dev->dev.parent;
01bcca68 4175
b5a02f50 4176 if (parent && parent->driver == &cxgb4_driver.driver) {
01bcca68
VP		 4177 switch (event) {
4178 case NETDEV_UP:
b5a02f50 4179 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
01bcca68
VP		 4180 break;
4181 case NETDEV_DOWN:
b5a02f50 4182 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
01bcca68
VP		 4183 break;
4184 default:
4185 break;
4186 }
4187 }
b5a02f50 4188 return NOTIFY_OK;
01bcca68
VP		 4189}
4190
b5a02f50 4191static bool inet6addr_registered;
01bcca68
VP		 4192static struct notifier_block cxgb4_inet6addr_notifier = {
4193 .notifier_call = cxgb4_inet6addr_handler
4194};
4195
01bcca68
VP		 4196static void update_clip(const struct adapter *adap)
4197{
4198 int i;
4199 struct net_device *dev;
4200 int ret;
4201
4202 rcu_read_lock();
4203
4204 for (i = 0; i < MAX_NPORTS; i++) {
4205 dev = adap->port[i];
4206 ret = 0;
4207
4208 if (dev)
b5a02f50 4209 ret = cxgb4_update_root_dev_clip(dev);
01bcca68
VP		 4210
4211 if (ret < 0)
4212 break;
4213 }
4214 rcu_read_unlock();
4215}
1bb60376 4216#endif /* IS_ENABLED(CONFIG_IPV6) */
01bcca68 4217
b8ff05a9
DM		 4218/**
4219 * cxgb_up - enable the adapter
4220 * @adap: adapter being enabled
4221 *
4222 * Called when the first port is enabled, this function performs the
4223 * actions necessary to make an adapter operational, such as completing
4224 * the initialization of HW modules, and enabling interrupts.
4225 *
4226 * Must be called with the rtnl lock held.
4227 */
4228static int cxgb_up(struct adapter *adap)
4229{
aaefae9b 4230 int err;
b8ff05a9 4231
aaefae9b
DM		 4232 err = setup_sge_queues(adap);
4233 if (err)
4234 goto out;
4235 err = setup_rss(adap);
4236 if (err)
4237 goto freeq;
b8ff05a9
DM		 4238
4239 if (adap->flags & USING_MSIX) {
aaefae9b 4240 name_msix_vecs(adap);
b8ff05a9
DM		 4241 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4242 adap->msix_info[0].desc, adap);
4243 if (err)
4244 goto irq_err;
4245
4246 err = request_msix_queue_irqs(adap);
4247 if (err) {
4248 free_irq(adap->msix_info[0].vec, adap);
4249 goto irq_err;
4250 }
4251 } else {
4252 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4253 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4254 adap->port[0]->name, adap);
b8ff05a9
DM		 4255 if (err)
4256 goto irq_err;
4257 }
4258 enable_rx(adap);
4259 t4_sge_start(adap);
4260 t4_intr_enable(adap);
aaefae9b 4261 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4262 notify_ulds(adap, CXGB4_STATE_UP);
1bb60376 4263#if IS_ENABLED(CONFIG_IPV6)
01bcca68 4264 update_clip(adap);
1bb60376 4265#endif
b8ff05a9
DM		 4266 out:
4267 return err;
4268 irq_err:
4269 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4270 freeq:
4271 t4_free_sge_resources(adap);
b8ff05a9
DM		 4272 goto out;
4273}
4274
4275static void cxgb_down(struct adapter *adapter)
4276{
b8ff05a9 4277 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4278 cancel_work_sync(&adapter->db_full_task);
4279 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4280 adapter->tid_release_task_busy = false;
204dc3c0 4281 adapter->tid_release_head = NULL;
b8ff05a9 4282
aaefae9b
DM		 4283 t4_sge_stop(adapter);
4284 t4_free_sge_resources(adapter);
4285 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4286}
4287
4288/*
4289 * net_device operations
4290 */
4291static int cxgb_open(struct net_device *dev)
4292{
4293 int err;
4294 struct port_info *pi = netdev_priv(dev);
4295 struct adapter *adapter = pi->adapter;
4296
6a3c869a
DM		 4297 netif_carrier_off(dev);
4298
aaefae9b
DM		 4299 if (!(adapter->flags & FULL_INIT_DONE)) {
4300 err = cxgb_up(adapter);
4301 if (err < 0)
4302 return err;
4303 }
b8ff05a9 4304
f68707b8
DM		 4305 err = link_start(dev);
4306 if (!err)
4307 netif_tx_start_all_queues(dev);
4308 return err;
b8ff05a9
DM		 4309}
4310
4311static int cxgb_close(struct net_device *dev)
4312{
b8ff05a9
DM		 4313 struct port_info *pi = netdev_priv(dev);
4314 struct adapter *adapter = pi->adapter;
4315
4316 netif_tx_stop_all_queues(dev);
4317 netif_carrier_off(dev);
060e0c75 4318 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4319}
4320
f2b7e78d
VP		 4321/* Return an error number if the indicated filter isn't writable ...
4322 */
4323static int writable_filter(struct filter_entry *f)
4324{
4325 if (f->locked)
4326 return -EPERM;
4327 if (f->pending)
4328 return -EBUSY;
4329
4330 return 0;
4331}
4332
4333/* Delete the filter at the specified index (if valid). The checks for all
4334 * the common problems with doing this like the filter being locked, currently
4335 * pending in another operation, etc.
4336 */
4337static int delete_filter(struct adapter *adapter, unsigned int fidx)
4338{
4339 struct filter_entry *f;
4340 int ret;
4341
dca4faeb 4342 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4343 return -EINVAL;
4344
4345 f = &adapter->tids.ftid_tab[fidx];
4346 ret = writable_filter(f);
4347 if (ret)
4348 return ret;
4349 if (f->valid)
4350 return del_filter_wr(adapter, fidx);
4351
4352 return 0;
4353}
4354
dca4faeb 4355int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4356 __be32 sip, __be16 sport, __be16 vlan,
4357 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4358{
4359 int ret;
4360 struct filter_entry *f;
4361 struct adapter *adap;
4362 int i;
4363 u8 *val;
4364
4365 adap = netdev2adap(dev);
4366
1cab775c 4367 /* Adjust stid to correct filter index */
470c60c4 4368 stid -= adap->tids.sftid_base;
1cab775c
VP		 4369 stid += adap->tids.nftids;
4370
dca4faeb
VP		 4371 /* Check to make sure the filter requested is writable ...
4372 */
4373 f = &adap->tids.ftid_tab[stid];
4374 ret = writable_filter(f);
4375 if (ret)
4376 return ret;
4377
4378 /* Clear out any old resources being used by the filter before
4379 * we start constructing the new filter.
4380 */
4381 if (f->valid)
4382 clear_filter(adap, f);
4383
4384 /* Clear out filter specifications */
4385 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4386 f->fs.val.lport = cpu_to_be16(sport);
4387 f->fs.mask.lport = ~0;
4388 val = (u8 *)&sip;
793dad94 4389 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4390 for (i = 0; i < 4; i++) {
4391 f->fs.val.lip[i] = val[i];
4392 f->fs.mask.lip[i] = ~0;
4393 }
0d804338 4394 if (adap->params.tp.vlan_pri_map & PORT_F) {
793dad94
VP		 4395 f->fs.val.iport = port;
4396 f->fs.mask.iport = mask;
4397 }
4398 }
dca4faeb 4399
0d804338 4400 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
7c89e555
KS		 4401 f->fs.val.proto = IPPROTO_TCP;
4402 f->fs.mask.proto = ~0;
4403 }
4404
dca4faeb
VP		 4405 f->fs.dirsteer = 1;
4406 f->fs.iq = queue;
4407 /* Mark filter as locked */
4408 f->locked = 1;
4409 f->fs.rpttid = 1;
4410
4411 ret = set_filter_wr(adap, stid);
4412 if (ret) {
4413 clear_filter(adap, f);
4414 return ret;
4415 }
4416
4417 return 0;
4418}
4419EXPORT_SYMBOL(cxgb4_create_server_filter);
4420
4421int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4422 unsigned int queue, bool ipv6)
4423{
4424 int ret;
4425 struct filter_entry *f;
4426 struct adapter *adap;
4427
4428 adap = netdev2adap(dev);
1cab775c
VP		 4429
4430 /* Adjust stid to correct filter index */
470c60c4 4431 stid -= adap->tids.sftid_base;
1cab775c
VP		 4432 stid += adap->tids.nftids;
4433
dca4faeb
VP		 4434 f = &adap->tids.ftid_tab[stid];
4435 /* Unlock the filter */
4436 f->locked = 0;
4437
4438 ret = delete_filter(adap, stid);
4439 if (ret)
4440 return ret;
4441
4442 return 0;
4443}
4444EXPORT_SYMBOL(cxgb4_remove_server_filter);
4445
f5152c90
DM		 4446static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4447 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4448{
4449 struct port_stats stats;
4450 struct port_info *p = netdev_priv(dev);
4451 struct adapter *adapter = p->adapter;
b8ff05a9 4452
9fe6cb58
GS		 4453 /* Block retrieving statistics during EEH error
4454 * recovery. Otherwise, the recovery might fail
4455 * and the PCI device will be removed permanently
4456 */
b8ff05a9 4457 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4458 if (!netif_device_present(dev)) {
4459 spin_unlock(&adapter->stats_lock);
4460 return ns;
4461 }
b8ff05a9
DM		 4462 t4_get_port_stats(adapter, p->tx_chan, &stats);
4463 spin_unlock(&adapter->stats_lock);
4464
4465 ns->tx_bytes = stats.tx_octets;
4466 ns->tx_packets = stats.tx_frames;
4467 ns->rx_bytes = stats.rx_octets;
4468 ns->rx_packets = stats.rx_frames;
4469 ns->multicast = stats.rx_mcast_frames;
4470
4471 /* detailed rx_errors */
4472 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4473 stats.rx_runt;
4474 ns->rx_over_errors = 0;
4475 ns->rx_crc_errors = stats.rx_fcs_err;
4476 ns->rx_frame_errors = stats.rx_symbol_err;
4477 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4478 stats.rx_ovflow2 + stats.rx_ovflow3 +
4479 stats.rx_trunc0 + stats.rx_trunc1 +
4480 stats.rx_trunc2 + stats.rx_trunc3;
4481 ns->rx_missed_errors = 0;
4482
4483 /* detailed tx_errors */
4484 ns->tx_aborted_errors = 0;
4485 ns->tx_carrier_errors = 0;
4486 ns->tx_fifo_errors = 0;
4487 ns->tx_heartbeat_errors = 0;
4488 ns->tx_window_errors = 0;
4489
4490 ns->tx_errors = stats.tx_error_frames;
4491 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4492 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4493 return ns;
4494}
4495
4496static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4497{
060e0c75 4498 unsigned int mbox;
b8ff05a9
DM		 4499 int ret = 0, prtad, devad;
4500 struct port_info *pi = netdev_priv(dev);
4501 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4502
4503 switch (cmd) {
4504 case SIOCGMIIPHY:
4505 if (pi->mdio_addr < 0)
4506 return -EOPNOTSUPP;
4507 data->phy_id = pi->mdio_addr;
4508 break;
4509 case SIOCGMIIREG:
4510 case SIOCSMIIREG:
4511 if (mdio_phy_id_is_c45(data->phy_id)) {
4512 prtad = mdio_phy_id_prtad(data->phy_id);
4513 devad = mdio_phy_id_devad(data->phy_id);
4514 } else if (data->phy_id < 32) {
4515 prtad = data->phy_id;
4516 devad = 0;
4517 data->reg_num &= 0x1f;
4518 } else
4519 return -EINVAL;
4520
060e0c75 4521 mbox = pi->adapter->fn;
b8ff05a9 4522 if (cmd == SIOCGMIIREG)
060e0c75 4523 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4524 data->reg_num, &data->val_out);
4525 else
060e0c75 4526 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4527 data->reg_num, data->val_in);
4528 break;
4529 default:
4530 return -EOPNOTSUPP;
4531 }
4532 return ret;
4533}
4534
4535static void cxgb_set_rxmode(struct net_device *dev)
4536{
4537 /* unfortunately we can't return errors to the stack */
4538 set_rxmode(dev, -1, false);
4539}
4540
4541static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4542{
4543 int ret;
4544 struct port_info *pi = netdev_priv(dev);
4545
4546 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4547 return -EINVAL;
060e0c75
DM		 4548 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4549 -1, -1, -1, true);
b8ff05a9
DM		 4550 if (!ret)
4551 dev->mtu = new_mtu;
4552 return ret;
4553}
4554
4555static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4556{
4557 int ret;
4558 struct sockaddr *addr = p;
4559 struct port_info *pi = netdev_priv(dev);
4560
4561 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4562 return -EADDRNOTAVAIL;
b8ff05a9 4563
060e0c75
DM		 4564 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4565 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4566 if (ret < 0)
4567 return ret;
4568
4569 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4570 pi->xact_addr_filt = ret;
4571 return 0;
4572}
4573
b8ff05a9
DM		 4574#ifdef CONFIG_NET_POLL_CONTROLLER
4575static void cxgb_netpoll(struct net_device *dev)
4576{
4577 struct port_info *pi = netdev_priv(dev);
4578 struct adapter *adap = pi->adapter;
4579
4580 if (adap->flags & USING_MSIX) {
4581 int i;
4582 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4583
4584 for (i = pi->nqsets; i; i--, rx++)
4585 t4_sge_intr_msix(0, &rx->rspq);
4586 } else
4587 t4_intr_handler(adap)(0, adap);
4588}
4589#endif
4590
4591static const struct net_device_ops cxgb4_netdev_ops = {
4592 .ndo_open = cxgb_open,
4593 .ndo_stop = cxgb_close,
4594 .ndo_start_xmit = t4_eth_xmit,
688848b1 4595 .ndo_select_queue = cxgb_select_queue,
9be793bf 4596 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4597 .ndo_set_rx_mode = cxgb_set_rxmode,
4598 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4599 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4600 .ndo_validate_addr = eth_validate_addr,
4601 .ndo_do_ioctl = cxgb_ioctl,
4602 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4603#ifdef CONFIG_NET_POLL_CONTROLLER
4604 .ndo_poll_controller = cxgb_netpoll,
4605#endif
84a200b3
VP		 4606#ifdef CONFIG_CHELSIO_T4_FCOE
4607 .ndo_fcoe_enable = cxgb_fcoe_enable,
4608 .ndo_fcoe_disable = cxgb_fcoe_disable,
4609#endif /* CONFIG_CHELSIO_T4_FCOE */
3a336cb1
HS		 4610#ifdef CONFIG_NET_RX_BUSY_POLL
4611 .ndo_busy_poll = cxgb_busy_poll,
4612#endif
4613
b8ff05a9
DM		 4614};
4615
4616void t4_fatal_err(struct adapter *adap)
4617{
f612b815 4618 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
b8ff05a9
DM		 4619 t4_intr_disable(adap);
4620 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4621}
4622
0abfd152
HS		 4623/* Return the specified PCI-E Configuration Space register from our Physical
4624 * Function. We try first via a Firmware LDST Command since we prefer to let
4625 * the firmware own all of these registers, but if that fails we go for it
4626 * directly ourselves.
4627 */
4628static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4629{
4630 struct fw_ldst_cmd ldst_cmd;
4631 u32 val;
4632 int ret;
4633
4634 /* Construct and send the Firmware LDST Command to retrieve the
4635 * specified PCI-E Configuration Space register.
4636 */
4637 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4638 ldst_cmd.op_to_addrspace =
e2ac9628
HS		 4639 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4640 FW_CMD_REQUEST_F |
4641 FW_CMD_READ_F |
5167865a 4642 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
0abfd152 4643 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
5167865a 4644 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
0abfd152 4645 ldst_cmd.u.pcie.ctrl_to_fn =
5167865a 4646 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
0abfd152
HS		 4647 ldst_cmd.u.pcie.r = reg;
4648 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4649 &ldst_cmd);
4650
4651 /* If the LDST Command suucceeded, exctract the returned register
4652 * value. Otherwise read it directly ourself.
4653 */
4654 if (ret == 0)
4655 val = ntohl(ldst_cmd.u.pcie.data[0]);
4656 else
4657 t4_hw_pci_read_cfg4(adap, reg, &val);
4658
4659 return val;
4660}
4661
b8ff05a9
DM		 4662static void setup_memwin(struct adapter *adap)
4663{
0abfd152 4664 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
b8ff05a9 4665
d14807dd 4666 if (is_t4(adap->params.chip)) {
0abfd152
HS		 4667 u32 bar0;
4668
4669 /* Truncation intentional: we only read the bottom 32-bits of
4670 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor
4671 * mechanism to read BAR0 instead of using
4672 * pci_resource_start() because we could be operating from
4673 * within a Virtual Machine which is trapping our accesses to
4674 * our Configuration Space and we need to set up the PCI-E
4675 * Memory Window decoders with the actual addresses which will
4676 * be coming across the PCI-E link.
4677 */
4678 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4679 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4680 adap->t4_bar0 = bar0;
4681
19dd37ba
SR		 4682 mem_win0_base = bar0 + MEMWIN0_BASE;
4683 mem_win1_base = bar0 + MEMWIN1_BASE;
4684 mem_win2_base = bar0 + MEMWIN2_BASE;
0abfd152 4685 mem_win2_aperture = MEMWIN2_APERTURE;
19dd37ba
SR		 4686 } else {
4687 /* For T5, only relative offset inside the PCIe BAR is passed */
4688 mem_win0_base = MEMWIN0_BASE;
0abfd152 4689 mem_win1_base = MEMWIN1_BASE;
19dd37ba 4690 mem_win2_base = MEMWIN2_BASE_T5;
0abfd152 4691 mem_win2_aperture = MEMWIN2_APERTURE_T5;
19dd37ba 4692 }
f061de42
HS		 4693 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4694 mem_win0_base | BIR_V(0) |
4695 WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4696 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4697 mem_win1_base | BIR_V(0) |
4698 WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4699 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4700 mem_win2_base | BIR_V(0) |
4701 WINDOW_V(ilog2(mem_win2_aperture) - 10));
4702 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
636f9d37
VP		 4703}
4704
4705static void setup_memwin_rdma(struct adapter *adap)
4706{
1ae970e0 4707 if (adap->vres.ocq.size) {
0abfd152
HS		 4708 u32 start;
4709 unsigned int sz_kb;
1ae970e0 4710
0abfd152
HS		 4711 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4712 start &= PCI_BASE_ADDRESS_MEM_MASK;
4713 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
1ae970e0
DM		 4714 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4715 t4_write_reg(adap,
f061de42
HS		 4716 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4717 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
1ae970e0 4718 t4_write_reg(adap,
f061de42 4719 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
1ae970e0
DM		 4720 adap->vres.ocq.start);
4721 t4_read_reg(adap,
f061de42 4722 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
1ae970e0 4723 }
b8ff05a9
DM		 4724}
4725
02b5fb8e
DM		 4726static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4727{
4728 u32 v;
4729 int ret;
4730
4731 /* get device capabilities */
4732 memset(c, 0, sizeof(*c));
e2ac9628
HS		 4733 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4734 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 4735 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4736 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4737 if (ret < 0)
4738 return ret;
4739
4740 /* select capabilities we'll be using */
4741 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4742 if (!vf_acls)
4743 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4744 else
4745 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4746 } else if (vf_acls) {
4747 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4748 return ret;
4749 }
e2ac9628
HS		 4750 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4751 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
060e0c75 4752 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4753 if (ret < 0)
4754 return ret;
4755
060e0c75 4756 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e 4757 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
b2e1a3f0
HS		 4758 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4759 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
02b5fb8e
DM		 4760 if (ret < 0)
4761 return ret;
4762
4b8e27a8
HS		 4763 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, adap->sge.egr_sz, 64,
4764 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF,
4765 FW_CMD_CAP_PF);
02b5fb8e
DM		 4766 if (ret < 0)
4767 return ret;
4768
4769 t4_sge_init(adap);
4770
02b5fb8e 4771 /* tweak some settings */
837e4a42 4772 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
0d804338 4773 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
837e4a42
HS		 4774 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4775 v = t4_read_reg(adap, TP_PIO_DATA_A);
4776 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
060e0c75 4777
dca4faeb
VP		 4778 /* first 4 Tx modulation queues point to consecutive Tx channels */
4779 adap->params.tp.tx_modq_map = 0xE4;
0d804338
HS		 4780 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4781 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
dca4faeb
VP		 4782
4783 /* associate each Tx modulation queue with consecutive Tx channels */
4784 v = 0x84218421;
837e4a42 4785 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4786 &v, 1, TP_TX_SCHED_HDR_A);
837e4a42 4787 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4788 &v, 1, TP_TX_SCHED_FIFO_A);
837e4a42 4789 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4790 &v, 1, TP_TX_SCHED_PCMD_A);
dca4faeb
VP		 4791
4792#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4793 if (is_offload(adap)) {
0d804338
HS		 4794 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4795 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4796 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4797 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4798 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4799 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4800 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4801 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4802 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4803 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
dca4faeb
VP		 4804 }
4805
060e0c75
DM		 4806 /* get basic stuff going */
4807 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4808}
4809
b8ff05a9
DM		 4810/*
4811 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4812 */
4813#define MAX_ATIDS 8192U
4814
636f9d37
VP		 4815/*
4816 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4817 *
4818 * If the firmware we're dealing with has Configuration File support, then
4819 * we use that to perform all configuration
4820 */
4821
4822/*
4823 * Tweak configuration based on module parameters, etc. Most of these have
4824 * defaults assigned to them by Firmware Configuration Files (if we're using
4825 * them) but need to be explicitly set if we're using hard-coded
4826 * initialization. But even in the case of using Firmware Configuration
4827 * Files, we'd like to expose the ability to change these via module
4828 * parameters so these are essentially common tweaks/settings for
4829 * Configuration Files and hard-coded initialization ...
4830 */
4831static int adap_init0_tweaks(struct adapter *adapter)
4832{
4833 /*
4834 * Fix up various Host-Dependent Parameters like Page Size, Cache
4835 * Line Size, etc. The firmware default is for a 4KB Page Size and
4836 * 64B Cache Line Size ...
4837 */
4838 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4839
4840 /*
4841 * Process module parameters which affect early initialization.
4842 */
4843 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4844 dev_err(&adapter->pdev->dev,
4845 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4846 rx_dma_offset);
4847 rx_dma_offset = 2;
4848 }
f612b815
HS		 4849 t4_set_reg_field(adapter, SGE_CONTROL_A,
4850 PKTSHIFT_V(PKTSHIFT_M),
4851 PKTSHIFT_V(rx_dma_offset));
636f9d37
VP		 4852
4853 /*
4854 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4855 * adds the pseudo header itself.
4856 */
837e4a42
HS		 4857 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
4858 CSUM_HAS_PSEUDO_HDR_F, 0);
636f9d37
VP		 4859
4860 return 0;
4861}
4862
4863/*
4864 * Attempt to initialize the adapter via a Firmware Configuration File.
4865 */
4866static int adap_init0_config(struct adapter *adapter, int reset)
4867{
4868 struct fw_caps_config_cmd caps_cmd;
4869 const struct firmware *cf;
4870 unsigned long mtype = 0, maddr = 0;
4871 u32 finiver, finicsum, cfcsum;
16e47624
HS		 4872 int ret;
4873 int config_issued = 0;
0a57a536 4874 char *fw_config_file, fw_config_file_path[256];
16e47624 4875 char *config_name = NULL;
636f9d37
VP		 4876
4877 /*
4878 * Reset device if necessary.
4879 */
4880 if (reset) {
4881 ret = t4_fw_reset(adapter, adapter->mbox,
0d804338 4882 PIORSTMODE_F | PIORST_F);
636f9d37
VP		 4883 if (ret < 0)
4884 goto bye;
4885 }
4886
4887 /*
4888 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4889 * then use that. Otherwise, use the configuration file stored
4890 * in the adapter flash ...
4891 */
d14807dd 4892 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 4893 case CHELSIO_T4:
16e47624 4894 fw_config_file = FW4_CFNAME;
0a57a536
SR		 4895 break;
4896 case CHELSIO_T5:
4897 fw_config_file = FW5_CFNAME;
4898 break;
4899 default:
4900 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4901 adapter->pdev->device);
4902 ret = -EINVAL;
4903 goto bye;
4904 }
4905
4906 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 4907 if (ret < 0) {
16e47624 4908 config_name = "On FLASH";
636f9d37
VP		 4909 mtype = FW_MEMTYPE_CF_FLASH;
4910 maddr = t4_flash_cfg_addr(adapter);
4911 } else {
4912 u32 params[7], val[7];
4913
16e47624
HS		 4914 sprintf(fw_config_file_path,
4915 "/lib/firmware/%s", fw_config_file);
4916 config_name = fw_config_file_path;
4917
636f9d37
VP		 4918 if (cf->size >= FLASH_CFG_MAX_SIZE)
4919 ret = -ENOMEM;
4920 else {
5167865a
HS		 4921 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
4922 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
636f9d37
VP		 4923 ret = t4_query_params(adapter, adapter->mbox,
4924 adapter->fn, 0, 1, params, val);
4925 if (ret == 0) {
4926 /*
fc5ab020 4927 * For t4_memory_rw() below addresses and
636f9d37
VP		 4928 * sizes have to be in terms of multiples of 4
4929 * bytes. So, if the Configuration File isn't
4930 * a multiple of 4 bytes in length we'll have
4931 * to write that out separately since we can't
4932 * guarantee that the bytes following the
4933 * residual byte in the buffer returned by
4934 * request_firmware() are zeroed out ...
4935 */
4936 size_t resid = cf->size & 0x3;
4937 size_t size = cf->size & ~0x3;
4938 __be32 *data = (__be32 *)cf->data;
4939
5167865a
HS		 4940 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
4941 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
636f9d37 4942
fc5ab020
HS		 4943 spin_lock(&adapter->win0_lock);
4944 ret = t4_memory_rw(adapter, 0, mtype, maddr,
4945 size, data, T4_MEMORY_WRITE);
636f9d37
VP		 4946 if (ret == 0 && resid != 0) {
4947 union {
4948 __be32 word;
4949 char buf[4];
4950 } last;
4951 int i;
4952
4953 last.word = data[size >> 2];
4954 for (i = resid; i < 4; i++)
4955 last.buf[i] = 0;
fc5ab020
HS		 4956 ret = t4_memory_rw(adapter, 0, mtype,
4957 maddr + size,
4958 4, &last.word,
4959 T4_MEMORY_WRITE);
636f9d37 4960 }
fc5ab020 4961 spin_unlock(&adapter->win0_lock);
636f9d37
VP		 4962 }
4963 }
4964
4965 release_firmware(cf);
4966 if (ret)
4967 goto bye;
4968 }
4969
4970 /*
4971 * Issue a Capability Configuration command to the firmware to get it
4972 * to parse the Configuration File. We don't use t4_fw_config_file()
4973 * because we want the ability to modify various features after we've
4974 * processed the configuration file ...
4975 */
4976 memset(&caps_cmd, 0, sizeof(caps_cmd));
4977 caps_cmd.op_to_write =
e2ac9628
HS		 4978 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4979 FW_CMD_REQUEST_F |
4980 FW_CMD_READ_F);
ce91a923 4981 caps_cmd.cfvalid_to_len16 =
5167865a
HS		 4982 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
4983 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
4984 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
636f9d37
VP		 4985 FW_LEN16(caps_cmd));
4986 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4987 &caps_cmd);
16e47624
HS		 4988
4989 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4990 * Configuration File in FLASH), our last gasp effort is to use the
4991 * Firmware Configuration File which is embedded in the firmware. A
4992 * very few early versions of the firmware didn't have one embedded
4993 * but we can ignore those.
4994 */
4995 if (ret == -ENOENT) {
4996 memset(&caps_cmd, 0, sizeof(caps_cmd));
4997 caps_cmd.op_to_write =
e2ac9628
HS		 4998 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4999 FW_CMD_REQUEST_F |
5000 FW_CMD_READ_F);
16e47624
HS		 5001 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5002 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
5003 sizeof(caps_cmd), &caps_cmd);
5004 config_name = "Firmware Default";
5005 }
5006
5007 config_issued = 1;
636f9d37
VP		 5008 if (ret < 0)
5009 goto bye;
5010
5011 finiver = ntohl(caps_cmd.finiver);
5012 finicsum = ntohl(caps_cmd.finicsum);
5013 cfcsum = ntohl(caps_cmd.cfcsum);
5014 if (finicsum != cfcsum)
5015 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
5016 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
5017 finicsum, cfcsum);
5018
636f9d37
VP		 5019 /*
5020 * And now tell the firmware to use the configuration we just loaded.
5021 */
5022 caps_cmd.op_to_write =
e2ac9628
HS		 5023 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5024 FW_CMD_REQUEST_F |
5025 FW_CMD_WRITE_F);
ce91a923 5026 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5027 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5028 NULL);
5029 if (ret < 0)
5030 goto bye;
5031
5032 /*
5033 * Tweak configuration based on system architecture, module
5034 * parameters, etc.
5035 */
5036 ret = adap_init0_tweaks(adapter);
5037 if (ret < 0)
5038 goto bye;
5039
5040 /*
5041 * And finally tell the firmware to initialize itself using the
5042 * parameters from the Configuration File.
5043 */
5044 ret = t4_fw_initialize(adapter, adapter->mbox);
5045 if (ret < 0)
5046 goto bye;
5047
06640310
HS		 5048 /* Emit Firmware Configuration File information and return
5049 * successfully.
636f9d37 5050 */
636f9d37 5051 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 5052 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5053 config_name, finiver, cfcsum);
636f9d37
VP		 5054 return 0;
5055
5056 /*
5057 * Something bad happened. Return the error ... (If the "error"
5058 * is that there's no Configuration File on the adapter we don't
5059 * want to issue a warning since this is fairly common.)
5060 */
5061bye:
16e47624
HS		 5062 if (config_issued && ret != -ENOENT)
5063 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5064 config_name, -ret);
636f9d37
VP		 5065 return ret;
5066}
5067
16e47624
HS		 5068static struct fw_info fw_info_array[] = {
5069 {
5070 .chip = CHELSIO_T4,
5071 .fs_name = FW4_CFNAME,
5072 .fw_mod_name = FW4_FNAME,
5073 .fw_hdr = {
5074 .chip = FW_HDR_CHIP_T4,
5075 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5076 .intfver_nic = FW_INTFVER(T4, NIC),
5077 .intfver_vnic = FW_INTFVER(T4, VNIC),
5078 .intfver_ri = FW_INTFVER(T4, RI),
5079 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5080 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5081 },
5082 }, {
5083 .chip = CHELSIO_T5,
5084 .fs_name = FW5_CFNAME,
5085 .fw_mod_name = FW5_FNAME,
5086 .fw_hdr = {
5087 .chip = FW_HDR_CHIP_T5,
5088 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5089 .intfver_nic = FW_INTFVER(T5, NIC),
5090 .intfver_vnic = FW_INTFVER(T5, VNIC),
5091 .intfver_ri = FW_INTFVER(T5, RI),
5092 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5093 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5094 },
5095 }
5096};
5097
5098static struct fw_info *find_fw_info(int chip)
5099{
5100 int i;
5101
5102 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5103 if (fw_info_array[i].chip == chip)
5104 return &fw_info_array[i];
5105 }
5106 return NULL;
5107}
5108
b8ff05a9
DM		 5109/*
5110 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5111 */
5112static int adap_init0(struct adapter *adap)
5113{
5114 int ret;
5115 u32 v, port_vec;
5116 enum dev_state state;
5117 u32 params[7], val[7];
9a4da2cd 5118 struct fw_caps_config_cmd caps_cmd;
dcf7b6f5 5119 int reset = 1;
b8ff05a9 5120
ae469b68
HS		 5121 /* Grab Firmware Device Log parameters as early as possible so we have
5122 * access to it for debugging, etc.
5123 */
5124 ret = t4_init_devlog_params(adap);
5125 if (ret < 0)
5126 return ret;
5127
666224d4
HS		 5128 /* Contact FW, advertising Master capability */
5129 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
b8ff05a9
DM		 5130 if (ret < 0) {
5131 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5132 ret);
5133 return ret;
5134 }
636f9d37
VP		 5135 if (ret == adap->mbox)
5136 adap->flags |= MASTER_PF;
b8ff05a9 5137
636f9d37
VP		 5138 /*
5139 * If we're the Master PF Driver and the device is uninitialized,
5140 * then let's consider upgrading the firmware ... (We always want
5141 * to check the firmware version number in order to A. get it for
5142 * later reporting and B. to warn if the currently loaded firmware
5143 * is excessively mismatched relative to the driver.)
5144 */
16e47624
HS		 5145 t4_get_fw_version(adap, &adap->params.fw_vers);
5146 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5147 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5148 struct fw_info *fw_info;
5149 struct fw_hdr *card_fw;
5150 const struct firmware *fw;
5151 const u8 *fw_data = NULL;
5152 unsigned int fw_size = 0;
5153
5154 /* This is the firmware whose headers the driver was compiled
5155 * against
5156 */
5157 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5158 if (fw_info == NULL) {
5159 dev_err(adap->pdev_dev,
5160 "unable to get firmware info for chip %d.\n",
5161 CHELSIO_CHIP_VERSION(adap->params.chip));
5162 return -EINVAL;
636f9d37 5163 }
16e47624
HS		 5164
5165 /* allocate memory to read the header of the firmware on the
5166 * card
5167 */
5168 card_fw = t4_alloc_mem(sizeof(*card_fw));
5169
5170 /* Get FW from from /lib/firmware/ */
5171 ret = request_firmware(&fw, fw_info->fw_mod_name,
5172 adap->pdev_dev);
5173 if (ret < 0) {
5174 dev_err(adap->pdev_dev,
5175 "unable to load firmware image %s, error %d\n",
5176 fw_info->fw_mod_name, ret);
5177 } else {
5178 fw_data = fw->data;
5179 fw_size = fw->size;
5180 }
5181
5182 /* upgrade FW logic */
5183 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5184 state, &reset);
5185
5186 /* Cleaning up */
0b5b6bee 5187 release_firmware(fw);
16e47624
HS		 5188 t4_free_mem(card_fw);
5189
636f9d37 5190 if (ret < 0)
16e47624 5191 goto bye;
636f9d37 5192 }
b8ff05a9 5193
636f9d37
VP		 5194 /*
5195 * Grab VPD parameters. This should be done after we establish a
5196 * connection to the firmware since some of the VPD parameters
5197 * (notably the Core Clock frequency) are retrieved via requests to
5198 * the firmware. On the other hand, we need these fairly early on
5199 * so we do this right after getting ahold of the firmware.
5200 */
5201 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5202 if (ret < 0)
5203 goto bye;
a0881cab 5204
636f9d37 5205 /*
13ee15d3
VP		 5206 * Find out what ports are available to us. Note that we need to do
5207 * this before calling adap_init0_no_config() since it needs nports
5208 * and portvec ...
636f9d37
VP		 5209 */
5210 v =
5167865a
HS		 5211 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5212 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
636f9d37 5213 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5214 if (ret < 0)
5215 goto bye;
5216
636f9d37
VP		 5217 adap->params.nports = hweight32(port_vec);
5218 adap->params.portvec = port_vec;
5219
06640310
HS		 5220 /* If the firmware is initialized already, emit a simply note to that
5221 * effect. Otherwise, it's time to try initializing the adapter.
636f9d37
VP		 5222 */
5223 if (state == DEV_STATE_INIT) {
5224 dev_info(adap->pdev_dev, "Coming up as %s: "\
5225 "Adapter already initialized\n",
5226 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
636f9d37
VP		 5227 } else {
5228 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5229 "Initializing adapter\n");
06640310
HS		 5230
5231 /* Find out whether we're dealing with a version of the
5232 * firmware which has configuration file support.
636f9d37 5233 */
06640310
HS		 5234 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5235 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5236 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5237 params, val);
13ee15d3 5238
06640310
HS		 5239 /* If the firmware doesn't support Configuration Files,
5240 * return an error.
5241 */
5242 if (ret < 0) {
5243 dev_err(adap->pdev_dev, "firmware doesn't support "
5244 "Firmware Configuration Files\n");
5245 goto bye;
5246 }
5247
5248 /* The firmware provides us with a memory buffer where we can
5249 * load a Configuration File from the host if we want to
5250 * override the Configuration File in flash.
5251 */
5252 ret = adap_init0_config(adap, reset);
5253 if (ret == -ENOENT) {
5254 dev_err(adap->pdev_dev, "no Configuration File "
5255 "present on adapter.\n");
5256 goto bye;
636f9d37
VP		 5257 }
5258 if (ret < 0) {
06640310
HS		 5259 dev_err(adap->pdev_dev, "could not initialize "
5260 "adapter, error %d\n", -ret);
636f9d37
VP		 5261 goto bye;
5262 }
5263 }
5264
06640310
HS		 5265 /* Give the SGE code a chance to pull in anything that it needs ...
5266 * Note that this must be called after we retrieve our VPD parameters
5267 * in order to know how to convert core ticks to seconds, etc.
636f9d37 5268 */
06640310
HS		 5269 ret = t4_sge_init(adap);
5270 if (ret < 0)
5271 goto bye;
636f9d37 5272
9a4da2cd
VP		 5273 if (is_bypass_device(adap->pdev->device))
5274 adap->params.bypass = 1;
5275
636f9d37
VP		 5276 /*
5277 * Grab some of our basic fundamental operating parameters.
5278 */
5279#define FW_PARAM_DEV(param) \
5167865a
HS		 5280 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5281 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
636f9d37 5282
b8ff05a9 5283#define FW_PARAM_PFVF(param) \
5167865a
HS		 5284 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5285 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
5286 FW_PARAMS_PARAM_Y_V(0) | \
5287 FW_PARAMS_PARAM_Z_V(0)
b8ff05a9 5288
636f9d37 5289 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5290 params[1] = FW_PARAM_PFVF(L2T_START);
5291 params[2] = FW_PARAM_PFVF(L2T_END);
5292 params[3] = FW_PARAM_PFVF(FILTER_START);
5293 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5294 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5295 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5296 if (ret < 0)
5297 goto bye;
636f9d37
VP		 5298 adap->sge.egr_start = val[0];
5299 adap->l2t_start = val[1];
5300 adap->l2t_end = val[2];
b8ff05a9
DM		 5301 adap->tids.ftid_base = val[3];
5302 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5303 adap->sge.ingr_start = val[5];
b8ff05a9 5304
4b8e27a8
HS		 5305 /* qids (ingress/egress) returned from firmware can be anywhere
5306 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END.
5307 * Hence driver needs to allocate memory for this range to
5308 * store the queue info. Get the highest IQFLINT/EQ index returned
5309 * in FW_EQ_*_CMD.alloc command.
5310 */
5311 params[0] = FW_PARAM_PFVF(EQ_END);
5312 params[1] = FW_PARAM_PFVF(IQFLINT_END);
5313 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5314 if (ret < 0)
5315 goto bye;
5316 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1;
5317 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1;
5318
5319 adap->sge.egr_map = kcalloc(adap->sge.egr_sz,
5320 sizeof(*adap->sge.egr_map), GFP_KERNEL);
5321 if (!adap->sge.egr_map) {
5322 ret = -ENOMEM;
5323 goto bye;
5324 }
5325
5326 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz,
5327 sizeof(*adap->sge.ingr_map), GFP_KERNEL);
5328 if (!adap->sge.ingr_map) {
5329 ret = -ENOMEM;
5330 goto bye;
5331 }
5332
5333 /* Allocate the memory for the vaious egress queue bitmaps
5334 * ie starving_fl and txq_maperr.
5335 */
5336 adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
5337 sizeof(long), GFP_KERNEL);
5338 if (!adap->sge.starving_fl) {
5339 ret = -ENOMEM;
5340 goto bye;
5341 }
5342
5343 adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
5344 sizeof(long), GFP_KERNEL);
5345 if (!adap->sge.txq_maperr) {
5346 ret = -ENOMEM;
5347 goto bye;
5348 }
5349
b5a02f50
AB		 5350 params[0] = FW_PARAM_PFVF(CLIP_START);
5351 params[1] = FW_PARAM_PFVF(CLIP_END);
5352 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5353 if (ret < 0)
5354 goto bye;
5355 adap->clipt_start = val[0];
5356 adap->clipt_end = val[1];
5357
636f9d37
VP		 5358 /* query params related to active filter region */
5359 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5360 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5361 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5362 /* If Active filter size is set we enable establishing
5363 * offload connection through firmware work request
5364 */
5365 if ((val[0] != val[1]) && (ret >= 0)) {
5366 adap->flags |= FW_OFLD_CONN;
5367 adap->tids.aftid_base = val[0];
5368 adap->tids.aftid_end = val[1];
5369 }
5370
b407a4a9
VP		 5371 /* If we're running on newer firmware, let it know that we're
5372 * prepared to deal with encapsulated CPL messages. Older
5373 * firmware won't understand this and we'll just get
5374 * unencapsulated messages ...
5375 */
5376 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5377 val[0] = 1;
5378 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5379
1ac0f095
KS		 5380 /*
5381 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5382 * capability. Earlier versions of the firmware didn't have the
5383 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5384 * permission to use ULPTX MEMWRITE DSGL.
5385 */
5386 if (is_t4(adap->params.chip)) {
5387 adap->params.ulptx_memwrite_dsgl = false;
5388 } else {
5389 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5390 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5391 1, params, val);
5392 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5393 }
5394
636f9d37
VP		 5395 /*
5396 * Get device capabilities so we can determine what resources we need
5397 * to manage.
5398 */
5399 memset(&caps_cmd, 0, sizeof(caps_cmd));
e2ac9628
HS		 5400 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5401 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 5402 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5403 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5404 &caps_cmd);
5405 if (ret < 0)
5406 goto bye;
5407
13ee15d3 5408 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5409 /* query offload-related parameters */
5410 params[0] = FW_PARAM_DEV(NTID);
5411 params[1] = FW_PARAM_PFVF(SERVER_START);
5412 params[2] = FW_PARAM_PFVF(SERVER_END);
5413 params[3] = FW_PARAM_PFVF(TDDP_START);
5414 params[4] = FW_PARAM_PFVF(TDDP_END);
5415 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5416 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5417 params, val);
b8ff05a9
DM		 5418 if (ret < 0)
5419 goto bye;
5420 adap->tids.ntids = val[0];
5421 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5422 adap->tids.stid_base = val[1];
5423 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37 5424 /*
dbedd44e 5425 * Setup server filter region. Divide the available filter
636f9d37
VP		 5426 * region into two parts. Regular filters get 1/3rd and server
5427 * filters get 2/3rd part. This is only enabled if workarond
5428 * path is enabled.
5429 * 1. For regular filters.
5430 * 2. Server filter: This are special filters which are used
5431 * to redirect SYN packets to offload queue.
5432 */
5433 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5434 adap->tids.sftid_base = adap->tids.ftid_base +
5435 DIV_ROUND_UP(adap->tids.nftids, 3);
5436 adap->tids.nsftids = adap->tids.nftids -
5437 DIV_ROUND_UP(adap->tids.nftids, 3);
5438 adap->tids.nftids = adap->tids.sftid_base -
5439 adap->tids.ftid_base;
5440 }
b8ff05a9
DM		 5441 adap->vres.ddp.start = val[3];
5442 adap->vres.ddp.size = val[4] - val[3] + 1;
5443 adap->params.ofldq_wr_cred = val[5];
636f9d37 5444
b8ff05a9
DM		 5445 adap->params.offload = 1;
5446 }
636f9d37 5447 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5448 params[0] = FW_PARAM_PFVF(STAG_START);
5449 params[1] = FW_PARAM_PFVF(STAG_END);
5450 params[2] = FW_PARAM_PFVF(RQ_START);
5451 params[3] = FW_PARAM_PFVF(RQ_END);
5452 params[4] = FW_PARAM_PFVF(PBL_START);
5453 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5454 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5455 params, val);
b8ff05a9
DM		 5456 if (ret < 0)
5457 goto bye;
5458 adap->vres.stag.start = val[0];
5459 adap->vres.stag.size = val[1] - val[0] + 1;
5460 adap->vres.rq.start = val[2];
5461 adap->vres.rq.size = val[3] - val[2] + 1;
5462 adap->vres.pbl.start = val[4];
5463 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5464
5465 params[0] = FW_PARAM_PFVF(SQRQ_START);
5466 params[1] = FW_PARAM_PFVF(SQRQ_END);
5467 params[2] = FW_PARAM_PFVF(CQ_START);
5468 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5469 params[4] = FW_PARAM_PFVF(OCQ_START);
5470 params[5] = FW_PARAM_PFVF(OCQ_END);
5c937dd3
HS		 5471 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5472 val);
a0881cab
DM		 5473 if (ret < 0)
5474 goto bye;
5475 adap->vres.qp.start = val[0];
5476 adap->vres.qp.size = val[1] - val[0] + 1;
5477 adap->vres.cq.start = val[2];
5478 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5479 adap->vres.ocq.start = val[4];
5480 adap->vres.ocq.size = val[5] - val[4] + 1;
4c2c5763
HS		 5481
5482 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5483 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5c937dd3
HS		 5484 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5485 val);
4c2c5763
HS		 5486 if (ret < 0) {
5487 adap->params.max_ordird_qp = 8;
5488 adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5489 ret = 0;
5490 } else {
5491 adap->params.max_ordird_qp = val[0];
5492 adap->params.max_ird_adapter = val[1];
5493 }
5494 dev_info(adap->pdev_dev,
5495 "max_ordird_qp %d max_ird_adapter %d\n",
5496 adap->params.max_ordird_qp,
5497 adap->params.max_ird_adapter);
b8ff05a9 5498 }
636f9d37 5499 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5500 params[0] = FW_PARAM_PFVF(ISCSI_START);
5501 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5502 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5503 params, val);
b8ff05a9
DM		 5504 if (ret < 0)
5505 goto bye;
5506 adap->vres.iscsi.start = val[0];
5507 adap->vres.iscsi.size = val[1] - val[0] + 1;
5508 }
5509#undef FW_PARAM_PFVF
5510#undef FW_PARAM_DEV
5511
92e7ae71
HS		 5512 /* The MTU/MSS Table is initialized by now, so load their values. If
5513 * we're initializing the adapter, then we'll make any modifications
5514 * we want to the MTU/MSS Table and also initialize the congestion
5515 * parameters.
636f9d37 5516 */
b8ff05a9 5517 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
92e7ae71
HS		 5518 if (state != DEV_STATE_INIT) {
5519 int i;
5520
5521 /* The default MTU Table contains values 1492 and 1500.
5522 * However, for TCP, it's better to have two values which are
5523 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5524 * This allows us to have a TCP Data Payload which is a
5525 * multiple of 8 regardless of what combination of TCP Options
5526 * are in use (always a multiple of 4 bytes) which is
5527 * important for performance reasons. For instance, if no
5528 * options are in use, then we have a 20-byte IP header and a
5529 * 20-byte TCP header. In this case, a 1500-byte MSS would
5530 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5531 * which is not a multiple of 8. So using an MSS of 1488 in
5532 * this case results in a TCP Data Payload of 1448 bytes which
5533 * is a multiple of 8. On the other hand, if 12-byte TCP Time
5534 * Stamps have been negotiated, then an MTU of 1500 bytes
5535 * results in a TCP Data Payload of 1448 bytes which, as
5536 * above, is a multiple of 8 bytes ...
5537 */
5538 for (i = 0; i < NMTUS; i++)
5539 if (adap->params.mtus[i] == 1492) {
5540 adap->params.mtus[i] = 1488;
5541 break;
5542 }
7ee9ff94 5543
92e7ae71
HS		 5544 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5545 adap->params.b_wnd);
5546 }
df64e4d3 5547 t4_init_sge_params(adap);
dcf7b6f5 5548 t4_init_tp_params(adap);
636f9d37 5549 adap->flags |= FW_OK;
b8ff05a9
DM		 5550 return 0;
5551
5552 /*
636f9d37
VP		 5553 * Something bad happened. If a command timed out or failed with EIO
5554 * FW does not operate within its spec or something catastrophic
5555 * happened to HW/FW, stop issuing commands.
b8ff05a9 5556 */
636f9d37 5557bye:
4b8e27a8
HS		 5558 kfree(adap->sge.egr_map);
5559 kfree(adap->sge.ingr_map);
5560 kfree(adap->sge.starving_fl);
5561 kfree(adap->sge.txq_maperr);
636f9d37
VP		 5562 if (ret != -ETIMEDOUT && ret != -EIO)
5563 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5564 return ret;
5565}
5566
204dc3c0
DM		 5567/* EEH callbacks */
5568
5569static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5570 pci_channel_state_t state)
5571{
5572 int i;
5573 struct adapter *adap = pci_get_drvdata(pdev);
5574
5575 if (!adap)
5576 goto out;
5577
5578 rtnl_lock();
5579 adap->flags &= ~FW_OK;
5580 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5581 spin_lock(&adap->stats_lock);
204dc3c0
DM		 5582 for_each_port(adap, i) {
5583 struct net_device *dev = adap->port[i];
5584
5585 netif_device_detach(dev);
5586 netif_carrier_off(dev);
5587 }
9fe6cb58 5588 spin_unlock(&adap->stats_lock);
b37987e8 5589 disable_interrupts(adap);
204dc3c0
DM		 5590 if (adap->flags & FULL_INIT_DONE)
5591 cxgb_down(adap);
5592 rtnl_unlock();
144be3d9
GS		 5593 if ((adap->flags & DEV_ENABLED)) {
5594 pci_disable_device(pdev);
5595 adap->flags &= ~DEV_ENABLED;
5596 }
204dc3c0
DM		 5597out: return state == pci_channel_io_perm_failure ?
5598 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5599}
5600
5601static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5602{
5603 int i, ret;
5604 struct fw_caps_config_cmd c;
5605 struct adapter *adap = pci_get_drvdata(pdev);
5606
5607 if (!adap) {
5608 pci_restore_state(pdev);
5609 pci_save_state(pdev);
5610 return PCI_ERS_RESULT_RECOVERED;
5611 }
5612
144be3d9
GS		 5613 if (!(adap->flags & DEV_ENABLED)) {
5614 if (pci_enable_device(pdev)) {
5615 dev_err(&pdev->dev, "Cannot reenable PCI "
5616 "device after reset\n");
5617 return PCI_ERS_RESULT_DISCONNECT;
5618 }
5619 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 5620 }
5621
5622 pci_set_master(pdev);
5623 pci_restore_state(pdev);
5624 pci_save_state(pdev);
5625 pci_cleanup_aer_uncorrect_error_status(pdev);
5626
8203b509 5627 if (t4_wait_dev_ready(adap->regs) < 0)
204dc3c0 5628 return PCI_ERS_RESULT_DISCONNECT;
777c2300 5629 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 5630 return PCI_ERS_RESULT_DISCONNECT;
5631 adap->flags |= FW_OK;
5632 if (adap_init1(adap, &c))
5633 return PCI_ERS_RESULT_DISCONNECT;
5634
5635 for_each_port(adap, i) {
5636 struct port_info *p = adap2pinfo(adap, i);
5637
060e0c75
DM		 5638 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5639 NULL, NULL);
204dc3c0
DM		 5640 if (ret < 0)
5641 return PCI_ERS_RESULT_DISCONNECT;
5642 p->viid = ret;
5643 p->xact_addr_filt = -1;
5644 }
5645
5646 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5647 adap->params.b_wnd);
1ae970e0 5648 setup_memwin(adap);
204dc3c0
DM		 5649 if (cxgb_up(adap))
5650 return PCI_ERS_RESULT_DISCONNECT;
5651 return PCI_ERS_RESULT_RECOVERED;
5652}
5653
5654static void eeh_resume(struct pci_dev *pdev)
5655{
5656 int i;
5657 struct adapter *adap = pci_get_drvdata(pdev);
5658
5659 if (!adap)
5660 return;
5661
5662 rtnl_lock();
5663 for_each_port(adap, i) {
5664 struct net_device *dev = adap->port[i];
5665
5666 if (netif_running(dev)) {
5667 link_start(dev);
5668 cxgb_set_rxmode(dev);
5669 }
5670 netif_device_attach(dev);
5671 }
5672 rtnl_unlock();
5673}
5674
3646f0e5 5675static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 5676 .error_detected = eeh_err_detected,
5677 .slot_reset = eeh_slot_reset,
5678 .resume = eeh_resume,
5679};
5680
57d8b764 5681static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 5682{
57d8b764
KS		 5683 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5684 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 5685}
5686
c887ad0e
HS		 5687static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
5688 unsigned int us, unsigned int cnt,
b8ff05a9
DM		 5689 unsigned int size, unsigned int iqe_size)
5690{
c887ad0e
HS		 5691 q->adap = adap;
5692 set_rspq_intr_params(q, us, cnt);
b8ff05a9
DM		 5693 q->iqe_len = iqe_size;
5694 q->size = size;
5695}
5696
5697/*
5698 * Perform default configuration of DMA queues depending on the number and type
5699 * of ports we found and the number of available CPUs. Most settings can be
5700 * modified by the admin prior to actual use.
5701 */
91744948 5702static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 5703{
5704 struct sge *s = &adap->sge;
688848b1
AB		 5705 int i, n10g = 0, qidx = 0;
5706#ifndef CONFIG_CHELSIO_T4_DCB
5707 int q10g = 0;
5708#endif
cf38be6d 5709 int ciq_size;
b8ff05a9
DM		 5710
5711 for_each_port(adap, i)
57d8b764 5712 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
688848b1
AB		 5713#ifdef CONFIG_CHELSIO_T4_DCB
5714 /* For Data Center Bridging support we need to be able to support up
5715 * to 8 Traffic Priorities; each of which will be assigned to its
5716 * own TX Queue in order to prevent Head-Of-Line Blocking.
5717 */
5718 if (adap->params.nports * 8 > MAX_ETH_QSETS) {
5719 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
5720 MAX_ETH_QSETS, adap->params.nports * 8);
5721 BUG_ON(1);
5722 }
b8ff05a9 5723
688848b1
AB		 5724 for_each_port(adap, i) {
5725 struct port_info *pi = adap2pinfo(adap, i);
5726
5727 pi->first_qset = qidx;
5728 pi->nqsets = 8;
5729 qidx += pi->nqsets;
5730 }
5731#else /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 5732 /*
5733 * We default to 1 queue per non-10G port and up to # of cores queues
5734 * per 10G port.
5735 */
5736 if (n10g)
5737 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 5738 if (q10g > netif_get_num_default_rss_queues())
5739 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 5740
5741 for_each_port(adap, i) {
5742 struct port_info *pi = adap2pinfo(adap, i);
5743
5744 pi->first_qset = qidx;
57d8b764 5745 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 5746 qidx += pi->nqsets;
5747 }
688848b1 5748#endif /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 5749
5750 s->ethqsets = qidx;
5751 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5752
5753 if (is_offload(adap)) {
5754 /*
5755 * For offload we use 1 queue/channel if all ports are up to 1G,
5756 * otherwise we divide all available queues amongst the channels
5757 * capped by the number of available cores.
5758 */
5759 if (n10g) {
5760 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5761 num_online_cpus());
5762 s->ofldqsets = roundup(i, adap->params.nports);
5763 } else
5764 s->ofldqsets = adap->params.nports;
5765 /* For RDMA one Rx queue per channel suffices */
5766 s->rdmaqs = adap->params.nports;
f36e58e5
HS		 5767 /* Try and allow at least 1 CIQ per cpu rounding down
5768 * to the number of ports, with a minimum of 1 per port.
5769 * A 2 port card in a 6 cpu system: 6 CIQs, 3 / port.
5770 * A 4 port card in a 6 cpu system: 4 CIQs, 1 / port.
5771 * A 4 port card in a 2 cpu system: 4 CIQs, 1 / port.
5772 */
5773 s->rdmaciqs = min_t(int, MAX_RDMA_CIQS, num_online_cpus());
5774 s->rdmaciqs = (s->rdmaciqs / adap->params.nports) *
5775 adap->params.nports;
5776 s->rdmaciqs = max_t(int, s->rdmaciqs, adap->params.nports);
b8ff05a9
DM		 5777 }
5778
5779 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5780 struct sge_eth_rxq *r = &s->ethrxq[i];
5781
c887ad0e 5782 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
b8ff05a9
DM		 5783 r->fl.size = 72;
5784 }
5785
5786 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5787 s->ethtxq[i].q.size = 1024;
5788
5789 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5790 s->ctrlq[i].q.size = 512;
5791
5792 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5793 s->ofldtxq[i].q.size = 1024;
5794
5795 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5796 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5797
c887ad0e 5798 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
b8ff05a9
DM		 5799 r->rspq.uld = CXGB4_ULD_ISCSI;
5800 r->fl.size = 72;
5801 }
5802
5803 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5804 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5805
c887ad0e 5806 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
b8ff05a9
DM		 5807 r->rspq.uld = CXGB4_ULD_RDMA;
5808 r->fl.size = 72;
5809 }
5810
cf38be6d
HS		 5811 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5812 if (ciq_size > SGE_MAX_IQ_SIZE) {
5813 CH_WARN(adap, "CIQ size too small for available IQs\n");
5814 ciq_size = SGE_MAX_IQ_SIZE;
5815 }
5816
5817 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5818 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5819
c887ad0e 5820 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
cf38be6d
HS		 5821 r->rspq.uld = CXGB4_ULD_RDMA;
5822 }
5823
c887ad0e
HS		 5824 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5825 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
b8ff05a9
DM		 5826}
5827
5828/*
5829 * Reduce the number of Ethernet queues across all ports to at most n.
5830 * n provides at least one queue per port.
5831 */
91744948 5832static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 5833{
5834 int i;
5835 struct port_info *pi;
5836
5837 while (n < adap->sge.ethqsets)
5838 for_each_port(adap, i) {
5839 pi = adap2pinfo(adap, i);
5840 if (pi->nqsets > 1) {
5841 pi->nqsets--;
5842 adap->sge.ethqsets--;
5843 if (adap->sge.ethqsets <= n)
5844 break;
5845 }
5846 }
5847
5848 n = 0;
5849 for_each_port(adap, i) {
5850 pi = adap2pinfo(adap, i);
5851 pi->first_qset = n;
5852 n += pi->nqsets;
5853 }
5854}
5855
5856/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5857#define EXTRA_VECS 2
5858
91744948 5859static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 5860{
5861 int ofld_need = 0;
f36e58e5 5862 int i, want, need, allocated;
b8ff05a9
DM		 5863 struct sge *s = &adap->sge;
5864 unsigned int nchan = adap->params.nports;
f36e58e5
HS		 5865 struct msix_entry *entries;
5866
5867 entries = kmalloc(sizeof(*entries) * (MAX_INGQ + 1),
5868 GFP_KERNEL);
5869 if (!entries)
5870 return -ENOMEM;
b8ff05a9 5871
f36e58e5 5872 for (i = 0; i < MAX_INGQ + 1; ++i)
b8ff05a9
DM		 5873 entries[i].entry = i;
5874
5875 want = s->max_ethqsets + EXTRA_VECS;
5876 if (is_offload(adap)) {
cf38be6d 5877 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
b8ff05a9 5878 /* need nchan for each possible ULD */
cf38be6d 5879 ofld_need = 3 * nchan;
b8ff05a9 5880 }
688848b1
AB		 5881#ifdef CONFIG_CHELSIO_T4_DCB
5882 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
5883 * each port.
5884 */
5885 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
5886#else
b8ff05a9 5887 need = adap->params.nports + EXTRA_VECS + ofld_need;
688848b1 5888#endif
f36e58e5
HS		 5889 allocated = pci_enable_msix_range(adap->pdev, entries, need, want);
5890 if (allocated < 0) {
5891 dev_info(adap->pdev_dev, "not enough MSI-X vectors left,"
5892 " not using MSI-X\n");
5893 kfree(entries);
5894 return allocated;
5895 }
b8ff05a9 5896
f36e58e5 5897 /* Distribute available vectors to the various queue groups.
c32ad224
AG		 5898 * Every group gets its minimum requirement and NIC gets top
5899 * priority for leftovers.
5900 */
f36e58e5 5901 i = allocated - EXTRA_VECS - ofld_need;
c32ad224
AG		 5902 if (i < s->max_ethqsets) {
5903 s->max_ethqsets = i;
5904 if (i < s->ethqsets)
5905 reduce_ethqs(adap, i);
5906 }
5907 if (is_offload(adap)) {
f36e58e5
HS		 5908 if (allocated < want) {
5909 s->rdmaqs = nchan;
5910 s->rdmaciqs = nchan;
5911 }
5912
5913 /* leftovers go to OFLD */
5914 i = allocated - EXTRA_VECS - s->max_ethqsets -
5915 s->rdmaqs - s->rdmaciqs;
c32ad224
AG		 5916 s->ofldqsets = (i / nchan) * nchan; /* round down */
5917 }
f36e58e5 5918 for (i = 0; i < allocated; ++i)
c32ad224
AG		 5919 adap->msix_info[i].vec = entries[i].vector;
5920
f36e58e5 5921 kfree(entries);
c32ad224 5922 return 0;
b8ff05a9
DM		 5923}
5924
5925#undef EXTRA_VECS
5926
91744948 5927static int init_rss(struct adapter *adap)
671b0060
DM		 5928{
5929 unsigned int i, j;
5930
5931 for_each_port(adap, i) {
5932 struct port_info *pi = adap2pinfo(adap, i);
5933
5934 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5935 if (!pi->rss)
5936 return -ENOMEM;
5937 for (j = 0; j < pi->rss_size; j++)
278bc429 5938 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 5939 }
5940 return 0;
5941}
5942
91744948 5943static void print_port_info(const struct net_device *dev)
b8ff05a9 5944{
b8ff05a9 5945 char buf[80];
118969ed 5946 char *bufp = buf;
f1a051b9 5947 const char *spd = "";
118969ed
DM		 5948 const struct port_info *pi = netdev_priv(dev);
5949 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 5950
5951 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5952 spd = " 2.5 GT/s";
5953 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5954 spd = " 5 GT/s";
d2e752db
RD		 5955 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5956 spd = " 8 GT/s";
b8ff05a9 5957
118969ed
DM		 5958 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5959 bufp += sprintf(bufp, "100/");
5960 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5961 bufp += sprintf(bufp, "1000/");
5962 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5963 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 5964 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5965 bufp += sprintf(bufp, "40G/");
118969ed
DM		 5966 if (bufp != buf)
5967 --bufp;
72aca4bf 5968 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 5969
5970 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 5971 adap->params.vpd.id,
d14807dd 5972 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 5973 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5974 (adap->flags & USING_MSIX) ? " MSI-X" :
5975 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 5976 netdev_info(dev, "S/N: %s, P/N: %s\n",
5977 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 5978}
5979
91744948 5980static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 5981{
e5c8ae5f 5982 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 5983}
5984
06546391
DM		 5985/*
5986 * Free the following resources:
5987 * - memory used for tables
5988 * - MSI/MSI-X
5989 * - net devices
5990 * - resources FW is holding for us
5991 */
5992static void free_some_resources(struct adapter *adapter)
5993{
5994 unsigned int i;
5995
5996 t4_free_mem(adapter->l2t);
5997 t4_free_mem(adapter->tids.tid_tab);
4b8e27a8
HS		 5998 kfree(adapter->sge.egr_map);
5999 kfree(adapter->sge.ingr_map);
6000 kfree(adapter->sge.starving_fl);
6001 kfree(adapter->sge.txq_maperr);
06546391
DM		 6002 disable_msi(adapter);
6003
6004 for_each_port(adapter, i)
671b0060
DM		 6005 if (adapter->port[i]) {
6006 kfree(adap2pinfo(adapter, i)->rss);
06546391 6007 free_netdev(adapter->port[i]);
671b0060 6008 }
06546391 6009 if (adapter->flags & FW_OK)
060e0c75 6010 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 6011}
6012
2ed28baa 6013#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 6014#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 6015 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 6016#define SEGMENT_SIZE 128
b8ff05a9 6017
1dd06ae8 6018static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 6019{
22adfe0a 6020 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 6021 struct port_info *pi;
c8f44aff 6022 bool highdma = false;
b8ff05a9 6023 struct adapter *adapter = NULL;
d6ce2628 6024 void __iomem *regs;
b8ff05a9
DM		 6025
6026 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
6027
6028 err = pci_request_regions(pdev, KBUILD_MODNAME);
6029 if (err) {
6030 /* Just info, some other driver may have claimed the device. */
6031 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
6032 return err;
6033 }
6034
b8ff05a9
DM		 6035 err = pci_enable_device(pdev);
6036 if (err) {
6037 dev_err(&pdev->dev, "cannot enable PCI device\n");
6038 goto out_release_regions;
6039 }
6040
d6ce2628
HS		 6041 regs = pci_ioremap_bar(pdev, 0);
6042 if (!regs) {
6043 dev_err(&pdev->dev, "cannot map device registers\n");
6044 err = -ENOMEM;
6045 goto out_disable_device;
6046 }
6047
8203b509
HS		 6048 err = t4_wait_dev_ready(regs);
6049 if (err < 0)
6050 goto out_unmap_bar0;
6051
d6ce2628 6052 /* We control everything through one PF */
0d804338 6053 func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
d6ce2628
HS		 6054 if (func != ent->driver_data) {
6055 iounmap(regs);
6056 pci_disable_device(pdev);
6057 pci_save_state(pdev); /* to restore SR-IOV later */
6058 goto sriov;
6059 }
6060
b8ff05a9 6061 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 6062 highdma = true;
b8ff05a9
DM		 6063 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
6064 if (err) {
6065 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
6066 "coherent allocations\n");
d6ce2628 6067 goto out_unmap_bar0;
b8ff05a9
DM		 6068 }
6069 } else {
6070 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
6071 if (err) {
6072 dev_err(&pdev->dev, "no usable DMA configuration\n");
d6ce2628 6073 goto out_unmap_bar0;
b8ff05a9
DM		 6074 }
6075 }
6076
6077 pci_enable_pcie_error_reporting(pdev);
ef306b50 6078 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 6079 pci_set_master(pdev);
6080 pci_save_state(pdev);
6081
6082 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
6083 if (!adapter) {
6084 err = -ENOMEM;
d6ce2628 6085 goto out_unmap_bar0;
b8ff05a9
DM		 6086 }
6087
29aaee65
AB		 6088 adapter->workq = create_singlethread_workqueue("cxgb4");
6089 if (!adapter->workq) {
6090 err = -ENOMEM;
6091 goto out_free_adapter;
6092 }
6093
144be3d9
GS		 6094 /* PCI device has been enabled */
6095 adapter->flags |= DEV_ENABLED;
6096
d6ce2628 6097 adapter->regs = regs;
b8ff05a9
DM		 6098 adapter->pdev = pdev;
6099 adapter->pdev_dev = &pdev->dev;
3069ee9b 6100 adapter->mbox = func;
060e0c75 6101 adapter->fn = func;
b8ff05a9
DM		 6102 adapter->msg_enable = dflt_msg_enable;
6103 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6104
6105 spin_lock_init(&adapter->stats_lock);
6106 spin_lock_init(&adapter->tid_release_lock);
e327c225 6107 spin_lock_init(&adapter->win0_lock);
b8ff05a9
DM		 6108
6109 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 6110 INIT_WORK(&adapter->db_full_task, process_db_full);
6111 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 6112
6113 err = t4_prep_adapter(adapter);
6114 if (err)
d6ce2628
HS		 6115 goto out_free_adapter;
6116
22adfe0a 6117
d14807dd 6118 if (!is_t4(adapter->params.chip)) {
f612b815
HS		 6119 s_qpp = (QUEUESPERPAGEPF0_S +
6120 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6121 adapter->fn);
6122 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6123 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
22adfe0a
SR		 6124 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6125
6126 /* Each segment size is 128B. Write coalescing is enabled only
6127 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6128 * queue is less no of segments that can be accommodated in
6129 * a page size.
6130 */
6131 if (qpp > num_seg) {
6132 dev_err(&pdev->dev,
6133 "Incorrect number of egress queues per page\n");
6134 err = -EINVAL;
d6ce2628 6135 goto out_free_adapter;
22adfe0a
SR		 6136 }
6137 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6138 pci_resource_len(pdev, 2));
6139 if (!adapter->bar2) {
6140 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6141 err = -ENOMEM;
d6ce2628 6142 goto out_free_adapter;
22adfe0a
SR		 6143 }
6144 }
6145
636f9d37 6146 setup_memwin(adapter);
b8ff05a9 6147 err = adap_init0(adapter);
636f9d37 6148 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6149 if (err)
6150 goto out_unmap_bar;
6151
6152 for_each_port(adapter, i) {
6153 struct net_device *netdev;
6154
6155 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6156 MAX_ETH_QSETS);
6157 if (!netdev) {
6158 err = -ENOMEM;
6159 goto out_free_dev;
6160 }
6161
6162 SET_NETDEV_DEV(netdev, &pdev->dev);
6163
6164 adapter->port[i] = netdev;
6165 pi = netdev_priv(netdev);
6166 pi->adapter = adapter;
6167 pi->xact_addr_filt = -1;
b8ff05a9 6168 pi->port_id = i;
b8ff05a9
DM		 6169 netdev->irq = pdev->irq;
6170
2ed28baa
MM		 6171 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6172 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6173 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6174 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6175 if (highdma)
6176 netdev->hw_features |= NETIF_F_HIGHDMA;
6177 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6178 netdev->vlan_features = netdev->features & VLAN_FEAT;
6179
01789349
JP		 6180 netdev->priv_flags |= IFF_UNICAST_FLT;
6181
b8ff05a9 6182 netdev->netdev_ops = &cxgb4_netdev_ops;
688848b1
AB		 6183#ifdef CONFIG_CHELSIO_T4_DCB
6184 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6185 cxgb4_dcb_state_init(netdev);
6186#endif
7ad24ea4 6187 netdev->ethtool_ops = &cxgb_ethtool_ops;
b8ff05a9
DM		 6188 }
6189
6190 pci_set_drvdata(pdev, adapter);
6191
6192 if (adapter->flags & FW_OK) {
060e0c75 6193 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6194 if (err)
6195 goto out_free_dev;
6196 }
6197
6198 /*
6199 * Configure queues and allocate tables now, they can be needed as
6200 * soon as the first register_netdev completes.
6201 */
6202 cfg_queues(adapter);
6203
6204 adapter->l2t = t4_init_l2t();
6205 if (!adapter->l2t) {
6206 /* We tolerate a lack of L2T, giving up some functionality */
6207 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6208 adapter->params.offload = 0;
6209 }
6210
b5a02f50
AB		 6211#if IS_ENABLED(CONFIG_IPV6)
6212 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
6213 adapter->clipt_end);
6214 if (!adapter->clipt) {
6215 /* We tolerate a lack of clip_table, giving up
6216 * some functionality
6217 */
6218 dev_warn(&pdev->dev,
6219 "could not allocate Clip table, continuing\n");
6220 adapter->params.offload = 0;
6221 }
6222#endif
b8ff05a9
DM		 6223 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6224 dev_warn(&pdev->dev, "could not allocate TID table, "
6225 "continuing\n");
6226 adapter->params.offload = 0;
6227 }
6228
f7cabcdd
DM		 6229 /* See what interrupts we'll be using */
6230 if (msi > 1 && enable_msix(adapter) == 0)
6231 adapter->flags |= USING_MSIX;
6232 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6233 adapter->flags |= USING_MSI;
6234
671b0060
DM		 6235 err = init_rss(adapter);
6236 if (err)
6237 goto out_free_dev;
6238
b8ff05a9
DM		 6239 /*
6240 * The card is now ready to go. If any errors occur during device
6241 * registration we do not fail the whole card but rather proceed only
6242 * with the ports we manage to register successfully. However we must
6243 * register at least one net device.
6244 */
6245 for_each_port(adapter, i) {
a57cabe0
DM		 6246 pi = adap2pinfo(adapter, i);
6247 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6248 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6249
b8ff05a9
DM		 6250 err = register_netdev(adapter->port[i]);
6251 if (err)
b1a3c2b6 6252 break;
b1a3c2b6
DM		 6253 adapter->chan_map[pi->tx_chan] = i;
6254 print_port_info(adapter->port[i]);
b8ff05a9 6255 }
b1a3c2b6 6256 if (i == 0) {
b8ff05a9
DM		 6257 dev_err(&pdev->dev, "could not register any net devices\n");
6258 goto out_free_dev;
6259 }
b1a3c2b6
DM		 6260 if (err) {
6261 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6262 err = 0;
6403eab1 6263 }
b8ff05a9
DM		 6264
6265 if (cxgb4_debugfs_root) {
6266 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6267 cxgb4_debugfs_root);
6268 setup_debugfs(adapter);
6269 }
6270
6482aa7c
DLR		 6271 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6272 pdev->needs_freset = 1;
6273
b8ff05a9
DM		 6274 if (is_offload(adapter))
6275 attach_ulds(adapter);
6276
8e1e6059 6277sriov:
b8ff05a9 6278#ifdef CONFIG_PCI_IOV
7d6727cf 6279 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6280 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6281 dev_info(&pdev->dev,
6282 "instantiated %u virtual functions\n",
6283 num_vf[func]);
6284#endif
6285 return 0;
6286
6287 out_free_dev:
06546391 6288 free_some_resources(adapter);
b8ff05a9 6289 out_unmap_bar:
d14807dd 6290 if (!is_t4(adapter->params.chip))
22adfe0a 6291 iounmap(adapter->bar2);
b8ff05a9 6292 out_free_adapter:
29aaee65
AB		 6293 if (adapter->workq)
6294 destroy_workqueue(adapter->workq);
6295
b8ff05a9 6296 kfree(adapter);
d6ce2628
HS		 6297 out_unmap_bar0:
6298 iounmap(regs);
b8ff05a9
DM		 6299 out_disable_device:
6300 pci_disable_pcie_error_reporting(pdev);
6301 pci_disable_device(pdev);
6302 out_release_regions:
6303 pci_release_regions(pdev);
b8ff05a9
DM		 6304 return err;
6305}
6306
91744948 6307static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6308{
6309 struct adapter *adapter = pci_get_drvdata(pdev);
6310
636f9d37 6311#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6312 pci_disable_sriov(pdev);
6313
636f9d37
VP		 6314#endif
6315
b8ff05a9
DM		 6316 if (adapter) {
6317 int i;
6318
29aaee65
AB		 6319 /* Tear down per-adapter Work Queue first since it can contain
6320 * references to our adapter data structure.
6321 */
6322 destroy_workqueue(adapter->workq);
6323
b8ff05a9
DM		 6324 if (is_offload(adapter))
6325 detach_ulds(adapter);
6326
b37987e8
HS		 6327 disable_interrupts(adapter);
6328
b8ff05a9 6329 for_each_port(adapter, i)
8f3a7676 6330 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6331 unregister_netdev(adapter->port[i]);
6332
9f16dc2e 6333 debugfs_remove_recursive(adapter->debugfs_root);
b8ff05a9 6334
f2b7e78d
VP		 6335 /* If we allocated filters, free up state associated with any
6336 * valid filters ...
6337 */
6338 if (adapter->tids.ftid_tab) {
6339 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6340 for (i = 0; i < (adapter->tids.nftids +
6341 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6342 if (f->valid)
6343 clear_filter(adapter, f);
6344 }
6345
aaefae9b
DM		 6346 if (adapter->flags & FULL_INIT_DONE)
6347 cxgb_down(adapter);
b8ff05a9 6348
06546391 6349 free_some_resources(adapter);
b5a02f50
AB		 6350#if IS_ENABLED(CONFIG_IPV6)
6351 t4_cleanup_clip_tbl(adapter);
6352#endif
b8ff05a9 6353 iounmap(adapter->regs);
d14807dd 6354 if (!is_t4(adapter->params.chip))
22adfe0a 6355 iounmap(adapter->bar2);
b8ff05a9 6356 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6357 if ((adapter->flags & DEV_ENABLED)) {
6358 pci_disable_device(pdev);
6359 adapter->flags &= ~DEV_ENABLED;
6360 }
b8ff05a9 6361 pci_release_regions(pdev);
ee9a33b2 6362 synchronize_rcu();
8b662fe7 6363 kfree(adapter);
a069ec91 6364 } else
b8ff05a9
DM		 6365 pci_release_regions(pdev);
6366}
6367
6368static struct pci_driver cxgb4_driver = {
6369 .name = KBUILD_MODNAME,
6370 .id_table = cxgb4_pci_tbl,
6371 .probe = init_one,
91744948 6372 .remove = remove_one,
687d705c 6373 .shutdown = remove_one,
204dc3c0 6374 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6375};
6376
6377static int __init cxgb4_init_module(void)
6378{
6379 int ret;
6380
6381 /* Debugfs support is optional, just warn if this fails */
6382 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6383 if (!cxgb4_debugfs_root)
428ac43f 6384 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6385
6386 ret = pci_register_driver(&cxgb4_driver);
29aaee65 6387 if (ret < 0)
b8ff05a9 6388 debugfs_remove(cxgb4_debugfs_root);
01bcca68 6389
1bb60376 6390#if IS_ENABLED(CONFIG_IPV6)
b5a02f50
AB		 6391 if (!inet6addr_registered) {
6392 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6393 inet6addr_registered = true;
6394 }
1bb60376 6395#endif
01bcca68 6396
b8ff05a9
DM		 6397 return ret;
6398}
6399
6400static void __exit cxgb4_cleanup_module(void)
6401{
1bb60376 6402#if IS_ENABLED(CONFIG_IPV6)
1793c798 6403 if (inet6addr_registered) {
b5a02f50
AB		 6404 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6405 inet6addr_registered = false;
6406 }
1bb60376 6407#endif
b8ff05a9
DM		 6408 pci_unregister_driver(&cxgb4_driver);
6409 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
6410}
6411
6412module_init(cxgb4_init_module);
6413module_exit(cxgb4_cleanup_module);
Ubuntu Artful kernel mirror