]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iw_cxgb4: don't truncate the recv window size
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
b8ff05a9
DM		 64#include <asm/uaccess.h>
65
66#include "cxgb4.h"
67#include "t4_regs.h"
68#include "t4_msg.h"
69#include "t4fw_api.h"
70#include "l2t.h"
71
01bcca68
VP		 72#include <../drivers/net/bonding/bonding.h>
73
74#ifdef DRV_VERSION
75#undef DRV_VERSION
76#endif
3a7f8554
SR		 77#define DRV_VERSION "2.0.0-ko"
78#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9
DM		 79
80/*
81 * Max interrupt hold-off timer value in us. Queues fall back to this value
82 * under extreme memory pressure so it's largish to give the system time to
83 * recover.
84 */
85#define MAX_SGE_TIMERVAL 200U
86
7ee9ff94 87enum {
13ee15d3
VP		 88 /*
89 * Physical Function provisioning constants.
90 */
91 PFRES_NVI = 4, /* # of Virtual Interfaces */
92 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */
93 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr
94 */
95 PFRES_NEQ = 256, /* # of egress queues */
96 PFRES_NIQ = 0, /* # of ingress queues */
97 PFRES_TC = 0, /* PCI-E traffic class */
98 PFRES_NEXACTF = 128, /* # of exact MPS filters */
99
100 PFRES_R_CAPS = FW_CMD_CAP_PF,
101 PFRES_WX_CAPS = FW_CMD_CAP_PF,
102
103#ifdef CONFIG_PCI_IOV
104 /*
105 * Virtual Function provisioning constants. We need two extra Ingress
106 * Queues with Interrupt capability to serve as the VF's Firmware
107 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
108 * neither will have Free Lists associated with them). For each
109 * Ethernet/Control Egress Queue and for each Free List, we need an
110 * Egress Context.
111 */
7ee9ff94
CL		 112 VFRES_NPORTS = 1, /* # of "ports" per VF */
113 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
114
115 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
116 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
117 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
7ee9ff94 118 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
13ee15d3 119 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
7ee9ff94
CL		 120 VFRES_TC = 0, /* PCI-E traffic class */
121 VFRES_NEXACTF = 16, /* # of exact MPS filters */
122
123 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
124 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
13ee15d3 125#endif
7ee9ff94
CL		 126};
127
128/*
129 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
130 * static and likely not to be useful in the long run. We really need to
131 * implement some form of persistent configuration which the firmware
132 * controls.
133 */
134static unsigned int pfvfres_pmask(struct adapter *adapter,
135 unsigned int pf, unsigned int vf)
136{
137 unsigned int portn, portvec;
138
139 /*
140 * Give PF's access to all of the ports.
141 */
142 if (vf == 0)
143 return FW_PFVF_CMD_PMASK_MASK;
144
145 /*
146 * For VFs, we'll assign them access to the ports based purely on the
147 * PF. We assign active ports in order, wrapping around if there are
148 * fewer active ports than PFs: e.g. active port[pf % nports].
149 * Unfortunately the adapter's port_info structs haven't been
150 * initialized yet so we have to compute this.
151 */
152 if (adapter->params.nports == 0)
153 return 0;
154
155 portn = pf % adapter->params.nports;
156 portvec = adapter->params.portvec;
157 for (;;) {
158 /*
159 * Isolate the lowest set bit in the port vector. If we're at
160 * the port number that we want, return that as the pmask.
161 * otherwise mask that bit out of the port vector and
162 * decrement our port number ...
163 */
164 unsigned int pmask = portvec ^ (portvec & (portvec-1));
165 if (portn == 0)
166 return pmask;
167 portn--;
168 portvec &= ~pmask;
169 }
170 /*NOTREACHED*/
171}
7ee9ff94 172
b8ff05a9
DM		 173enum {
174 MAX_TXQ_ENTRIES = 16384,
175 MAX_CTRL_TXQ_ENTRIES = 1024,
176 MAX_RSPQ_ENTRIES = 16384,
177 MAX_RX_BUFFERS = 16384,
178 MIN_TXQ_ENTRIES = 32,
179 MIN_CTRL_TXQ_ENTRIES = 32,
180 MIN_RSPQ_ENTRIES = 128,
181 MIN_FL_ENTRIES = 16
182};
183
f2b7e78d
VP		 184/* Host shadow copy of ingress filter entry. This is in host native format
185 * and doesn't match the ordering or bit order, etc. of the hardware of the
186 * firmware command. The use of bit-field structure elements is purely to
187 * remind ourselves of the field size limitations and save memory in the case
188 * where the filter table is large.
189 */
190struct filter_entry {
191 /* Administrative fields for filter.
192 */
193 u32 valid:1; /* filter allocated and valid */
194 u32 locked:1; /* filter is administratively locked */
195
196 u32 pending:1; /* filter action is pending firmware reply */
197 u32 smtidx:8; /* Source MAC Table index for smac */
198 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
199
200 /* The filter itself. Most of this is a straight copy of information
201 * provided by the extended ioctl(). Some fields are translated to
202 * internal forms -- for instance the Ingress Queue ID passed in from
203 * the ioctl() is translated into the Absolute Ingress Queue ID.
204 */
205 struct ch_filter_specification fs;
206};
207
b8ff05a9
DM		 208#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
209 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
210 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
211
060e0c75 212#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
b8ff05a9
DM		 213
214static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
060e0c75 215 CH_DEVICE(0xa000, 0), /* PE10K */
ccea790e
DM		 216 CH_DEVICE(0x4001, -1),
217 CH_DEVICE(0x4002, -1),
218 CH_DEVICE(0x4003, -1),
219 CH_DEVICE(0x4004, -1),
220 CH_DEVICE(0x4005, -1),
221 CH_DEVICE(0x4006, -1),
222 CH_DEVICE(0x4007, -1),
223 CH_DEVICE(0x4008, -1),
224 CH_DEVICE(0x4009, -1),
225 CH_DEVICE(0x400a, -1),
226 CH_DEVICE(0x4401, 4),
227 CH_DEVICE(0x4402, 4),
228 CH_DEVICE(0x4403, 4),
229 CH_DEVICE(0x4404, 4),
230 CH_DEVICE(0x4405, 4),
231 CH_DEVICE(0x4406, 4),
232 CH_DEVICE(0x4407, 4),
233 CH_DEVICE(0x4408, 4),
234 CH_DEVICE(0x4409, 4),
235 CH_DEVICE(0x440a, 4),
f637d577
VP		 236 CH_DEVICE(0x440d, 4),
237 CH_DEVICE(0x440e, 4),
9ef603a0
VP		 238 CH_DEVICE(0x5001, 4),
239 CH_DEVICE(0x5002, 4),
240 CH_DEVICE(0x5003, 4),
241 CH_DEVICE(0x5004, 4),
242 CH_DEVICE(0x5005, 4),
243 CH_DEVICE(0x5006, 4),
244 CH_DEVICE(0x5007, 4),
245 CH_DEVICE(0x5008, 4),
246 CH_DEVICE(0x5009, 4),
247 CH_DEVICE(0x500A, 4),
248 CH_DEVICE(0x500B, 4),
249 CH_DEVICE(0x500C, 4),
250 CH_DEVICE(0x500D, 4),
251 CH_DEVICE(0x500E, 4),
252 CH_DEVICE(0x500F, 4),
253 CH_DEVICE(0x5010, 4),
254 CH_DEVICE(0x5011, 4),
255 CH_DEVICE(0x5012, 4),
256 CH_DEVICE(0x5013, 4),
f0a8e6de
HS		 257 CH_DEVICE(0x5014, 4),
258 CH_DEVICE(0x5015, 4),
0183aa62
HS		 259 CH_DEVICE(0x5080, 4),
260 CH_DEVICE(0x5081, 4),
261 CH_DEVICE(0x5082, 4),
262 CH_DEVICE(0x5083, 4),
263 CH_DEVICE(0x5084, 4),
264 CH_DEVICE(0x5085, 4),
9ef603a0
VP		 265 CH_DEVICE(0x5401, 4),
266 CH_DEVICE(0x5402, 4),
267 CH_DEVICE(0x5403, 4),
268 CH_DEVICE(0x5404, 4),
269 CH_DEVICE(0x5405, 4),
270 CH_DEVICE(0x5406, 4),
271 CH_DEVICE(0x5407, 4),
272 CH_DEVICE(0x5408, 4),
273 CH_DEVICE(0x5409, 4),
274 CH_DEVICE(0x540A, 4),
275 CH_DEVICE(0x540B, 4),
276 CH_DEVICE(0x540C, 4),
277 CH_DEVICE(0x540D, 4),
278 CH_DEVICE(0x540E, 4),
279 CH_DEVICE(0x540F, 4),
280 CH_DEVICE(0x5410, 4),
281 CH_DEVICE(0x5411, 4),
282 CH_DEVICE(0x5412, 4),
283 CH_DEVICE(0x5413, 4),
f0a8e6de
HS		 284 CH_DEVICE(0x5414, 4),
285 CH_DEVICE(0x5415, 4),
0183aa62
HS		 286 CH_DEVICE(0x5480, 4),
287 CH_DEVICE(0x5481, 4),
288 CH_DEVICE(0x5482, 4),
289 CH_DEVICE(0x5483, 4),
290 CH_DEVICE(0x5484, 4),
291 CH_DEVICE(0x5485, 4),
b8ff05a9
DM		 292 { 0, }
293};
294
16e47624 295#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 296#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 297#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 298#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 299
300MODULE_DESCRIPTION(DRV_DESC);
301MODULE_AUTHOR("Chelsio Communications");
302MODULE_LICENSE("Dual BSD/GPL");
303MODULE_VERSION(DRV_VERSION);
304MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 305MODULE_FIRMWARE(FW4_FNAME);
0a57a536 306MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 307
636f9d37
VP		 308/*
309 * Normally we're willing to become the firmware's Master PF but will be happy
310 * if another PF has already become the Master and initialized the adapter.
311 * Setting "force_init" will cause this driver to forcibly establish itself as
312 * the Master PF and initialize the adapter.
313 */
314static uint force_init;
315
316module_param(force_init, uint, 0644);
317MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
318
13ee15d3
VP		 319/*
320 * Normally if the firmware we connect to has Configuration File support, we
321 * use that and only fall back to the old Driver-based initialization if the
322 * Configuration File fails for some reason. If force_old_init is set, then
323 * we'll always use the old Driver-based initialization sequence.
324 */
325static uint force_old_init;
326
327module_param(force_old_init, uint, 0644);
328MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
329
b8ff05a9
DM		 330static int dflt_msg_enable = DFLT_MSG_ENABLE;
331
332module_param(dflt_msg_enable, int, 0644);
333MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
334
335/*
336 * The driver uses the best interrupt scheme available on a platform in the
337 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
338 * of these schemes the driver may consider as follows:
339 *
340 * msi = 2: choose from among all three options
341 * msi = 1: only consider MSI and INTx interrupts
342 * msi = 0: force INTx interrupts
343 */
344static int msi = 2;
345
346module_param(msi, int, 0644);
347MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
348
349/*
350 * Queue interrupt hold-off timer values. Queues default to the first of these
351 * upon creation.
352 */
353static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
354
355module_param_array(intr_holdoff, uint, NULL, 0644);
356MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
357 "0..4 in microseconds");
358
359static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
360
361module_param_array(intr_cnt, uint, NULL, 0644);
362MODULE_PARM_DESC(intr_cnt,
363 "thresholds 1..3 for queue interrupt packet counters");
364
636f9d37
VP		 365/*
366 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
367 * offset by 2 bytes in order to have the IP headers line up on 4-byte
368 * boundaries. This is a requirement for many architectures which will throw
369 * a machine check fault if an attempt is made to access one of the 4-byte IP
370 * header fields on a non-4-byte boundary. And it's a major performance issue
371 * even on some architectures which allow it like some implementations of the
372 * x86 ISA. However, some architectures don't mind this and for some very
373 * edge-case performance sensitive applications (like forwarding large volumes
374 * of small packets), setting this DMA offset to 0 will decrease the number of
375 * PCI-E Bus transfers enough to measurably affect performance.
376 */
377static int rx_dma_offset = 2;
378
eb939922 379static bool vf_acls;
b8ff05a9
DM		 380
381#ifdef CONFIG_PCI_IOV
382module_param(vf_acls, bool, 0644);
383MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
384
7d6727cf
SR		 385/* Configure the number of PCI-E Virtual Function which are to be instantiated
386 * on SR-IOV Capable Physical Functions.
0a57a536 387 */
7d6727cf 388static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 389
390module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 391MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 392#endif
393
13ee15d3
VP		 394/*
395 * The filter TCAM has a fixed portion and a variable portion. The fixed
396 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
397 * ports. The variable portion is 36 bits which can include things like Exact
398 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
399 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
400 * far exceed the 36-bit budget for this "compressed" header portion of the
401 * filter. Thus, we have a scarce resource which must be carefully managed.
402 *
403 * By default we set this up to mostly match the set of filter matching
404 * capabilities of T3 but with accommodations for some of T4's more
405 * interesting features:
406 *
407 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
408 * [Inner] VLAN (17), Port (3), FCoE (1) }
409 */
410enum {
411 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
412 TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
413 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
414};
415
416static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
417
f2b7e78d
VP		 418module_param(tp_vlan_pri_map, uint, 0644);
419MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
420
b8ff05a9
DM		 421static struct dentry *cxgb4_debugfs_root;
422
423static LIST_HEAD(adapter_list);
424static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 425/* Adapter list to be accessed from atomic context */
426static LIST_HEAD(adap_rcu_list);
427static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 428static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
429static const char *uld_str[] = { "RDMA", "iSCSI" };
430
431static void link_report(struct net_device *dev)
432{
433 if (!netif_carrier_ok(dev))
434 netdev_info(dev, "link down\n");
435 else {
436 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
437
438 const char *s = "10Mbps";
439 const struct port_info *p = netdev_priv(dev);
440
441 switch (p->link_cfg.speed) {
e8b39015 442 case 10000:
b8ff05a9
DM		 443 s = "10Gbps";
444 break;
e8b39015 445 case 1000:
b8ff05a9
DM		 446 s = "1000Mbps";
447 break;
e8b39015 448 case 100:
b8ff05a9
DM		 449 s = "100Mbps";
450 break;
e8b39015 451 case 40000:
72aca4bf
KS		 452 s = "40Gbps";
453 break;
b8ff05a9
DM		 454 }
455
456 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
457 fc[p->link_cfg.fc]);
458 }
459}
460
461void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
462{
463 struct net_device *dev = adapter->port[port_id];
464
465 /* Skip changes from disabled ports. */
466 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
467 if (link_stat)
468 netif_carrier_on(dev);
469 else
470 netif_carrier_off(dev);
471
472 link_report(dev);
473 }
474}
475
476void t4_os_portmod_changed(const struct adapter *adap, int port_id)
477{
478 static const char *mod_str[] = {
a0881cab 479 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 480 };
481
482 const struct net_device *dev = adap->port[port_id];
483 const struct port_info *pi = netdev_priv(dev);
484
485 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
486 netdev_info(dev, "port module unplugged\n");
a0881cab 487 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 488 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
489}
490
491/*
492 * Configure the exact and hash address filters to handle a port's multicast
493 * and secondary unicast MAC addresses.
494 */
495static int set_addr_filters(const struct net_device *dev, bool sleep)
496{
497 u64 mhash = 0;
498 u64 uhash = 0;
499 bool free = true;
500 u16 filt_idx[7];
501 const u8 *addr[7];
502 int ret, naddr = 0;
b8ff05a9
DM		 503 const struct netdev_hw_addr *ha;
504 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 505 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 506 const struct port_info *pi = netdev_priv(dev);
060e0c75 507 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 508
509 /* first do the secondary unicast addresses */
510 netdev_for_each_uc_addr(ha, dev) {
511 addr[naddr++] = ha->addr;
512 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 513 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 514 naddr, addr, filt_idx, &uhash, sleep);
515 if (ret < 0)
516 return ret;
517
518 free = false;
519 naddr = 0;
520 }
521 }
522
523 /* next set up the multicast addresses */
4a35ecf8
DM		 524 netdev_for_each_mc_addr(ha, dev) {
525 addr[naddr++] = ha->addr;
526 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 527 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 528 naddr, addr, filt_idx, &mhash, sleep);
529 if (ret < 0)
530 return ret;
531
532 free = false;
533 naddr = 0;
534 }
535 }
536
060e0c75 537 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 538 uhash | mhash, sleep);
539}
540
3069ee9b
VP		 541int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
542module_param(dbfifo_int_thresh, int, 0644);
543MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
544
404d9e3f
VP		 545/*
546 * usecs to sleep while draining the dbfifo
547 */
548static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 549module_param(dbfifo_drain_delay, int, 0644);
550MODULE_PARM_DESC(dbfifo_drain_delay,
551 "usecs to sleep while draining the dbfifo");
552
b8ff05a9
DM		 553/*
554 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
555 * If @mtu is -1 it is left unchanged.
556 */
557static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
558{
559 int ret;
560 struct port_info *pi = netdev_priv(dev);
561
562 ret = set_addr_filters(dev, sleep_ok);
563 if (ret == 0)
060e0c75 564 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 565 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 566 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 567 sleep_ok);
568 return ret;
569}
570
3069ee9b
VP		 571static struct workqueue_struct *workq;
572
b8ff05a9
DM		 573/**
574 * link_start - enable a port
575 * @dev: the port to enable
576 *
577 * Performs the MAC and PHY actions needed to enable a port.
578 */
579static int link_start(struct net_device *dev)
580{
581 int ret;
582 struct port_info *pi = netdev_priv(dev);
060e0c75 583 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 584
585 /*
586 * We do not set address filters and promiscuity here, the stack does
587 * that step explicitly.
588 */
060e0c75 589 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 590 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 591 if (ret == 0) {
060e0c75 592 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 593 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 594 true);
b8ff05a9
DM		 595 if (ret >= 0) {
596 pi->xact_addr_filt = ret;
597 ret = 0;
598 }
599 }
600 if (ret == 0)
060e0c75
DM		 601 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
602 &pi->link_cfg);
b8ff05a9 603 if (ret == 0)
060e0c75 604 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
b8ff05a9
DM		 605 return ret;
606}
607
f2b7e78d
VP		 608/* Clear a filter and release any of its resources that we own. This also
609 * clears the filter's "pending" status.
610 */
611static void clear_filter(struct adapter *adap, struct filter_entry *f)
612{
613 /* If the new or old filter have loopback rewriteing rules then we'll
614 * need to free any existing Layer Two Table (L2T) entries of the old
615 * filter rule. The firmware will handle freeing up any Source MAC
616 * Table (SMT) entries used for rewriting Source MAC Addresses in
617 * loopback rules.
618 */
619 if (f->l2t)
620 cxgb4_l2t_release(f->l2t);
621
622 /* The zeroing of the filter rule below clears the filter valid,
623 * pending, locked flags, l2t pointer, etc. so it's all we need for
624 * this operation.
625 */
626 memset(f, 0, sizeof(*f));
627}
628
629/* Handle a filter write/deletion reply.
630 */
631static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
632{
633 unsigned int idx = GET_TID(rpl);
634 unsigned int nidx = idx - adap->tids.ftid_base;
635 unsigned int ret;
636 struct filter_entry *f;
637
638 if (idx >= adap->tids.ftid_base && nidx <
639 (adap->tids.nftids + adap->tids.nsftids)) {
640 idx = nidx;
641 ret = GET_TCB_COOKIE(rpl->cookie);
642 f = &adap->tids.ftid_tab[idx];
643
644 if (ret == FW_FILTER_WR_FLT_DELETED) {
645 /* Clear the filter when we get confirmation from the
646 * hardware that the filter has been deleted.
647 */
648 clear_filter(adap, f);
649 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
650 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
651 idx);
652 clear_filter(adap, f);
653 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
654 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
655 f->pending = 0; /* asynchronous setup completed */
656 f->valid = 1;
657 } else {
658 /* Something went wrong. Issue a warning about the
659 * problem and clear everything out.
660 */
661 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
662 idx, ret);
663 clear_filter(adap, f);
664 }
665 }
666}
667
668/* Response queue handler for the FW event queue.
b8ff05a9
DM		 669 */
670static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
671 const struct pkt_gl *gl)
672{
673 u8 opcode = ((const struct rss_header *)rsp)->opcode;
674
675 rsp++; /* skip RSS header */
b407a4a9
VP		 676
677 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
678 */
679 if (unlikely(opcode == CPL_FW4_MSG &&
680 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
681 rsp++;
682 opcode = ((const struct rss_header *)rsp)->opcode;
683 rsp++;
684 if (opcode != CPL_SGE_EGR_UPDATE) {
685 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
686 , opcode);
687 goto out;
688 }
689 }
690
b8ff05a9
DM		 691 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
692 const struct cpl_sge_egr_update *p = (void *)rsp;
693 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 694 struct sge_txq *txq;
b8ff05a9 695
e46dab4d 696 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 697 txq->restarts++;
e46dab4d 698 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 699 struct sge_eth_txq *eq;
700
701 eq = container_of(txq, struct sge_eth_txq, q);
702 netif_tx_wake_queue(eq->txq);
703 } else {
704 struct sge_ofld_txq *oq;
705
706 oq = container_of(txq, struct sge_ofld_txq, q);
707 tasklet_schedule(&oq->qresume_tsk);
708 }
709 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
710 const struct cpl_fw6_msg *p = (void *)rsp;
711
712 if (p->type == 0)
713 t4_handle_fw_rpl(q->adap, p->data);
714 } else if (opcode == CPL_L2T_WRITE_RPL) {
715 const struct cpl_l2t_write_rpl *p = (void *)rsp;
716
717 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 718 } else if (opcode == CPL_SET_TCB_RPL) {
719 const struct cpl_set_tcb_rpl *p = (void *)rsp;
720
721 filter_rpl(q->adap, p);
b8ff05a9
DM		 722 } else
723 dev_err(q->adap->pdev_dev,
724 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 725out:
b8ff05a9
DM		 726 return 0;
727}
728
729/**
730 * uldrx_handler - response queue handler for ULD queues
731 * @q: the response queue that received the packet
732 * @rsp: the response queue descriptor holding the offload message
733 * @gl: the gather list of packet fragments
734 *
735 * Deliver an ingress offload packet to a ULD. All processing is done by
736 * the ULD, we just maintain statistics.
737 */
738static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
739 const struct pkt_gl *gl)
740{
741 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
742
b407a4a9
VP		 743 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
744 */
745 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
746 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
747 rsp += 2;
748
b8ff05a9
DM		 749 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
750 rxq->stats.nomem++;
751 return -1;
752 }
753 if (gl == NULL)
754 rxq->stats.imm++;
755 else if (gl == CXGB4_MSG_AN)
756 rxq->stats.an++;
757 else
758 rxq->stats.pkts++;
759 return 0;
760}
761
762static void disable_msi(struct adapter *adapter)
763{
764 if (adapter->flags & USING_MSIX) {
765 pci_disable_msix(adapter->pdev);
766 adapter->flags &= ~USING_MSIX;
767 } else if (adapter->flags & USING_MSI) {
768 pci_disable_msi(adapter->pdev);
769 adapter->flags &= ~USING_MSI;
770 }
771}
772
773/*
774 * Interrupt handler for non-data events used with MSI-X.
775 */
776static irqreturn_t t4_nondata_intr(int irq, void *cookie)
777{
778 struct adapter *adap = cookie;
779
780 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
781 if (v & PFSW) {
782 adap->swintr = 1;
783 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
784 }
785 t4_slow_intr_handler(adap);
786 return IRQ_HANDLED;
787}
788
789/*
790 * Name the MSI-X interrupts.
791 */
792static void name_msix_vecs(struct adapter *adap)
793{
ba27816c 794 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 795
796 /* non-data interrupts */
b1a3c2b6 797 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 798
799 /* FW events */
b1a3c2b6
DM		 800 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
801 adap->port[0]->name);
b8ff05a9
DM		 802
803 /* Ethernet queues */
804 for_each_port(adap, j) {
805 struct net_device *d = adap->port[j];
806 const struct port_info *pi = netdev_priv(d);
807
ba27816c 808 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 809 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
810 d->name, i);
b8ff05a9
DM		 811 }
812
813 /* offload queues */
ba27816c
DM		 814 for_each_ofldrxq(&adap->sge, i)
815 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 816 adap->port[0]->name, i);
ba27816c
DM		 817
818 for_each_rdmarxq(&adap->sge, i)
819 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 820 adap->port[0]->name, i);
cf38be6d
HS		 821
822 for_each_rdmaciq(&adap->sge, i)
823 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
824 adap->port[0]->name, i);
b8ff05a9
DM		 825}
826
827static int request_msix_queue_irqs(struct adapter *adap)
828{
829 struct sge *s = &adap->sge;
cf38be6d
HS		 830 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
831 int msi_index = 2;
b8ff05a9
DM		 832
833 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
834 adap->msix_info[1].desc, &s->fw_evtq);
835 if (err)
836 return err;
837
838 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 839 err = request_irq(adap->msix_info[msi_index].vec,
840 t4_sge_intr_msix, 0,
841 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 842 &s->ethrxq[ethqidx].rspq);
843 if (err)
844 goto unwind;
404d9e3f 845 msi_index++;
b8ff05a9
DM		 846 }
847 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 848 err = request_irq(adap->msix_info[msi_index].vec,
849 t4_sge_intr_msix, 0,
850 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 851 &s->ofldrxq[ofldqidx].rspq);
852 if (err)
853 goto unwind;
404d9e3f 854 msi_index++;
b8ff05a9
DM		 855 }
856 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 857 err = request_irq(adap->msix_info[msi_index].vec,
858 t4_sge_intr_msix, 0,
859 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 860 &s->rdmarxq[rdmaqidx].rspq);
861 if (err)
862 goto unwind;
404d9e3f 863 msi_index++;
b8ff05a9 864 }
cf38be6d
HS		 865 for_each_rdmaciq(s, rdmaciqqidx) {
866 err = request_irq(adap->msix_info[msi_index].vec,
867 t4_sge_intr_msix, 0,
868 adap->msix_info[msi_index].desc,
869 &s->rdmaciq[rdmaciqqidx].rspq);
870 if (err)
871 goto unwind;
872 msi_index++;
873 }
b8ff05a9
DM		 874 return 0;
875
876unwind:
cf38be6d
HS		 877 while (--rdmaciqqidx >= 0)
878 free_irq(adap->msix_info[--msi_index].vec,
879 &s->rdmaciq[rdmaciqqidx].rspq);
b8ff05a9 880 while (--rdmaqidx >= 0)
404d9e3f 881 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 882 &s->rdmarxq[rdmaqidx].rspq);
883 while (--ofldqidx >= 0)
404d9e3f 884 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 885 &s->ofldrxq[ofldqidx].rspq);
886 while (--ethqidx >= 0)
404d9e3f
VP		 887 free_irq(adap->msix_info[--msi_index].vec,
888 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 889 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
890 return err;
891}
892
893static void free_msix_queue_irqs(struct adapter *adap)
894{
404d9e3f 895 int i, msi_index = 2;
b8ff05a9
DM		 896 struct sge *s = &adap->sge;
897
898 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
899 for_each_ethrxq(s, i)
404d9e3f 900 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 901 for_each_ofldrxq(s, i)
404d9e3f 902 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 903 for_each_rdmarxq(s, i)
404d9e3f 904 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
cf38be6d
HS		 905 for_each_rdmaciq(s, i)
906 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
b8ff05a9
DM		 907}
908
671b0060
DM		 909/**
910 * write_rss - write the RSS table for a given port
911 * @pi: the port
912 * @queues: array of queue indices for RSS
913 *
914 * Sets up the portion of the HW RSS table for the port's VI to distribute
915 * packets to the Rx queues in @queues.
916 */
917static int write_rss(const struct port_info *pi, const u16 *queues)
918{
919 u16 *rss;
920 int i, err;
921 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
922
923 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
924 if (!rss)
925 return -ENOMEM;
926
927 /* map the queue indices to queue ids */
928 for (i = 0; i < pi->rss_size; i++, queues++)
929 rss[i] = q[*queues].rspq.abs_id;
930
060e0c75
DM		 931 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
932 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 933 kfree(rss);
934 return err;
935}
936
b8ff05a9
DM		 937/**
938 * setup_rss - configure RSS
939 * @adap: the adapter
940 *
671b0060 941 * Sets up RSS for each port.
b8ff05a9
DM		 942 */
943static int setup_rss(struct adapter *adap)
944{
671b0060 945 int i, err;
b8ff05a9
DM		 946
947 for_each_port(adap, i) {
948 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 949
671b0060 950 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 951 if (err)
952 return err;
953 }
954 return 0;
955}
956
e46dab4d
DM		 957/*
958 * Return the channel of the ingress queue with the given qid.
959 */
960static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
961{
962 qid -= p->ingr_start;
963 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
964}
965
b8ff05a9
DM		 966/*
967 * Wait until all NAPI handlers are descheduled.
968 */
969static void quiesce_rx(struct adapter *adap)
970{
971 int i;
972
973 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
974 struct sge_rspq *q = adap->sge.ingr_map[i];
975
976 if (q && q->handler)
977 napi_disable(&q->napi);
978 }
979}
980
981/*
982 * Enable NAPI scheduling and interrupt generation for all Rx queues.
983 */
984static void enable_rx(struct adapter *adap)
985{
986 int i;
987
988 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
989 struct sge_rspq *q = adap->sge.ingr_map[i];
990
991 if (!q)
992 continue;
993 if (q->handler)
994 napi_enable(&q->napi);
995 /* 0-increment GTS to start the timer and enable interrupts */
996 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
997 SEINTARM(q->intr_params) |
998 INGRESSQID(q->cntxt_id));
999 }
1000}
1001
1002/**
1003 * setup_sge_queues - configure SGE Tx/Rx/response queues
1004 * @adap: the adapter
1005 *
1006 * Determines how many sets of SGE queues to use and initializes them.
1007 * We support multiple queue sets per port if we have MSI-X, otherwise
1008 * just one queue set per port.
1009 */
1010static int setup_sge_queues(struct adapter *adap)
1011{
1012 int err, msi_idx, i, j;
1013 struct sge *s = &adap->sge;
1014
1015 bitmap_zero(s->starving_fl, MAX_EGRQ);
1016 bitmap_zero(s->txq_maperr, MAX_EGRQ);
1017
1018 if (adap->flags & USING_MSIX)
1019 msi_idx = 1; /* vector 0 is for non-queue interrupts */
1020 else {
1021 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1022 NULL, NULL);
1023 if (err)
1024 return err;
1025 msi_idx = -((int)s->intrq.abs_id + 1);
1026 }
1027
1028 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1029 msi_idx, NULL, fwevtq_handler);
1030 if (err) {
1031freeout: t4_free_sge_resources(adap);
1032 return err;
1033 }
1034
1035 for_each_port(adap, i) {
1036 struct net_device *dev = adap->port[i];
1037 struct port_info *pi = netdev_priv(dev);
1038 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1039 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1040
1041 for (j = 0; j < pi->nqsets; j++, q++) {
1042 if (msi_idx > 0)
1043 msi_idx++;
1044 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1045 msi_idx, &q->fl,
1046 t4_ethrx_handler);
1047 if (err)
1048 goto freeout;
1049 q->rspq.idx = j;
1050 memset(&q->stats, 0, sizeof(q->stats));
1051 }
1052 for (j = 0; j < pi->nqsets; j++, t++) {
1053 err = t4_sge_alloc_eth_txq(adap, t, dev,
1054 netdev_get_tx_queue(dev, j),
1055 s->fw_evtq.cntxt_id);
1056 if (err)
1057 goto freeout;
1058 }
1059 }
1060
1061 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1062 for_each_ofldrxq(s, i) {
1063 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1064 struct net_device *dev = adap->port[i / j];
1065
1066 if (msi_idx > 0)
1067 msi_idx++;
1068 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
cf38be6d
HS		 1069 q->fl.size ? &q->fl : NULL,
1070 uldrx_handler);
b8ff05a9
DM		 1071 if (err)
1072 goto freeout;
1073 memset(&q->stats, 0, sizeof(q->stats));
1074 s->ofld_rxq[i] = q->rspq.abs_id;
1075 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1076 s->fw_evtq.cntxt_id);
1077 if (err)
1078 goto freeout;
1079 }
1080
1081 for_each_rdmarxq(s, i) {
1082 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1083
1084 if (msi_idx > 0)
1085 msi_idx++;
1086 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
cf38be6d
HS		 1087 msi_idx, q->fl.size ? &q->fl : NULL,
1088 uldrx_handler);
b8ff05a9
DM		 1089 if (err)
1090 goto freeout;
1091 memset(&q->stats, 0, sizeof(q->stats));
1092 s->rdma_rxq[i] = q->rspq.abs_id;
1093 }
1094
cf38be6d
HS		 1095 for_each_rdmaciq(s, i) {
1096 struct sge_ofld_rxq *q = &s->rdmaciq[i];
1097
1098 if (msi_idx > 0)
1099 msi_idx++;
1100 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1101 msi_idx, q->fl.size ? &q->fl : NULL,
1102 uldrx_handler);
1103 if (err)
1104 goto freeout;
1105 memset(&q->stats, 0, sizeof(q->stats));
1106 s->rdma_ciq[i] = q->rspq.abs_id;
1107 }
1108
b8ff05a9
DM		 1109 for_each_port(adap, i) {
1110 /*
1111 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1112 * have RDMA queues, and that's the right value.
1113 */
1114 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1115 s->fw_evtq.cntxt_id,
1116 s->rdmarxq[i].rspq.cntxt_id);
1117 if (err)
1118 goto freeout;
1119 }
1120
1121 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1122 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1123 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1124 return 0;
1125}
1126
b8ff05a9
DM		 1127/*
1128 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1129 * The allocated memory is cleared.
1130 */
1131void *t4_alloc_mem(size_t size)
1132{
8be04b93 1133 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1134
1135 if (!p)
89bf67f1 1136 p = vzalloc(size);
b8ff05a9
DM		 1137 return p;
1138}
1139
1140/*
1141 * Free memory allocated through alloc_mem().
1142 */
31b9c19b 1143static void t4_free_mem(void *addr)
b8ff05a9
DM		 1144{
1145 if (is_vmalloc_addr(addr))
1146 vfree(addr);
1147 else
1148 kfree(addr);
1149}
1150
f2b7e78d
VP		 1151/* Send a Work Request to write the filter at a specified index. We construct
1152 * a Firmware Filter Work Request to have the work done and put the indicated
1153 * filter into "pending" mode which will prevent any further actions against
1154 * it till we get a reply from the firmware on the completion status of the
1155 * request.
1156 */
1157static int set_filter_wr(struct adapter *adapter, int fidx)
1158{
1159 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1160 struct sk_buff *skb;
1161 struct fw_filter_wr *fwr;
1162 unsigned int ftid;
1163
1164 /* If the new filter requires loopback Destination MAC and/or VLAN
1165 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1166 * the filter.
1167 */
1168 if (f->fs.newdmac || f->fs.newvlan) {
1169 /* allocate L2T entry for new filter */
1170 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1171 if (f->l2t == NULL)
1172 return -EAGAIN;
1173 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1174 f->fs.eport, f->fs.dmac)) {
1175 cxgb4_l2t_release(f->l2t);
1176 f->l2t = NULL;
1177 return -ENOMEM;
1178 }
1179 }
1180
1181 ftid = adapter->tids.ftid_base + fidx;
1182
1183 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1184 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1185 memset(fwr, 0, sizeof(*fwr));
1186
1187 /* It would be nice to put most of the following in t4_hw.c but most
1188 * of the work is translating the cxgbtool ch_filter_specification
1189 * into the Work Request and the definition of that structure is
1190 * currently in cxgbtool.h which isn't appropriate to pull into the
1191 * common code. We may eventually try to come up with a more neutral
1192 * filter specification structure but for now it's easiest to simply
1193 * put this fairly direct code in line ...
1194 */
1195 fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1196 fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1197 fwr->tid_to_iq =
1198 htonl(V_FW_FILTER_WR_TID(ftid) |
1199 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1200 V_FW_FILTER_WR_NOREPLY(0) |
1201 V_FW_FILTER_WR_IQ(f->fs.iq));
1202 fwr->del_filter_to_l2tix =
1203 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1204 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1205 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1206 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1207 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1208 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1209 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1210 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1211 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1212 f->fs.newvlan == VLAN_REWRITE) |
1213 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1214 f->fs.newvlan == VLAN_REWRITE) |
1215 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1216 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1217 V_FW_FILTER_WR_PRIO(f->fs.prio) |
1218 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1219 fwr->ethtype = htons(f->fs.val.ethtype);
1220 fwr->ethtypem = htons(f->fs.mask.ethtype);
1221 fwr->frag_to_ovlan_vldm =
1222 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1223 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1224 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1225 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1226 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1227 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1228 fwr->smac_sel = 0;
1229 fwr->rx_chan_rx_rpl_iq =
1230 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1231 V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1232 fwr->maci_to_matchtypem =
1233 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1234 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1235 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1236 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1237 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1238 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1239 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1240 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1241 fwr->ptcl = f->fs.val.proto;
1242 fwr->ptclm = f->fs.mask.proto;
1243 fwr->ttyp = f->fs.val.tos;
1244 fwr->ttypm = f->fs.mask.tos;
1245 fwr->ivlan = htons(f->fs.val.ivlan);
1246 fwr->ivlanm = htons(f->fs.mask.ivlan);
1247 fwr->ovlan = htons(f->fs.val.ovlan);
1248 fwr->ovlanm = htons(f->fs.mask.ovlan);
1249 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1250 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1251 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1252 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1253 fwr->lp = htons(f->fs.val.lport);
1254 fwr->lpm = htons(f->fs.mask.lport);
1255 fwr->fp = htons(f->fs.val.fport);
1256 fwr->fpm = htons(f->fs.mask.fport);
1257 if (f->fs.newsmac)
1258 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1259
1260 /* Mark the filter as "pending" and ship off the Filter Work Request.
1261 * When we get the Work Request Reply we'll clear the pending status.
1262 */
1263 f->pending = 1;
1264 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1265 t4_ofld_send(adapter, skb);
1266 return 0;
1267}
1268
1269/* Delete the filter at a specified index.
1270 */
1271static int del_filter_wr(struct adapter *adapter, int fidx)
1272{
1273 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1274 struct sk_buff *skb;
1275 struct fw_filter_wr *fwr;
1276 unsigned int len, ftid;
1277
1278 len = sizeof(*fwr);
1279 ftid = adapter->tids.ftid_base + fidx;
1280
1281 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1282 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1283 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1284
1285 /* Mark the filter as "pending" and ship off the Filter Work Request.
1286 * When we get the Work Request Reply we'll clear the pending status.
1287 */
1288 f->pending = 1;
1289 t4_mgmt_tx(adapter, skb);
1290 return 0;
1291}
1292
b8ff05a9
DM		 1293static inline int is_offload(const struct adapter *adap)
1294{
1295 return adap->params.offload;
1296}
1297
1298/*
1299 * Implementation of ethtool operations.
1300 */
1301
1302static u32 get_msglevel(struct net_device *dev)
1303{
1304 return netdev2adap(dev)->msg_enable;
1305}
1306
1307static void set_msglevel(struct net_device *dev, u32 val)
1308{
1309 netdev2adap(dev)->msg_enable = val;
1310}
1311
1312static char stats_strings[][ETH_GSTRING_LEN] = {
1313 "TxOctetsOK ",
1314 "TxFramesOK ",
1315 "TxBroadcastFrames ",
1316 "TxMulticastFrames ",
1317 "TxUnicastFrames ",
1318 "TxErrorFrames ",
1319
1320 "TxFrames64 ",
1321 "TxFrames65To127 ",
1322 "TxFrames128To255 ",
1323 "TxFrames256To511 ",
1324 "TxFrames512To1023 ",
1325 "TxFrames1024To1518 ",
1326 "TxFrames1519ToMax ",
1327
1328 "TxFramesDropped ",
1329 "TxPauseFrames ",
1330 "TxPPP0Frames ",
1331 "TxPPP1Frames ",
1332 "TxPPP2Frames ",
1333 "TxPPP3Frames ",
1334 "TxPPP4Frames ",
1335 "TxPPP5Frames ",
1336 "TxPPP6Frames ",
1337 "TxPPP7Frames ",
1338
1339 "RxOctetsOK ",
1340 "RxFramesOK ",
1341 "RxBroadcastFrames ",
1342 "RxMulticastFrames ",
1343 "RxUnicastFrames ",
1344
1345 "RxFramesTooLong ",
1346 "RxJabberErrors ",
1347 "RxFCSErrors ",
1348 "RxLengthErrors ",
1349 "RxSymbolErrors ",
1350 "RxRuntFrames ",
1351
1352 "RxFrames64 ",
1353 "RxFrames65To127 ",
1354 "RxFrames128To255 ",
1355 "RxFrames256To511 ",
1356 "RxFrames512To1023 ",
1357 "RxFrames1024To1518 ",
1358 "RxFrames1519ToMax ",
1359
1360 "RxPauseFrames ",
1361 "RxPPP0Frames ",
1362 "RxPPP1Frames ",
1363 "RxPPP2Frames ",
1364 "RxPPP3Frames ",
1365 "RxPPP4Frames ",
1366 "RxPPP5Frames ",
1367 "RxPPP6Frames ",
1368 "RxPPP7Frames ",
1369
1370 "RxBG0FramesDropped ",
1371 "RxBG1FramesDropped ",
1372 "RxBG2FramesDropped ",
1373 "RxBG3FramesDropped ",
1374 "RxBG0FramesTrunc ",
1375 "RxBG1FramesTrunc ",
1376 "RxBG2FramesTrunc ",
1377 "RxBG3FramesTrunc ",
1378
1379 "TSO ",
1380 "TxCsumOffload ",
1381 "RxCsumGood ",
1382 "VLANextractions ",
1383 "VLANinsertions ",
4a6346d4
DM		 1384 "GROpackets ",
1385 "GROmerged ",
22adfe0a
SR		 1386 "WriteCoalSuccess ",
1387 "WriteCoalFail ",
b8ff05a9
DM		 1388};
1389
1390static int get_sset_count(struct net_device *dev, int sset)
1391{
1392 switch (sset) {
1393 case ETH_SS_STATS:
1394 return ARRAY_SIZE(stats_strings);
1395 default:
1396 return -EOPNOTSUPP;
1397 }
1398}
1399
1400#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1401#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1402
1403static int get_regs_len(struct net_device *dev)
1404{
251f9e88 1405 struct adapter *adap = netdev2adap(dev);
d14807dd 1406 if (is_t4(adap->params.chip))
251f9e88
SR		 1407 return T4_REGMAP_SIZE;
1408 else
1409 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1410}
1411
1412static int get_eeprom_len(struct net_device *dev)
1413{
1414 return EEPROMSIZE;
1415}
1416
1417static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1418{
1419 struct adapter *adapter = netdev2adap(dev);
1420
23020ab3
RJ		 1421 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1422 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1423 strlcpy(info->bus_info, pci_name(adapter->pdev),
1424 sizeof(info->bus_info));
b8ff05a9 1425
84b40501 1426 if (adapter->params.fw_vers)
b8ff05a9
DM		 1427 snprintf(info->fw_version, sizeof(info->fw_version),
1428 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1429 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1430 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1431 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1432 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1433 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1434 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1435 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1436 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1437}
1438
1439static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1440{
1441 if (stringset == ETH_SS_STATS)
1442 memcpy(data, stats_strings, sizeof(stats_strings));
1443}
1444
1445/*
1446 * port stats maintained per queue of the port. They should be in the same
1447 * order as in stats_strings above.
1448 */
1449struct queue_port_stats {
1450 u64 tso;
1451 u64 tx_csum;
1452 u64 rx_csum;
1453 u64 vlan_ex;
1454 u64 vlan_ins;
4a6346d4
DM		 1455 u64 gro_pkts;
1456 u64 gro_merged;
b8ff05a9
DM		 1457};
1458
1459static void collect_sge_port_stats(const struct adapter *adap,
1460 const struct port_info *p, struct queue_port_stats *s)
1461{
1462 int i;
1463 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1464 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1465
1466 memset(s, 0, sizeof(*s));
1467 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1468 s->tso += tx->tso;
1469 s->tx_csum += tx->tx_cso;
1470 s->rx_csum += rx->stats.rx_cso;
1471 s->vlan_ex += rx->stats.vlan_ex;
1472 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1473 s->gro_pkts += rx->stats.lro_pkts;
1474 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1475 }
1476}
1477
1478static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1479 u64 *data)
1480{
1481 struct port_info *pi = netdev_priv(dev);
1482 struct adapter *adapter = pi->adapter;
22adfe0a 1483 u32 val1, val2;
b8ff05a9
DM		 1484
1485 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1486
1487 data += sizeof(struct port_stats) / sizeof(u64);
1488 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1489 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1490 if (!is_t4(adapter->params.chip)) {
22adfe0a
SR		 1491 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1492 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1493 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1494 *data = val1 - val2;
1495 data++;
1496 *data = val2;
1497 data++;
1498 } else {
1499 memset(data, 0, 2 * sizeof(u64));
1500 *data += 2;
1501 }
b8ff05a9
DM		 1502}
1503
1504/*
1505 * Return a version number to identify the type of adapter. The scheme is:
1506 * - bits 0..9: chip version
1507 * - bits 10..15: chip revision
835bb606 1508 * - bits 16..23: register dump version
b8ff05a9
DM		 1509 */
1510static inline unsigned int mk_adap_vers(const struct adapter *ap)
1511{
d14807dd
HS		 1512 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1513 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1514}
1515
1516static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1517 unsigned int end)
1518{
1519 u32 *p = buf + start;
1520
1521 for ( ; start <= end; start += sizeof(u32))
1522 *p++ = t4_read_reg(ap, start);
1523}
1524
1525static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1526 void *buf)
1527{
251f9e88 1528 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1529 0x1008, 0x1108,
1530 0x1180, 0x11b4,
1531 0x11fc, 0x123c,
1532 0x1300, 0x173c,
1533 0x1800, 0x18fc,
1534 0x3000, 0x30d8,
1535 0x30e0, 0x5924,
1536 0x5960, 0x59d4,
1537 0x5a00, 0x5af8,
1538 0x6000, 0x6098,
1539 0x6100, 0x6150,
1540 0x6200, 0x6208,
1541 0x6240, 0x6248,
1542 0x6280, 0x6338,
1543 0x6370, 0x638c,
1544 0x6400, 0x643c,
1545 0x6500, 0x6524,
1546 0x6a00, 0x6a38,
1547 0x6a60, 0x6a78,
1548 0x6b00, 0x6b84,
1549 0x6bf0, 0x6c84,
1550 0x6cf0, 0x6d84,
1551 0x6df0, 0x6e84,
1552 0x6ef0, 0x6f84,
1553 0x6ff0, 0x7084,
1554 0x70f0, 0x7184,
1555 0x71f0, 0x7284,
1556 0x72f0, 0x7384,
1557 0x73f0, 0x7450,
1558 0x7500, 0x7530,
1559 0x7600, 0x761c,
1560 0x7680, 0x76cc,
1561 0x7700, 0x7798,
1562 0x77c0, 0x77fc,
1563 0x7900, 0x79fc,
1564 0x7b00, 0x7c38,
1565 0x7d00, 0x7efc,
1566 0x8dc0, 0x8e1c,
1567 0x8e30, 0x8e78,
1568 0x8ea0, 0x8f6c,
1569 0x8fc0, 0x9074,
1570 0x90fc, 0x90fc,
1571 0x9400, 0x9458,
1572 0x9600, 0x96bc,
1573 0x9800, 0x9808,
1574 0x9820, 0x983c,
1575 0x9850, 0x9864,
1576 0x9c00, 0x9c6c,
1577 0x9c80, 0x9cec,
1578 0x9d00, 0x9d6c,
1579 0x9d80, 0x9dec,
1580 0x9e00, 0x9e6c,
1581 0x9e80, 0x9eec,
1582 0x9f00, 0x9f6c,
1583 0x9f80, 0x9fec,
1584 0xd004, 0xd03c,
1585 0xdfc0, 0xdfe0,
1586 0xe000, 0xea7c,
1587 0xf000, 0x11190,
835bb606
DM		 1588 0x19040, 0x1906c,
1589 0x19078, 0x19080,
1590 0x1908c, 0x19124,
b8ff05a9
DM		 1591 0x19150, 0x191b0,
1592 0x191d0, 0x191e8,
1593 0x19238, 0x1924c,
1594 0x193f8, 0x19474,
1595 0x19490, 0x194f8,
1596 0x19800, 0x19f30,
1597 0x1a000, 0x1a06c,
1598 0x1a0b0, 0x1a120,
1599 0x1a128, 0x1a138,
1600 0x1a190, 0x1a1c4,
1601 0x1a1fc, 0x1a1fc,
1602 0x1e040, 0x1e04c,
835bb606 1603 0x1e284, 0x1e28c,
b8ff05a9
DM		 1604 0x1e2c0, 0x1e2c0,
1605 0x1e2e0, 0x1e2e0,
1606 0x1e300, 0x1e384,
1607 0x1e3c0, 0x1e3c8,
1608 0x1e440, 0x1e44c,
835bb606 1609 0x1e684, 0x1e68c,
b8ff05a9
DM		 1610 0x1e6c0, 0x1e6c0,
1611 0x1e6e0, 0x1e6e0,
1612 0x1e700, 0x1e784,
1613 0x1e7c0, 0x1e7c8,
1614 0x1e840, 0x1e84c,
835bb606 1615 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1616 0x1eac0, 0x1eac0,
1617 0x1eae0, 0x1eae0,
1618 0x1eb00, 0x1eb84,
1619 0x1ebc0, 0x1ebc8,
1620 0x1ec40, 0x1ec4c,
835bb606 1621 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1622 0x1eec0, 0x1eec0,
1623 0x1eee0, 0x1eee0,
1624 0x1ef00, 0x1ef84,
1625 0x1efc0, 0x1efc8,
1626 0x1f040, 0x1f04c,
835bb606 1627 0x1f284, 0x1f28c,
b8ff05a9
DM		 1628 0x1f2c0, 0x1f2c0,
1629 0x1f2e0, 0x1f2e0,
1630 0x1f300, 0x1f384,
1631 0x1f3c0, 0x1f3c8,
1632 0x1f440, 0x1f44c,
835bb606 1633 0x1f684, 0x1f68c,
b8ff05a9
DM		 1634 0x1f6c0, 0x1f6c0,
1635 0x1f6e0, 0x1f6e0,
1636 0x1f700, 0x1f784,
1637 0x1f7c0, 0x1f7c8,
1638 0x1f840, 0x1f84c,
835bb606 1639 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1640 0x1fac0, 0x1fac0,
1641 0x1fae0, 0x1fae0,
1642 0x1fb00, 0x1fb84,
1643 0x1fbc0, 0x1fbc8,
1644 0x1fc40, 0x1fc4c,
835bb606 1645 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1646 0x1fec0, 0x1fec0,
1647 0x1fee0, 0x1fee0,
1648 0x1ff00, 0x1ff84,
1649 0x1ffc0, 0x1ffc8,
1650 0x20000, 0x2002c,
1651 0x20100, 0x2013c,
1652 0x20190, 0x201c8,
1653 0x20200, 0x20318,
1654 0x20400, 0x20528,
1655 0x20540, 0x20614,
1656 0x21000, 0x21040,
1657 0x2104c, 0x21060,
1658 0x210c0, 0x210ec,
1659 0x21200, 0x21268,
1660 0x21270, 0x21284,
1661 0x212fc, 0x21388,
1662 0x21400, 0x21404,
1663 0x21500, 0x21518,
1664 0x2152c, 0x2153c,
1665 0x21550, 0x21554,
1666 0x21600, 0x21600,
1667 0x21608, 0x21628,
1668 0x21630, 0x2163c,
1669 0x21700, 0x2171c,
1670 0x21780, 0x2178c,
1671 0x21800, 0x21c38,
1672 0x21c80, 0x21d7c,
1673 0x21e00, 0x21e04,
1674 0x22000, 0x2202c,
1675 0x22100, 0x2213c,
1676 0x22190, 0x221c8,
1677 0x22200, 0x22318,
1678 0x22400, 0x22528,
1679 0x22540, 0x22614,
1680 0x23000, 0x23040,
1681 0x2304c, 0x23060,
1682 0x230c0, 0x230ec,
1683 0x23200, 0x23268,
1684 0x23270, 0x23284,
1685 0x232fc, 0x23388,
1686 0x23400, 0x23404,
1687 0x23500, 0x23518,
1688 0x2352c, 0x2353c,
1689 0x23550, 0x23554,
1690 0x23600, 0x23600,
1691 0x23608, 0x23628,
1692 0x23630, 0x2363c,
1693 0x23700, 0x2371c,
1694 0x23780, 0x2378c,
1695 0x23800, 0x23c38,
1696 0x23c80, 0x23d7c,
1697 0x23e00, 0x23e04,
1698 0x24000, 0x2402c,
1699 0x24100, 0x2413c,
1700 0x24190, 0x241c8,
1701 0x24200, 0x24318,
1702 0x24400, 0x24528,
1703 0x24540, 0x24614,
1704 0x25000, 0x25040,
1705 0x2504c, 0x25060,
1706 0x250c0, 0x250ec,
1707 0x25200, 0x25268,
1708 0x25270, 0x25284,
1709 0x252fc, 0x25388,
1710 0x25400, 0x25404,
1711 0x25500, 0x25518,
1712 0x2552c, 0x2553c,
1713 0x25550, 0x25554,
1714 0x25600, 0x25600,
1715 0x25608, 0x25628,
1716 0x25630, 0x2563c,
1717 0x25700, 0x2571c,
1718 0x25780, 0x2578c,
1719 0x25800, 0x25c38,
1720 0x25c80, 0x25d7c,
1721 0x25e00, 0x25e04,
1722 0x26000, 0x2602c,
1723 0x26100, 0x2613c,
1724 0x26190, 0x261c8,
1725 0x26200, 0x26318,
1726 0x26400, 0x26528,
1727 0x26540, 0x26614,
1728 0x27000, 0x27040,
1729 0x2704c, 0x27060,
1730 0x270c0, 0x270ec,
1731 0x27200, 0x27268,
1732 0x27270, 0x27284,
1733 0x272fc, 0x27388,
1734 0x27400, 0x27404,
1735 0x27500, 0x27518,
1736 0x2752c, 0x2753c,
1737 0x27550, 0x27554,
1738 0x27600, 0x27600,
1739 0x27608, 0x27628,
1740 0x27630, 0x2763c,
1741 0x27700, 0x2771c,
1742 0x27780, 0x2778c,
1743 0x27800, 0x27c38,
1744 0x27c80, 0x27d7c,
1745 0x27e00, 0x27e04
1746 };
1747
251f9e88
SR		 1748 static const unsigned int t5_reg_ranges[] = {
1749 0x1008, 0x1148,
1750 0x1180, 0x11b4,
1751 0x11fc, 0x123c,
1752 0x1280, 0x173c,
1753 0x1800, 0x18fc,
1754 0x3000, 0x3028,
1755 0x3060, 0x30d8,
1756 0x30e0, 0x30fc,
1757 0x3140, 0x357c,
1758 0x35a8, 0x35cc,
1759 0x35ec, 0x35ec,
1760 0x3600, 0x5624,
1761 0x56cc, 0x575c,
1762 0x580c, 0x5814,
1763 0x5890, 0x58bc,
1764 0x5940, 0x59dc,
1765 0x59fc, 0x5a18,
1766 0x5a60, 0x5a9c,
1767 0x5b9c, 0x5bfc,
1768 0x6000, 0x6040,
1769 0x6058, 0x614c,
1770 0x7700, 0x7798,
1771 0x77c0, 0x78fc,
1772 0x7b00, 0x7c54,
1773 0x7d00, 0x7efc,
1774 0x8dc0, 0x8de0,
1775 0x8df8, 0x8e84,
1776 0x8ea0, 0x8f84,
1777 0x8fc0, 0x90f8,
1778 0x9400, 0x9470,
1779 0x9600, 0x96f4,
1780 0x9800, 0x9808,
1781 0x9820, 0x983c,
1782 0x9850, 0x9864,
1783 0x9c00, 0x9c6c,
1784 0x9c80, 0x9cec,
1785 0x9d00, 0x9d6c,
1786 0x9d80, 0x9dec,
1787 0x9e00, 0x9e6c,
1788 0x9e80, 0x9eec,
1789 0x9f00, 0x9f6c,
1790 0x9f80, 0xa020,
1791 0xd004, 0xd03c,
1792 0xdfc0, 0xdfe0,
1793 0xe000, 0x11088,
1794 0x1109c, 0x1117c,
1795 0x11190, 0x11204,
1796 0x19040, 0x1906c,
1797 0x19078, 0x19080,
1798 0x1908c, 0x19124,
1799 0x19150, 0x191b0,
1800 0x191d0, 0x191e8,
1801 0x19238, 0x19290,
1802 0x193f8, 0x19474,
1803 0x19490, 0x194cc,
1804 0x194f0, 0x194f8,
1805 0x19c00, 0x19c60,
1806 0x19c94, 0x19e10,
1807 0x19e50, 0x19f34,
1808 0x19f40, 0x19f50,
1809 0x19f90, 0x19fe4,
1810 0x1a000, 0x1a06c,
1811 0x1a0b0, 0x1a120,
1812 0x1a128, 0x1a138,
1813 0x1a190, 0x1a1c4,
1814 0x1a1fc, 0x1a1fc,
1815 0x1e008, 0x1e00c,
1816 0x1e040, 0x1e04c,
1817 0x1e284, 0x1e290,
1818 0x1e2c0, 0x1e2c0,
1819 0x1e2e0, 0x1e2e0,
1820 0x1e300, 0x1e384,
1821 0x1e3c0, 0x1e3c8,
1822 0x1e408, 0x1e40c,
1823 0x1e440, 0x1e44c,
1824 0x1e684, 0x1e690,
1825 0x1e6c0, 0x1e6c0,
1826 0x1e6e0, 0x1e6e0,
1827 0x1e700, 0x1e784,
1828 0x1e7c0, 0x1e7c8,
1829 0x1e808, 0x1e80c,
1830 0x1e840, 0x1e84c,
1831 0x1ea84, 0x1ea90,
1832 0x1eac0, 0x1eac0,
1833 0x1eae0, 0x1eae0,
1834 0x1eb00, 0x1eb84,
1835 0x1ebc0, 0x1ebc8,
1836 0x1ec08, 0x1ec0c,
1837 0x1ec40, 0x1ec4c,
1838 0x1ee84, 0x1ee90,
1839 0x1eec0, 0x1eec0,
1840 0x1eee0, 0x1eee0,
1841 0x1ef00, 0x1ef84,
1842 0x1efc0, 0x1efc8,
1843 0x1f008, 0x1f00c,
1844 0x1f040, 0x1f04c,
1845 0x1f284, 0x1f290,
1846 0x1f2c0, 0x1f2c0,
1847 0x1f2e0, 0x1f2e0,
1848 0x1f300, 0x1f384,
1849 0x1f3c0, 0x1f3c8,
1850 0x1f408, 0x1f40c,
1851 0x1f440, 0x1f44c,
1852 0x1f684, 0x1f690,
1853 0x1f6c0, 0x1f6c0,
1854 0x1f6e0, 0x1f6e0,
1855 0x1f700, 0x1f784,
1856 0x1f7c0, 0x1f7c8,
1857 0x1f808, 0x1f80c,
1858 0x1f840, 0x1f84c,
1859 0x1fa84, 0x1fa90,
1860 0x1fac0, 0x1fac0,
1861 0x1fae0, 0x1fae0,
1862 0x1fb00, 0x1fb84,
1863 0x1fbc0, 0x1fbc8,
1864 0x1fc08, 0x1fc0c,
1865 0x1fc40, 0x1fc4c,
1866 0x1fe84, 0x1fe90,
1867 0x1fec0, 0x1fec0,
1868 0x1fee0, 0x1fee0,
1869 0x1ff00, 0x1ff84,
1870 0x1ffc0, 0x1ffc8,
1871 0x30000, 0x30030,
1872 0x30100, 0x30144,
1873 0x30190, 0x301d0,
1874 0x30200, 0x30318,
1875 0x30400, 0x3052c,
1876 0x30540, 0x3061c,
1877 0x30800, 0x30834,
1878 0x308c0, 0x30908,
1879 0x30910, 0x309ac,
1880 0x30a00, 0x30a04,
1881 0x30a0c, 0x30a2c,
1882 0x30a44, 0x30a50,
1883 0x30a74, 0x30c24,
1884 0x30d08, 0x30d14,
1885 0x30d1c, 0x30d20,
1886 0x30d3c, 0x30d50,
1887 0x31200, 0x3120c,
1888 0x31220, 0x31220,
1889 0x31240, 0x31240,
1890 0x31600, 0x31600,
1891 0x31608, 0x3160c,
1892 0x31a00, 0x31a1c,
1893 0x31e04, 0x31e20,
1894 0x31e38, 0x31e3c,
1895 0x31e80, 0x31e80,
1896 0x31e88, 0x31ea8,
1897 0x31eb0, 0x31eb4,
1898 0x31ec8, 0x31ed4,
1899 0x31fb8, 0x32004,
1900 0x32208, 0x3223c,
1901 0x32600, 0x32630,
1902 0x32a00, 0x32abc,
1903 0x32b00, 0x32b70,
1904 0x33000, 0x33048,
1905 0x33060, 0x3309c,
1906 0x330f0, 0x33148,
1907 0x33160, 0x3319c,
1908 0x331f0, 0x332e4,
1909 0x332f8, 0x333e4,
1910 0x333f8, 0x33448,
1911 0x33460, 0x3349c,
1912 0x334f0, 0x33548,
1913 0x33560, 0x3359c,
1914 0x335f0, 0x336e4,
1915 0x336f8, 0x337e4,
1916 0x337f8, 0x337fc,
1917 0x33814, 0x33814,
1918 0x3382c, 0x3382c,
1919 0x33880, 0x3388c,
1920 0x338e8, 0x338ec,
1921 0x33900, 0x33948,
1922 0x33960, 0x3399c,
1923 0x339f0, 0x33ae4,
1924 0x33af8, 0x33b10,
1925 0x33b28, 0x33b28,
1926 0x33b3c, 0x33b50,
1927 0x33bf0, 0x33c10,
1928 0x33c28, 0x33c28,
1929 0x33c3c, 0x33c50,
1930 0x33cf0, 0x33cfc,
1931 0x34000, 0x34030,
1932 0x34100, 0x34144,
1933 0x34190, 0x341d0,
1934 0x34200, 0x34318,
1935 0x34400, 0x3452c,
1936 0x34540, 0x3461c,
1937 0x34800, 0x34834,
1938 0x348c0, 0x34908,
1939 0x34910, 0x349ac,
1940 0x34a00, 0x34a04,
1941 0x34a0c, 0x34a2c,
1942 0x34a44, 0x34a50,
1943 0x34a74, 0x34c24,
1944 0x34d08, 0x34d14,
1945 0x34d1c, 0x34d20,
1946 0x34d3c, 0x34d50,
1947 0x35200, 0x3520c,
1948 0x35220, 0x35220,
1949 0x35240, 0x35240,
1950 0x35600, 0x35600,
1951 0x35608, 0x3560c,
1952 0x35a00, 0x35a1c,
1953 0x35e04, 0x35e20,
1954 0x35e38, 0x35e3c,
1955 0x35e80, 0x35e80,
1956 0x35e88, 0x35ea8,
1957 0x35eb0, 0x35eb4,
1958 0x35ec8, 0x35ed4,
1959 0x35fb8, 0x36004,
1960 0x36208, 0x3623c,
1961 0x36600, 0x36630,
1962 0x36a00, 0x36abc,
1963 0x36b00, 0x36b70,
1964 0x37000, 0x37048,
1965 0x37060, 0x3709c,
1966 0x370f0, 0x37148,
1967 0x37160, 0x3719c,
1968 0x371f0, 0x372e4,
1969 0x372f8, 0x373e4,
1970 0x373f8, 0x37448,
1971 0x37460, 0x3749c,
1972 0x374f0, 0x37548,
1973 0x37560, 0x3759c,
1974 0x375f0, 0x376e4,
1975 0x376f8, 0x377e4,
1976 0x377f8, 0x377fc,
1977 0x37814, 0x37814,
1978 0x3782c, 0x3782c,
1979 0x37880, 0x3788c,
1980 0x378e8, 0x378ec,
1981 0x37900, 0x37948,
1982 0x37960, 0x3799c,
1983 0x379f0, 0x37ae4,
1984 0x37af8, 0x37b10,
1985 0x37b28, 0x37b28,
1986 0x37b3c, 0x37b50,
1987 0x37bf0, 0x37c10,
1988 0x37c28, 0x37c28,
1989 0x37c3c, 0x37c50,
1990 0x37cf0, 0x37cfc,
1991 0x38000, 0x38030,
1992 0x38100, 0x38144,
1993 0x38190, 0x381d0,
1994 0x38200, 0x38318,
1995 0x38400, 0x3852c,
1996 0x38540, 0x3861c,
1997 0x38800, 0x38834,
1998 0x388c0, 0x38908,
1999 0x38910, 0x389ac,
2000 0x38a00, 0x38a04,
2001 0x38a0c, 0x38a2c,
2002 0x38a44, 0x38a50,
2003 0x38a74, 0x38c24,
2004 0x38d08, 0x38d14,
2005 0x38d1c, 0x38d20,
2006 0x38d3c, 0x38d50,
2007 0x39200, 0x3920c,
2008 0x39220, 0x39220,
2009 0x39240, 0x39240,
2010 0x39600, 0x39600,
2011 0x39608, 0x3960c,
2012 0x39a00, 0x39a1c,
2013 0x39e04, 0x39e20,
2014 0x39e38, 0x39e3c,
2015 0x39e80, 0x39e80,
2016 0x39e88, 0x39ea8,
2017 0x39eb0, 0x39eb4,
2018 0x39ec8, 0x39ed4,
2019 0x39fb8, 0x3a004,
2020 0x3a208, 0x3a23c,
2021 0x3a600, 0x3a630,
2022 0x3aa00, 0x3aabc,
2023 0x3ab00, 0x3ab70,
2024 0x3b000, 0x3b048,
2025 0x3b060, 0x3b09c,
2026 0x3b0f0, 0x3b148,
2027 0x3b160, 0x3b19c,
2028 0x3b1f0, 0x3b2e4,
2029 0x3b2f8, 0x3b3e4,
2030 0x3b3f8, 0x3b448,
2031 0x3b460, 0x3b49c,
2032 0x3b4f0, 0x3b548,
2033 0x3b560, 0x3b59c,
2034 0x3b5f0, 0x3b6e4,
2035 0x3b6f8, 0x3b7e4,
2036 0x3b7f8, 0x3b7fc,
2037 0x3b814, 0x3b814,
2038 0x3b82c, 0x3b82c,
2039 0x3b880, 0x3b88c,
2040 0x3b8e8, 0x3b8ec,
2041 0x3b900, 0x3b948,
2042 0x3b960, 0x3b99c,
2043 0x3b9f0, 0x3bae4,
2044 0x3baf8, 0x3bb10,
2045 0x3bb28, 0x3bb28,
2046 0x3bb3c, 0x3bb50,
2047 0x3bbf0, 0x3bc10,
2048 0x3bc28, 0x3bc28,
2049 0x3bc3c, 0x3bc50,
2050 0x3bcf0, 0x3bcfc,
2051 0x3c000, 0x3c030,
2052 0x3c100, 0x3c144,
2053 0x3c190, 0x3c1d0,
2054 0x3c200, 0x3c318,
2055 0x3c400, 0x3c52c,
2056 0x3c540, 0x3c61c,
2057 0x3c800, 0x3c834,
2058 0x3c8c0, 0x3c908,
2059 0x3c910, 0x3c9ac,
2060 0x3ca00, 0x3ca04,
2061 0x3ca0c, 0x3ca2c,
2062 0x3ca44, 0x3ca50,
2063 0x3ca74, 0x3cc24,
2064 0x3cd08, 0x3cd14,
2065 0x3cd1c, 0x3cd20,
2066 0x3cd3c, 0x3cd50,
2067 0x3d200, 0x3d20c,
2068 0x3d220, 0x3d220,
2069 0x3d240, 0x3d240,
2070 0x3d600, 0x3d600,
2071 0x3d608, 0x3d60c,
2072 0x3da00, 0x3da1c,
2073 0x3de04, 0x3de20,
2074 0x3de38, 0x3de3c,
2075 0x3de80, 0x3de80,
2076 0x3de88, 0x3dea8,
2077 0x3deb0, 0x3deb4,
2078 0x3dec8, 0x3ded4,
2079 0x3dfb8, 0x3e004,
2080 0x3e208, 0x3e23c,
2081 0x3e600, 0x3e630,
2082 0x3ea00, 0x3eabc,
2083 0x3eb00, 0x3eb70,
2084 0x3f000, 0x3f048,
2085 0x3f060, 0x3f09c,
2086 0x3f0f0, 0x3f148,
2087 0x3f160, 0x3f19c,
2088 0x3f1f0, 0x3f2e4,
2089 0x3f2f8, 0x3f3e4,
2090 0x3f3f8, 0x3f448,
2091 0x3f460, 0x3f49c,
2092 0x3f4f0, 0x3f548,
2093 0x3f560, 0x3f59c,
2094 0x3f5f0, 0x3f6e4,
2095 0x3f6f8, 0x3f7e4,
2096 0x3f7f8, 0x3f7fc,
2097 0x3f814, 0x3f814,
2098 0x3f82c, 0x3f82c,
2099 0x3f880, 0x3f88c,
2100 0x3f8e8, 0x3f8ec,
2101 0x3f900, 0x3f948,
2102 0x3f960, 0x3f99c,
2103 0x3f9f0, 0x3fae4,
2104 0x3faf8, 0x3fb10,
2105 0x3fb28, 0x3fb28,
2106 0x3fb3c, 0x3fb50,
2107 0x3fbf0, 0x3fc10,
2108 0x3fc28, 0x3fc28,
2109 0x3fc3c, 0x3fc50,
2110 0x3fcf0, 0x3fcfc,
2111 0x40000, 0x4000c,
2112 0x40040, 0x40068,
2113 0x40080, 0x40144,
2114 0x40180, 0x4018c,
2115 0x40200, 0x40298,
2116 0x402ac, 0x4033c,
2117 0x403f8, 0x403fc,
c1f49e3e 2118 0x41304, 0x413c4,
251f9e88
SR		 2119 0x41400, 0x4141c,
2120 0x41480, 0x414d0,
2121 0x44000, 0x44078,
2122 0x440c0, 0x44278,
2123 0x442c0, 0x44478,
2124 0x444c0, 0x44678,
2125 0x446c0, 0x44878,
2126 0x448c0, 0x449fc,
2127 0x45000, 0x45068,
2128 0x45080, 0x45084,
2129 0x450a0, 0x450b0,
2130 0x45200, 0x45268,
2131 0x45280, 0x45284,
2132 0x452a0, 0x452b0,
2133 0x460c0, 0x460e4,
2134 0x47000, 0x4708c,
2135 0x47200, 0x47250,
2136 0x47400, 0x47420,
2137 0x47600, 0x47618,
2138 0x47800, 0x47814,
2139 0x48000, 0x4800c,
2140 0x48040, 0x48068,
2141 0x48080, 0x48144,
2142 0x48180, 0x4818c,
2143 0x48200, 0x48298,
2144 0x482ac, 0x4833c,
2145 0x483f8, 0x483fc,
c1f49e3e 2146 0x49304, 0x493c4,
251f9e88
SR		 2147 0x49400, 0x4941c,
2148 0x49480, 0x494d0,
2149 0x4c000, 0x4c078,
2150 0x4c0c0, 0x4c278,
2151 0x4c2c0, 0x4c478,
2152 0x4c4c0, 0x4c678,
2153 0x4c6c0, 0x4c878,
2154 0x4c8c0, 0x4c9fc,
2155 0x4d000, 0x4d068,
2156 0x4d080, 0x4d084,
2157 0x4d0a0, 0x4d0b0,
2158 0x4d200, 0x4d268,
2159 0x4d280, 0x4d284,
2160 0x4d2a0, 0x4d2b0,
2161 0x4e0c0, 0x4e0e4,
2162 0x4f000, 0x4f08c,
2163 0x4f200, 0x4f250,
2164 0x4f400, 0x4f420,
2165 0x4f600, 0x4f618,
2166 0x4f800, 0x4f814,
2167 0x50000, 0x500cc,
2168 0x50400, 0x50400,
2169 0x50800, 0x508cc,
2170 0x50c00, 0x50c00,
2171 0x51000, 0x5101c,
2172 0x51300, 0x51308,
2173 };
2174
b8ff05a9
DM		 2175 int i;
2176 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2177 static const unsigned int *reg_ranges;
2178 int arr_size = 0, buf_size = 0;
2179
d14807dd 2180 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2181 reg_ranges = &t4_reg_ranges[0];
2182 arr_size = ARRAY_SIZE(t4_reg_ranges);
2183 buf_size = T4_REGMAP_SIZE;
2184 } else {
2185 reg_ranges = &t5_reg_ranges[0];
2186 arr_size = ARRAY_SIZE(t5_reg_ranges);
2187 buf_size = T5_REGMAP_SIZE;
2188 }
b8ff05a9
DM		 2189
2190 regs->version = mk_adap_vers(ap);
2191
251f9e88
SR		 2192 memset(buf, 0, buf_size);
2193 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2194 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2195}
2196
2197static int restart_autoneg(struct net_device *dev)
2198{
2199 struct port_info *p = netdev_priv(dev);
2200
2201 if (!netif_running(dev))
2202 return -EAGAIN;
2203 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2204 return -EINVAL;
060e0c75 2205 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2206 return 0;
2207}
2208
c5e06360
DM		 2209static int identify_port(struct net_device *dev,
2210 enum ethtool_phys_id_state state)
b8ff05a9 2211{
c5e06360 2212 unsigned int val;
060e0c75
DM		 2213 struct adapter *adap = netdev2adap(dev);
2214
c5e06360
DM		 2215 if (state == ETHTOOL_ID_ACTIVE)
2216 val = 0xffff;
2217 else if (state == ETHTOOL_ID_INACTIVE)
2218 val = 0;
2219 else
2220 return -EINVAL;
b8ff05a9 2221
c5e06360 2222 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2223}
2224
2225static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
2226{
2227 unsigned int v = 0;
2228
a0881cab
DM		 2229 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2230 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2231 v |= SUPPORTED_TP;
2232 if (caps & FW_PORT_CAP_SPEED_100M)
2233 v |= SUPPORTED_100baseT_Full;
2234 if (caps & FW_PORT_CAP_SPEED_1G)
2235 v |= SUPPORTED_1000baseT_Full;
2236 if (caps & FW_PORT_CAP_SPEED_10G)
2237 v |= SUPPORTED_10000baseT_Full;
2238 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2239 v |= SUPPORTED_Backplane;
2240 if (caps & FW_PORT_CAP_SPEED_1G)
2241 v |= SUPPORTED_1000baseKX_Full;
2242 if (caps & FW_PORT_CAP_SPEED_10G)
2243 v |= SUPPORTED_10000baseKX4_Full;
2244 } else if (type == FW_PORT_TYPE_KR)
2245 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2246 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2247 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2248 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2249 else if (type == FW_PORT_TYPE_BP4_AP)
2250 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2251 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2252 SUPPORTED_10000baseKX4_Full;
a0881cab
DM		 2253 else if (type == FW_PORT_TYPE_FIBER_XFI ||
2254 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9 2255 v |= SUPPORTED_FIBRE;
72aca4bf
KS		 2256 else if (type == FW_PORT_TYPE_BP40_BA)
2257 v |= SUPPORTED_40000baseSR4_Full;
b8ff05a9
DM		 2258
2259 if (caps & FW_PORT_CAP_ANEG)
2260 v |= SUPPORTED_Autoneg;
2261 return v;
2262}
2263
2264static unsigned int to_fw_linkcaps(unsigned int caps)
2265{
2266 unsigned int v = 0;
2267
2268 if (caps & ADVERTISED_100baseT_Full)
2269 v |= FW_PORT_CAP_SPEED_100M;
2270 if (caps & ADVERTISED_1000baseT_Full)
2271 v |= FW_PORT_CAP_SPEED_1G;
2272 if (caps & ADVERTISED_10000baseT_Full)
2273 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2274 if (caps & ADVERTISED_40000baseSR4_Full)
2275 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2276 return v;
2277}
2278
2279static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2280{
2281 const struct port_info *p = netdev_priv(dev);
2282
2283 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2284 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2285 p->port_type == FW_PORT_TYPE_BT_XAUI)
2286 cmd->port = PORT_TP;
a0881cab
DM		 2287 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2288 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2289 cmd->port = PORT_FIBRE;
3e00a509
HS		 2290 else if (p->port_type == FW_PORT_TYPE_SFP ||
2291 p->port_type == FW_PORT_TYPE_QSFP_10G ||
2292 p->port_type == FW_PORT_TYPE_QSFP) {
2293 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2294 p->mod_type == FW_PORT_MOD_TYPE_SR ||
2295 p->mod_type == FW_PORT_MOD_TYPE_ER ||
2296 p->mod_type == FW_PORT_MOD_TYPE_LRM)
2297 cmd->port = PORT_FIBRE;
2298 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2299 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
a0881cab
DM		 2300 cmd->port = PORT_DA;
2301 else
3e00a509 2302 cmd->port = PORT_OTHER;
a0881cab 2303 } else
b8ff05a9
DM		 2304 cmd->port = PORT_OTHER;
2305
2306 if (p->mdio_addr >= 0) {
2307 cmd->phy_address = p->mdio_addr;
2308 cmd->transceiver = XCVR_EXTERNAL;
2309 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2310 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2311 } else {
2312 cmd->phy_address = 0; /* not really, but no better option */
2313 cmd->transceiver = XCVR_INTERNAL;
2314 cmd->mdio_support = 0;
2315 }
2316
2317 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2318 cmd->advertising = from_fw_linkcaps(p->port_type,
2319 p->link_cfg.advertising);
70739497
DD		 2320 ethtool_cmd_speed_set(cmd,
2321 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2322 cmd->duplex = DUPLEX_FULL;
2323 cmd->autoneg = p->link_cfg.autoneg;
2324 cmd->maxtxpkt = 0;
2325 cmd->maxrxpkt = 0;
2326 return 0;
2327}
2328
2329static unsigned int speed_to_caps(int speed)
2330{
e8b39015 2331 if (speed == 100)
b8ff05a9 2332 return FW_PORT_CAP_SPEED_100M;
e8b39015 2333 if (speed == 1000)
b8ff05a9 2334 return FW_PORT_CAP_SPEED_1G;
e8b39015 2335 if (speed == 10000)
b8ff05a9 2336 return FW_PORT_CAP_SPEED_10G;
e8b39015 2337 if (speed == 40000)
72aca4bf 2338 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2339 return 0;
2340}
2341
2342static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2343{
2344 unsigned int cap;
2345 struct port_info *p = netdev_priv(dev);
2346 struct link_config *lc = &p->link_cfg;
25db0338 2347 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2348
2349 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2350 return -EINVAL;
2351
2352 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2353 /*
2354 * PHY offers a single speed. See if that's what's
2355 * being requested.
2356 */
2357 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2358 (lc->supported & speed_to_caps(speed)))
2359 return 0;
b8ff05a9
DM		 2360 return -EINVAL;
2361 }
2362
2363 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2364 cap = speed_to_caps(speed);
b8ff05a9 2365
72aca4bf 2366 if (!(lc->supported & cap) ||
e8b39015
BH		 2367 (speed == 1000) ||
2368 (speed == 10000) ||
72aca4bf 2369 (speed == 40000))
b8ff05a9
DM		 2370 return -EINVAL;
2371 lc->requested_speed = cap;
2372 lc->advertising = 0;
2373 } else {
2374 cap = to_fw_linkcaps(cmd->advertising);
2375 if (!(lc->supported & cap))
2376 return -EINVAL;
2377 lc->requested_speed = 0;
2378 lc->advertising = cap | FW_PORT_CAP_ANEG;
2379 }
2380 lc->autoneg = cmd->autoneg;
2381
2382 if (netif_running(dev))
060e0c75
DM		 2383 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2384 lc);
b8ff05a9
DM		 2385 return 0;
2386}
2387
2388static void get_pauseparam(struct net_device *dev,
2389 struct ethtool_pauseparam *epause)
2390{
2391 struct port_info *p = netdev_priv(dev);
2392
2393 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2394 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2395 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2396}
2397
2398static int set_pauseparam(struct net_device *dev,
2399 struct ethtool_pauseparam *epause)
2400{
2401 struct port_info *p = netdev_priv(dev);
2402 struct link_config *lc = &p->link_cfg;
2403
2404 if (epause->autoneg == AUTONEG_DISABLE)
2405 lc->requested_fc = 0;
2406 else if (lc->supported & FW_PORT_CAP_ANEG)
2407 lc->requested_fc = PAUSE_AUTONEG;
2408 else
2409 return -EINVAL;
2410
2411 if (epause->rx_pause)
2412 lc->requested_fc |= PAUSE_RX;
2413 if (epause->tx_pause)
2414 lc->requested_fc |= PAUSE_TX;
2415 if (netif_running(dev))
060e0c75
DM		 2416 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2417 lc);
b8ff05a9
DM		 2418 return 0;
2419}
2420
b8ff05a9
DM		 2421static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2422{
2423 const struct port_info *pi = netdev_priv(dev);
2424 const struct sge *s = &pi->adapter->sge;
2425
2426 e->rx_max_pending = MAX_RX_BUFFERS;
2427 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2428 e->rx_jumbo_max_pending = 0;
2429 e->tx_max_pending = MAX_TXQ_ENTRIES;
2430
2431 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2432 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2433 e->rx_jumbo_pending = 0;
2434 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2435}
2436
2437static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2438{
2439 int i;
2440 const struct port_info *pi = netdev_priv(dev);
2441 struct adapter *adapter = pi->adapter;
2442 struct sge *s = &adapter->sge;
2443
2444 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2445 e->tx_pending > MAX_TXQ_ENTRIES ||
2446 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2447 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2448 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2449 return -EINVAL;
2450
2451 if (adapter->flags & FULL_INIT_DONE)
2452 return -EBUSY;
2453
2454 for (i = 0; i < pi->nqsets; ++i) {
2455 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2456 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2457 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2458 }
2459 return 0;
2460}
2461
2462static int closest_timer(const struct sge *s, int time)
2463{
2464 int i, delta, match = 0, min_delta = INT_MAX;
2465
2466 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2467 delta = time - s->timer_val[i];
2468 if (delta < 0)
2469 delta = -delta;
2470 if (delta < min_delta) {
2471 min_delta = delta;
2472 match = i;
2473 }
2474 }
2475 return match;
2476}
2477
2478static int closest_thres(const struct sge *s, int thres)
2479{
2480 int i, delta, match = 0, min_delta = INT_MAX;
2481
2482 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2483 delta = thres - s->counter_val[i];
2484 if (delta < 0)
2485 delta = -delta;
2486 if (delta < min_delta) {
2487 min_delta = delta;
2488 match = i;
2489 }
2490 }
2491 return match;
2492}
2493
2494/*
2495 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2496 */
2497static unsigned int qtimer_val(const struct adapter *adap,
2498 const struct sge_rspq *q)
2499{
2500 unsigned int idx = q->intr_params >> 1;
2501
2502 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2503}
2504
2505/**
2506 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
2507 * @adap: the adapter
2508 * @q: the Rx queue
2509 * @us: the hold-off time in us, or 0 to disable timer
2510 * @cnt: the hold-off packet count, or 0 to disable counter
2511 *
2512 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2513 * one of the two needs to be enabled for the queue to generate interrupts.
2514 */
2515static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
2516 unsigned int us, unsigned int cnt)
2517{
2518 if ((us | cnt) == 0)
2519 cnt = 1;
2520
2521 if (cnt) {
2522 int err;
2523 u32 v, new_idx;
2524
2525 new_idx = closest_thres(&adap->sge, cnt);
2526 if (q->desc && q->pktcnt_idx != new_idx) {
2527 /* the queue has already been created, update it */
2528 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2529 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2530 FW_PARAMS_PARAM_YZ(q->cntxt_id);
060e0c75
DM		 2531 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2532 &new_idx);
b8ff05a9
DM		 2533 if (err)
2534 return err;
2535 }
2536 q->pktcnt_idx = new_idx;
2537 }
2538
2539 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2540 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2541 return 0;
2542}
2543
2544static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2545{
2546 const struct port_info *pi = netdev_priv(dev);
2547 struct adapter *adap = pi->adapter;
d4fc9dc2
TLSC		 2548 struct sge_rspq *q;
2549 int i;
2550 int r = 0;
2551
2552 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2553 q = &adap->sge.ethrxq[i].rspq;
2554 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2555 c->rx_max_coalesced_frames);
2556 if (r) {
2557 dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2558 break;
2559 }
2560 }
2561 return r;
b8ff05a9
DM		 2562}
2563
2564static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2565{
2566 const struct port_info *pi = netdev_priv(dev);
2567 const struct adapter *adap = pi->adapter;
2568 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2569
2570 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2571 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2572 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2573 return 0;
2574}
2575
1478b3ee
DM		 2576/**
2577 * eeprom_ptov - translate a physical EEPROM address to virtual
2578 * @phys_addr: the physical EEPROM address
2579 * @fn: the PCI function number
2580 * @sz: size of function-specific area
2581 *
2582 * Translate a physical EEPROM address to virtual. The first 1K is
2583 * accessed through virtual addresses starting at 31K, the rest is
2584 * accessed through virtual addresses starting at 0.
2585 *
2586 * The mapping is as follows:
2587 * [0..1K) -> [31K..32K)
2588 * [1K..1K+A) -> [31K-A..31K)
2589 * [1K+A..ES) -> [0..ES-A-1K)
2590 *
2591 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2592 */
1478b3ee 2593static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2594{
1478b3ee 2595 fn *= sz;
b8ff05a9
DM		 2596 if (phys_addr < 1024)
2597 return phys_addr + (31 << 10);
1478b3ee
DM		 2598 if (phys_addr < 1024 + fn)
2599 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2600 if (phys_addr < EEPROMSIZE)
1478b3ee 2601 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2602 return -EINVAL;
2603}
2604
2605/*
2606 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2607 */
2608static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2609{
1478b3ee 2610 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2611
2612 if (vaddr >= 0)
2613 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2614 return vaddr < 0 ? vaddr : 0;
2615}
2616
2617static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2618{
1478b3ee 2619 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2620
2621 if (vaddr >= 0)
2622 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2623 return vaddr < 0 ? vaddr : 0;
2624}
2625
2626#define EEPROM_MAGIC 0x38E2F10C
2627
2628static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2629 u8 *data)
2630{
2631 int i, err = 0;
2632 struct adapter *adapter = netdev2adap(dev);
2633
2634 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2635 if (!buf)
2636 return -ENOMEM;
2637
2638 e->magic = EEPROM_MAGIC;
2639 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2640 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2641
2642 if (!err)
2643 memcpy(data, buf + e->offset, e->len);
2644 kfree(buf);
2645 return err;
2646}
2647
2648static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2649 u8 *data)
2650{
2651 u8 *buf;
2652 int err = 0;
2653 u32 aligned_offset, aligned_len, *p;
2654 struct adapter *adapter = netdev2adap(dev);
2655
2656 if (eeprom->magic != EEPROM_MAGIC)
2657 return -EINVAL;
2658
2659 aligned_offset = eeprom->offset & ~3;
2660 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2661
1478b3ee
DM		 2662 if (adapter->fn > 0) {
2663 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2664
2665 if (aligned_offset < start ||
2666 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2667 return -EPERM;
2668 }
2669
b8ff05a9
DM		 2670 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2671 /*
2672 * RMW possibly needed for first or last words.
2673 */
2674 buf = kmalloc(aligned_len, GFP_KERNEL);
2675 if (!buf)
2676 return -ENOMEM;
2677 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2678 if (!err && aligned_len > 4)
2679 err = eeprom_rd_phys(adapter,
2680 aligned_offset + aligned_len - 4,
2681 (u32 *)&buf[aligned_len - 4]);
2682 if (err)
2683 goto out;
2684 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2685 } else
2686 buf = data;
2687
2688 err = t4_seeprom_wp(adapter, false);
2689 if (err)
2690 goto out;
2691
2692 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2693 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2694 aligned_offset += 4;
2695 }
2696
2697 if (!err)
2698 err = t4_seeprom_wp(adapter, true);
2699out:
2700 if (buf != data)
2701 kfree(buf);
2702 return err;
2703}
2704
2705static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2706{
2707 int ret;
2708 const struct firmware *fw;
2709 struct adapter *adap = netdev2adap(netdev);
2710
2711 ef->data[sizeof(ef->data) - 1] = '\0';
2712 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2713 if (ret < 0)
2714 return ret;
2715
2716 ret = t4_load_fw(adap, fw->data, fw->size);
2717 release_firmware(fw);
2718 if (!ret)
2719 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2720 return ret;
2721}
2722
2723#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2724#define BCAST_CRC 0xa0ccc1a6
2725
2726static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2727{
2728 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2729 wol->wolopts = netdev2adap(dev)->wol;
2730 memset(&wol->sopass, 0, sizeof(wol->sopass));
2731}
2732
2733static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2734{
2735 int err = 0;
2736 struct port_info *pi = netdev_priv(dev);
2737
2738 if (wol->wolopts & ~WOL_SUPPORTED)
2739 return -EINVAL;
2740 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2741 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2742 if (wol->wolopts & WAKE_BCAST) {
2743 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2744 ~0ULL, 0, false);
2745 if (!err)
2746 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2747 ~6ULL, ~0ULL, BCAST_CRC, true);
2748 } else
2749 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2750 return err;
2751}
2752
c8f44aff 2753static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2754{
2ed28baa 2755 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2756 netdev_features_t changed = dev->features ^ features;
19ecae2c 2757 int err;
19ecae2c 2758
f646968f 2759 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2760 return 0;
19ecae2c 2761
2ed28baa
MM		 2762 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2763 -1, -1, -1,
f646968f 2764 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2765 if (unlikely(err))
f646968f 2766 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2767 return err;
87b6cf51
DM		 2768}
2769
7850f63f 2770static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2771{
2772 const struct port_info *pi = netdev_priv(dev);
671b0060 2773
7850f63f
BH		 2774 return pi->rss_size;
2775}
2776
fe62d001 2777static int get_rss_table(struct net_device *dev, u32 *p, u8 *key)
7850f63f
BH		 2778{
2779 const struct port_info *pi = netdev_priv(dev);
2780 unsigned int n = pi->rss_size;
2781
671b0060 2782 while (n--)
7850f63f 2783 p[n] = pi->rss[n];
671b0060
DM		 2784 return 0;
2785}
2786
fe62d001 2787static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key)
671b0060
DM		 2788{
2789 unsigned int i;
2790 struct port_info *pi = netdev_priv(dev);
2791
7850f63f
BH		 2792 for (i = 0; i < pi->rss_size; i++)
2793 pi->rss[i] = p[i];
671b0060
DM		 2794 if (pi->adapter->flags & FULL_INIT_DONE)
2795 return write_rss(pi, pi->rss);
2796 return 0;
2797}
2798
2799static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2800 u32 *rules)
671b0060 2801{
f796564a
DM		 2802 const struct port_info *pi = netdev_priv(dev);
2803
671b0060 2804 switch (info->cmd) {
f796564a
DM		 2805 case ETHTOOL_GRXFH: {
2806 unsigned int v = pi->rss_mode;
2807
2808 info->data = 0;
2809 switch (info->flow_type) {
2810 case TCP_V4_FLOW:
2811 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2812 info->data = RXH_IP_SRC | RXH_IP_DST |
2813 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2814 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2815 info->data = RXH_IP_SRC | RXH_IP_DST;
2816 break;
2817 case UDP_V4_FLOW:
2818 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2819 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2820 info->data = RXH_IP_SRC | RXH_IP_DST |
2821 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2822 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2823 info->data = RXH_IP_SRC | RXH_IP_DST;
2824 break;
2825 case SCTP_V4_FLOW:
2826 case AH_ESP_V4_FLOW:
2827 case IPV4_FLOW:
2828 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2829 info->data = RXH_IP_SRC | RXH_IP_DST;
2830 break;
2831 case TCP_V6_FLOW:
2832 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2833 info->data = RXH_IP_SRC | RXH_IP_DST |
2834 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2835 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2836 info->data = RXH_IP_SRC | RXH_IP_DST;
2837 break;
2838 case UDP_V6_FLOW:
2839 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2840 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2841 info->data = RXH_IP_SRC | RXH_IP_DST |
2842 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2843 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2844 info->data = RXH_IP_SRC | RXH_IP_DST;
2845 break;
2846 case SCTP_V6_FLOW:
2847 case AH_ESP_V6_FLOW:
2848 case IPV6_FLOW:
2849 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2850 info->data = RXH_IP_SRC | RXH_IP_DST;
2851 break;
2852 }
2853 return 0;
2854 }
671b0060 2855 case ETHTOOL_GRXRINGS:
f796564a 2856 info->data = pi->nqsets;
671b0060
DM		 2857 return 0;
2858 }
2859 return -EOPNOTSUPP;
2860}
2861
9b07be4b 2862static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 2863 .get_settings = get_settings,
2864 .set_settings = set_settings,
2865 .get_drvinfo = get_drvinfo,
2866 .get_msglevel = get_msglevel,
2867 .set_msglevel = set_msglevel,
2868 .get_ringparam = get_sge_param,
2869 .set_ringparam = set_sge_param,
2870 .get_coalesce = get_coalesce,
2871 .set_coalesce = set_coalesce,
2872 .get_eeprom_len = get_eeprom_len,
2873 .get_eeprom = get_eeprom,
2874 .set_eeprom = set_eeprom,
2875 .get_pauseparam = get_pauseparam,
2876 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2877 .get_link = ethtool_op_get_link,
2878 .get_strings = get_strings,
c5e06360 2879 .set_phys_id = identify_port,
b8ff05a9
DM		 2880 .nway_reset = restart_autoneg,
2881 .get_sset_count = get_sset_count,
2882 .get_ethtool_stats = get_stats,
2883 .get_regs_len = get_regs_len,
2884 .get_regs = get_regs,
2885 .get_wol = get_wol,
2886 .set_wol = set_wol,
671b0060 2887 .get_rxnfc = get_rxnfc,
7850f63f 2888 .get_rxfh_indir_size = get_rss_table_size,
fe62d001
BH		 2889 .get_rxfh = get_rss_table,
2890 .set_rxfh = set_rss_table,
b8ff05a9
DM		 2891 .flash_device = set_flash,
2892};
2893
2894/*
2895 * debugfs support
2896 */
b8ff05a9
DM		 2897static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2898 loff_t *ppos)
2899{
2900 loff_t pos = *ppos;
496ad9aa 2901 loff_t avail = file_inode(file)->i_size;
b8ff05a9
DM		 2902 unsigned int mem = (uintptr_t)file->private_data & 3;
2903 struct adapter *adap = file->private_data - mem;
2904
2905 if (pos < 0)
2906 return -EINVAL;
2907 if (pos >= avail)
2908 return 0;
2909 if (count > avail - pos)
2910 count = avail - pos;
2911
2912 while (count) {
2913 size_t len;
2914 int ret, ofst;
2915 __be32 data[16];
2916
19dd37ba
SR		 2917 if ((mem == MEM_MC) || (mem == MEM_MC1))
2918 ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);
b8ff05a9
DM		 2919 else
2920 ret = t4_edc_read(adap, mem, pos, data, NULL);
2921 if (ret)
2922 return ret;
2923
2924 ofst = pos % sizeof(data);
2925 len = min(count, sizeof(data) - ofst);
2926 if (copy_to_user(buf, (u8 *)data + ofst, len))
2927 return -EFAULT;
2928
2929 buf += len;
2930 pos += len;
2931 count -= len;
2932 }
2933 count = pos - *ppos;
2934 *ppos = pos;
2935 return count;
2936}
2937
2938static const struct file_operations mem_debugfs_fops = {
2939 .owner = THIS_MODULE,
234e3405 2940 .open = simple_open,
b8ff05a9 2941 .read = mem_read,
6038f373 2942 .llseek = default_llseek,
b8ff05a9
DM		 2943};
2944
91744948 2945static void add_debugfs_mem(struct adapter *adap, const char *name,
1dd06ae8 2946 unsigned int idx, unsigned int size_mb)
b8ff05a9
DM		 2947{
2948 struct dentry *de;
2949
2950 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2951 (void *)adap + idx, &mem_debugfs_fops);
2952 if (de && de->d_inode)
2953 de->d_inode->i_size = size_mb << 20;
2954}
2955
91744948 2956static int setup_debugfs(struct adapter *adap)
b8ff05a9
DM		 2957{
2958 int i;
19dd37ba 2959 u32 size;
b8ff05a9
DM		 2960
2961 if (IS_ERR_OR_NULL(adap->debugfs_root))
2962 return -1;
2963
2964 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
19dd37ba
SR		 2965 if (i & EDRAM0_ENABLE) {
2966 size = t4_read_reg(adap, MA_EDRAM0_BAR);
2967 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));
2968 }
2969 if (i & EDRAM1_ENABLE) {
2970 size = t4_read_reg(adap, MA_EDRAM1_BAR);
2971 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));
2972 }
d14807dd 2973 if (is_t4(adap->params.chip)) {
19dd37ba
SR		 2974 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2975 if (i & EXT_MEM_ENABLE)
2976 add_debugfs_mem(adap, "mc", MEM_MC,
2977 EXT_MEM_SIZE_GET(size));
2978 } else {
2979 if (i & EXT_MEM_ENABLE) {
2980 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2981 add_debugfs_mem(adap, "mc0", MEM_MC0,
2982 EXT_MEM_SIZE_GET(size));
2983 }
2984 if (i & EXT_MEM1_ENABLE) {
2985 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);
2986 add_debugfs_mem(adap, "mc1", MEM_MC1,
2987 EXT_MEM_SIZE_GET(size));
2988 }
2989 }
b8ff05a9
DM		 2990 if (adap->l2t)
2991 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2992 &t4_l2t_fops);
2993 return 0;
2994}
2995
2996/*
2997 * upper-layer driver support
2998 */
2999
3000/*
3001 * Allocate an active-open TID and set it to the supplied value.
3002 */
3003int cxgb4_alloc_atid(struct tid_info *t, void *data)
3004{
3005 int atid = -1;
3006
3007 spin_lock_bh(&t->atid_lock);
3008 if (t->afree) {
3009 union aopen_entry *p = t->afree;
3010
f2b7e78d 3011 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 3012 t->afree = p->next;
3013 p->data = data;
3014 t->atids_in_use++;
3015 }
3016 spin_unlock_bh(&t->atid_lock);
3017 return atid;
3018}
3019EXPORT_SYMBOL(cxgb4_alloc_atid);
3020
3021/*
3022 * Release an active-open TID.
3023 */
3024void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3025{
f2b7e78d 3026 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 3027
3028 spin_lock_bh(&t->atid_lock);
3029 p->next = t->afree;
3030 t->afree = p;
3031 t->atids_in_use--;
3032 spin_unlock_bh(&t->atid_lock);
3033}
3034EXPORT_SYMBOL(cxgb4_free_atid);
3035
3036/*
3037 * Allocate a server TID and set it to the supplied value.
3038 */
3039int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3040{
3041 int stid;
3042
3043 spin_lock_bh(&t->stid_lock);
3044 if (family == PF_INET) {
3045 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3046 if (stid < t->nstids)
3047 __set_bit(stid, t->stid_bmap);
3048 else
3049 stid = -1;
3050 } else {
3051 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3052 if (stid < 0)
3053 stid = -1;
3054 }
3055 if (stid >= 0) {
3056 t->stid_tab[stid].data = data;
3057 stid += t->stid_base;
15f63b74
KS		 3058 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3059 * This is equivalent to 4 TIDs. With CLIP enabled it
3060 * needs 2 TIDs.
3061 */
3062 if (family == PF_INET)
3063 t->stids_in_use++;
3064 else
3065 t->stids_in_use += 4;
b8ff05a9
DM		 3066 }
3067 spin_unlock_bh(&t->stid_lock);
3068 return stid;
3069}
3070EXPORT_SYMBOL(cxgb4_alloc_stid);
3071
dca4faeb
VP		 3072/* Allocate a server filter TID and set it to the supplied value.
3073 */
3074int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3075{
3076 int stid;
3077
3078 spin_lock_bh(&t->stid_lock);
3079 if (family == PF_INET) {
3080 stid = find_next_zero_bit(t->stid_bmap,
3081 t->nstids + t->nsftids, t->nstids);
3082 if (stid < (t->nstids + t->nsftids))
3083 __set_bit(stid, t->stid_bmap);
3084 else
3085 stid = -1;
3086 } else {
3087 stid = -1;
3088 }
3089 if (stid >= 0) {
3090 t->stid_tab[stid].data = data;
470c60c4
KS		 3091 stid -= t->nstids;
3092 stid += t->sftid_base;
dca4faeb
VP		 3093 t->stids_in_use++;
3094 }
3095 spin_unlock_bh(&t->stid_lock);
3096 return stid;
3097}
3098EXPORT_SYMBOL(cxgb4_alloc_sftid);
3099
3100/* Release a server TID.
b8ff05a9
DM		 3101 */
3102void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3103{
470c60c4
KS		 3104 /* Is it a server filter TID? */
3105 if (t->nsftids && (stid >= t->sftid_base)) {
3106 stid -= t->sftid_base;
3107 stid += t->nstids;
3108 } else {
3109 stid -= t->stid_base;
3110 }
3111
b8ff05a9
DM		 3112 spin_lock_bh(&t->stid_lock);
3113 if (family == PF_INET)
3114 __clear_bit(stid, t->stid_bmap);
3115 else
3116 bitmap_release_region(t->stid_bmap, stid, 2);
3117 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3118 if (family == PF_INET)
3119 t->stids_in_use--;
3120 else
3121 t->stids_in_use -= 4;
b8ff05a9
DM		 3122 spin_unlock_bh(&t->stid_lock);
3123}
3124EXPORT_SYMBOL(cxgb4_free_stid);
3125
3126/*
3127 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3128 */
3129static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3130 unsigned int tid)
3131{
3132 struct cpl_tid_release *req;
3133
3134 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3135 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3136 INIT_TP_WR(req, tid);
3137 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3138}
3139
3140/*
3141 * Queue a TID release request and if necessary schedule a work queue to
3142 * process it.
3143 */
31b9c19b 3144static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3145 unsigned int tid)
b8ff05a9
DM		 3146{
3147 void **p = &t->tid_tab[tid];
3148 struct adapter *adap = container_of(t, struct adapter, tids);
3149
3150 spin_lock_bh(&adap->tid_release_lock);
3151 *p = adap->tid_release_head;
3152 /* Low 2 bits encode the Tx channel number */
3153 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3154 if (!adap->tid_release_task_busy) {
3155 adap->tid_release_task_busy = true;
3069ee9b 3156 queue_work(workq, &adap->tid_release_task);
b8ff05a9
DM		 3157 }
3158 spin_unlock_bh(&adap->tid_release_lock);
3159}
b8ff05a9
DM		 3160
3161/*
3162 * Process the list of pending TID release requests.
3163 */
3164static void process_tid_release_list(struct work_struct *work)
3165{
3166 struct sk_buff *skb;
3167 struct adapter *adap;
3168
3169 adap = container_of(work, struct adapter, tid_release_task);
3170
3171 spin_lock_bh(&adap->tid_release_lock);
3172 while (adap->tid_release_head) {
3173 void **p = adap->tid_release_head;
3174 unsigned int chan = (uintptr_t)p & 3;
3175 p = (void *)p - chan;
3176
3177 adap->tid_release_head = *p;
3178 *p = NULL;
3179 spin_unlock_bh(&adap->tid_release_lock);
3180
3181 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3182 GFP_KERNEL)))
3183 schedule_timeout_uninterruptible(1);
3184
3185 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3186 t4_ofld_send(adap, skb);
3187 spin_lock_bh(&adap->tid_release_lock);
3188 }
3189 adap->tid_release_task_busy = false;
3190 spin_unlock_bh(&adap->tid_release_lock);
3191}
3192
3193/*
3194 * Release a TID and inform HW. If we are unable to allocate the release
3195 * message we defer to a work queue.
3196 */
3197void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3198{
3199 void *old;
3200 struct sk_buff *skb;
3201 struct adapter *adap = container_of(t, struct adapter, tids);
3202
3203 old = t->tid_tab[tid];
3204 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3205 if (likely(skb)) {
3206 t->tid_tab[tid] = NULL;
3207 mk_tid_release(skb, chan, tid);
3208 t4_ofld_send(adap, skb);
3209 } else
3210 cxgb4_queue_tid_release(t, chan, tid);
3211 if (old)
3212 atomic_dec(&t->tids_in_use);
3213}
3214EXPORT_SYMBOL(cxgb4_remove_tid);
3215
3216/*
3217 * Allocate and initialize the TID tables. Returns 0 on success.
3218 */
3219static int tid_init(struct tid_info *t)
3220{
3221 size_t size;
f2b7e78d 3222 unsigned int stid_bmap_size;
b8ff05a9 3223 unsigned int natids = t->natids;
b6f8eaec 3224 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3225
dca4faeb 3226 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3227 size = t->ntids * sizeof(*t->tid_tab) +
3228 natids * sizeof(*t->atid_tab) +
b8ff05a9 3229 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3230 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3231 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3232 t->nftids * sizeof(*t->ftid_tab) +
3233 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3234
b8ff05a9
DM		 3235 t->tid_tab = t4_alloc_mem(size);
3236 if (!t->tid_tab)
3237 return -ENOMEM;
3238
3239 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3240 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3241 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3242 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3243 spin_lock_init(&t->stid_lock);
3244 spin_lock_init(&t->atid_lock);
3245
3246 t->stids_in_use = 0;
3247 t->afree = NULL;
3248 t->atids_in_use = 0;
3249 atomic_set(&t->tids_in_use, 0);
3250
3251 /* Setup the free list for atid_tab and clear the stid bitmap. */
3252 if (natids) {
3253 while (--natids)
3254 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3255 t->afree = t->atid_tab;
3256 }
dca4faeb 3257 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3258 /* Reserve stid 0 for T4/T5 adapters */
3259 if (!t->stid_base &&
3260 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3261 __set_bit(0, t->stid_bmap);
3262
b8ff05a9
DM		 3263 return 0;
3264}
3265
01bcca68
VP		 3266static int cxgb4_clip_get(const struct net_device *dev,
3267 const struct in6_addr *lip)
3268{
3269 struct adapter *adap;
3270 struct fw_clip_cmd c;
3271
3272 adap = netdev2adap(dev);
3273 memset(&c, 0, sizeof(c));
3274 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3275 FW_CMD_REQUEST | FW_CMD_WRITE);
3276 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
12f2a479
JP		 3277 c.ip_hi = *(__be64 *)(lip->s6_addr);
3278 c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
01bcca68
VP		 3279 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3280}
3281
3282static int cxgb4_clip_release(const struct net_device *dev,
3283 const struct in6_addr *lip)
3284{
3285 struct adapter *adap;
3286 struct fw_clip_cmd c;
3287
3288 adap = netdev2adap(dev);
3289 memset(&c, 0, sizeof(c));
3290 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3291 FW_CMD_REQUEST | FW_CMD_READ);
3292 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
12f2a479
JP		 3293 c.ip_hi = *(__be64 *)(lip->s6_addr);
3294 c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
01bcca68
VP		 3295 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3296}
3297
b8ff05a9
DM		 3298/**
3299 * cxgb4_create_server - create an IP server
3300 * @dev: the device
3301 * @stid: the server TID
3302 * @sip: local IP address to bind server to
3303 * @sport: the server's TCP port
3304 * @queue: queue to direct messages from this server to
3305 *
3306 * Create an IP server for the given port and address.
3307 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3308 */
3309int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3310 __be32 sip, __be16 sport, __be16 vlan,
3311 unsigned int queue)
b8ff05a9
DM		 3312{
3313 unsigned int chan;
3314 struct sk_buff *skb;
3315 struct adapter *adap;
3316 struct cpl_pass_open_req *req;
80f40c1f 3317 int ret;
b8ff05a9
DM		 3318
3319 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3320 if (!skb)
3321 return -ENOMEM;
3322
3323 adap = netdev2adap(dev);
3324 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3325 INIT_TP_WR(req, 0);
3326 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3327 req->local_port = sport;
3328 req->peer_port = htons(0);
3329 req->local_ip = sip;
3330 req->peer_ip = htonl(0);
e46dab4d 3331 chan = rxq_to_chan(&adap->sge, queue);
b8ff05a9
DM		 3332 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3333 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3334 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
80f40c1f
VP		 3335 ret = t4_mgmt_tx(adap, skb);
3336 return net_xmit_eval(ret);
b8ff05a9
DM		 3337}
3338EXPORT_SYMBOL(cxgb4_create_server);
3339
80f40c1f
VP		 3340/* cxgb4_create_server6 - create an IPv6 server
3341 * @dev: the device
3342 * @stid: the server TID
3343 * @sip: local IPv6 address to bind server to
3344 * @sport: the server's TCP port
3345 * @queue: queue to direct messages from this server to
3346 *
3347 * Create an IPv6 server for the given port and address.
3348 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3349 */
3350int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3351 const struct in6_addr *sip, __be16 sport,
3352 unsigned int queue)
3353{
3354 unsigned int chan;
3355 struct sk_buff *skb;
3356 struct adapter *adap;
3357 struct cpl_pass_open_req6 *req;
3358 int ret;
3359
3360 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3361 if (!skb)
3362 return -ENOMEM;
3363
3364 adap = netdev2adap(dev);
3365 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3366 INIT_TP_WR(req, 0);
3367 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3368 req->local_port = sport;
3369 req->peer_port = htons(0);
3370 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3371 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3372 req->peer_ip_hi = cpu_to_be64(0);
3373 req->peer_ip_lo = cpu_to_be64(0);
3374 chan = rxq_to_chan(&adap->sge, queue);
3375 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3376 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3377 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3378 ret = t4_mgmt_tx(adap, skb);
3379 return net_xmit_eval(ret);
3380}
3381EXPORT_SYMBOL(cxgb4_create_server6);
3382
3383int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3384 unsigned int queue, bool ipv6)
3385{
3386 struct sk_buff *skb;
3387 struct adapter *adap;
3388 struct cpl_close_listsvr_req *req;
3389 int ret;
3390
3391 adap = netdev2adap(dev);
3392
3393 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3394 if (!skb)
3395 return -ENOMEM;
3396
3397 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3398 INIT_TP_WR(req, 0);
3399 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3400 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3401 LISTSVR_IPV6(0)) | QUEUENO(queue));
3402 ret = t4_mgmt_tx(adap, skb);
3403 return net_xmit_eval(ret);
3404}
3405EXPORT_SYMBOL(cxgb4_remove_server);
3406
b8ff05a9
DM		 3407/**
3408 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3409 * @mtus: the HW MTU table
3410 * @mtu: the target MTU
3411 * @idx: index of selected entry in the MTU table
3412 *
3413 * Returns the index and the value in the HW MTU table that is closest to
3414 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3415 * table, in which case that smallest available value is selected.
3416 */
3417unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3418 unsigned int *idx)
3419{
3420 unsigned int i = 0;
3421
3422 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3423 ++i;
3424 if (idx)
3425 *idx = i;
3426 return mtus[i];
3427}
3428EXPORT_SYMBOL(cxgb4_best_mtu);
3429
92e7ae71
HS		 3430/**
3431 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3432 * @mtus: the HW MTU table
3433 * @header_size: Header Size
3434 * @data_size_max: maximum Data Segment Size
3435 * @data_size_align: desired Data Segment Size Alignment (2^N)
3436 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3437 *
3438 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3439 * MTU Table based solely on a Maximum MTU parameter, we break that
3440 * parameter up into a Header Size and Maximum Data Segment Size, and
3441 * provide a desired Data Segment Size Alignment. If we find an MTU in
3442 * the Hardware MTU Table which will result in a Data Segment Size with
3443 * the requested alignment _and_ that MTU isn't "too far" from the
3444 * closest MTU, then we'll return that rather than the closest MTU.
3445 */
3446unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3447 unsigned short header_size,
3448 unsigned short data_size_max,
3449 unsigned short data_size_align,
3450 unsigned int *mtu_idxp)
3451{
3452 unsigned short max_mtu = header_size + data_size_max;
3453 unsigned short data_size_align_mask = data_size_align - 1;
3454 int mtu_idx, aligned_mtu_idx;
3455
3456 /* Scan the MTU Table till we find an MTU which is larger than our
3457 * Maximum MTU or we reach the end of the table. Along the way,
3458 * record the last MTU found, if any, which will result in a Data
3459 * Segment Length matching the requested alignment.
3460 */
3461 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3462 unsigned short data_size = mtus[mtu_idx] - header_size;
3463
3464 /* If this MTU minus the Header Size would result in a
3465 * Data Segment Size of the desired alignment, remember it.
3466 */
3467 if ((data_size & data_size_align_mask) == 0)
3468 aligned_mtu_idx = mtu_idx;
3469
3470 /* If we're not at the end of the Hardware MTU Table and the
3471 * next element is larger than our Maximum MTU, drop out of
3472 * the loop.
3473 */
3474 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3475 break;
3476 }
3477
3478 /* If we fell out of the loop because we ran to the end of the table,
3479 * then we just have to use the last [largest] entry.
3480 */
3481 if (mtu_idx == NMTUS)
3482 mtu_idx--;
3483
3484 /* If we found an MTU which resulted in the requested Data Segment
3485 * Length alignment and that's "not far" from the largest MTU which is
3486 * less than or equal to the maximum MTU, then use that.
3487 */
3488 if (aligned_mtu_idx >= 0 &&
3489 mtu_idx - aligned_mtu_idx <= 1)
3490 mtu_idx = aligned_mtu_idx;
3491
3492 /* If the caller has passed in an MTU Index pointer, pass the
3493 * MTU Index back. Return the MTU value.
3494 */
3495 if (mtu_idxp)
3496 *mtu_idxp = mtu_idx;
3497 return mtus[mtu_idx];
3498}
3499EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3500
b8ff05a9
DM		 3501/**
3502 * cxgb4_port_chan - get the HW channel of a port
3503 * @dev: the net device for the port
3504 *
3505 * Return the HW Tx channel of the given port.
3506 */
3507unsigned int cxgb4_port_chan(const struct net_device *dev)
3508{
3509 return netdev2pinfo(dev)->tx_chan;
3510}
3511EXPORT_SYMBOL(cxgb4_port_chan);
3512
881806bc
VP		 3513unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3514{
3515 struct adapter *adap = netdev2adap(dev);
2cc301d2 3516 u32 v1, v2, lp_count, hp_count;
881806bc 3517
2cc301d2
SR		 3518 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3519 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
d14807dd 3520 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3521 lp_count = G_LP_COUNT(v1);
3522 hp_count = G_HP_COUNT(v1);
3523 } else {
3524 lp_count = G_LP_COUNT_T5(v1);
3525 hp_count = G_HP_COUNT_T5(v2);
3526 }
3527 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3528}
3529EXPORT_SYMBOL(cxgb4_dbfifo_count);
3530
b8ff05a9
DM		 3531/**
3532 * cxgb4_port_viid - get the VI id of a port
3533 * @dev: the net device for the port
3534 *
3535 * Return the VI id of the given port.
3536 */
3537unsigned int cxgb4_port_viid(const struct net_device *dev)
3538{
3539 return netdev2pinfo(dev)->viid;
3540}
3541EXPORT_SYMBOL(cxgb4_port_viid);
3542
3543/**
3544 * cxgb4_port_idx - get the index of a port
3545 * @dev: the net device for the port
3546 *
3547 * Return the index of the given port.
3548 */
3549unsigned int cxgb4_port_idx(const struct net_device *dev)
3550{
3551 return netdev2pinfo(dev)->port_id;
3552}
3553EXPORT_SYMBOL(cxgb4_port_idx);
3554
b8ff05a9
DM		 3555void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3556 struct tp_tcp_stats *v6)
3557{
3558 struct adapter *adap = pci_get_drvdata(pdev);
3559
3560 spin_lock(&adap->stats_lock);
3561 t4_tp_get_tcp_stats(adap, v4, v6);
3562 spin_unlock(&adap->stats_lock);
3563}
3564EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3565
3566void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3567 const unsigned int *pgsz_order)
3568{
3569 struct adapter *adap = netdev2adap(dev);
3570
3571 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3572 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3573 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3574 HPZ3(pgsz_order[3]));
3575}
3576EXPORT_SYMBOL(cxgb4_iscsi_init);
3577
3069ee9b
VP		 3578int cxgb4_flush_eq_cache(struct net_device *dev)
3579{
3580 struct adapter *adap = netdev2adap(dev);
3581 int ret;
3582
3583 ret = t4_fwaddrspace_write(adap, adap->mbox,
3584 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3585 return ret;
3586}
3587EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3588
3589static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3590{
3591 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3592 __be64 indices;
3593 int ret;
3594
3595 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
3596 if (!ret) {
404d9e3f
VP		 3597 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3598 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3599 }
3600 return ret;
3601}
3602
3603int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3604 u16 size)
3605{
3606 struct adapter *adap = netdev2adap(dev);
3607 u16 hw_pidx, hw_cidx;
3608 int ret;
3609
3610 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3611 if (ret)
3612 goto out;
3613
3614 if (pidx != hw_pidx) {
3615 u16 delta;
3616
3617 if (pidx >= hw_pidx)
3618 delta = pidx - hw_pidx;
3619 else
3620 delta = size - hw_pidx + pidx;
3621 wmb();
840f3000
VP		 3622 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3623 QID(qid) | PIDX(delta));
3069ee9b
VP		 3624 }
3625out:
3626 return ret;
3627}
3628EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3629
3cbdb928
VP		 3630void cxgb4_disable_db_coalescing(struct net_device *dev)
3631{
3632 struct adapter *adap;
3633
3634 adap = netdev2adap(dev);
3635 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3636 F_NOCOALESCE);
3637}
3638EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3639
3640void cxgb4_enable_db_coalescing(struct net_device *dev)
3641{
3642 struct adapter *adap;
3643
3644 adap = netdev2adap(dev);
3645 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3646}
3647EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3648
b8ff05a9
DM		 3649static struct pci_driver cxgb4_driver;
3650
3651static void check_neigh_update(struct neighbour *neigh)
3652{
3653 const struct device *parent;
3654 const struct net_device *netdev = neigh->dev;
3655
3656 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3657 netdev = vlan_dev_real_dev(netdev);
3658 parent = netdev->dev.parent;
3659 if (parent && parent->driver == &cxgb4_driver.driver)
3660 t4_l2t_update(dev_get_drvdata(parent), neigh);
3661}
3662
3663static int netevent_cb(struct notifier_block *nb, unsigned long event,
3664 void *data)
3665{
3666 switch (event) {
3667 case NETEVENT_NEIGH_UPDATE:
3668 check_neigh_update(data);
3669 break;
b8ff05a9
DM		 3670 case NETEVENT_REDIRECT:
3671 default:
3672 break;
3673 }
3674 return 0;
3675}
3676
3677static bool netevent_registered;
3678static struct notifier_block cxgb4_netevent_nb = {
3679 .notifier_call = netevent_cb
3680};
3681
3069ee9b
VP		 3682static void drain_db_fifo(struct adapter *adap, int usecs)
3683{
2cc301d2 3684 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3685
3686 do {
2cc301d2
SR		 3687 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3688 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
d14807dd 3689 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3690 lp_count = G_LP_COUNT(v1);
3691 hp_count = G_HP_COUNT(v1);
3692 } else {
3693 lp_count = G_LP_COUNT_T5(v1);
3694 hp_count = G_HP_COUNT_T5(v2);
3695 }
3696
3697 if (lp_count == 0 && hp_count == 0)
3698 break;
3069ee9b
VP		 3699 set_current_state(TASK_UNINTERRUPTIBLE);
3700 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3701 } while (1);
3702}
3703
3704static void disable_txq_db(struct sge_txq *q)
3705{
05eb2389
SW		 3706 unsigned long flags;
3707
3708 spin_lock_irqsave(&q->db_lock, flags);
3069ee9b 3709 q->db_disabled = 1;
05eb2389 3710 spin_unlock_irqrestore(&q->db_lock, flags);
3069ee9b
VP		 3711}
3712
05eb2389 3713static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3069ee9b
VP		 3714{
3715 spin_lock_irq(&q->db_lock);
05eb2389
SW		 3716 if (q->db_pidx_inc) {
3717 /* Make sure that all writes to the TX descriptors
3718 * are committed before we tell HW about them.
3719 */
3720 wmb();
3721 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3722 QID(q->cntxt_id) | PIDX(q->db_pidx_inc));
3723 q->db_pidx_inc = 0;
3724 }
3069ee9b
VP		 3725 q->db_disabled = 0;
3726 spin_unlock_irq(&q->db_lock);
3727}
3728
3729static void disable_dbs(struct adapter *adap)
3730{
3731 int i;
3732
3733 for_each_ethrxq(&adap->sge, i)
3734 disable_txq_db(&adap->sge.ethtxq[i].q);
3735 for_each_ofldrxq(&adap->sge, i)
3736 disable_txq_db(&adap->sge.ofldtxq[i].q);
3737 for_each_port(adap, i)
3738 disable_txq_db(&adap->sge.ctrlq[i].q);
3739}
3740
3741static void enable_dbs(struct adapter *adap)
3742{
3743 int i;
3744
3745 for_each_ethrxq(&adap->sge, i)
05eb2389 3746 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3069ee9b 3747 for_each_ofldrxq(&adap->sge, i)
05eb2389 3748 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3069ee9b 3749 for_each_port(adap, i)
05eb2389
SW		 3750 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3751}
3752
3753static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3754{
3755 if (adap->uld_handle[CXGB4_ULD_RDMA])
3756 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3757 cmd);
3758}
3759
3760static void process_db_full(struct work_struct *work)
3761{
3762 struct adapter *adap;
3763
3764 adap = container_of(work, struct adapter, db_full_task);
3765
3766 drain_db_fifo(adap, dbfifo_drain_delay);
3767 enable_dbs(adap);
3768 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3769 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3770 DBFIFO_HP_INT | DBFIFO_LP_INT,
3771 DBFIFO_HP_INT | DBFIFO_LP_INT);
3069ee9b
VP		 3772}
3773
3774static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3775{
3776 u16 hw_pidx, hw_cidx;
3777 int ret;
3778
05eb2389 3779 spin_lock_irq(&q->db_lock);
3069ee9b
VP		 3780 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3781 if (ret)
3782 goto out;
3783 if (q->db_pidx != hw_pidx) {
3784 u16 delta;
3785
3786 if (q->db_pidx >= hw_pidx)
3787 delta = q->db_pidx - hw_pidx;
3788 else
3789 delta = q->size - hw_pidx + q->db_pidx;
3790 wmb();
840f3000
VP		 3791 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3792 QID(q->cntxt_id) | PIDX(delta));
3069ee9b
VP		 3793 }
3794out:
3795 q->db_disabled = 0;
05eb2389
SW		 3796 q->db_pidx_inc = 0;
3797 spin_unlock_irq(&q->db_lock);
3069ee9b
VP		 3798 if (ret)
3799 CH_WARN(adap, "DB drop recovery failed.\n");
3800}
3801static void recover_all_queues(struct adapter *adap)
3802{
3803 int i;
3804
3805 for_each_ethrxq(&adap->sge, i)
3806 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3807 for_each_ofldrxq(&adap->sge, i)
3808 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3809 for_each_port(adap, i)
3810 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3811}
3812
881806bc
VP		 3813static void process_db_drop(struct work_struct *work)
3814{
3815 struct adapter *adap;
881806bc 3816
3069ee9b 3817 adap = container_of(work, struct adapter, db_drop_task);
881806bc 3818
d14807dd 3819 if (is_t4(adap->params.chip)) {
05eb2389 3820 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3821 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
05eb2389 3822 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3823 recover_all_queues(adap);
05eb2389 3824 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3825 enable_dbs(adap);
05eb2389 3826 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2cc301d2
SR		 3827 } else {
3828 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3829 u16 qid = (dropped_db >> 15) & 0x1ffff;
3830 u16 pidx_inc = dropped_db & 0x1fff;
3831 unsigned int s_qpp;
3832 unsigned short udb_density;
3833 unsigned long qpshift;
3834 int page;
3835 u32 udb;
3836
3837 dev_warn(adap->pdev_dev,
3838 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3839 dropped_db, qid,
3840 (dropped_db >> 14) & 1,
3841 (dropped_db >> 13) & 1,
3842 pidx_inc);
3843
3844 drain_db_fifo(adap, 1);
3845
3846 s_qpp = QUEUESPERPAGEPF1 * adap->fn;
3847 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,
3848 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
3849 qpshift = PAGE_SHIFT - ilog2(udb_density);
3850 udb = qid << qpshift;
3851 udb &= PAGE_MASK;
3852 page = udb / PAGE_SIZE;
3853 udb += (qid - (page * udb_density)) * 128;
3854
3855 writel(PIDX(pidx_inc), adap->bar2 + udb + 8);
3856
3857 /* Re-enable BAR2 WC */
3858 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3859 }
3860
3069ee9b 3861 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
881806bc
VP		 3862}
3863
3864void t4_db_full(struct adapter *adap)
3865{
d14807dd 3866 if (is_t4(adap->params.chip)) {
05eb2389
SW		 3867 disable_dbs(adap);
3868 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2cc301d2
SR		 3869 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3870 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3871 queue_work(workq, &adap->db_full_task);
3872 }
881806bc
VP		 3873}
3874
3875void t4_db_dropped(struct adapter *adap)
3876{
05eb2389
SW		 3877 if (is_t4(adap->params.chip)) {
3878 disable_dbs(adap);
3879 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3880 }
3881 queue_work(workq, &adap->db_drop_task);
881806bc
VP		 3882}
3883
b8ff05a9
DM		 3884static void uld_attach(struct adapter *adap, unsigned int uld)
3885{
3886 void *handle;
3887 struct cxgb4_lld_info lli;
dca4faeb 3888 unsigned short i;
b8ff05a9
DM		 3889
3890 lli.pdev = adap->pdev;
3891 lli.l2t = adap->l2t;
3892 lli.tids = &adap->tids;
3893 lli.ports = adap->port;
3894 lli.vr = &adap->vres;
3895 lli.mtus = adap->params.mtus;
3896 if (uld == CXGB4_ULD_RDMA) {
3897 lli.rxq_ids = adap->sge.rdma_rxq;
cf38be6d 3898 lli.ciq_ids = adap->sge.rdma_ciq;
b8ff05a9 3899 lli.nrxq = adap->sge.rdmaqs;
cf38be6d 3900 lli.nciq = adap->sge.rdmaciqs;
b8ff05a9
DM		 3901 } else if (uld == CXGB4_ULD_ISCSI) {
3902 lli.rxq_ids = adap->sge.ofld_rxq;
3903 lli.nrxq = adap->sge.ofldqsets;
3904 }
3905 lli.ntxq = adap->sge.ofldqsets;
3906 lli.nchan = adap->params.nports;
3907 lli.nports = adap->params.nports;
3908 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 3909 lli.adapter_type = adap->params.chip;
b8ff05a9
DM		 3910 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3911 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 3912 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3913 (adap->fn * 4));
b8ff05a9 3914 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 3915 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3916 (adap->fn * 4));
dcf7b6f5 3917 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 3918 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3919 for (i = 0; i < NCHAN; i++)
3920 lli.tx_modq[i] = i;
b8ff05a9
DM		 3921 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3922 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3923 lli.fw_vers = adap->params.fw_vers;
3069ee9b 3924 lli.dbfifo_int_thresh = dbfifo_int_thresh;
dca4faeb
VP		 3925 lli.sge_pktshift = adap->sge.pktshift;
3926 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
1ac0f095 3927 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 3928
3929 handle = ulds[uld].add(&lli);
3930 if (IS_ERR(handle)) {
3931 dev_warn(adap->pdev_dev,
3932 "could not attach to the %s driver, error %ld\n",
3933 uld_str[uld], PTR_ERR(handle));
3934 return;
3935 }
3936
3937 adap->uld_handle[uld] = handle;
3938
3939 if (!netevent_registered) {
3940 register_netevent_notifier(&cxgb4_netevent_nb);
3941 netevent_registered = true;
3942 }
e29f5dbc
DM		 3943
3944 if (adap->flags & FULL_INIT_DONE)
3945 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 3946}
3947
3948static void attach_ulds(struct adapter *adap)
3949{
3950 unsigned int i;
3951
01bcca68
VP		 3952 spin_lock(&adap_rcu_lock);
3953 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
3954 spin_unlock(&adap_rcu_lock);
3955
b8ff05a9
DM		 3956 mutex_lock(&uld_mutex);
3957 list_add_tail(&adap->list_node, &adapter_list);
3958 for (i = 0; i < CXGB4_ULD_MAX; i++)
3959 if (ulds[i].add)
3960 uld_attach(adap, i);
3961 mutex_unlock(&uld_mutex);
3962}
3963
3964static void detach_ulds(struct adapter *adap)
3965{
3966 unsigned int i;
3967
3968 mutex_lock(&uld_mutex);
3969 list_del(&adap->list_node);
3970 for (i = 0; i < CXGB4_ULD_MAX; i++)
3971 if (adap->uld_handle[i]) {
3972 ulds[i].state_change(adap->uld_handle[i],
3973 CXGB4_STATE_DETACH);
3974 adap->uld_handle[i] = NULL;
3975 }
3976 if (netevent_registered && list_empty(&adapter_list)) {
3977 unregister_netevent_notifier(&cxgb4_netevent_nb);
3978 netevent_registered = false;
3979 }
3980 mutex_unlock(&uld_mutex);
01bcca68
VP		 3981
3982 spin_lock(&adap_rcu_lock);
3983 list_del_rcu(&adap->rcu_node);
3984 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 3985}
3986
3987static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3988{
3989 unsigned int i;
3990
3991 mutex_lock(&uld_mutex);
3992 for (i = 0; i < CXGB4_ULD_MAX; i++)
3993 if (adap->uld_handle[i])
3994 ulds[i].state_change(adap->uld_handle[i], new_state);
3995 mutex_unlock(&uld_mutex);
3996}
3997
3998/**
3999 * cxgb4_register_uld - register an upper-layer driver
4000 * @type: the ULD type
4001 * @p: the ULD methods
4002 *
4003 * Registers an upper-layer driver with this driver and notifies the ULD
4004 * about any presently available devices that support its type. Returns
4005 * %-EBUSY if a ULD of the same type is already registered.
4006 */
4007int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4008{
4009 int ret = 0;
4010 struct adapter *adap;
4011
4012 if (type >= CXGB4_ULD_MAX)
4013 return -EINVAL;
4014 mutex_lock(&uld_mutex);
4015 if (ulds[type].add) {
4016 ret = -EBUSY;
4017 goto out;
4018 }
4019 ulds[type] = *p;
4020 list_for_each_entry(adap, &adapter_list, list_node)
4021 uld_attach(adap, type);
4022out: mutex_unlock(&uld_mutex);
4023 return ret;
4024}
4025EXPORT_SYMBOL(cxgb4_register_uld);
4026
4027/**
4028 * cxgb4_unregister_uld - unregister an upper-layer driver
4029 * @type: the ULD type
4030 *
4031 * Unregisters an existing upper-layer driver.
4032 */
4033int cxgb4_unregister_uld(enum cxgb4_uld type)
4034{
4035 struct adapter *adap;
4036
4037 if (type >= CXGB4_ULD_MAX)
4038 return -EINVAL;
4039 mutex_lock(&uld_mutex);
4040 list_for_each_entry(adap, &adapter_list, list_node)
4041 adap->uld_handle[type] = NULL;
4042 ulds[type].add = NULL;
4043 mutex_unlock(&uld_mutex);
4044 return 0;
4045}
4046EXPORT_SYMBOL(cxgb4_unregister_uld);
4047
01bcca68
VP		 4048/* Check if netdev on which event is occured belongs to us or not. Return
4049 * suceess (1) if it belongs otherwise failure (0).
4050 */
4051static int cxgb4_netdev(struct net_device *netdev)
4052{
4053 struct adapter *adap;
4054 int i;
4055
4056 spin_lock(&adap_rcu_lock);
4057 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
4058 for (i = 0; i < MAX_NPORTS; i++)
4059 if (adap->port[i] == netdev) {
4060 spin_unlock(&adap_rcu_lock);
4061 return 1;
4062 }
4063 spin_unlock(&adap_rcu_lock);
4064 return 0;
4065}
4066
4067static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
4068 unsigned long event)
4069{
4070 int ret = NOTIFY_DONE;
4071
4072 rcu_read_lock();
4073 if (cxgb4_netdev(event_dev)) {
4074 switch (event) {
4075 case NETDEV_UP:
4076 ret = cxgb4_clip_get(event_dev,
4077 (const struct in6_addr *)ifa->addr.s6_addr);
4078 if (ret < 0) {
4079 rcu_read_unlock();
4080 return ret;
4081 }
4082 ret = NOTIFY_OK;
4083 break;
4084 case NETDEV_DOWN:
4085 cxgb4_clip_release(event_dev,
4086 (const struct in6_addr *)ifa->addr.s6_addr);
4087 ret = NOTIFY_OK;
4088 break;
4089 default:
4090 break;
4091 }
4092 }
4093 rcu_read_unlock();
4094 return ret;
4095}
4096
4097static int cxgb4_inet6addr_handler(struct notifier_block *this,
4098 unsigned long event, void *data)
4099{
4100 struct inet6_ifaddr *ifa = data;
4101 struct net_device *event_dev;
4102 int ret = NOTIFY_DONE;
01bcca68 4103 struct bonding *bond = netdev_priv(ifa->idev->dev);
9caff1e7 4104 struct list_head *iter;
01bcca68
VP		 4105 struct slave *slave;
4106 struct pci_dev *first_pdev = NULL;
4107
4108 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
4109 event_dev = vlan_dev_real_dev(ifa->idev->dev);
4110 ret = clip_add(event_dev, ifa, event);
4111 } else if (ifa->idev->dev->flags & IFF_MASTER) {
4112 /* It is possible that two different adapters are bonded in one
4113 * bond. We need to find such different adapters and add clip
4114 * in all of them only once.
4115 */
4116 read_lock(&bond->lock);
9caff1e7 4117 bond_for_each_slave(bond, slave, iter) {
01bcca68
VP		 4118 if (!first_pdev) {
4119 ret = clip_add(slave->dev, ifa, event);
4120 /* If clip_add is success then only initialize
4121 * first_pdev since it means it is our device
4122 */
4123 if (ret == NOTIFY_OK)
4124 first_pdev = to_pci_dev(
4125 slave->dev->dev.parent);
4126 } else if (first_pdev !=
4127 to_pci_dev(slave->dev->dev.parent))
4128 ret = clip_add(slave->dev, ifa, event);
4129 }
4130 read_unlock(&bond->lock);
4131 } else
4132 ret = clip_add(ifa->idev->dev, ifa, event);
4133
4134 return ret;
4135}
4136
4137static struct notifier_block cxgb4_inet6addr_notifier = {
4138 .notifier_call = cxgb4_inet6addr_handler
4139};
4140
4141/* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4142 * a physical device.
4143 * The physical device reference is needed to send the actul CLIP command.
4144 */
4145static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
4146{
4147 struct inet6_dev *idev = NULL;
4148 struct inet6_ifaddr *ifa;
4149 int ret = 0;
4150
4151 idev = __in6_dev_get(root_dev);
4152 if (!idev)
4153 return ret;
4154
4155 read_lock_bh(&idev->lock);
4156 list_for_each_entry(ifa, &idev->addr_list, if_list) {
4157 ret = cxgb4_clip_get(dev,
4158 (const struct in6_addr *)ifa->addr.s6_addr);
4159 if (ret < 0)
4160 break;
4161 }
4162 read_unlock_bh(&idev->lock);
4163
4164 return ret;
4165}
4166
4167static int update_root_dev_clip(struct net_device *dev)
4168{
4169 struct net_device *root_dev = NULL;
4170 int i, ret = 0;
4171
4172 /* First populate the real net device's IPv6 addresses */
4173 ret = update_dev_clip(dev, dev);
4174 if (ret)
4175 return ret;
4176
4177 /* Parse all bond and vlan devices layered on top of the physical dev */
4178 for (i = 0; i < VLAN_N_VID; i++) {
f06c7f9f 4179 root_dev = __vlan_find_dev_deep_rcu(dev, htons(ETH_P_8021Q), i);
01bcca68
VP		 4180 if (!root_dev)
4181 continue;
4182
4183 ret = update_dev_clip(root_dev, dev);
4184 if (ret)
4185 break;
4186 }
4187 return ret;
4188}
4189
4190static void update_clip(const struct adapter *adap)
4191{
4192 int i;
4193 struct net_device *dev;
4194 int ret;
4195
4196 rcu_read_lock();
4197
4198 for (i = 0; i < MAX_NPORTS; i++) {
4199 dev = adap->port[i];
4200 ret = 0;
4201
4202 if (dev)
4203 ret = update_root_dev_clip(dev);
4204
4205 if (ret < 0)
4206 break;
4207 }
4208 rcu_read_unlock();
4209}
4210
b8ff05a9
DM		 4211/**
4212 * cxgb_up - enable the adapter
4213 * @adap: adapter being enabled
4214 *
4215 * Called when the first port is enabled, this function performs the
4216 * actions necessary to make an adapter operational, such as completing
4217 * the initialization of HW modules, and enabling interrupts.
4218 *
4219 * Must be called with the rtnl lock held.
4220 */
4221static int cxgb_up(struct adapter *adap)
4222{
aaefae9b 4223 int err;
b8ff05a9 4224
aaefae9b
DM		 4225 err = setup_sge_queues(adap);
4226 if (err)
4227 goto out;
4228 err = setup_rss(adap);
4229 if (err)
4230 goto freeq;
b8ff05a9
DM		 4231
4232 if (adap->flags & USING_MSIX) {
aaefae9b 4233 name_msix_vecs(adap);
b8ff05a9
DM		 4234 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4235 adap->msix_info[0].desc, adap);
4236 if (err)
4237 goto irq_err;
4238
4239 err = request_msix_queue_irqs(adap);
4240 if (err) {
4241 free_irq(adap->msix_info[0].vec, adap);
4242 goto irq_err;
4243 }
4244 } else {
4245 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4246 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4247 adap->port[0]->name, adap);
b8ff05a9
DM		 4248 if (err)
4249 goto irq_err;
4250 }
4251 enable_rx(adap);
4252 t4_sge_start(adap);
4253 t4_intr_enable(adap);
aaefae9b 4254 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4255 notify_ulds(adap, CXGB4_STATE_UP);
01bcca68 4256 update_clip(adap);
b8ff05a9
DM		 4257 out:
4258 return err;
4259 irq_err:
4260 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4261 freeq:
4262 t4_free_sge_resources(adap);
b8ff05a9
DM		 4263 goto out;
4264}
4265
4266static void cxgb_down(struct adapter *adapter)
4267{
4268 t4_intr_disable(adapter);
4269 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4270 cancel_work_sync(&adapter->db_full_task);
4271 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4272 adapter->tid_release_task_busy = false;
204dc3c0 4273 adapter->tid_release_head = NULL;
b8ff05a9
DM		 4274
4275 if (adapter->flags & USING_MSIX) {
4276 free_msix_queue_irqs(adapter);
4277 free_irq(adapter->msix_info[0].vec, adapter);
4278 } else
4279 free_irq(adapter->pdev->irq, adapter);
4280 quiesce_rx(adapter);
aaefae9b
DM		 4281 t4_sge_stop(adapter);
4282 t4_free_sge_resources(adapter);
4283 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4284}
4285
4286/*
4287 * net_device operations
4288 */
4289static int cxgb_open(struct net_device *dev)
4290{
4291 int err;
4292 struct port_info *pi = netdev_priv(dev);
4293 struct adapter *adapter = pi->adapter;
4294
6a3c869a
DM		 4295 netif_carrier_off(dev);
4296
aaefae9b
DM		 4297 if (!(adapter->flags & FULL_INIT_DONE)) {
4298 err = cxgb_up(adapter);
4299 if (err < 0)
4300 return err;
4301 }
b8ff05a9 4302
f68707b8
DM		 4303 err = link_start(dev);
4304 if (!err)
4305 netif_tx_start_all_queues(dev);
4306 return err;
b8ff05a9
DM		 4307}
4308
4309static int cxgb_close(struct net_device *dev)
4310{
b8ff05a9
DM		 4311 struct port_info *pi = netdev_priv(dev);
4312 struct adapter *adapter = pi->adapter;
4313
4314 netif_tx_stop_all_queues(dev);
4315 netif_carrier_off(dev);
060e0c75 4316 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4317}
4318
f2b7e78d
VP		 4319/* Return an error number if the indicated filter isn't writable ...
4320 */
4321static int writable_filter(struct filter_entry *f)
4322{
4323 if (f->locked)
4324 return -EPERM;
4325 if (f->pending)
4326 return -EBUSY;
4327
4328 return 0;
4329}
4330
4331/* Delete the filter at the specified index (if valid). The checks for all
4332 * the common problems with doing this like the filter being locked, currently
4333 * pending in another operation, etc.
4334 */
4335static int delete_filter(struct adapter *adapter, unsigned int fidx)
4336{
4337 struct filter_entry *f;
4338 int ret;
4339
dca4faeb 4340 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4341 return -EINVAL;
4342
4343 f = &adapter->tids.ftid_tab[fidx];
4344 ret = writable_filter(f);
4345 if (ret)
4346 return ret;
4347 if (f->valid)
4348 return del_filter_wr(adapter, fidx);
4349
4350 return 0;
4351}
4352
dca4faeb 4353int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4354 __be32 sip, __be16 sport, __be16 vlan,
4355 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4356{
4357 int ret;
4358 struct filter_entry *f;
4359 struct adapter *adap;
4360 int i;
4361 u8 *val;
4362
4363 adap = netdev2adap(dev);
4364
1cab775c 4365 /* Adjust stid to correct filter index */
470c60c4 4366 stid -= adap->tids.sftid_base;
1cab775c
VP		 4367 stid += adap->tids.nftids;
4368
dca4faeb
VP		 4369 /* Check to make sure the filter requested is writable ...
4370 */
4371 f = &adap->tids.ftid_tab[stid];
4372 ret = writable_filter(f);
4373 if (ret)
4374 return ret;
4375
4376 /* Clear out any old resources being used by the filter before
4377 * we start constructing the new filter.
4378 */
4379 if (f->valid)
4380 clear_filter(adap, f);
4381
4382 /* Clear out filter specifications */
4383 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4384 f->fs.val.lport = cpu_to_be16(sport);
4385 f->fs.mask.lport = ~0;
4386 val = (u8 *)&sip;
793dad94 4387 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4388 for (i = 0; i < 4; i++) {
4389 f->fs.val.lip[i] = val[i];
4390 f->fs.mask.lip[i] = ~0;
4391 }
dcf7b6f5 4392 if (adap->params.tp.vlan_pri_map & F_PORT) {
793dad94
VP		 4393 f->fs.val.iport = port;
4394 f->fs.mask.iport = mask;
4395 }
4396 }
dca4faeb 4397
dcf7b6f5 4398 if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {
7c89e555
KS		 4399 f->fs.val.proto = IPPROTO_TCP;
4400 f->fs.mask.proto = ~0;
4401 }
4402
dca4faeb
VP		 4403 f->fs.dirsteer = 1;
4404 f->fs.iq = queue;
4405 /* Mark filter as locked */
4406 f->locked = 1;
4407 f->fs.rpttid = 1;
4408
4409 ret = set_filter_wr(adap, stid);
4410 if (ret) {
4411 clear_filter(adap, f);
4412 return ret;
4413 }
4414
4415 return 0;
4416}
4417EXPORT_SYMBOL(cxgb4_create_server_filter);
4418
4419int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4420 unsigned int queue, bool ipv6)
4421{
4422 int ret;
4423 struct filter_entry *f;
4424 struct adapter *adap;
4425
4426 adap = netdev2adap(dev);
1cab775c
VP		 4427
4428 /* Adjust stid to correct filter index */
470c60c4 4429 stid -= adap->tids.sftid_base;
1cab775c
VP		 4430 stid += adap->tids.nftids;
4431
dca4faeb
VP		 4432 f = &adap->tids.ftid_tab[stid];
4433 /* Unlock the filter */
4434 f->locked = 0;
4435
4436 ret = delete_filter(adap, stid);
4437 if (ret)
4438 return ret;
4439
4440 return 0;
4441}
4442EXPORT_SYMBOL(cxgb4_remove_server_filter);
4443
f5152c90
DM		 4444static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4445 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4446{
4447 struct port_stats stats;
4448 struct port_info *p = netdev_priv(dev);
4449 struct adapter *adapter = p->adapter;
b8ff05a9 4450
9fe6cb58
GS		 4451 /* Block retrieving statistics during EEH error
4452 * recovery. Otherwise, the recovery might fail
4453 * and the PCI device will be removed permanently
4454 */
b8ff05a9 4455 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4456 if (!netif_device_present(dev)) {
4457 spin_unlock(&adapter->stats_lock);
4458 return ns;
4459 }
b8ff05a9
DM		 4460 t4_get_port_stats(adapter, p->tx_chan, &stats);
4461 spin_unlock(&adapter->stats_lock);
4462
4463 ns->tx_bytes = stats.tx_octets;
4464 ns->tx_packets = stats.tx_frames;
4465 ns->rx_bytes = stats.rx_octets;
4466 ns->rx_packets = stats.rx_frames;
4467 ns->multicast = stats.rx_mcast_frames;
4468
4469 /* detailed rx_errors */
4470 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4471 stats.rx_runt;
4472 ns->rx_over_errors = 0;
4473 ns->rx_crc_errors = stats.rx_fcs_err;
4474 ns->rx_frame_errors = stats.rx_symbol_err;
4475 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4476 stats.rx_ovflow2 + stats.rx_ovflow3 +
4477 stats.rx_trunc0 + stats.rx_trunc1 +
4478 stats.rx_trunc2 + stats.rx_trunc3;
4479 ns->rx_missed_errors = 0;
4480
4481 /* detailed tx_errors */
4482 ns->tx_aborted_errors = 0;
4483 ns->tx_carrier_errors = 0;
4484 ns->tx_fifo_errors = 0;
4485 ns->tx_heartbeat_errors = 0;
4486 ns->tx_window_errors = 0;
4487
4488 ns->tx_errors = stats.tx_error_frames;
4489 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4490 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4491 return ns;
4492}
4493
4494static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4495{
060e0c75 4496 unsigned int mbox;
b8ff05a9
DM		 4497 int ret = 0, prtad, devad;
4498 struct port_info *pi = netdev_priv(dev);
4499 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4500
4501 switch (cmd) {
4502 case SIOCGMIIPHY:
4503 if (pi->mdio_addr < 0)
4504 return -EOPNOTSUPP;
4505 data->phy_id = pi->mdio_addr;
4506 break;
4507 case SIOCGMIIREG:
4508 case SIOCSMIIREG:
4509 if (mdio_phy_id_is_c45(data->phy_id)) {
4510 prtad = mdio_phy_id_prtad(data->phy_id);
4511 devad = mdio_phy_id_devad(data->phy_id);
4512 } else if (data->phy_id < 32) {
4513 prtad = data->phy_id;
4514 devad = 0;
4515 data->reg_num &= 0x1f;
4516 } else
4517 return -EINVAL;
4518
060e0c75 4519 mbox = pi->adapter->fn;
b8ff05a9 4520 if (cmd == SIOCGMIIREG)
060e0c75 4521 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4522 data->reg_num, &data->val_out);
4523 else
060e0c75 4524 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4525 data->reg_num, data->val_in);
4526 break;
4527 default:
4528 return -EOPNOTSUPP;
4529 }
4530 return ret;
4531}
4532
4533static void cxgb_set_rxmode(struct net_device *dev)
4534{
4535 /* unfortunately we can't return errors to the stack */
4536 set_rxmode(dev, -1, false);
4537}
4538
4539static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4540{
4541 int ret;
4542 struct port_info *pi = netdev_priv(dev);
4543
4544 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4545 return -EINVAL;
060e0c75
DM		 4546 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4547 -1, -1, -1, true);
b8ff05a9
DM		 4548 if (!ret)
4549 dev->mtu = new_mtu;
4550 return ret;
4551}
4552
4553static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4554{
4555 int ret;
4556 struct sockaddr *addr = p;
4557 struct port_info *pi = netdev_priv(dev);
4558
4559 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4560 return -EADDRNOTAVAIL;
b8ff05a9 4561
060e0c75
DM		 4562 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4563 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4564 if (ret < 0)
4565 return ret;
4566
4567 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4568 pi->xact_addr_filt = ret;
4569 return 0;
4570}
4571
b8ff05a9
DM		 4572#ifdef CONFIG_NET_POLL_CONTROLLER
4573static void cxgb_netpoll(struct net_device *dev)
4574{
4575 struct port_info *pi = netdev_priv(dev);
4576 struct adapter *adap = pi->adapter;
4577
4578 if (adap->flags & USING_MSIX) {
4579 int i;
4580 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4581
4582 for (i = pi->nqsets; i; i--, rx++)
4583 t4_sge_intr_msix(0, &rx->rspq);
4584 } else
4585 t4_intr_handler(adap)(0, adap);
4586}
4587#endif
4588
4589static const struct net_device_ops cxgb4_netdev_ops = {
4590 .ndo_open = cxgb_open,
4591 .ndo_stop = cxgb_close,
4592 .ndo_start_xmit = t4_eth_xmit,
9be793bf 4593 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4594 .ndo_set_rx_mode = cxgb_set_rxmode,
4595 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4596 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4597 .ndo_validate_addr = eth_validate_addr,
4598 .ndo_do_ioctl = cxgb_ioctl,
4599 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4600#ifdef CONFIG_NET_POLL_CONTROLLER
4601 .ndo_poll_controller = cxgb_netpoll,
4602#endif
4603};
4604
4605void t4_fatal_err(struct adapter *adap)
4606{
4607 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4608 t4_intr_disable(adap);
4609 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4610}
4611
4612static void setup_memwin(struct adapter *adap)
4613{
19dd37ba 4614 u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;
b8ff05a9
DM		 4615
4616 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
d14807dd 4617 if (is_t4(adap->params.chip)) {
19dd37ba
SR		 4618 mem_win0_base = bar0 + MEMWIN0_BASE;
4619 mem_win1_base = bar0 + MEMWIN1_BASE;
4620 mem_win2_base = bar0 + MEMWIN2_BASE;
4621 } else {
4622 /* For T5, only relative offset inside the PCIe BAR is passed */
4623 mem_win0_base = MEMWIN0_BASE;
4624 mem_win1_base = MEMWIN1_BASE_T5;
4625 mem_win2_base = MEMWIN2_BASE_T5;
4626 }
b8ff05a9 4627 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
19dd37ba 4628 mem_win0_base | BIR(0) |
b8ff05a9
DM		 4629 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4630 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
19dd37ba 4631 mem_win1_base | BIR(0) |
b8ff05a9
DM		 4632 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4633 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
19dd37ba 4634 mem_win2_base | BIR(0) |
b8ff05a9 4635 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
636f9d37
VP		 4636}
4637
4638static void setup_memwin_rdma(struct adapter *adap)
4639{
1ae970e0
DM		 4640 if (adap->vres.ocq.size) {
4641 unsigned int start, sz_kb;
4642
4643 start = pci_resource_start(adap->pdev, 2) +
4644 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4645 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4646 t4_write_reg(adap,
4647 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4648 start | BIR(1) | WINDOW(ilog2(sz_kb)));
4649 t4_write_reg(adap,
4650 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4651 adap->vres.ocq.start);
4652 t4_read_reg(adap,
4653 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4654 }
b8ff05a9
DM		 4655}
4656
02b5fb8e
DM		 4657static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4658{
4659 u32 v;
4660 int ret;
4661
4662 /* get device capabilities */
4663 memset(c, 0, sizeof(*c));
4664 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4665 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 4666 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4667 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4668 if (ret < 0)
4669 return ret;
4670
4671 /* select capabilities we'll be using */
4672 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4673 if (!vf_acls)
4674 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4675 else
4676 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4677 } else if (vf_acls) {
4678 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4679 return ret;
4680 }
4681 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4682 FW_CMD_REQUEST | FW_CMD_WRITE);
060e0c75 4683 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4684 if (ret < 0)
4685 return ret;
4686
060e0c75 4687 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e
DM		 4688 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4689 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4690 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
4691 if (ret < 0)
4692 return ret;
4693
060e0c75
DM		 4694 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4695 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 4696 if (ret < 0)
4697 return ret;
4698
4699 t4_sge_init(adap);
4700
02b5fb8e
DM		 4701 /* tweak some settings */
4702 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4703 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4704 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4705 v = t4_read_reg(adap, TP_PIO_DATA);
4706 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
060e0c75 4707
dca4faeb
VP		 4708 /* first 4 Tx modulation queues point to consecutive Tx channels */
4709 adap->params.tp.tx_modq_map = 0xE4;
4710 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4711 V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4712
4713 /* associate each Tx modulation queue with consecutive Tx channels */
4714 v = 0x84218421;
4715 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4716 &v, 1, A_TP_TX_SCHED_HDR);
4717 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4718 &v, 1, A_TP_TX_SCHED_FIFO);
4719 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4720 &v, 1, A_TP_TX_SCHED_PCMD);
4721
4722#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4723 if (is_offload(adap)) {
4724 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4725 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4726 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4727 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4728 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4729 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4730 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4731 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4732 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4733 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4734 }
4735
060e0c75
DM		 4736 /* get basic stuff going */
4737 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4738}
4739
b8ff05a9
DM		 4740/*
4741 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4742 */
4743#define MAX_ATIDS 8192U
4744
636f9d37
VP		 4745/*
4746 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4747 *
4748 * If the firmware we're dealing with has Configuration File support, then
4749 * we use that to perform all configuration
4750 */
4751
4752/*
4753 * Tweak configuration based on module parameters, etc. Most of these have
4754 * defaults assigned to them by Firmware Configuration Files (if we're using
4755 * them) but need to be explicitly set if we're using hard-coded
4756 * initialization. But even in the case of using Firmware Configuration
4757 * Files, we'd like to expose the ability to change these via module
4758 * parameters so these are essentially common tweaks/settings for
4759 * Configuration Files and hard-coded initialization ...
4760 */
4761static int adap_init0_tweaks(struct adapter *adapter)
4762{
4763 /*
4764 * Fix up various Host-Dependent Parameters like Page Size, Cache
4765 * Line Size, etc. The firmware default is for a 4KB Page Size and
4766 * 64B Cache Line Size ...
4767 */
4768 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4769
4770 /*
4771 * Process module parameters which affect early initialization.
4772 */
4773 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4774 dev_err(&adapter->pdev->dev,
4775 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4776 rx_dma_offset);
4777 rx_dma_offset = 2;
4778 }
4779 t4_set_reg_field(adapter, SGE_CONTROL,
4780 PKTSHIFT_MASK,
4781 PKTSHIFT(rx_dma_offset));
4782
4783 /*
4784 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4785 * adds the pseudo header itself.
4786 */
4787 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
4788 CSUM_HAS_PSEUDO_HDR, 0);
4789
4790 return 0;
4791}
4792
4793/*
4794 * Attempt to initialize the adapter via a Firmware Configuration File.
4795 */
4796static int adap_init0_config(struct adapter *adapter, int reset)
4797{
4798 struct fw_caps_config_cmd caps_cmd;
4799 const struct firmware *cf;
4800 unsigned long mtype = 0, maddr = 0;
4801 u32 finiver, finicsum, cfcsum;
16e47624
HS		 4802 int ret;
4803 int config_issued = 0;
0a57a536 4804 char *fw_config_file, fw_config_file_path[256];
16e47624 4805 char *config_name = NULL;
636f9d37
VP		 4806
4807 /*
4808 * Reset device if necessary.
4809 */
4810 if (reset) {
4811 ret = t4_fw_reset(adapter, adapter->mbox,
4812 PIORSTMODE | PIORST);
4813 if (ret < 0)
4814 goto bye;
4815 }
4816
4817 /*
4818 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4819 * then use that. Otherwise, use the configuration file stored
4820 * in the adapter flash ...
4821 */
d14807dd 4822 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 4823 case CHELSIO_T4:
16e47624 4824 fw_config_file = FW4_CFNAME;
0a57a536
SR		 4825 break;
4826 case CHELSIO_T5:
4827 fw_config_file = FW5_CFNAME;
4828 break;
4829 default:
4830 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4831 adapter->pdev->device);
4832 ret = -EINVAL;
4833 goto bye;
4834 }
4835
4836 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 4837 if (ret < 0) {
16e47624 4838 config_name = "On FLASH";
636f9d37
VP		 4839 mtype = FW_MEMTYPE_CF_FLASH;
4840 maddr = t4_flash_cfg_addr(adapter);
4841 } else {
4842 u32 params[7], val[7];
4843
16e47624
HS		 4844 sprintf(fw_config_file_path,
4845 "/lib/firmware/%s", fw_config_file);
4846 config_name = fw_config_file_path;
4847
636f9d37
VP		 4848 if (cf->size >= FLASH_CFG_MAX_SIZE)
4849 ret = -ENOMEM;
4850 else {
4851 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4852 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4853 ret = t4_query_params(adapter, adapter->mbox,
4854 adapter->fn, 0, 1, params, val);
4855 if (ret == 0) {
4856 /*
4857 * For t4_memory_write() below addresses and
4858 * sizes have to be in terms of multiples of 4
4859 * bytes. So, if the Configuration File isn't
4860 * a multiple of 4 bytes in length we'll have
4861 * to write that out separately since we can't
4862 * guarantee that the bytes following the
4863 * residual byte in the buffer returned by
4864 * request_firmware() are zeroed out ...
4865 */
4866 size_t resid = cf->size & 0x3;
4867 size_t size = cf->size & ~0x3;
4868 __be32 *data = (__be32 *)cf->data;
4869
4870 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
4871 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
4872
4873 ret = t4_memory_write(adapter, mtype, maddr,
4874 size, data);
4875 if (ret == 0 && resid != 0) {
4876 union {
4877 __be32 word;
4878 char buf[4];
4879 } last;
4880 int i;
4881
4882 last.word = data[size >> 2];
4883 for (i = resid; i < 4; i++)
4884 last.buf[i] = 0;
4885 ret = t4_memory_write(adapter, mtype,
4886 maddr + size,
4887 4, &last.word);
4888 }
4889 }
4890 }
4891
4892 release_firmware(cf);
4893 if (ret)
4894 goto bye;
4895 }
4896
4897 /*
4898 * Issue a Capability Configuration command to the firmware to get it
4899 * to parse the Configuration File. We don't use t4_fw_config_file()
4900 * because we want the ability to modify various features after we've
4901 * processed the configuration file ...
4902 */
4903 memset(&caps_cmd, 0, sizeof(caps_cmd));
4904 caps_cmd.op_to_write =
4905 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4906 FW_CMD_REQUEST |
4907 FW_CMD_READ);
ce91a923 4908 caps_cmd.cfvalid_to_len16 =
636f9d37
VP		 4909 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
4910 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4911 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
4912 FW_LEN16(caps_cmd));
4913 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4914 &caps_cmd);
16e47624
HS		 4915
4916 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4917 * Configuration File in FLASH), our last gasp effort is to use the
4918 * Firmware Configuration File which is embedded in the firmware. A
4919 * very few early versions of the firmware didn't have one embedded
4920 * but we can ignore those.
4921 */
4922 if (ret == -ENOENT) {
4923 memset(&caps_cmd, 0, sizeof(caps_cmd));
4924 caps_cmd.op_to_write =
4925 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4926 FW_CMD_REQUEST |
4927 FW_CMD_READ);
4928 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4929 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4930 sizeof(caps_cmd), &caps_cmd);
4931 config_name = "Firmware Default";
4932 }
4933
4934 config_issued = 1;
636f9d37
VP		 4935 if (ret < 0)
4936 goto bye;
4937
4938 finiver = ntohl(caps_cmd.finiver);
4939 finicsum = ntohl(caps_cmd.finicsum);
4940 cfcsum = ntohl(caps_cmd.cfcsum);
4941 if (finicsum != cfcsum)
4942 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4943 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4944 finicsum, cfcsum);
4945
636f9d37
VP		 4946 /*
4947 * And now tell the firmware to use the configuration we just loaded.
4948 */
4949 caps_cmd.op_to_write =
4950 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4951 FW_CMD_REQUEST |
4952 FW_CMD_WRITE);
ce91a923 4953 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 4954 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4955 NULL);
4956 if (ret < 0)
4957 goto bye;
4958
4959 /*
4960 * Tweak configuration based on system architecture, module
4961 * parameters, etc.
4962 */
4963 ret = adap_init0_tweaks(adapter);
4964 if (ret < 0)
4965 goto bye;
4966
4967 /*
4968 * And finally tell the firmware to initialize itself using the
4969 * parameters from the Configuration File.
4970 */
4971 ret = t4_fw_initialize(adapter, adapter->mbox);
4972 if (ret < 0)
4973 goto bye;
4974
4975 /*
4976 * Return successfully and note that we're operating with parameters
4977 * not supplied by the driver, rather than from hard-wired
4978 * initialization constants burried in the driver.
4979 */
4980 adapter->flags |= USING_SOFT_PARAMS;
4981 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 4982 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
4983 config_name, finiver, cfcsum);
636f9d37
VP		 4984 return 0;
4985
4986 /*
4987 * Something bad happened. Return the error ... (If the "error"
4988 * is that there's no Configuration File on the adapter we don't
4989 * want to issue a warning since this is fairly common.)
4990 */
4991bye:
16e47624
HS		 4992 if (config_issued && ret != -ENOENT)
4993 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
4994 config_name, -ret);
636f9d37
VP		 4995 return ret;
4996}
4997
13ee15d3
VP		 4998/*
4999 * Attempt to initialize the adapter via hard-coded, driver supplied
5000 * parameters ...
5001 */
5002static int adap_init0_no_config(struct adapter *adapter, int reset)
5003{
5004 struct sge *s = &adapter->sge;
5005 struct fw_caps_config_cmd caps_cmd;
5006 u32 v;
5007 int i, ret;
5008
5009 /*
5010 * Reset device if necessary
5011 */
5012 if (reset) {
5013 ret = t4_fw_reset(adapter, adapter->mbox,
5014 PIORSTMODE | PIORST);
5015 if (ret < 0)
5016 goto bye;
5017 }
5018
5019 /*
5020 * Get device capabilities and select which we'll be using.
5021 */
5022 memset(&caps_cmd, 0, sizeof(caps_cmd));
5023 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5024 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 5025 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
13ee15d3
VP		 5026 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5027 &caps_cmd);
5028 if (ret < 0)
5029 goto bye;
5030
13ee15d3
VP		 5031 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
5032 if (!vf_acls)
5033 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
5034 else
5035 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
5036 } else if (vf_acls) {
5037 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
5038 goto bye;
5039 }
5040 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5041 FW_CMD_REQUEST | FW_CMD_WRITE);
5042 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5043 NULL);
5044 if (ret < 0)
5045 goto bye;
5046
5047 /*
5048 * Tweak configuration based on system architecture, module
5049 * parameters, etc.
5050 */
5051 ret = adap_init0_tweaks(adapter);
5052 if (ret < 0)
5053 goto bye;
5054
5055 /*
5056 * Select RSS Global Mode we want to use. We use "Basic Virtual"
5057 * mode which maps each Virtual Interface to its own section of
5058 * the RSS Table and we turn on all map and hash enables ...
5059 */
5060 adapter->flags |= RSS_TNLALLLOOKUP;
5061 ret = t4_config_glbl_rss(adapter, adapter->mbox,
5062 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
5063 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
5064 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
5065 ((adapter->flags & RSS_TNLALLLOOKUP) ?
5066 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
5067 if (ret < 0)
5068 goto bye;
5069
5070 /*
5071 * Set up our own fundamental resource provisioning ...
5072 */
5073 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
5074 PFRES_NEQ, PFRES_NETHCTRL,
5075 PFRES_NIQFLINT, PFRES_NIQ,
5076 PFRES_TC, PFRES_NVI,
5077 FW_PFVF_CMD_CMASK_MASK,
5078 pfvfres_pmask(adapter, adapter->fn, 0),
5079 PFRES_NEXACTF,
5080 PFRES_R_CAPS, PFRES_WX_CAPS);
5081 if (ret < 0)
5082 goto bye;
5083
5084 /*
5085 * Perform low level SGE initialization. We need to do this before we
5086 * send the firmware the INITIALIZE command because that will cause
5087 * any other PF Drivers which are waiting for the Master
5088 * Initialization to proceed forward.
5089 */
5090 for (i = 0; i < SGE_NTIMERS - 1; i++)
5091 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
5092 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
5093 s->counter_val[0] = 1;
5094 for (i = 1; i < SGE_NCOUNTERS; i++)
5095 s->counter_val[i] = min(intr_cnt[i - 1],
5096 THRESHOLD_0_GET(THRESHOLD_0_MASK));
5097 t4_sge_init(adapter);
5098
5099#ifdef CONFIG_PCI_IOV
5100 /*
5101 * Provision resource limits for Virtual Functions. We currently
5102 * grant them all the same static resource limits except for the Port
5103 * Access Rights Mask which we're assigning based on the PF. All of
5104 * the static provisioning stuff for both the PF and VF really needs
5105 * to be managed in a persistent manner for each device which the
5106 * firmware controls.
5107 */
5108 {
5109 int pf, vf;
5110
7d6727cf 5111 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
13ee15d3
VP		 5112 if (num_vf[pf] <= 0)
5113 continue;
5114
5115 /* VF numbering starts at 1! */
5116 for (vf = 1; vf <= num_vf[pf]; vf++) {
5117 ret = t4_cfg_pfvf(adapter, adapter->mbox,
5118 pf, vf,
5119 VFRES_NEQ, VFRES_NETHCTRL,
5120 VFRES_NIQFLINT, VFRES_NIQ,
5121 VFRES_TC, VFRES_NVI,
1f1e4958 5122 FW_PFVF_CMD_CMASK_MASK,
13ee15d3
VP		 5123 pfvfres_pmask(
5124 adapter, pf, vf),
5125 VFRES_NEXACTF,
5126 VFRES_R_CAPS, VFRES_WX_CAPS);
5127 if (ret < 0)
5128 dev_warn(adapter->pdev_dev,
5129 "failed to "\
5130 "provision pf/vf=%d/%d; "
5131 "err=%d\n", pf, vf, ret);
5132 }
5133 }
5134 }
5135#endif
5136
5137 /*
5138 * Set up the default filter mode. Later we'll want to implement this
5139 * via a firmware command, etc. ... This needs to be done before the
5140 * firmare initialization command ... If the selected set of fields
5141 * isn't equal to the default value, we'll need to make sure that the
5142 * field selections will fit in the 36-bit budget.
5143 */
5144 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
404d9e3f 5145 int j, bits = 0;
13ee15d3 5146
404d9e3f
VP		 5147 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
5148 switch (tp_vlan_pri_map & (1 << j)) {
13ee15d3
VP		 5149 case 0:
5150 /* compressed filter field not enabled */
5151 break;
5152 case FCOE_MASK:
5153 bits += 1;
5154 break;
5155 case PORT_MASK:
5156 bits += 3;
5157 break;
5158 case VNIC_ID_MASK:
5159 bits += 17;
5160 break;
5161 case VLAN_MASK:
5162 bits += 17;
5163 break;
5164 case TOS_MASK:
5165 bits += 8;
5166 break;
5167 case PROTOCOL_MASK:
5168 bits += 8;
5169 break;
5170 case ETHERTYPE_MASK:
5171 bits += 16;
5172 break;
5173 case MACMATCH_MASK:
5174 bits += 9;
5175 break;
5176 case MPSHITTYPE_MASK:
5177 bits += 3;
5178 break;
5179 case FRAGMENTATION_MASK:
5180 bits += 1;
5181 break;
5182 }
5183
5184 if (bits > 36) {
5185 dev_err(adapter->pdev_dev,
5186 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5187 " using %#x\n", tp_vlan_pri_map, bits,
5188 TP_VLAN_PRI_MAP_DEFAULT);
5189 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5190 }
5191 }
5192 v = tp_vlan_pri_map;
5193 t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
5194 &v, 1, TP_VLAN_PRI_MAP);
5195
5196 /*
5197 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5198 * to support any of the compressed filter fields above. Newer
5199 * versions of the firmware do this automatically but it doesn't hurt
5200 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5201 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5202 * since the firmware automatically turns this on and off when we have
5203 * a non-zero number of filters active (since it does have a
5204 * performance impact).
5205 */
5206 if (tp_vlan_pri_map)
5207 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
5208 FIVETUPLELOOKUP_MASK,
5209 FIVETUPLELOOKUP_MASK);
5210
5211 /*
5212 * Tweak some settings.
5213 */
5214 t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
5215 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5216 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5217 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5218
5219 /*
5220 * Get basic stuff going by issuing the Firmware Initialize command.
5221 * Note that this _must_ be after all PFVF commands ...
5222 */
5223 ret = t4_fw_initialize(adapter, adapter->mbox);
5224 if (ret < 0)
5225 goto bye;
5226
5227 /*
5228 * Return successfully!
5229 */
5230 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5231 "driver parameters\n");
5232 return 0;
5233
5234 /*
5235 * Something bad happened. Return the error ...
5236 */
5237bye:
5238 return ret;
5239}
5240
16e47624
HS		 5241static struct fw_info fw_info_array[] = {
5242 {
5243 .chip = CHELSIO_T4,
5244 .fs_name = FW4_CFNAME,
5245 .fw_mod_name = FW4_FNAME,
5246 .fw_hdr = {
5247 .chip = FW_HDR_CHIP_T4,
5248 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5249 .intfver_nic = FW_INTFVER(T4, NIC),
5250 .intfver_vnic = FW_INTFVER(T4, VNIC),
5251 .intfver_ri = FW_INTFVER(T4, RI),
5252 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5253 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5254 },
5255 }, {
5256 .chip = CHELSIO_T5,
5257 .fs_name = FW5_CFNAME,
5258 .fw_mod_name = FW5_FNAME,
5259 .fw_hdr = {
5260 .chip = FW_HDR_CHIP_T5,
5261 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5262 .intfver_nic = FW_INTFVER(T5, NIC),
5263 .intfver_vnic = FW_INTFVER(T5, VNIC),
5264 .intfver_ri = FW_INTFVER(T5, RI),
5265 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5266 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5267 },
5268 }
5269};
5270
5271static struct fw_info *find_fw_info(int chip)
5272{
5273 int i;
5274
5275 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5276 if (fw_info_array[i].chip == chip)
5277 return &fw_info_array[i];
5278 }
5279 return NULL;
5280}
5281
b8ff05a9
DM		 5282/*
5283 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5284 */
5285static int adap_init0(struct adapter *adap)
5286{
5287 int ret;
5288 u32 v, port_vec;
5289 enum dev_state state;
5290 u32 params[7], val[7];
9a4da2cd 5291 struct fw_caps_config_cmd caps_cmd;
dcf7b6f5 5292 int reset = 1;
b8ff05a9 5293
636f9d37
VP		 5294 /*
5295 * Contact FW, advertising Master capability (and potentially forcing
5296 * ourselves as the Master PF if our module parameter force_init is
5297 * set).
5298 */
5299 ret = t4_fw_hello(adap, adap->mbox, adap->fn,
5300 force_init ? MASTER_MUST : MASTER_MAY,
5301 &state);
b8ff05a9
DM		 5302 if (ret < 0) {
5303 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5304 ret);
5305 return ret;
5306 }
636f9d37
VP		 5307 if (ret == adap->mbox)
5308 adap->flags |= MASTER_PF;
5309 if (force_init && state == DEV_STATE_INIT)
5310 state = DEV_STATE_UNINIT;
b8ff05a9 5311
636f9d37
VP		 5312 /*
5313 * If we're the Master PF Driver and the device is uninitialized,
5314 * then let's consider upgrading the firmware ... (We always want
5315 * to check the firmware version number in order to A. get it for
5316 * later reporting and B. to warn if the currently loaded firmware
5317 * is excessively mismatched relative to the driver.)
5318 */
16e47624
HS		 5319 t4_get_fw_version(adap, &adap->params.fw_vers);
5320 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5321 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5322 struct fw_info *fw_info;
5323 struct fw_hdr *card_fw;
5324 const struct firmware *fw;
5325 const u8 *fw_data = NULL;
5326 unsigned int fw_size = 0;
5327
5328 /* This is the firmware whose headers the driver was compiled
5329 * against
5330 */
5331 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5332 if (fw_info == NULL) {
5333 dev_err(adap->pdev_dev,
5334 "unable to get firmware info for chip %d.\n",
5335 CHELSIO_CHIP_VERSION(adap->params.chip));
5336 return -EINVAL;
636f9d37 5337 }
16e47624
HS		 5338
5339 /* allocate memory to read the header of the firmware on the
5340 * card
5341 */
5342 card_fw = t4_alloc_mem(sizeof(*card_fw));
5343
5344 /* Get FW from from /lib/firmware/ */
5345 ret = request_firmware(&fw, fw_info->fw_mod_name,
5346 adap->pdev_dev);
5347 if (ret < 0) {
5348 dev_err(adap->pdev_dev,
5349 "unable to load firmware image %s, error %d\n",
5350 fw_info->fw_mod_name, ret);
5351 } else {
5352 fw_data = fw->data;
5353 fw_size = fw->size;
5354 }
5355
5356 /* upgrade FW logic */
5357 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5358 state, &reset);
5359
5360 /* Cleaning up */
5361 if (fw != NULL)
5362 release_firmware(fw);
5363 t4_free_mem(card_fw);
5364
636f9d37 5365 if (ret < 0)
16e47624 5366 goto bye;
636f9d37 5367 }
b8ff05a9 5368
636f9d37
VP		 5369 /*
5370 * Grab VPD parameters. This should be done after we establish a
5371 * connection to the firmware since some of the VPD parameters
5372 * (notably the Core Clock frequency) are retrieved via requests to
5373 * the firmware. On the other hand, we need these fairly early on
5374 * so we do this right after getting ahold of the firmware.
5375 */
5376 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5377 if (ret < 0)
5378 goto bye;
a0881cab 5379
636f9d37 5380 /*
13ee15d3
VP		 5381 * Find out what ports are available to us. Note that we need to do
5382 * this before calling adap_init0_no_config() since it needs nports
5383 * and portvec ...
636f9d37
VP		 5384 */
5385 v =
5386 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5387 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
5388 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5389 if (ret < 0)
5390 goto bye;
5391
636f9d37
VP		 5392 adap->params.nports = hweight32(port_vec);
5393 adap->params.portvec = port_vec;
5394
5395 /*
5396 * If the firmware is initialized already (and we're not forcing a
5397 * master initialization), note that we're living with existing
5398 * adapter parameters. Otherwise, it's time to try initializing the
5399 * adapter ...
5400 */
5401 if (state == DEV_STATE_INIT) {
5402 dev_info(adap->pdev_dev, "Coming up as %s: "\
5403 "Adapter already initialized\n",
5404 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5405 adap->flags |= USING_SOFT_PARAMS;
5406 } else {
5407 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5408 "Initializing adapter\n");
636f9d37
VP		 5409
5410 /*
5411 * If the firmware doesn't support Configuration
5412 * Files warn user and exit,
5413 */
5414 if (ret < 0)
13ee15d3 5415 dev_warn(adap->pdev_dev, "Firmware doesn't support "
636f9d37 5416 "configuration file.\n");
13ee15d3
VP		 5417 if (force_old_init)
5418 ret = adap_init0_no_config(adap, reset);
636f9d37
VP		 5419 else {
5420 /*
13ee15d3
VP		 5421 * Find out whether we're dealing with a version of
5422 * the firmware which has configuration file support.
636f9d37 5423 */
13ee15d3
VP		 5424 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5425 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
5426 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5427 params, val);
636f9d37 5428
13ee15d3
VP		 5429 /*
5430 * If the firmware doesn't support Configuration
5431 * Files, use the old Driver-based, hard-wired
5432 * initialization. Otherwise, try using the
5433 * Configuration File support and fall back to the
5434 * Driver-based initialization if there's no
5435 * Configuration File found.
5436 */
5437 if (ret < 0)
5438 ret = adap_init0_no_config(adap, reset);
5439 else {
5440 /*
5441 * The firmware provides us with a memory
5442 * buffer where we can load a Configuration
5443 * File from the host if we want to override
5444 * the Configuration File in flash.
5445 */
5446
5447 ret = adap_init0_config(adap, reset);
5448 if (ret == -ENOENT) {
5449 dev_info(adap->pdev_dev,
5450 "No Configuration File present "
16e47624 5451 "on adapter. Using hard-wired "
13ee15d3
VP		 5452 "configuration parameters.\n");
5453 ret = adap_init0_no_config(adap, reset);
5454 }
636f9d37
VP		 5455 }
5456 }
5457 if (ret < 0) {
5458 dev_err(adap->pdev_dev,
5459 "could not initialize adapter, error %d\n",
5460 -ret);
5461 goto bye;
5462 }
5463 }
5464
5465 /*
5466 * If we're living with non-hard-coded parameters (either from a
5467 * Firmware Configuration File or values programmed by a different PF
5468 * Driver), give the SGE code a chance to pull in anything that it
5469 * needs ... Note that this must be called after we retrieve our VPD
5470 * parameters in order to know how to convert core ticks to seconds.
5471 */
5472 if (adap->flags & USING_SOFT_PARAMS) {
5473 ret = t4_sge_init(adap);
5474 if (ret < 0)
5475 goto bye;
5476 }
5477
9a4da2cd
VP		 5478 if (is_bypass_device(adap->pdev->device))
5479 adap->params.bypass = 1;
5480
636f9d37
VP		 5481 /*
5482 * Grab some of our basic fundamental operating parameters.
5483 */
5484#define FW_PARAM_DEV(param) \
5485 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5486 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5487
b8ff05a9 5488#define FW_PARAM_PFVF(param) \
636f9d37
VP		 5489 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5490 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
5491 FW_PARAMS_PARAM_Y(0) | \
5492 FW_PARAMS_PARAM_Z(0)
b8ff05a9 5493
636f9d37 5494 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5495 params[1] = FW_PARAM_PFVF(L2T_START);
5496 params[2] = FW_PARAM_PFVF(L2T_END);
5497 params[3] = FW_PARAM_PFVF(FILTER_START);
5498 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5499 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5500 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5501 if (ret < 0)
5502 goto bye;
636f9d37
VP		 5503 adap->sge.egr_start = val[0];
5504 adap->l2t_start = val[1];
5505 adap->l2t_end = val[2];
b8ff05a9
DM		 5506 adap->tids.ftid_base = val[3];
5507 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5508 adap->sge.ingr_start = val[5];
b8ff05a9 5509
636f9d37
VP		 5510 /* query params related to active filter region */
5511 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5512 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5513 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5514 /* If Active filter size is set we enable establishing
5515 * offload connection through firmware work request
5516 */
5517 if ((val[0] != val[1]) && (ret >= 0)) {
5518 adap->flags |= FW_OFLD_CONN;
5519 adap->tids.aftid_base = val[0];
5520 adap->tids.aftid_end = val[1];
5521 }
5522
b407a4a9
VP		 5523 /* If we're running on newer firmware, let it know that we're
5524 * prepared to deal with encapsulated CPL messages. Older
5525 * firmware won't understand this and we'll just get
5526 * unencapsulated messages ...
5527 */
5528 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5529 val[0] = 1;
5530 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5531
1ac0f095
KS		 5532 /*
5533 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5534 * capability. Earlier versions of the firmware didn't have the
5535 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5536 * permission to use ULPTX MEMWRITE DSGL.
5537 */
5538 if (is_t4(adap->params.chip)) {
5539 adap->params.ulptx_memwrite_dsgl = false;
5540 } else {
5541 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5542 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5543 1, params, val);
5544 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5545 }
5546
636f9d37
VP		 5547 /*
5548 * Get device capabilities so we can determine what resources we need
5549 * to manage.
5550 */
5551 memset(&caps_cmd, 0, sizeof(caps_cmd));
9a4da2cd 5552 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
13ee15d3 5553 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 5554 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5555 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5556 &caps_cmd);
5557 if (ret < 0)
5558 goto bye;
5559
13ee15d3 5560 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5561 /* query offload-related parameters */
5562 params[0] = FW_PARAM_DEV(NTID);
5563 params[1] = FW_PARAM_PFVF(SERVER_START);
5564 params[2] = FW_PARAM_PFVF(SERVER_END);
5565 params[3] = FW_PARAM_PFVF(TDDP_START);
5566 params[4] = FW_PARAM_PFVF(TDDP_END);
5567 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5568 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5569 params, val);
b8ff05a9
DM		 5570 if (ret < 0)
5571 goto bye;
5572 adap->tids.ntids = val[0];
5573 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5574 adap->tids.stid_base = val[1];
5575 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37
VP		 5576 /*
5577 * Setup server filter region. Divide the availble filter
5578 * region into two parts. Regular filters get 1/3rd and server
5579 * filters get 2/3rd part. This is only enabled if workarond
5580 * path is enabled.
5581 * 1. For regular filters.
5582 * 2. Server filter: This are special filters which are used
5583 * to redirect SYN packets to offload queue.
5584 */
5585 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5586 adap->tids.sftid_base = adap->tids.ftid_base +
5587 DIV_ROUND_UP(adap->tids.nftids, 3);
5588 adap->tids.nsftids = adap->tids.nftids -
5589 DIV_ROUND_UP(adap->tids.nftids, 3);
5590 adap->tids.nftids = adap->tids.sftid_base -
5591 adap->tids.ftid_base;
5592 }
b8ff05a9
DM		 5593 adap->vres.ddp.start = val[3];
5594 adap->vres.ddp.size = val[4] - val[3] + 1;
5595 adap->params.ofldq_wr_cred = val[5];
636f9d37 5596
b8ff05a9
DM		 5597 adap->params.offload = 1;
5598 }
636f9d37 5599 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5600 params[0] = FW_PARAM_PFVF(STAG_START);
5601 params[1] = FW_PARAM_PFVF(STAG_END);
5602 params[2] = FW_PARAM_PFVF(RQ_START);
5603 params[3] = FW_PARAM_PFVF(RQ_END);
5604 params[4] = FW_PARAM_PFVF(PBL_START);
5605 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5606 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5607 params, val);
b8ff05a9
DM		 5608 if (ret < 0)
5609 goto bye;
5610 adap->vres.stag.start = val[0];
5611 adap->vres.stag.size = val[1] - val[0] + 1;
5612 adap->vres.rq.start = val[2];
5613 adap->vres.rq.size = val[3] - val[2] + 1;
5614 adap->vres.pbl.start = val[4];
5615 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5616
5617 params[0] = FW_PARAM_PFVF(SQRQ_START);
5618 params[1] = FW_PARAM_PFVF(SQRQ_END);
5619 params[2] = FW_PARAM_PFVF(CQ_START);
5620 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5621 params[4] = FW_PARAM_PFVF(OCQ_START);
5622 params[5] = FW_PARAM_PFVF(OCQ_END);
636f9d37 5623 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
a0881cab
DM		 5624 if (ret < 0)
5625 goto bye;
5626 adap->vres.qp.start = val[0];
5627 adap->vres.qp.size = val[1] - val[0] + 1;
5628 adap->vres.cq.start = val[2];
5629 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5630 adap->vres.ocq.start = val[4];
5631 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9 5632 }
636f9d37 5633 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5634 params[0] = FW_PARAM_PFVF(ISCSI_START);
5635 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5636 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5637 params, val);
b8ff05a9
DM		 5638 if (ret < 0)
5639 goto bye;
5640 adap->vres.iscsi.start = val[0];
5641 adap->vres.iscsi.size = val[1] - val[0] + 1;
5642 }
5643#undef FW_PARAM_PFVF
5644#undef FW_PARAM_DEV
5645
92e7ae71
HS		 5646 /* The MTU/MSS Table is initialized by now, so load their values. If
5647 * we're initializing the adapter, then we'll make any modifications
5648 * we want to the MTU/MSS Table and also initialize the congestion
5649 * parameters.
636f9d37 5650 */
b8ff05a9 5651 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
92e7ae71
HS		 5652 if (state != DEV_STATE_INIT) {
5653 int i;
5654
5655 /* The default MTU Table contains values 1492 and 1500.
5656 * However, for TCP, it's better to have two values which are
5657 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5658 * This allows us to have a TCP Data Payload which is a
5659 * multiple of 8 regardless of what combination of TCP Options
5660 * are in use (always a multiple of 4 bytes) which is
5661 * important for performance reasons. For instance, if no
5662 * options are in use, then we have a 20-byte IP header and a
5663 * 20-byte TCP header. In this case, a 1500-byte MSS would
5664 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5665 * which is not a multiple of 8. So using an MSS of 1488 in
5666 * this case results in a TCP Data Payload of 1448 bytes which
5667 * is a multiple of 8. On the other hand, if 12-byte TCP Time
5668 * Stamps have been negotiated, then an MTU of 1500 bytes
5669 * results in a TCP Data Payload of 1448 bytes which, as
5670 * above, is a multiple of 8 bytes ...
5671 */
5672 for (i = 0; i < NMTUS; i++)
5673 if (adap->params.mtus[i] == 1492) {
5674 adap->params.mtus[i] = 1488;
5675 break;
5676 }
7ee9ff94 5677
92e7ae71
HS		 5678 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5679 adap->params.b_wnd);
5680 }
dcf7b6f5 5681 t4_init_tp_params(adap);
636f9d37 5682 adap->flags |= FW_OK;
b8ff05a9
DM		 5683 return 0;
5684
5685 /*
636f9d37
VP		 5686 * Something bad happened. If a command timed out or failed with EIO
5687 * FW does not operate within its spec or something catastrophic
5688 * happened to HW/FW, stop issuing commands.
b8ff05a9 5689 */
636f9d37
VP		 5690bye:
5691 if (ret != -ETIMEDOUT && ret != -EIO)
5692 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5693 return ret;
5694}
5695
204dc3c0
DM		 5696/* EEH callbacks */
5697
5698static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5699 pci_channel_state_t state)
5700{
5701 int i;
5702 struct adapter *adap = pci_get_drvdata(pdev);
5703
5704 if (!adap)
5705 goto out;
5706
5707 rtnl_lock();
5708 adap->flags &= ~FW_OK;
5709 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5710 spin_lock(&adap->stats_lock);
204dc3c0
DM		 5711 for_each_port(adap, i) {
5712 struct net_device *dev = adap->port[i];
5713
5714 netif_device_detach(dev);
5715 netif_carrier_off(dev);
5716 }
9fe6cb58 5717 spin_unlock(&adap->stats_lock);
204dc3c0
DM		 5718 if (adap->flags & FULL_INIT_DONE)
5719 cxgb_down(adap);
5720 rtnl_unlock();
144be3d9
GS		 5721 if ((adap->flags & DEV_ENABLED)) {
5722 pci_disable_device(pdev);
5723 adap->flags &= ~DEV_ENABLED;
5724 }
204dc3c0
DM		 5725out: return state == pci_channel_io_perm_failure ?
5726 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5727}
5728
5729static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5730{
5731 int i, ret;
5732 struct fw_caps_config_cmd c;
5733 struct adapter *adap = pci_get_drvdata(pdev);
5734
5735 if (!adap) {
5736 pci_restore_state(pdev);
5737 pci_save_state(pdev);
5738 return PCI_ERS_RESULT_RECOVERED;
5739 }
5740
144be3d9
GS		 5741 if (!(adap->flags & DEV_ENABLED)) {
5742 if (pci_enable_device(pdev)) {
5743 dev_err(&pdev->dev, "Cannot reenable PCI "
5744 "device after reset\n");
5745 return PCI_ERS_RESULT_DISCONNECT;
5746 }
5747 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 5748 }
5749
5750 pci_set_master(pdev);
5751 pci_restore_state(pdev);
5752 pci_save_state(pdev);
5753 pci_cleanup_aer_uncorrect_error_status(pdev);
5754
5755 if (t4_wait_dev_ready(adap) < 0)
5756 return PCI_ERS_RESULT_DISCONNECT;
777c2300 5757 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 5758 return PCI_ERS_RESULT_DISCONNECT;
5759 adap->flags |= FW_OK;
5760 if (adap_init1(adap, &c))
5761 return PCI_ERS_RESULT_DISCONNECT;
5762
5763 for_each_port(adap, i) {
5764 struct port_info *p = adap2pinfo(adap, i);
5765
060e0c75
DM		 5766 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5767 NULL, NULL);
204dc3c0
DM		 5768 if (ret < 0)
5769 return PCI_ERS_RESULT_DISCONNECT;
5770 p->viid = ret;
5771 p->xact_addr_filt = -1;
5772 }
5773
5774 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5775 adap->params.b_wnd);
1ae970e0 5776 setup_memwin(adap);
204dc3c0
DM		 5777 if (cxgb_up(adap))
5778 return PCI_ERS_RESULT_DISCONNECT;
5779 return PCI_ERS_RESULT_RECOVERED;
5780}
5781
5782static void eeh_resume(struct pci_dev *pdev)
5783{
5784 int i;
5785 struct adapter *adap = pci_get_drvdata(pdev);
5786
5787 if (!adap)
5788 return;
5789
5790 rtnl_lock();
5791 for_each_port(adap, i) {
5792 struct net_device *dev = adap->port[i];
5793
5794 if (netif_running(dev)) {
5795 link_start(dev);
5796 cxgb_set_rxmode(dev);
5797 }
5798 netif_device_attach(dev);
5799 }
5800 rtnl_unlock();
5801}
5802
3646f0e5 5803static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 5804 .error_detected = eeh_err_detected,
5805 .slot_reset = eeh_slot_reset,
5806 .resume = eeh_resume,
5807};
5808
57d8b764 5809static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 5810{
57d8b764
KS		 5811 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5812 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 5813}
5814
5815static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
5816 unsigned int size, unsigned int iqe_size)
5817{
5818 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
5819 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
5820 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
5821 q->iqe_len = iqe_size;
5822 q->size = size;
5823}
5824
5825/*
5826 * Perform default configuration of DMA queues depending on the number and type
5827 * of ports we found and the number of available CPUs. Most settings can be
5828 * modified by the admin prior to actual use.
5829 */
91744948 5830static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 5831{
5832 struct sge *s = &adap->sge;
5833 int i, q10g = 0, n10g = 0, qidx = 0;
cf38be6d 5834 int ciq_size;
b8ff05a9
DM		 5835
5836 for_each_port(adap, i)
57d8b764 5837 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
b8ff05a9
DM		 5838
5839 /*
5840 * We default to 1 queue per non-10G port and up to # of cores queues
5841 * per 10G port.
5842 */
5843 if (n10g)
5844 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 5845 if (q10g > netif_get_num_default_rss_queues())
5846 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 5847
5848 for_each_port(adap, i) {
5849 struct port_info *pi = adap2pinfo(adap, i);
5850
5851 pi->first_qset = qidx;
57d8b764 5852 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 5853 qidx += pi->nqsets;
5854 }
5855
5856 s->ethqsets = qidx;
5857 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5858
5859 if (is_offload(adap)) {
5860 /*
5861 * For offload we use 1 queue/channel if all ports are up to 1G,
5862 * otherwise we divide all available queues amongst the channels
5863 * capped by the number of available cores.
5864 */
5865 if (n10g) {
5866 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5867 num_online_cpus());
5868 s->ofldqsets = roundup(i, adap->params.nports);
5869 } else
5870 s->ofldqsets = adap->params.nports;
5871 /* For RDMA one Rx queue per channel suffices */
5872 s->rdmaqs = adap->params.nports;
cf38be6d 5873 s->rdmaciqs = adap->params.nports;
b8ff05a9
DM		 5874 }
5875
5876 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5877 struct sge_eth_rxq *r = &s->ethrxq[i];
5878
5879 init_rspq(&r->rspq, 0, 0, 1024, 64);
5880 r->fl.size = 72;
5881 }
5882
5883 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5884 s->ethtxq[i].q.size = 1024;
5885
5886 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5887 s->ctrlq[i].q.size = 512;
5888
5889 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5890 s->ofldtxq[i].q.size = 1024;
5891
5892 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5893 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5894
5895 init_rspq(&r->rspq, 0, 0, 1024, 64);
5896 r->rspq.uld = CXGB4_ULD_ISCSI;
5897 r->fl.size = 72;
5898 }
5899
5900 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5901 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5902
5903 init_rspq(&r->rspq, 0, 0, 511, 64);
5904 r->rspq.uld = CXGB4_ULD_RDMA;
5905 r->fl.size = 72;
5906 }
5907
cf38be6d
HS		 5908 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5909 if (ciq_size > SGE_MAX_IQ_SIZE) {
5910 CH_WARN(adap, "CIQ size too small for available IQs\n");
5911 ciq_size = SGE_MAX_IQ_SIZE;
5912 }
5913
5914 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5915 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5916
5917 init_rspq(&r->rspq, 0, 0, ciq_size, 64);
5918 r->rspq.uld = CXGB4_ULD_RDMA;
5919 }
5920
b8ff05a9
DM		 5921 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
5922 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
5923}
5924
5925/*
5926 * Reduce the number of Ethernet queues across all ports to at most n.
5927 * n provides at least one queue per port.
5928 */
91744948 5929static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 5930{
5931 int i;
5932 struct port_info *pi;
5933
5934 while (n < adap->sge.ethqsets)
5935 for_each_port(adap, i) {
5936 pi = adap2pinfo(adap, i);
5937 if (pi->nqsets > 1) {
5938 pi->nqsets--;
5939 adap->sge.ethqsets--;
5940 if (adap->sge.ethqsets <= n)
5941 break;
5942 }
5943 }
5944
5945 n = 0;
5946 for_each_port(adap, i) {
5947 pi = adap2pinfo(adap, i);
5948 pi->first_qset = n;
5949 n += pi->nqsets;
5950 }
5951}
5952
5953/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5954#define EXTRA_VECS 2
5955
91744948 5956static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 5957{
5958 int ofld_need = 0;
c32ad224 5959 int i, want, need;
b8ff05a9
DM		 5960 struct sge *s = &adap->sge;
5961 unsigned int nchan = adap->params.nports;
5962 struct msix_entry entries[MAX_INGQ + 1];
5963
5964 for (i = 0; i < ARRAY_SIZE(entries); ++i)
5965 entries[i].entry = i;
5966
5967 want = s->max_ethqsets + EXTRA_VECS;
5968 if (is_offload(adap)) {
cf38be6d 5969 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
b8ff05a9 5970 /* need nchan for each possible ULD */
cf38be6d 5971 ofld_need = 3 * nchan;
b8ff05a9
DM		 5972 }
5973 need = adap->params.nports + EXTRA_VECS + ofld_need;
5974
c32ad224
AG		 5975 want = pci_enable_msix_range(adap->pdev, entries, need, want);
5976 if (want < 0)
5977 return want;
b8ff05a9 5978
c32ad224
AG		 5979 /*
5980 * Distribute available vectors to the various queue groups.
5981 * Every group gets its minimum requirement and NIC gets top
5982 * priority for leftovers.
5983 */
5984 i = want - EXTRA_VECS - ofld_need;
5985 if (i < s->max_ethqsets) {
5986 s->max_ethqsets = i;
5987 if (i < s->ethqsets)
5988 reduce_ethqs(adap, i);
5989 }
5990 if (is_offload(adap)) {
5991 i = want - EXTRA_VECS - s->max_ethqsets;
5992 i -= ofld_need - nchan;
5993 s->ofldqsets = (i / nchan) * nchan; /* round down */
5994 }
5995 for (i = 0; i < want; ++i)
5996 adap->msix_info[i].vec = entries[i].vector;
5997
5998 return 0;
b8ff05a9
DM		 5999}
6000
6001#undef EXTRA_VECS
6002
91744948 6003static int init_rss(struct adapter *adap)
671b0060
DM		 6004{
6005 unsigned int i, j;
6006
6007 for_each_port(adap, i) {
6008 struct port_info *pi = adap2pinfo(adap, i);
6009
6010 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
6011 if (!pi->rss)
6012 return -ENOMEM;
6013 for (j = 0; j < pi->rss_size; j++)
278bc429 6014 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 6015 }
6016 return 0;
6017}
6018
91744948 6019static void print_port_info(const struct net_device *dev)
b8ff05a9 6020{
b8ff05a9 6021 char buf[80];
118969ed 6022 char *bufp = buf;
f1a051b9 6023 const char *spd = "";
118969ed
DM		 6024 const struct port_info *pi = netdev_priv(dev);
6025 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 6026
6027 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
6028 spd = " 2.5 GT/s";
6029 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
6030 spd = " 5 GT/s";
d2e752db
RD		 6031 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
6032 spd = " 8 GT/s";
b8ff05a9 6033
118969ed
DM		 6034 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
6035 bufp += sprintf(bufp, "100/");
6036 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
6037 bufp += sprintf(bufp, "1000/");
6038 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
6039 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 6040 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
6041 bufp += sprintf(bufp, "40G/");
118969ed
DM		 6042 if (bufp != buf)
6043 --bufp;
72aca4bf 6044 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 6045
6046 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 6047 adap->params.vpd.id,
d14807dd 6048 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 6049 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
6050 (adap->flags & USING_MSIX) ? " MSI-X" :
6051 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 6052 netdev_info(dev, "S/N: %s, P/N: %s\n",
6053 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 6054}
6055
91744948 6056static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 6057{
e5c8ae5f 6058 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 6059}
6060
06546391
DM		 6061/*
6062 * Free the following resources:
6063 * - memory used for tables
6064 * - MSI/MSI-X
6065 * - net devices
6066 * - resources FW is holding for us
6067 */
6068static void free_some_resources(struct adapter *adapter)
6069{
6070 unsigned int i;
6071
6072 t4_free_mem(adapter->l2t);
6073 t4_free_mem(adapter->tids.tid_tab);
6074 disable_msi(adapter);
6075
6076 for_each_port(adapter, i)
671b0060
DM		 6077 if (adapter->port[i]) {
6078 kfree(adap2pinfo(adapter, i)->rss);
06546391 6079 free_netdev(adapter->port[i]);
671b0060 6080 }
06546391 6081 if (adapter->flags & FW_OK)
060e0c75 6082 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 6083}
6084
2ed28baa 6085#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 6086#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 6087 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 6088#define SEGMENT_SIZE 128
b8ff05a9 6089
1dd06ae8 6090static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 6091{
22adfe0a 6092 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 6093 struct port_info *pi;
c8f44aff 6094 bool highdma = false;
b8ff05a9
DM		 6095 struct adapter *adapter = NULL;
6096
6097 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
6098
6099 err = pci_request_regions(pdev, KBUILD_MODNAME);
6100 if (err) {
6101 /* Just info, some other driver may have claimed the device. */
6102 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
6103 return err;
6104 }
6105
060e0c75 6106 /* We control everything through one PF */
b8ff05a9 6107 func = PCI_FUNC(pdev->devfn);
060e0c75 6108 if (func != ent->driver_data) {
204dc3c0 6109 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 6110 goto sriov;
204dc3c0 6111 }
b8ff05a9
DM		 6112
6113 err = pci_enable_device(pdev);
6114 if (err) {
6115 dev_err(&pdev->dev, "cannot enable PCI device\n");
6116 goto out_release_regions;
6117 }
6118
6119 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 6120 highdma = true;
b8ff05a9
DM		 6121 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
6122 if (err) {
6123 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
6124 "coherent allocations\n");
6125 goto out_disable_device;
6126 }
6127 } else {
6128 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
6129 if (err) {
6130 dev_err(&pdev->dev, "no usable DMA configuration\n");
6131 goto out_disable_device;
6132 }
6133 }
6134
6135 pci_enable_pcie_error_reporting(pdev);
ef306b50 6136 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 6137 pci_set_master(pdev);
6138 pci_save_state(pdev);
6139
6140 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
6141 if (!adapter) {
6142 err = -ENOMEM;
6143 goto out_disable_device;
6144 }
6145
144be3d9
GS		 6146 /* PCI device has been enabled */
6147 adapter->flags |= DEV_ENABLED;
6148
b8ff05a9
DM		 6149 adapter->regs = pci_ioremap_bar(pdev, 0);
6150 if (!adapter->regs) {
6151 dev_err(&pdev->dev, "cannot map device registers\n");
6152 err = -ENOMEM;
6153 goto out_free_adapter;
6154 }
6155
6156 adapter->pdev = pdev;
6157 adapter->pdev_dev = &pdev->dev;
3069ee9b 6158 adapter->mbox = func;
060e0c75 6159 adapter->fn = func;
b8ff05a9
DM		 6160 adapter->msg_enable = dflt_msg_enable;
6161 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6162
6163 spin_lock_init(&adapter->stats_lock);
6164 spin_lock_init(&adapter->tid_release_lock);
6165
6166 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 6167 INIT_WORK(&adapter->db_full_task, process_db_full);
6168 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 6169
6170 err = t4_prep_adapter(adapter);
6171 if (err)
22adfe0a
SR		 6172 goto out_unmap_bar0;
6173
d14807dd 6174 if (!is_t4(adapter->params.chip)) {
22adfe0a
SR		 6175 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
6176 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
6177 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
6178 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6179
6180 /* Each segment size is 128B. Write coalescing is enabled only
6181 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6182 * queue is less no of segments that can be accommodated in
6183 * a page size.
6184 */
6185 if (qpp > num_seg) {
6186 dev_err(&pdev->dev,
6187 "Incorrect number of egress queues per page\n");
6188 err = -EINVAL;
6189 goto out_unmap_bar0;
6190 }
6191 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6192 pci_resource_len(pdev, 2));
6193 if (!adapter->bar2) {
6194 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6195 err = -ENOMEM;
6196 goto out_unmap_bar0;
6197 }
6198 }
6199
636f9d37 6200 setup_memwin(adapter);
b8ff05a9 6201 err = adap_init0(adapter);
636f9d37 6202 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6203 if (err)
6204 goto out_unmap_bar;
6205
6206 for_each_port(adapter, i) {
6207 struct net_device *netdev;
6208
6209 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6210 MAX_ETH_QSETS);
6211 if (!netdev) {
6212 err = -ENOMEM;
6213 goto out_free_dev;
6214 }
6215
6216 SET_NETDEV_DEV(netdev, &pdev->dev);
6217
6218 adapter->port[i] = netdev;
6219 pi = netdev_priv(netdev);
6220 pi->adapter = adapter;
6221 pi->xact_addr_filt = -1;
b8ff05a9 6222 pi->port_id = i;
b8ff05a9
DM		 6223 netdev->irq = pdev->irq;
6224
2ed28baa
MM		 6225 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6226 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6227 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6228 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6229 if (highdma)
6230 netdev->hw_features |= NETIF_F_HIGHDMA;
6231 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6232 netdev->vlan_features = netdev->features & VLAN_FEAT;
6233
01789349
JP		 6234 netdev->priv_flags |= IFF_UNICAST_FLT;
6235
b8ff05a9 6236 netdev->netdev_ops = &cxgb4_netdev_ops;
7ad24ea4 6237 netdev->ethtool_ops = &cxgb_ethtool_ops;
b8ff05a9
DM		 6238 }
6239
6240 pci_set_drvdata(pdev, adapter);
6241
6242 if (adapter->flags & FW_OK) {
060e0c75 6243 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6244 if (err)
6245 goto out_free_dev;
6246 }
6247
6248 /*
6249 * Configure queues and allocate tables now, they can be needed as
6250 * soon as the first register_netdev completes.
6251 */
6252 cfg_queues(adapter);
6253
6254 adapter->l2t = t4_init_l2t();
6255 if (!adapter->l2t) {
6256 /* We tolerate a lack of L2T, giving up some functionality */
6257 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6258 adapter->params.offload = 0;
6259 }
6260
6261 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6262 dev_warn(&pdev->dev, "could not allocate TID table, "
6263 "continuing\n");
6264 adapter->params.offload = 0;
6265 }
6266
f7cabcdd
DM		 6267 /* See what interrupts we'll be using */
6268 if (msi > 1 && enable_msix(adapter) == 0)
6269 adapter->flags |= USING_MSIX;
6270 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6271 adapter->flags |= USING_MSI;
6272
671b0060
DM		 6273 err = init_rss(adapter);
6274 if (err)
6275 goto out_free_dev;
6276
b8ff05a9
DM		 6277 /*
6278 * The card is now ready to go. If any errors occur during device
6279 * registration we do not fail the whole card but rather proceed only
6280 * with the ports we manage to register successfully. However we must
6281 * register at least one net device.
6282 */
6283 for_each_port(adapter, i) {
a57cabe0
DM		 6284 pi = adap2pinfo(adapter, i);
6285 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6286 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6287
b8ff05a9
DM		 6288 err = register_netdev(adapter->port[i]);
6289 if (err)
b1a3c2b6 6290 break;
b1a3c2b6
DM		 6291 adapter->chan_map[pi->tx_chan] = i;
6292 print_port_info(adapter->port[i]);
b8ff05a9 6293 }
b1a3c2b6 6294 if (i == 0) {
b8ff05a9
DM		 6295 dev_err(&pdev->dev, "could not register any net devices\n");
6296 goto out_free_dev;
6297 }
b1a3c2b6
DM		 6298 if (err) {
6299 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6300 err = 0;
6403eab1 6301 }
b8ff05a9
DM		 6302
6303 if (cxgb4_debugfs_root) {
6304 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6305 cxgb4_debugfs_root);
6306 setup_debugfs(adapter);
6307 }
6308
6482aa7c
DLR		 6309 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6310 pdev->needs_freset = 1;
6311
b8ff05a9
DM		 6312 if (is_offload(adapter))
6313 attach_ulds(adapter);
6314
b8ff05a9
DM		 6315sriov:
6316#ifdef CONFIG_PCI_IOV
7d6727cf 6317 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6318 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6319 dev_info(&pdev->dev,
6320 "instantiated %u virtual functions\n",
6321 num_vf[func]);
6322#endif
6323 return 0;
6324
6325 out_free_dev:
06546391 6326 free_some_resources(adapter);
b8ff05a9 6327 out_unmap_bar:
d14807dd 6328 if (!is_t4(adapter->params.chip))
22adfe0a
SR		 6329 iounmap(adapter->bar2);
6330 out_unmap_bar0:
b8ff05a9
DM		 6331 iounmap(adapter->regs);
6332 out_free_adapter:
6333 kfree(adapter);
6334 out_disable_device:
6335 pci_disable_pcie_error_reporting(pdev);
6336 pci_disable_device(pdev);
6337 out_release_regions:
6338 pci_release_regions(pdev);
b8ff05a9
DM		 6339 return err;
6340}
6341
91744948 6342static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6343{
6344 struct adapter *adapter = pci_get_drvdata(pdev);
6345
636f9d37 6346#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6347 pci_disable_sriov(pdev);
6348
636f9d37
VP		 6349#endif
6350
b8ff05a9
DM		 6351 if (adapter) {
6352 int i;
6353
6354 if (is_offload(adapter))
6355 detach_ulds(adapter);
6356
6357 for_each_port(adapter, i)
8f3a7676 6358 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6359 unregister_netdev(adapter->port[i]);
6360
6361 if (adapter->debugfs_root)
6362 debugfs_remove_recursive(adapter->debugfs_root);
6363
f2b7e78d
VP		 6364 /* If we allocated filters, free up state associated with any
6365 * valid filters ...
6366 */
6367 if (adapter->tids.ftid_tab) {
6368 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6369 for (i = 0; i < (adapter->tids.nftids +
6370 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6371 if (f->valid)
6372 clear_filter(adapter, f);
6373 }
6374
aaefae9b
DM		 6375 if (adapter->flags & FULL_INIT_DONE)
6376 cxgb_down(adapter);
b8ff05a9 6377
06546391 6378 free_some_resources(adapter);
b8ff05a9 6379 iounmap(adapter->regs);
d14807dd 6380 if (!is_t4(adapter->params.chip))
22adfe0a 6381 iounmap(adapter->bar2);
b8ff05a9 6382 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6383 if ((adapter->flags & DEV_ENABLED)) {
6384 pci_disable_device(pdev);
6385 adapter->flags &= ~DEV_ENABLED;
6386 }
b8ff05a9 6387 pci_release_regions(pdev);
8b662fe7 6388 kfree(adapter);
a069ec91 6389 } else
b8ff05a9
DM		 6390 pci_release_regions(pdev);
6391}
6392
6393static struct pci_driver cxgb4_driver = {
6394 .name = KBUILD_MODNAME,
6395 .id_table = cxgb4_pci_tbl,
6396 .probe = init_one,
91744948 6397 .remove = remove_one,
687d705c 6398 .shutdown = remove_one,
204dc3c0 6399 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6400};
6401
6402static int __init cxgb4_init_module(void)
6403{
6404 int ret;
6405
3069ee9b
VP		 6406 workq = create_singlethread_workqueue("cxgb4");
6407 if (!workq)
6408 return -ENOMEM;
6409
b8ff05a9
DM		 6410 /* Debugfs support is optional, just warn if this fails */
6411 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6412 if (!cxgb4_debugfs_root)
428ac43f 6413 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6414
6415 ret = pci_register_driver(&cxgb4_driver);
73a695f8 6416 if (ret < 0) {
b8ff05a9 6417 debugfs_remove(cxgb4_debugfs_root);
73a695f8
WY		 6418 destroy_workqueue(workq);
6419 }
01bcca68
VP		 6420
6421 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6422
b8ff05a9
DM		 6423 return ret;
6424}
6425
6426static void __exit cxgb4_cleanup_module(void)
6427{
01bcca68 6428 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
b8ff05a9
DM		 6429 pci_unregister_driver(&cxgb4_driver);
6430 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3069ee9b
VP		 6431 flush_workqueue(workq);
6432 destroy_workqueue(workq);
b8ff05a9
DM		 6433}
6434
6435module_init(cxgb4_init_module);
6436module_exit(cxgb4_cleanup_module);
ubuntu-zesty-kernel mirror