2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <net/addrconf.h>
64 #include <asm/uaccess.h>
70 #include "cxgb4_dcb.h"
73 #include <../drivers/net/bonding/bonding.h>
78 #define DRV_VERSION "2.0.0-ko"
79 #define DRV_DESC "Chelsio T4/T5 Network Driver"
82 * Max interrupt hold-off timer value in us. Queues fall back to this value
83 * under extreme memory pressure so it's largish to give the system time to
86 #define MAX_SGE_TIMERVAL 200U
90 * Physical Function provisioning constants.
92 PFRES_NVI
= 4, /* # of Virtual Interfaces */
93 PFRES_NETHCTRL
= 128, /* # of EQs used for ETH or CTRL Qs */
94 PFRES_NIQFLINT
= 128, /* # of ingress Qs/w Free List(s)/intr
96 PFRES_NEQ
= 256, /* # of egress queues */
97 PFRES_NIQ
= 0, /* # of ingress queues */
98 PFRES_TC
= 0, /* PCI-E traffic class */
99 PFRES_NEXACTF
= 128, /* # of exact MPS filters */
101 PFRES_R_CAPS
= FW_CMD_CAP_PF
,
102 PFRES_WX_CAPS
= FW_CMD_CAP_PF
,
104 #ifdef CONFIG_PCI_IOV
106 * Virtual Function provisioning constants. We need two extra Ingress
107 * Queues with Interrupt capability to serve as the VF's Firmware
108 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
109 * neither will have Free Lists associated with them). For each
110 * Ethernet/Control Egress Queue and for each Free List, we need an
113 VFRES_NPORTS
= 1, /* # of "ports" per VF */
114 VFRES_NQSETS
= 2, /* # of "Queue Sets" per VF */
116 VFRES_NVI
= VFRES_NPORTS
, /* # of Virtual Interfaces */
117 VFRES_NETHCTRL
= VFRES_NQSETS
, /* # of EQs used for ETH or CTRL Qs */
118 VFRES_NIQFLINT
= VFRES_NQSETS
+2,/* # of ingress Qs/w Free List(s)/intr */
119 VFRES_NEQ
= VFRES_NQSETS
*2, /* # of egress queues */
120 VFRES_NIQ
= 0, /* # of non-fl/int ingress queues */
121 VFRES_TC
= 0, /* PCI-E traffic class */
122 VFRES_NEXACTF
= 16, /* # of exact MPS filters */
124 VFRES_R_CAPS
= FW_CMD_CAP_DMAQ
|FW_CMD_CAP_VF
|FW_CMD_CAP_PORT
,
125 VFRES_WX_CAPS
= FW_CMD_CAP_DMAQ
|FW_CMD_CAP_VF
,
130 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
131 * static and likely not to be useful in the long run. We really need to
132 * implement some form of persistent configuration which the firmware
135 static unsigned int pfvfres_pmask(struct adapter
*adapter
,
136 unsigned int pf
, unsigned int vf
)
138 unsigned int portn
, portvec
;
141 * Give PF's access to all of the ports.
144 return FW_PFVF_CMD_PMASK_MASK
;
147 * For VFs, we'll assign them access to the ports based purely on the
148 * PF. We assign active ports in order, wrapping around if there are
149 * fewer active ports than PFs: e.g. active port[pf % nports].
150 * Unfortunately the adapter's port_info structs haven't been
151 * initialized yet so we have to compute this.
153 if (adapter
->params
.nports
== 0)
156 portn
= pf
% adapter
->params
.nports
;
157 portvec
= adapter
->params
.portvec
;
160 * Isolate the lowest set bit in the port vector. If we're at
161 * the port number that we want, return that as the pmask.
162 * otherwise mask that bit out of the port vector and
163 * decrement our port number ...
165 unsigned int pmask
= portvec
^ (portvec
& (portvec
-1));
175 MAX_TXQ_ENTRIES
= 16384,
176 MAX_CTRL_TXQ_ENTRIES
= 1024,
177 MAX_RSPQ_ENTRIES
= 16384,
178 MAX_RX_BUFFERS
= 16384,
179 MIN_TXQ_ENTRIES
= 32,
180 MIN_CTRL_TXQ_ENTRIES
= 32,
181 MIN_RSPQ_ENTRIES
= 128,
185 /* Host shadow copy of ingress filter entry. This is in host native format
186 * and doesn't match the ordering or bit order, etc. of the hardware of the
187 * firmware command. The use of bit-field structure elements is purely to
188 * remind ourselves of the field size limitations and save memory in the case
189 * where the filter table is large.
191 struct filter_entry
{
192 /* Administrative fields for filter.
194 u32 valid
:1; /* filter allocated and valid */
195 u32 locked
:1; /* filter is administratively locked */
197 u32 pending
:1; /* filter action is pending firmware reply */
198 u32 smtidx
:8; /* Source MAC Table index for smac */
199 struct l2t_entry
*l2t
; /* Layer Two Table entry for dmac */
201 /* The filter itself. Most of this is a straight copy of information
202 * provided by the extended ioctl(). Some fields are translated to
203 * internal forms -- for instance the Ingress Queue ID passed in from
204 * the ioctl() is translated into the Absolute Ingress Queue ID.
206 struct ch_filter_specification fs
;
209 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
210 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
211 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
213 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
215 static const struct pci_device_id cxgb4_pci_tbl
[] = {
216 CH_DEVICE(0xa000, 0), /* PE10K */
217 CH_DEVICE(0x4001, -1),
218 CH_DEVICE(0x4002, -1),
219 CH_DEVICE(0x4003, -1),
220 CH_DEVICE(0x4004, -1),
221 CH_DEVICE(0x4005, -1),
222 CH_DEVICE(0x4006, -1),
223 CH_DEVICE(0x4007, -1),
224 CH_DEVICE(0x4008, -1),
225 CH_DEVICE(0x4009, -1),
226 CH_DEVICE(0x400a, -1),
227 CH_DEVICE(0x400d, -1),
228 CH_DEVICE(0x400e, -1),
229 CH_DEVICE(0x4080, -1),
230 CH_DEVICE(0x4081, -1),
231 CH_DEVICE(0x4082, -1),
232 CH_DEVICE(0x4083, -1),
233 CH_DEVICE(0x4084, -1),
234 CH_DEVICE(0x4085, -1),
235 CH_DEVICE(0x4086, -1),
236 CH_DEVICE(0x4087, -1),
237 CH_DEVICE(0x4088, -1),
238 CH_DEVICE(0x4401, 4),
239 CH_DEVICE(0x4402, 4),
240 CH_DEVICE(0x4403, 4),
241 CH_DEVICE(0x4404, 4),
242 CH_DEVICE(0x4405, 4),
243 CH_DEVICE(0x4406, 4),
244 CH_DEVICE(0x4407, 4),
245 CH_DEVICE(0x4408, 4),
246 CH_DEVICE(0x4409, 4),
247 CH_DEVICE(0x440a, 4),
248 CH_DEVICE(0x440d, 4),
249 CH_DEVICE(0x440e, 4),
250 CH_DEVICE(0x4480, 4),
251 CH_DEVICE(0x4481, 4),
252 CH_DEVICE(0x4482, 4),
253 CH_DEVICE(0x4483, 4),
254 CH_DEVICE(0x4484, 4),
255 CH_DEVICE(0x4485, 4),
256 CH_DEVICE(0x4486, 4),
257 CH_DEVICE(0x4487, 4),
258 CH_DEVICE(0x4488, 4),
259 CH_DEVICE(0x5001, 4),
260 CH_DEVICE(0x5002, 4),
261 CH_DEVICE(0x5003, 4),
262 CH_DEVICE(0x5004, 4),
263 CH_DEVICE(0x5005, 4),
264 CH_DEVICE(0x5006, 4),
265 CH_DEVICE(0x5007, 4),
266 CH_DEVICE(0x5008, 4),
267 CH_DEVICE(0x5009, 4),
268 CH_DEVICE(0x500A, 4),
269 CH_DEVICE(0x500B, 4),
270 CH_DEVICE(0x500C, 4),
271 CH_DEVICE(0x500D, 4),
272 CH_DEVICE(0x500E, 4),
273 CH_DEVICE(0x500F, 4),
274 CH_DEVICE(0x5010, 4),
275 CH_DEVICE(0x5011, 4),
276 CH_DEVICE(0x5012, 4),
277 CH_DEVICE(0x5013, 4),
278 CH_DEVICE(0x5014, 4),
279 CH_DEVICE(0x5015, 4),
280 CH_DEVICE(0x5080, 4),
281 CH_DEVICE(0x5081, 4),
282 CH_DEVICE(0x5082, 4),
283 CH_DEVICE(0x5083, 4),
284 CH_DEVICE(0x5084, 4),
285 CH_DEVICE(0x5085, 4),
286 CH_DEVICE(0x5401, 4),
287 CH_DEVICE(0x5402, 4),
288 CH_DEVICE(0x5403, 4),
289 CH_DEVICE(0x5404, 4),
290 CH_DEVICE(0x5405, 4),
291 CH_DEVICE(0x5406, 4),
292 CH_DEVICE(0x5407, 4),
293 CH_DEVICE(0x5408, 4),
294 CH_DEVICE(0x5409, 4),
295 CH_DEVICE(0x540A, 4),
296 CH_DEVICE(0x540B, 4),
297 CH_DEVICE(0x540C, 4),
298 CH_DEVICE(0x540D, 4),
299 CH_DEVICE(0x540E, 4),
300 CH_DEVICE(0x540F, 4),
301 CH_DEVICE(0x5410, 4),
302 CH_DEVICE(0x5411, 4),
303 CH_DEVICE(0x5412, 4),
304 CH_DEVICE(0x5413, 4),
305 CH_DEVICE(0x5414, 4),
306 CH_DEVICE(0x5415, 4),
307 CH_DEVICE(0x5480, 4),
308 CH_DEVICE(0x5481, 4),
309 CH_DEVICE(0x5482, 4),
310 CH_DEVICE(0x5483, 4),
311 CH_DEVICE(0x5484, 4),
312 CH_DEVICE(0x5485, 4),
316 #define FW4_FNAME "cxgb4/t4fw.bin"
317 #define FW5_FNAME "cxgb4/t5fw.bin"
318 #define FW4_CFNAME "cxgb4/t4-config.txt"
319 #define FW5_CFNAME "cxgb4/t5-config.txt"
321 MODULE_DESCRIPTION(DRV_DESC
);
322 MODULE_AUTHOR("Chelsio Communications");
323 MODULE_LICENSE("Dual BSD/GPL");
324 MODULE_VERSION(DRV_VERSION
);
325 MODULE_DEVICE_TABLE(pci
, cxgb4_pci_tbl
);
326 MODULE_FIRMWARE(FW4_FNAME
);
327 MODULE_FIRMWARE(FW5_FNAME
);
330 * Normally we're willing to become the firmware's Master PF but will be happy
331 * if another PF has already become the Master and initialized the adapter.
332 * Setting "force_init" will cause this driver to forcibly establish itself as
333 * the Master PF and initialize the adapter.
335 static uint force_init
;
337 module_param(force_init
, uint
, 0644);
338 MODULE_PARM_DESC(force_init
, "Forcibly become Master PF and initialize adapter");
341 * Normally if the firmware we connect to has Configuration File support, we
342 * use that and only fall back to the old Driver-based initialization if the
343 * Configuration File fails for some reason. If force_old_init is set, then
344 * we'll always use the old Driver-based initialization sequence.
346 static uint force_old_init
;
348 module_param(force_old_init
, uint
, 0644);
349 MODULE_PARM_DESC(force_old_init
, "Force old initialization sequence");
351 static int dflt_msg_enable
= DFLT_MSG_ENABLE
;
353 module_param(dflt_msg_enable
, int, 0644);
354 MODULE_PARM_DESC(dflt_msg_enable
, "Chelsio T4 default message enable bitmap");
357 * The driver uses the best interrupt scheme available on a platform in the
358 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
359 * of these schemes the driver may consider as follows:
361 * msi = 2: choose from among all three options
362 * msi = 1: only consider MSI and INTx interrupts
363 * msi = 0: force INTx interrupts
367 module_param(msi
, int, 0644);
368 MODULE_PARM_DESC(msi
, "whether to use INTx (0), MSI (1) or MSI-X (2)");
371 * Queue interrupt hold-off timer values. Queues default to the first of these
374 static unsigned int intr_holdoff
[SGE_NTIMERS
- 1] = { 5, 10, 20, 50, 100 };
376 module_param_array(intr_holdoff
, uint
, NULL
, 0644);
377 MODULE_PARM_DESC(intr_holdoff
, "values for queue interrupt hold-off timers "
378 "0..4 in microseconds");
380 static unsigned int intr_cnt
[SGE_NCOUNTERS
- 1] = { 4, 8, 16 };
382 module_param_array(intr_cnt
, uint
, NULL
, 0644);
383 MODULE_PARM_DESC(intr_cnt
,
384 "thresholds 1..3 for queue interrupt packet counters");
387 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
388 * offset by 2 bytes in order to have the IP headers line up on 4-byte
389 * boundaries. This is a requirement for many architectures which will throw
390 * a machine check fault if an attempt is made to access one of the 4-byte IP
391 * header fields on a non-4-byte boundary. And it's a major performance issue
392 * even on some architectures which allow it like some implementations of the
393 * x86 ISA. However, some architectures don't mind this and for some very
394 * edge-case performance sensitive applications (like forwarding large volumes
395 * of small packets), setting this DMA offset to 0 will decrease the number of
396 * PCI-E Bus transfers enough to measurably affect performance.
398 static int rx_dma_offset
= 2;
402 #ifdef CONFIG_PCI_IOV
403 module_param(vf_acls
, bool, 0644);
404 MODULE_PARM_DESC(vf_acls
, "if set enable virtualization L2 ACL enforcement");
406 /* Configure the number of PCI-E Virtual Function which are to be instantiated
407 * on SR-IOV Capable Physical Functions.
409 static unsigned int num_vf
[NUM_OF_PF_WITH_SRIOV
];
411 module_param_array(num_vf
, uint
, NULL
, 0644);
412 MODULE_PARM_DESC(num_vf
, "number of VFs for each of PFs 0-3");
415 /* TX Queue select used to determine what algorithm to use for selecting TX
416 * queue. Select between the kernel provided function (select_queue=0) or user
417 * cxgb_select_queue function (select_queue=1)
419 * Default: select_queue=0
421 static int select_queue
;
422 module_param(select_queue
, int, 0644);
423 MODULE_PARM_DESC(select_queue
,
424 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
427 * The filter TCAM has a fixed portion and a variable portion. The fixed
428 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
429 * ports. The variable portion is 36 bits which can include things like Exact
430 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
431 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
432 * far exceed the 36-bit budget for this "compressed" header portion of the
433 * filter. Thus, we have a scarce resource which must be carefully managed.
435 * By default we set this up to mostly match the set of filter matching
436 * capabilities of T3 but with accommodations for some of T4's more
437 * interesting features:
439 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
440 * [Inner] VLAN (17), Port (3), FCoE (1) }
443 TP_VLAN_PRI_MAP_DEFAULT
= HW_TPL_FR_MT_PR_IV_P_FC
,
444 TP_VLAN_PRI_MAP_FIRST
= FCOE_SHIFT
,
445 TP_VLAN_PRI_MAP_LAST
= FRAGMENTATION_SHIFT
,
448 static unsigned int tp_vlan_pri_map
= TP_VLAN_PRI_MAP_DEFAULT
;
450 module_param(tp_vlan_pri_map
, uint
, 0644);
451 MODULE_PARM_DESC(tp_vlan_pri_map
, "global compressed filter configuration");
453 static struct dentry
*cxgb4_debugfs_root
;
455 static LIST_HEAD(adapter_list
);
456 static DEFINE_MUTEX(uld_mutex
);
457 /* Adapter list to be accessed from atomic context */
458 static LIST_HEAD(adap_rcu_list
);
459 static DEFINE_SPINLOCK(adap_rcu_lock
);
460 static struct cxgb4_uld_info ulds
[CXGB4_ULD_MAX
];
461 static const char *uld_str
[] = { "RDMA", "iSCSI" };
463 static void link_report(struct net_device
*dev
)
465 if (!netif_carrier_ok(dev
))
466 netdev_info(dev
, "link down\n");
468 static const char *fc
[] = { "no", "Rx", "Tx", "Tx/Rx" };
470 const char *s
= "10Mbps";
471 const struct port_info
*p
= netdev_priv(dev
);
473 switch (p
->link_cfg
.speed
) {
488 netdev_info(dev
, "link up, %s, full-duplex, %s PAUSE\n", s
,
493 #ifdef CONFIG_CHELSIO_T4_DCB
494 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */
495 static void dcb_tx_queue_prio_enable(struct net_device
*dev
, int enable
)
497 struct port_info
*pi
= netdev_priv(dev
);
498 struct adapter
*adap
= pi
->adapter
;
499 struct sge_eth_txq
*txq
= &adap
->sge
.ethtxq
[pi
->first_qset
];
502 /* We use a simple mapping of Port TX Queue Index to DCB
503 * Priority when we're enabling DCB.
505 for (i
= 0; i
< pi
->nqsets
; i
++, txq
++) {
509 name
= (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ
) |
510 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH
) |
511 FW_PARAMS_PARAM_YZ(txq
->q
.cntxt_id
));
512 value
= enable
? i
: 0xffffffff;
514 /* Since we can be called while atomic (from "interrupt
515 * level") we need to issue the Set Parameters Commannd
516 * without sleeping (timeout < 0).
518 err
= t4_set_params_nosleep(adap
, adap
->mbox
, adap
->fn
, 0, 1,
522 dev_err(adap
->pdev_dev
,
523 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
524 enable
? "set" : "unset", pi
->port_id
, i
, -err
);
526 txq
->dcb_prio
= value
;
529 #endif /* CONFIG_CHELSIO_T4_DCB */
531 void t4_os_link_changed(struct adapter
*adapter
, int port_id
, int link_stat
)
533 struct net_device
*dev
= adapter
->port
[port_id
];
535 /* Skip changes from disabled ports. */
536 if (netif_running(dev
) && link_stat
!= netif_carrier_ok(dev
)) {
538 netif_carrier_on(dev
);
540 #ifdef CONFIG_CHELSIO_T4_DCB
541 cxgb4_dcb_state_init(dev
);
542 dcb_tx_queue_prio_enable(dev
, false);
543 #endif /* CONFIG_CHELSIO_T4_DCB */
544 netif_carrier_off(dev
);
551 void t4_os_portmod_changed(const struct adapter
*adap
, int port_id
)
553 static const char *mod_str
[] = {
554 NULL
, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
557 const struct net_device
*dev
= adap
->port
[port_id
];
558 const struct port_info
*pi
= netdev_priv(dev
);
560 if (pi
->mod_type
== FW_PORT_MOD_TYPE_NONE
)
561 netdev_info(dev
, "port module unplugged\n");
562 else if (pi
->mod_type
< ARRAY_SIZE(mod_str
))
563 netdev_info(dev
, "%s module inserted\n", mod_str
[pi
->mod_type
]);
567 * Configure the exact and hash address filters to handle a port's multicast
568 * and secondary unicast MAC addresses.
570 static int set_addr_filters(const struct net_device
*dev
, bool sleep
)
578 const struct netdev_hw_addr
*ha
;
579 int uc_cnt
= netdev_uc_count(dev
);
580 int mc_cnt
= netdev_mc_count(dev
);
581 const struct port_info
*pi
= netdev_priv(dev
);
582 unsigned int mb
= pi
->adapter
->fn
;
584 /* first do the secondary unicast addresses */
585 netdev_for_each_uc_addr(ha
, dev
) {
586 addr
[naddr
++] = ha
->addr
;
587 if (--uc_cnt
== 0 || naddr
>= ARRAY_SIZE(addr
)) {
588 ret
= t4_alloc_mac_filt(pi
->adapter
, mb
, pi
->viid
, free
,
589 naddr
, addr
, filt_idx
, &uhash
, sleep
);
598 /* next set up the multicast addresses */
599 netdev_for_each_mc_addr(ha
, dev
) {
600 addr
[naddr
++] = ha
->addr
;
601 if (--mc_cnt
== 0 || naddr
>= ARRAY_SIZE(addr
)) {
602 ret
= t4_alloc_mac_filt(pi
->adapter
, mb
, pi
->viid
, free
,
603 naddr
, addr
, filt_idx
, &mhash
, sleep
);
612 return t4_set_addr_hash(pi
->adapter
, mb
, pi
->viid
, uhash
!= 0,
613 uhash
| mhash
, sleep
);
616 int dbfifo_int_thresh
= 10; /* 10 == 640 entry threshold */
617 module_param(dbfifo_int_thresh
, int, 0644);
618 MODULE_PARM_DESC(dbfifo_int_thresh
, "doorbell fifo interrupt threshold");
621 * usecs to sleep while draining the dbfifo
623 static int dbfifo_drain_delay
= 1000;
624 module_param(dbfifo_drain_delay
, int, 0644);
625 MODULE_PARM_DESC(dbfifo_drain_delay
,
626 "usecs to sleep while draining the dbfifo");
629 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
630 * If @mtu is -1 it is left unchanged.
632 static int set_rxmode(struct net_device
*dev
, int mtu
, bool sleep_ok
)
635 struct port_info
*pi
= netdev_priv(dev
);
637 ret
= set_addr_filters(dev
, sleep_ok
);
639 ret
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, mtu
,
640 (dev
->flags
& IFF_PROMISC
) ? 1 : 0,
641 (dev
->flags
& IFF_ALLMULTI
) ? 1 : 0, 1, -1,
647 * link_start - enable a port
648 * @dev: the port to enable
650 * Performs the MAC and PHY actions needed to enable a port.
652 static int link_start(struct net_device
*dev
)
655 struct port_info
*pi
= netdev_priv(dev
);
656 unsigned int mb
= pi
->adapter
->fn
;
659 * We do not set address filters and promiscuity here, the stack does
660 * that step explicitly.
662 ret
= t4_set_rxmode(pi
->adapter
, mb
, pi
->viid
, dev
->mtu
, -1, -1, -1,
663 !!(dev
->features
& NETIF_F_HW_VLAN_CTAG_RX
), true);
665 ret
= t4_change_mac(pi
->adapter
, mb
, pi
->viid
,
666 pi
->xact_addr_filt
, dev
->dev_addr
, true,
669 pi
->xact_addr_filt
= ret
;
674 ret
= t4_link_start(pi
->adapter
, mb
, pi
->tx_chan
,
678 ret
= t4_enable_vi_params(pi
->adapter
, mb
, pi
->viid
, true,
679 true, CXGB4_DCB_ENABLED
);
686 int cxgb4_dcb_enabled(const struct net_device
*dev
)
688 #ifdef CONFIG_CHELSIO_T4_DCB
689 struct port_info
*pi
= netdev_priv(dev
);
691 return pi
->dcb
.state
== CXGB4_DCB_STATE_FW_ALLSYNCED
;
696 EXPORT_SYMBOL(cxgb4_dcb_enabled
);
698 #ifdef CONFIG_CHELSIO_T4_DCB
699 /* Handle a Data Center Bridging update message from the firmware. */
700 static void dcb_rpl(struct adapter
*adap
, const struct fw_port_cmd
*pcmd
)
702 int port
= FW_PORT_CMD_PORTID_GET(ntohl(pcmd
->op_to_portid
));
703 struct net_device
*dev
= adap
->port
[port
];
704 int old_dcb_enabled
= cxgb4_dcb_enabled(dev
);
707 cxgb4_dcb_handle_fw_update(adap
, pcmd
);
708 new_dcb_enabled
= cxgb4_dcb_enabled(dev
);
710 /* If the DCB has become enabled or disabled on the port then we're
711 * going to need to set up/tear down DCB Priority parameters for the
712 * TX Queues associated with the port.
714 if (new_dcb_enabled
!= old_dcb_enabled
)
715 dcb_tx_queue_prio_enable(dev
, new_dcb_enabled
);
717 #endif /* CONFIG_CHELSIO_T4_DCB */
719 /* Clear a filter and release any of its resources that we own. This also
720 * clears the filter's "pending" status.
722 static void clear_filter(struct adapter
*adap
, struct filter_entry
*f
)
724 /* If the new or old filter have loopback rewriteing rules then we'll
725 * need to free any existing Layer Two Table (L2T) entries of the old
726 * filter rule. The firmware will handle freeing up any Source MAC
727 * Table (SMT) entries used for rewriting Source MAC Addresses in
731 cxgb4_l2t_release(f
->l2t
);
733 /* The zeroing of the filter rule below clears the filter valid,
734 * pending, locked flags, l2t pointer, etc. so it's all we need for
737 memset(f
, 0, sizeof(*f
));
740 /* Handle a filter write/deletion reply.
742 static void filter_rpl(struct adapter
*adap
, const struct cpl_set_tcb_rpl
*rpl
)
744 unsigned int idx
= GET_TID(rpl
);
745 unsigned int nidx
= idx
- adap
->tids
.ftid_base
;
747 struct filter_entry
*f
;
749 if (idx
>= adap
->tids
.ftid_base
&& nidx
<
750 (adap
->tids
.nftids
+ adap
->tids
.nsftids
)) {
752 ret
= GET_TCB_COOKIE(rpl
->cookie
);
753 f
= &adap
->tids
.ftid_tab
[idx
];
755 if (ret
== FW_FILTER_WR_FLT_DELETED
) {
756 /* Clear the filter when we get confirmation from the
757 * hardware that the filter has been deleted.
759 clear_filter(adap
, f
);
760 } else if (ret
== FW_FILTER_WR_SMT_TBL_FULL
) {
761 dev_err(adap
->pdev_dev
, "filter %u setup failed due to full SMT\n",
763 clear_filter(adap
, f
);
764 } else if (ret
== FW_FILTER_WR_FLT_ADDED
) {
765 f
->smtidx
= (be64_to_cpu(rpl
->oldval
) >> 24) & 0xff;
766 f
->pending
= 0; /* asynchronous setup completed */
769 /* Something went wrong. Issue a warning about the
770 * problem and clear everything out.
772 dev_err(adap
->pdev_dev
, "filter %u setup failed with error %u\n",
774 clear_filter(adap
, f
);
779 /* Response queue handler for the FW event queue.
781 static int fwevtq_handler(struct sge_rspq
*q
, const __be64
*rsp
,
782 const struct pkt_gl
*gl
)
784 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
786 rsp
++; /* skip RSS header */
788 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
790 if (unlikely(opcode
== CPL_FW4_MSG
&&
791 ((const struct cpl_fw4_msg
*)rsp
)->type
== FW_TYPE_RSSCPL
)) {
793 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
795 if (opcode
!= CPL_SGE_EGR_UPDATE
) {
796 dev_err(q
->adap
->pdev_dev
, "unexpected FW4/CPL %#x on FW event queue\n"
802 if (likely(opcode
== CPL_SGE_EGR_UPDATE
)) {
803 const struct cpl_sge_egr_update
*p
= (void *)rsp
;
804 unsigned int qid
= EGR_QID(ntohl(p
->opcode_qid
));
807 txq
= q
->adap
->sge
.egr_map
[qid
- q
->adap
->sge
.egr_start
];
809 if ((u8
*)txq
< (u8
*)q
->adap
->sge
.ofldtxq
) {
810 struct sge_eth_txq
*eq
;
812 eq
= container_of(txq
, struct sge_eth_txq
, q
);
813 netif_tx_wake_queue(eq
->txq
);
815 struct sge_ofld_txq
*oq
;
817 oq
= container_of(txq
, struct sge_ofld_txq
, q
);
818 tasklet_schedule(&oq
->qresume_tsk
);
820 } else if (opcode
== CPL_FW6_MSG
|| opcode
== CPL_FW4_MSG
) {
821 const struct cpl_fw6_msg
*p
= (void *)rsp
;
823 #ifdef CONFIG_CHELSIO_T4_DCB
824 const struct fw_port_cmd
*pcmd
= (const void *)p
->data
;
825 unsigned int cmd
= FW_CMD_OP_GET(ntohl(pcmd
->op_to_portid
));
826 unsigned int action
=
827 FW_PORT_CMD_ACTION_GET(ntohl(pcmd
->action_to_len16
));
829 if (cmd
== FW_PORT_CMD
&&
830 action
== FW_PORT_ACTION_GET_PORT_INFO
) {
831 int port
= FW_PORT_CMD_PORTID_GET(
832 be32_to_cpu(pcmd
->op_to_portid
));
833 struct net_device
*dev
= q
->adap
->port
[port
];
834 int state_input
= ((pcmd
->u
.info
.dcbxdis_pkd
&
836 ? CXGB4_DCB_INPUT_FW_DISABLED
837 : CXGB4_DCB_INPUT_FW_ENABLED
);
839 cxgb4_dcb_state_fsm(dev
, state_input
);
842 if (cmd
== FW_PORT_CMD
&&
843 action
== FW_PORT_ACTION_L2_DCB_CFG
)
844 dcb_rpl(q
->adap
, pcmd
);
848 t4_handle_fw_rpl(q
->adap
, p
->data
);
849 } else if (opcode
== CPL_L2T_WRITE_RPL
) {
850 const struct cpl_l2t_write_rpl
*p
= (void *)rsp
;
852 do_l2t_write_rpl(q
->adap
, p
);
853 } else if (opcode
== CPL_SET_TCB_RPL
) {
854 const struct cpl_set_tcb_rpl
*p
= (void *)rsp
;
856 filter_rpl(q
->adap
, p
);
858 dev_err(q
->adap
->pdev_dev
,
859 "unexpected CPL %#x on FW event queue\n", opcode
);
865 * uldrx_handler - response queue handler for ULD queues
866 * @q: the response queue that received the packet
867 * @rsp: the response queue descriptor holding the offload message
868 * @gl: the gather list of packet fragments
870 * Deliver an ingress offload packet to a ULD. All processing is done by
871 * the ULD, we just maintain statistics.
873 static int uldrx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
874 const struct pkt_gl
*gl
)
876 struct sge_ofld_rxq
*rxq
= container_of(q
, struct sge_ofld_rxq
, rspq
);
878 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
880 if (((const struct rss_header
*)rsp
)->opcode
== CPL_FW4_MSG
&&
881 ((const struct cpl_fw4_msg
*)(rsp
+ 1))->type
== FW_TYPE_RSSCPL
)
884 if (ulds
[q
->uld
].rx_handler(q
->adap
->uld_handle
[q
->uld
], rsp
, gl
)) {
890 else if (gl
== CXGB4_MSG_AN
)
897 static void disable_msi(struct adapter
*adapter
)
899 if (adapter
->flags
& USING_MSIX
) {
900 pci_disable_msix(adapter
->pdev
);
901 adapter
->flags
&= ~USING_MSIX
;
902 } else if (adapter
->flags
& USING_MSI
) {
903 pci_disable_msi(adapter
->pdev
);
904 adapter
->flags
&= ~USING_MSI
;
909 * Interrupt handler for non-data events used with MSI-X.
911 static irqreturn_t
t4_nondata_intr(int irq
, void *cookie
)
913 struct adapter
*adap
= cookie
;
915 u32 v
= t4_read_reg(adap
, MYPF_REG(PL_PF_INT_CAUSE
));
918 t4_write_reg(adap
, MYPF_REG(PL_PF_INT_CAUSE
), v
);
920 t4_slow_intr_handler(adap
);
925 * Name the MSI-X interrupts.
927 static void name_msix_vecs(struct adapter
*adap
)
929 int i
, j
, msi_idx
= 2, n
= sizeof(adap
->msix_info
[0].desc
);
931 /* non-data interrupts */
932 snprintf(adap
->msix_info
[0].desc
, n
, "%s", adap
->port
[0]->name
);
935 snprintf(adap
->msix_info
[1].desc
, n
, "%s-FWeventq",
936 adap
->port
[0]->name
);
938 /* Ethernet queues */
939 for_each_port(adap
, j
) {
940 struct net_device
*d
= adap
->port
[j
];
941 const struct port_info
*pi
= netdev_priv(d
);
943 for (i
= 0; i
< pi
->nqsets
; i
++, msi_idx
++)
944 snprintf(adap
->msix_info
[msi_idx
].desc
, n
, "%s-Rx%d",
949 for_each_ofldrxq(&adap
->sge
, i
)
950 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-ofld%d",
951 adap
->port
[0]->name
, i
);
953 for_each_rdmarxq(&adap
->sge
, i
)
954 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-rdma%d",
955 adap
->port
[0]->name
, i
);
957 for_each_rdmaciq(&adap
->sge
, i
)
958 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-rdma-ciq%d",
959 adap
->port
[0]->name
, i
);
962 static int request_msix_queue_irqs(struct adapter
*adap
)
964 struct sge
*s
= &adap
->sge
;
965 int err
, ethqidx
, ofldqidx
= 0, rdmaqidx
= 0, rdmaciqqidx
= 0;
968 err
= request_irq(adap
->msix_info
[1].vec
, t4_sge_intr_msix
, 0,
969 adap
->msix_info
[1].desc
, &s
->fw_evtq
);
973 for_each_ethrxq(s
, ethqidx
) {
974 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
976 adap
->msix_info
[msi_index
].desc
,
977 &s
->ethrxq
[ethqidx
].rspq
);
982 for_each_ofldrxq(s
, ofldqidx
) {
983 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
985 adap
->msix_info
[msi_index
].desc
,
986 &s
->ofldrxq
[ofldqidx
].rspq
);
991 for_each_rdmarxq(s
, rdmaqidx
) {
992 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
994 adap
->msix_info
[msi_index
].desc
,
995 &s
->rdmarxq
[rdmaqidx
].rspq
);
1000 for_each_rdmaciq(s
, rdmaciqqidx
) {
1001 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
1002 t4_sge_intr_msix
, 0,
1003 adap
->msix_info
[msi_index
].desc
,
1004 &s
->rdmaciq
[rdmaciqqidx
].rspq
);
1012 while (--rdmaciqqidx
>= 0)
1013 free_irq(adap
->msix_info
[--msi_index
].vec
,
1014 &s
->rdmaciq
[rdmaciqqidx
].rspq
);
1015 while (--rdmaqidx
>= 0)
1016 free_irq(adap
->msix_info
[--msi_index
].vec
,
1017 &s
->rdmarxq
[rdmaqidx
].rspq
);
1018 while (--ofldqidx
>= 0)
1019 free_irq(adap
->msix_info
[--msi_index
].vec
,
1020 &s
->ofldrxq
[ofldqidx
].rspq
);
1021 while (--ethqidx
>= 0)
1022 free_irq(adap
->msix_info
[--msi_index
].vec
,
1023 &s
->ethrxq
[ethqidx
].rspq
);
1024 free_irq(adap
->msix_info
[1].vec
, &s
->fw_evtq
);
1028 static void free_msix_queue_irqs(struct adapter
*adap
)
1030 int i
, msi_index
= 2;
1031 struct sge
*s
= &adap
->sge
;
1033 free_irq(adap
->msix_info
[1].vec
, &s
->fw_evtq
);
1034 for_each_ethrxq(s
, i
)
1035 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->ethrxq
[i
].rspq
);
1036 for_each_ofldrxq(s
, i
)
1037 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->ofldrxq
[i
].rspq
);
1038 for_each_rdmarxq(s
, i
)
1039 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->rdmarxq
[i
].rspq
);
1040 for_each_rdmaciq(s
, i
)
1041 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->rdmaciq
[i
].rspq
);
1045 * write_rss - write the RSS table for a given port
1047 * @queues: array of queue indices for RSS
1049 * Sets up the portion of the HW RSS table for the port's VI to distribute
1050 * packets to the Rx queues in @queues.
1052 static int write_rss(const struct port_info
*pi
, const u16
*queues
)
1056 const struct sge_eth_rxq
*q
= &pi
->adapter
->sge
.ethrxq
[pi
->first_qset
];
1058 rss
= kmalloc(pi
->rss_size
* sizeof(u16
), GFP_KERNEL
);
1062 /* map the queue indices to queue ids */
1063 for (i
= 0; i
< pi
->rss_size
; i
++, queues
++)
1064 rss
[i
] = q
[*queues
].rspq
.abs_id
;
1066 err
= t4_config_rss_range(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, 0,
1067 pi
->rss_size
, rss
, pi
->rss_size
);
1073 * setup_rss - configure RSS
1074 * @adap: the adapter
1076 * Sets up RSS for each port.
1078 static int setup_rss(struct adapter
*adap
)
1082 for_each_port(adap
, i
) {
1083 const struct port_info
*pi
= adap2pinfo(adap
, i
);
1085 err
= write_rss(pi
, pi
->rss
);
1093 * Return the channel of the ingress queue with the given qid.
1095 static unsigned int rxq_to_chan(const struct sge
*p
, unsigned int qid
)
1097 qid
-= p
->ingr_start
;
1098 return netdev2pinfo(p
->ingr_map
[qid
]->netdev
)->tx_chan
;
1102 * Wait until all NAPI handlers are descheduled.
1104 static void quiesce_rx(struct adapter
*adap
)
1108 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ingr_map
); i
++) {
1109 struct sge_rspq
*q
= adap
->sge
.ingr_map
[i
];
1111 if (q
&& q
->handler
)
1112 napi_disable(&q
->napi
);
1117 * Enable NAPI scheduling and interrupt generation for all Rx queues.
1119 static void enable_rx(struct adapter
*adap
)
1123 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ingr_map
); i
++) {
1124 struct sge_rspq
*q
= adap
->sge
.ingr_map
[i
];
1129 napi_enable(&q
->napi
);
1130 /* 0-increment GTS to start the timer and enable interrupts */
1131 t4_write_reg(adap
, MYPF_REG(SGE_PF_GTS
),
1132 SEINTARM(q
->intr_params
) |
1133 INGRESSQID(q
->cntxt_id
));
1138 * setup_sge_queues - configure SGE Tx/Rx/response queues
1139 * @adap: the adapter
1141 * Determines how many sets of SGE queues to use and initializes them.
1142 * We support multiple queue sets per port if we have MSI-X, otherwise
1143 * just one queue set per port.
1145 static int setup_sge_queues(struct adapter
*adap
)
1147 int err
, msi_idx
, i
, j
;
1148 struct sge
*s
= &adap
->sge
;
1150 bitmap_zero(s
->starving_fl
, MAX_EGRQ
);
1151 bitmap_zero(s
->txq_maperr
, MAX_EGRQ
);
1153 if (adap
->flags
& USING_MSIX
)
1154 msi_idx
= 1; /* vector 0 is for non-queue interrupts */
1156 err
= t4_sge_alloc_rxq(adap
, &s
->intrq
, false, adap
->port
[0], 0,
1160 msi_idx
= -((int)s
->intrq
.abs_id
+ 1);
1163 err
= t4_sge_alloc_rxq(adap
, &s
->fw_evtq
, true, adap
->port
[0],
1164 msi_idx
, NULL
, fwevtq_handler
);
1166 freeout
: t4_free_sge_resources(adap
);
1170 for_each_port(adap
, i
) {
1171 struct net_device
*dev
= adap
->port
[i
];
1172 struct port_info
*pi
= netdev_priv(dev
);
1173 struct sge_eth_rxq
*q
= &s
->ethrxq
[pi
->first_qset
];
1174 struct sge_eth_txq
*t
= &s
->ethtxq
[pi
->first_qset
];
1176 for (j
= 0; j
< pi
->nqsets
; j
++, q
++) {
1179 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, dev
,
1185 memset(&q
->stats
, 0, sizeof(q
->stats
));
1187 for (j
= 0; j
< pi
->nqsets
; j
++, t
++) {
1188 err
= t4_sge_alloc_eth_txq(adap
, t
, dev
,
1189 netdev_get_tx_queue(dev
, j
),
1190 s
->fw_evtq
.cntxt_id
);
1196 j
= s
->ofldqsets
/ adap
->params
.nports
; /* ofld queues per channel */
1197 for_each_ofldrxq(s
, i
) {
1198 struct sge_ofld_rxq
*q
= &s
->ofldrxq
[i
];
1199 struct net_device
*dev
= adap
->port
[i
/ j
];
1203 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, dev
, msi_idx
,
1204 q
->fl
.size
? &q
->fl
: NULL
,
1208 memset(&q
->stats
, 0, sizeof(q
->stats
));
1209 s
->ofld_rxq
[i
] = q
->rspq
.abs_id
;
1210 err
= t4_sge_alloc_ofld_txq(adap
, &s
->ofldtxq
[i
], dev
,
1211 s
->fw_evtq
.cntxt_id
);
1216 for_each_rdmarxq(s
, i
) {
1217 struct sge_ofld_rxq
*q
= &s
->rdmarxq
[i
];
1221 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, adap
->port
[i
],
1222 msi_idx
, q
->fl
.size
? &q
->fl
: NULL
,
1226 memset(&q
->stats
, 0, sizeof(q
->stats
));
1227 s
->rdma_rxq
[i
] = q
->rspq
.abs_id
;
1230 for_each_rdmaciq(s
, i
) {
1231 struct sge_ofld_rxq
*q
= &s
->rdmaciq
[i
];
1235 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, adap
->port
[i
],
1236 msi_idx
, q
->fl
.size
? &q
->fl
: NULL
,
1240 memset(&q
->stats
, 0, sizeof(q
->stats
));
1241 s
->rdma_ciq
[i
] = q
->rspq
.abs_id
;
1244 for_each_port(adap
, i
) {
1246 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1247 * have RDMA queues, and that's the right value.
1249 err
= t4_sge_alloc_ctrl_txq(adap
, &s
->ctrlq
[i
], adap
->port
[i
],
1250 s
->fw_evtq
.cntxt_id
,
1251 s
->rdmarxq
[i
].rspq
.cntxt_id
);
1256 t4_write_reg(adap
, MPS_TRC_RSS_CONTROL
,
1257 RSSCONTROL(netdev2pinfo(adap
->port
[0])->tx_chan
) |
1258 QUEUENUMBER(s
->ethrxq
[0].rspq
.abs_id
));
1263 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1264 * The allocated memory is cleared.
1266 void *t4_alloc_mem(size_t size
)
1268 void *p
= kzalloc(size
, GFP_KERNEL
| __GFP_NOWARN
);
1276 * Free memory allocated through alloc_mem().
1278 static void t4_free_mem(void *addr
)
1280 if (is_vmalloc_addr(addr
))
1286 /* Send a Work Request to write the filter at a specified index. We construct
1287 * a Firmware Filter Work Request to have the work done and put the indicated
1288 * filter into "pending" mode which will prevent any further actions against
1289 * it till we get a reply from the firmware on the completion status of the
1292 static int set_filter_wr(struct adapter
*adapter
, int fidx
)
1294 struct filter_entry
*f
= &adapter
->tids
.ftid_tab
[fidx
];
1295 struct sk_buff
*skb
;
1296 struct fw_filter_wr
*fwr
;
1299 /* If the new filter requires loopback Destination MAC and/or VLAN
1300 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1303 if (f
->fs
.newdmac
|| f
->fs
.newvlan
) {
1304 /* allocate L2T entry for new filter */
1305 f
->l2t
= t4_l2t_alloc_switching(adapter
->l2t
);
1308 if (t4_l2t_set_switching(adapter
, f
->l2t
, f
->fs
.vlan
,
1309 f
->fs
.eport
, f
->fs
.dmac
)) {
1310 cxgb4_l2t_release(f
->l2t
);
1316 ftid
= adapter
->tids
.ftid_base
+ fidx
;
1318 skb
= alloc_skb(sizeof(*fwr
), GFP_KERNEL
| __GFP_NOFAIL
);
1319 fwr
= (struct fw_filter_wr
*)__skb_put(skb
, sizeof(*fwr
));
1320 memset(fwr
, 0, sizeof(*fwr
));
1322 /* It would be nice to put most of the following in t4_hw.c but most
1323 * of the work is translating the cxgbtool ch_filter_specification
1324 * into the Work Request and the definition of that structure is
1325 * currently in cxgbtool.h which isn't appropriate to pull into the
1326 * common code. We may eventually try to come up with a more neutral
1327 * filter specification structure but for now it's easiest to simply
1328 * put this fairly direct code in line ...
1330 fwr
->op_pkd
= htonl(FW_WR_OP(FW_FILTER_WR
));
1331 fwr
->len16_pkd
= htonl(FW_WR_LEN16(sizeof(*fwr
)/16));
1333 htonl(V_FW_FILTER_WR_TID(ftid
) |
1334 V_FW_FILTER_WR_RQTYPE(f
->fs
.type
) |
1335 V_FW_FILTER_WR_NOREPLY(0) |
1336 V_FW_FILTER_WR_IQ(f
->fs
.iq
));
1337 fwr
->del_filter_to_l2tix
=
1338 htonl(V_FW_FILTER_WR_RPTTID(f
->fs
.rpttid
) |
1339 V_FW_FILTER_WR_DROP(f
->fs
.action
== FILTER_DROP
) |
1340 V_FW_FILTER_WR_DIRSTEER(f
->fs
.dirsteer
) |
1341 V_FW_FILTER_WR_MASKHASH(f
->fs
.maskhash
) |
1342 V_FW_FILTER_WR_DIRSTEERHASH(f
->fs
.dirsteerhash
) |
1343 V_FW_FILTER_WR_LPBK(f
->fs
.action
== FILTER_SWITCH
) |
1344 V_FW_FILTER_WR_DMAC(f
->fs
.newdmac
) |
1345 V_FW_FILTER_WR_SMAC(f
->fs
.newsmac
) |
1346 V_FW_FILTER_WR_INSVLAN(f
->fs
.newvlan
== VLAN_INSERT
||
1347 f
->fs
.newvlan
== VLAN_REWRITE
) |
1348 V_FW_FILTER_WR_RMVLAN(f
->fs
.newvlan
== VLAN_REMOVE
||
1349 f
->fs
.newvlan
== VLAN_REWRITE
) |
1350 V_FW_FILTER_WR_HITCNTS(f
->fs
.hitcnts
) |
1351 V_FW_FILTER_WR_TXCHAN(f
->fs
.eport
) |
1352 V_FW_FILTER_WR_PRIO(f
->fs
.prio
) |
1353 V_FW_FILTER_WR_L2TIX(f
->l2t
? f
->l2t
->idx
: 0));
1354 fwr
->ethtype
= htons(f
->fs
.val
.ethtype
);
1355 fwr
->ethtypem
= htons(f
->fs
.mask
.ethtype
);
1356 fwr
->frag_to_ovlan_vldm
=
1357 (V_FW_FILTER_WR_FRAG(f
->fs
.val
.frag
) |
1358 V_FW_FILTER_WR_FRAGM(f
->fs
.mask
.frag
) |
1359 V_FW_FILTER_WR_IVLAN_VLD(f
->fs
.val
.ivlan_vld
) |
1360 V_FW_FILTER_WR_OVLAN_VLD(f
->fs
.val
.ovlan_vld
) |
1361 V_FW_FILTER_WR_IVLAN_VLDM(f
->fs
.mask
.ivlan_vld
) |
1362 V_FW_FILTER_WR_OVLAN_VLDM(f
->fs
.mask
.ovlan_vld
));
1364 fwr
->rx_chan_rx_rpl_iq
=
1365 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1366 V_FW_FILTER_WR_RX_RPL_IQ(adapter
->sge
.fw_evtq
.abs_id
));
1367 fwr
->maci_to_matchtypem
=
1368 htonl(V_FW_FILTER_WR_MACI(f
->fs
.val
.macidx
) |
1369 V_FW_FILTER_WR_MACIM(f
->fs
.mask
.macidx
) |
1370 V_FW_FILTER_WR_FCOE(f
->fs
.val
.fcoe
) |
1371 V_FW_FILTER_WR_FCOEM(f
->fs
.mask
.fcoe
) |
1372 V_FW_FILTER_WR_PORT(f
->fs
.val
.iport
) |
1373 V_FW_FILTER_WR_PORTM(f
->fs
.mask
.iport
) |
1374 V_FW_FILTER_WR_MATCHTYPE(f
->fs
.val
.matchtype
) |
1375 V_FW_FILTER_WR_MATCHTYPEM(f
->fs
.mask
.matchtype
));
1376 fwr
->ptcl
= f
->fs
.val
.proto
;
1377 fwr
->ptclm
= f
->fs
.mask
.proto
;
1378 fwr
->ttyp
= f
->fs
.val
.tos
;
1379 fwr
->ttypm
= f
->fs
.mask
.tos
;
1380 fwr
->ivlan
= htons(f
->fs
.val
.ivlan
);
1381 fwr
->ivlanm
= htons(f
->fs
.mask
.ivlan
);
1382 fwr
->ovlan
= htons(f
->fs
.val
.ovlan
);
1383 fwr
->ovlanm
= htons(f
->fs
.mask
.ovlan
);
1384 memcpy(fwr
->lip
, f
->fs
.val
.lip
, sizeof(fwr
->lip
));
1385 memcpy(fwr
->lipm
, f
->fs
.mask
.lip
, sizeof(fwr
->lipm
));
1386 memcpy(fwr
->fip
, f
->fs
.val
.fip
, sizeof(fwr
->fip
));
1387 memcpy(fwr
->fipm
, f
->fs
.mask
.fip
, sizeof(fwr
->fipm
));
1388 fwr
->lp
= htons(f
->fs
.val
.lport
);
1389 fwr
->lpm
= htons(f
->fs
.mask
.lport
);
1390 fwr
->fp
= htons(f
->fs
.val
.fport
);
1391 fwr
->fpm
= htons(f
->fs
.mask
.fport
);
1393 memcpy(fwr
->sma
, f
->fs
.smac
, sizeof(fwr
->sma
));
1395 /* Mark the filter as "pending" and ship off the Filter Work Request.
1396 * When we get the Work Request Reply we'll clear the pending status.
1399 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, f
->fs
.val
.iport
& 0x3);
1400 t4_ofld_send(adapter
, skb
);
1404 /* Delete the filter at a specified index.
1406 static int del_filter_wr(struct adapter
*adapter
, int fidx
)
1408 struct filter_entry
*f
= &adapter
->tids
.ftid_tab
[fidx
];
1409 struct sk_buff
*skb
;
1410 struct fw_filter_wr
*fwr
;
1411 unsigned int len
, ftid
;
1414 ftid
= adapter
->tids
.ftid_base
+ fidx
;
1416 skb
= alloc_skb(len
, GFP_KERNEL
| __GFP_NOFAIL
);
1417 fwr
= (struct fw_filter_wr
*)__skb_put(skb
, len
);
1418 t4_mk_filtdelwr(ftid
, fwr
, adapter
->sge
.fw_evtq
.abs_id
);
1420 /* Mark the filter as "pending" and ship off the Filter Work Request.
1421 * When we get the Work Request Reply we'll clear the pending status.
1424 t4_mgmt_tx(adapter
, skb
);
1428 static u16
cxgb_select_queue(struct net_device
*dev
, struct sk_buff
*skb
,
1429 void *accel_priv
, select_queue_fallback_t fallback
)
1433 #ifdef CONFIG_CHELSIO_T4_DCB
1434 /* If a Data Center Bridging has been successfully negotiated on this
1435 * link then we'll use the skb's priority to map it to a TX Queue.
1436 * The skb's priority is determined via the VLAN Tag Priority Code
1439 if (cxgb4_dcb_enabled(dev
)) {
1443 err
= vlan_get_tag(skb
, &vlan_tci
);
1444 if (unlikely(err
)) {
1445 if (net_ratelimit())
1447 "TX Packet without VLAN Tag on DCB Link\n");
1450 txq
= (vlan_tci
& VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
;
1454 #endif /* CONFIG_CHELSIO_T4_DCB */
1457 txq
= (skb_rx_queue_recorded(skb
)
1458 ? skb_get_rx_queue(skb
)
1459 : smp_processor_id());
1461 while (unlikely(txq
>= dev
->real_num_tx_queues
))
1462 txq
-= dev
->real_num_tx_queues
;
1467 return fallback(dev
, skb
) % dev
->real_num_tx_queues
;
1470 static inline int is_offload(const struct adapter
*adap
)
1472 return adap
->params
.offload
;
1476 * Implementation of ethtool operations.
1479 static u32
get_msglevel(struct net_device
*dev
)
1481 return netdev2adap(dev
)->msg_enable
;
1484 static void set_msglevel(struct net_device
*dev
, u32 val
)
1486 netdev2adap(dev
)->msg_enable
= val
;
1489 static char stats_strings
[][ETH_GSTRING_LEN
] = {
1492 "TxBroadcastFrames ",
1493 "TxMulticastFrames ",
1499 "TxFrames128To255 ",
1500 "TxFrames256To511 ",
1501 "TxFrames512To1023 ",
1502 "TxFrames1024To1518 ",
1503 "TxFrames1519ToMax ",
1518 "RxBroadcastFrames ",
1519 "RxMulticastFrames ",
1531 "RxFrames128To255 ",
1532 "RxFrames256To511 ",
1533 "RxFrames512To1023 ",
1534 "RxFrames1024To1518 ",
1535 "RxFrames1519ToMax ",
1547 "RxBG0FramesDropped ",
1548 "RxBG1FramesDropped ",
1549 "RxBG2FramesDropped ",
1550 "RxBG3FramesDropped ",
1551 "RxBG0FramesTrunc ",
1552 "RxBG1FramesTrunc ",
1553 "RxBG2FramesTrunc ",
1554 "RxBG3FramesTrunc ",
1563 "WriteCoalSuccess ",
1567 static int get_sset_count(struct net_device
*dev
, int sset
)
1571 return ARRAY_SIZE(stats_strings
);
1577 #define T4_REGMAP_SIZE (160 * 1024)
1578 #define T5_REGMAP_SIZE (332 * 1024)
1580 static int get_regs_len(struct net_device
*dev
)
1582 struct adapter
*adap
= netdev2adap(dev
);
1583 if (is_t4(adap
->params
.chip
))
1584 return T4_REGMAP_SIZE
;
1586 return T5_REGMAP_SIZE
;
1589 static int get_eeprom_len(struct net_device
*dev
)
1594 static void get_drvinfo(struct net_device
*dev
, struct ethtool_drvinfo
*info
)
1596 struct adapter
*adapter
= netdev2adap(dev
);
1598 strlcpy(info
->driver
, KBUILD_MODNAME
, sizeof(info
->driver
));
1599 strlcpy(info
->version
, DRV_VERSION
, sizeof(info
->version
));
1600 strlcpy(info
->bus_info
, pci_name(adapter
->pdev
),
1601 sizeof(info
->bus_info
));
1603 if (adapter
->params
.fw_vers
)
1604 snprintf(info
->fw_version
, sizeof(info
->fw_version
),
1605 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1606 FW_HDR_FW_VER_MAJOR_GET(adapter
->params
.fw_vers
),
1607 FW_HDR_FW_VER_MINOR_GET(adapter
->params
.fw_vers
),
1608 FW_HDR_FW_VER_MICRO_GET(adapter
->params
.fw_vers
),
1609 FW_HDR_FW_VER_BUILD_GET(adapter
->params
.fw_vers
),
1610 FW_HDR_FW_VER_MAJOR_GET(adapter
->params
.tp_vers
),
1611 FW_HDR_FW_VER_MINOR_GET(adapter
->params
.tp_vers
),
1612 FW_HDR_FW_VER_MICRO_GET(adapter
->params
.tp_vers
),
1613 FW_HDR_FW_VER_BUILD_GET(adapter
->params
.tp_vers
));
1616 static void get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
1618 if (stringset
== ETH_SS_STATS
)
1619 memcpy(data
, stats_strings
, sizeof(stats_strings
));
1623 * port stats maintained per queue of the port. They should be in the same
1624 * order as in stats_strings above.
1626 struct queue_port_stats
{
1636 static void collect_sge_port_stats(const struct adapter
*adap
,
1637 const struct port_info
*p
, struct queue_port_stats
*s
)
1640 const struct sge_eth_txq
*tx
= &adap
->sge
.ethtxq
[p
->first_qset
];
1641 const struct sge_eth_rxq
*rx
= &adap
->sge
.ethrxq
[p
->first_qset
];
1643 memset(s
, 0, sizeof(*s
));
1644 for (i
= 0; i
< p
->nqsets
; i
++, rx
++, tx
++) {
1646 s
->tx_csum
+= tx
->tx_cso
;
1647 s
->rx_csum
+= rx
->stats
.rx_cso
;
1648 s
->vlan_ex
+= rx
->stats
.vlan_ex
;
1649 s
->vlan_ins
+= tx
->vlan_ins
;
1650 s
->gro_pkts
+= rx
->stats
.lro_pkts
;
1651 s
->gro_merged
+= rx
->stats
.lro_merged
;
1655 static void get_stats(struct net_device
*dev
, struct ethtool_stats
*stats
,
1658 struct port_info
*pi
= netdev_priv(dev
);
1659 struct adapter
*adapter
= pi
->adapter
;
1662 t4_get_port_stats(adapter
, pi
->tx_chan
, (struct port_stats
*)data
);
1664 data
+= sizeof(struct port_stats
) / sizeof(u64
);
1665 collect_sge_port_stats(adapter
, pi
, (struct queue_port_stats
*)data
);
1666 data
+= sizeof(struct queue_port_stats
) / sizeof(u64
);
1667 if (!is_t4(adapter
->params
.chip
)) {
1668 t4_write_reg(adapter
, SGE_STAT_CFG
, STATSOURCE_T5(7));
1669 val1
= t4_read_reg(adapter
, SGE_STAT_TOTAL
);
1670 val2
= t4_read_reg(adapter
, SGE_STAT_MATCH
);
1671 *data
= val1
- val2
;
1676 memset(data
, 0, 2 * sizeof(u64
));
1682 * Return a version number to identify the type of adapter. The scheme is:
1683 * - bits 0..9: chip version
1684 * - bits 10..15: chip revision
1685 * - bits 16..23: register dump version
1687 static inline unsigned int mk_adap_vers(const struct adapter
*ap
)
1689 return CHELSIO_CHIP_VERSION(ap
->params
.chip
) |
1690 (CHELSIO_CHIP_RELEASE(ap
->params
.chip
) << 10) | (1 << 16);
1693 static void reg_block_dump(struct adapter
*ap
, void *buf
, unsigned int start
,
1696 u32
*p
= buf
+ start
;
1698 for ( ; start
<= end
; start
+= sizeof(u32
))
1699 *p
++ = t4_read_reg(ap
, start
);
1702 static void get_regs(struct net_device
*dev
, struct ethtool_regs
*regs
,
1705 static const unsigned int t4_reg_ranges
[] = {
1925 static const unsigned int t5_reg_ranges
[] = {
2353 struct adapter
*ap
= netdev2adap(dev
);
2354 static const unsigned int *reg_ranges
;
2355 int arr_size
= 0, buf_size
= 0;
2357 if (is_t4(ap
->params
.chip
)) {
2358 reg_ranges
= &t4_reg_ranges
[0];
2359 arr_size
= ARRAY_SIZE(t4_reg_ranges
);
2360 buf_size
= T4_REGMAP_SIZE
;
2362 reg_ranges
= &t5_reg_ranges
[0];
2363 arr_size
= ARRAY_SIZE(t5_reg_ranges
);
2364 buf_size
= T5_REGMAP_SIZE
;
2367 regs
->version
= mk_adap_vers(ap
);
2369 memset(buf
, 0, buf_size
);
2370 for (i
= 0; i
< arr_size
; i
+= 2)
2371 reg_block_dump(ap
, buf
, reg_ranges
[i
], reg_ranges
[i
+ 1]);
2374 static int restart_autoneg(struct net_device
*dev
)
2376 struct port_info
*p
= netdev_priv(dev
);
2378 if (!netif_running(dev
))
2380 if (p
->link_cfg
.autoneg
!= AUTONEG_ENABLE
)
2382 t4_restart_aneg(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
);
2386 static int identify_port(struct net_device
*dev
,
2387 enum ethtool_phys_id_state state
)
2390 struct adapter
*adap
= netdev2adap(dev
);
2392 if (state
== ETHTOOL_ID_ACTIVE
)
2394 else if (state
== ETHTOOL_ID_INACTIVE
)
2399 return t4_identify_port(adap
, adap
->fn
, netdev2pinfo(dev
)->viid
, val
);
2402 static unsigned int from_fw_linkcaps(unsigned int type
, unsigned int caps
)
2406 if (type
== FW_PORT_TYPE_BT_SGMII
|| type
== FW_PORT_TYPE_BT_XFI
||
2407 type
== FW_PORT_TYPE_BT_XAUI
) {
2409 if (caps
& FW_PORT_CAP_SPEED_100M
)
2410 v
|= SUPPORTED_100baseT_Full
;
2411 if (caps
& FW_PORT_CAP_SPEED_1G
)
2412 v
|= SUPPORTED_1000baseT_Full
;
2413 if (caps
& FW_PORT_CAP_SPEED_10G
)
2414 v
|= SUPPORTED_10000baseT_Full
;
2415 } else if (type
== FW_PORT_TYPE_KX4
|| type
== FW_PORT_TYPE_KX
) {
2416 v
|= SUPPORTED_Backplane
;
2417 if (caps
& FW_PORT_CAP_SPEED_1G
)
2418 v
|= SUPPORTED_1000baseKX_Full
;
2419 if (caps
& FW_PORT_CAP_SPEED_10G
)
2420 v
|= SUPPORTED_10000baseKX4_Full
;
2421 } else if (type
== FW_PORT_TYPE_KR
)
2422 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseKR_Full
;
2423 else if (type
== FW_PORT_TYPE_BP_AP
)
2424 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseR_FEC
|
2425 SUPPORTED_10000baseKR_Full
| SUPPORTED_1000baseKX_Full
;
2426 else if (type
== FW_PORT_TYPE_BP4_AP
)
2427 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseR_FEC
|
2428 SUPPORTED_10000baseKR_Full
| SUPPORTED_1000baseKX_Full
|
2429 SUPPORTED_10000baseKX4_Full
;
2430 else if (type
== FW_PORT_TYPE_FIBER_XFI
||
2431 type
== FW_PORT_TYPE_FIBER_XAUI
|| type
== FW_PORT_TYPE_SFP
)
2432 v
|= SUPPORTED_FIBRE
;
2433 else if (type
== FW_PORT_TYPE_BP40_BA
)
2434 v
|= SUPPORTED_40000baseSR4_Full
;
2436 if (caps
& FW_PORT_CAP_ANEG
)
2437 v
|= SUPPORTED_Autoneg
;
2441 static unsigned int to_fw_linkcaps(unsigned int caps
)
2445 if (caps
& ADVERTISED_100baseT_Full
)
2446 v
|= FW_PORT_CAP_SPEED_100M
;
2447 if (caps
& ADVERTISED_1000baseT_Full
)
2448 v
|= FW_PORT_CAP_SPEED_1G
;
2449 if (caps
& ADVERTISED_10000baseT_Full
)
2450 v
|= FW_PORT_CAP_SPEED_10G
;
2451 if (caps
& ADVERTISED_40000baseSR4_Full
)
2452 v
|= FW_PORT_CAP_SPEED_40G
;
2456 static int get_settings(struct net_device
*dev
, struct ethtool_cmd
*cmd
)
2458 const struct port_info
*p
= netdev_priv(dev
);
2460 if (p
->port_type
== FW_PORT_TYPE_BT_SGMII
||
2461 p
->port_type
== FW_PORT_TYPE_BT_XFI
||
2462 p
->port_type
== FW_PORT_TYPE_BT_XAUI
)
2463 cmd
->port
= PORT_TP
;
2464 else if (p
->port_type
== FW_PORT_TYPE_FIBER_XFI
||
2465 p
->port_type
== FW_PORT_TYPE_FIBER_XAUI
)
2466 cmd
->port
= PORT_FIBRE
;
2467 else if (p
->port_type
== FW_PORT_TYPE_SFP
||
2468 p
->port_type
== FW_PORT_TYPE_QSFP_10G
||
2469 p
->port_type
== FW_PORT_TYPE_QSFP
) {
2470 if (p
->mod_type
== FW_PORT_MOD_TYPE_LR
||
2471 p
->mod_type
== FW_PORT_MOD_TYPE_SR
||
2472 p
->mod_type
== FW_PORT_MOD_TYPE_ER
||
2473 p
->mod_type
== FW_PORT_MOD_TYPE_LRM
)
2474 cmd
->port
= PORT_FIBRE
;
2475 else if (p
->mod_type
== FW_PORT_MOD_TYPE_TWINAX_PASSIVE
||
2476 p
->mod_type
== FW_PORT_MOD_TYPE_TWINAX_ACTIVE
)
2477 cmd
->port
= PORT_DA
;
2479 cmd
->port
= PORT_OTHER
;
2481 cmd
->port
= PORT_OTHER
;
2483 if (p
->mdio_addr
>= 0) {
2484 cmd
->phy_address
= p
->mdio_addr
;
2485 cmd
->transceiver
= XCVR_EXTERNAL
;
2486 cmd
->mdio_support
= p
->port_type
== FW_PORT_TYPE_BT_SGMII
?
2487 MDIO_SUPPORTS_C22
: MDIO_SUPPORTS_C45
;
2489 cmd
->phy_address
= 0; /* not really, but no better option */
2490 cmd
->transceiver
= XCVR_INTERNAL
;
2491 cmd
->mdio_support
= 0;
2494 cmd
->supported
= from_fw_linkcaps(p
->port_type
, p
->link_cfg
.supported
);
2495 cmd
->advertising
= from_fw_linkcaps(p
->port_type
,
2496 p
->link_cfg
.advertising
);
2497 ethtool_cmd_speed_set(cmd
,
2498 netif_carrier_ok(dev
) ? p
->link_cfg
.speed
: 0);
2499 cmd
->duplex
= DUPLEX_FULL
;
2500 cmd
->autoneg
= p
->link_cfg
.autoneg
;
2506 static unsigned int speed_to_caps(int speed
)
2509 return FW_PORT_CAP_SPEED_100M
;
2511 return FW_PORT_CAP_SPEED_1G
;
2513 return FW_PORT_CAP_SPEED_10G
;
2515 return FW_PORT_CAP_SPEED_40G
;
2519 static int set_settings(struct net_device
*dev
, struct ethtool_cmd
*cmd
)
2522 struct port_info
*p
= netdev_priv(dev
);
2523 struct link_config
*lc
= &p
->link_cfg
;
2524 u32 speed
= ethtool_cmd_speed(cmd
);
2526 if (cmd
->duplex
!= DUPLEX_FULL
) /* only full-duplex supported */
2529 if (!(lc
->supported
& FW_PORT_CAP_ANEG
)) {
2531 * PHY offers a single speed. See if that's what's
2534 if (cmd
->autoneg
== AUTONEG_DISABLE
&&
2535 (lc
->supported
& speed_to_caps(speed
)))
2540 if (cmd
->autoneg
== AUTONEG_DISABLE
) {
2541 cap
= speed_to_caps(speed
);
2543 if (!(lc
->supported
& cap
) ||
2548 lc
->requested_speed
= cap
;
2549 lc
->advertising
= 0;
2551 cap
= to_fw_linkcaps(cmd
->advertising
);
2552 if (!(lc
->supported
& cap
))
2554 lc
->requested_speed
= 0;
2555 lc
->advertising
= cap
| FW_PORT_CAP_ANEG
;
2557 lc
->autoneg
= cmd
->autoneg
;
2559 if (netif_running(dev
))
2560 return t4_link_start(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
,
2565 static void get_pauseparam(struct net_device
*dev
,
2566 struct ethtool_pauseparam
*epause
)
2568 struct port_info
*p
= netdev_priv(dev
);
2570 epause
->autoneg
= (p
->link_cfg
.requested_fc
& PAUSE_AUTONEG
) != 0;
2571 epause
->rx_pause
= (p
->link_cfg
.fc
& PAUSE_RX
) != 0;
2572 epause
->tx_pause
= (p
->link_cfg
.fc
& PAUSE_TX
) != 0;
2575 static int set_pauseparam(struct net_device
*dev
,
2576 struct ethtool_pauseparam
*epause
)
2578 struct port_info
*p
= netdev_priv(dev
);
2579 struct link_config
*lc
= &p
->link_cfg
;
2581 if (epause
->autoneg
== AUTONEG_DISABLE
)
2582 lc
->requested_fc
= 0;
2583 else if (lc
->supported
& FW_PORT_CAP_ANEG
)
2584 lc
->requested_fc
= PAUSE_AUTONEG
;
2588 if (epause
->rx_pause
)
2589 lc
->requested_fc
|= PAUSE_RX
;
2590 if (epause
->tx_pause
)
2591 lc
->requested_fc
|= PAUSE_TX
;
2592 if (netif_running(dev
))
2593 return t4_link_start(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
,
2598 static void get_sge_param(struct net_device
*dev
, struct ethtool_ringparam
*e
)
2600 const struct port_info
*pi
= netdev_priv(dev
);
2601 const struct sge
*s
= &pi
->adapter
->sge
;
2603 e
->rx_max_pending
= MAX_RX_BUFFERS
;
2604 e
->rx_mini_max_pending
= MAX_RSPQ_ENTRIES
;
2605 e
->rx_jumbo_max_pending
= 0;
2606 e
->tx_max_pending
= MAX_TXQ_ENTRIES
;
2608 e
->rx_pending
= s
->ethrxq
[pi
->first_qset
].fl
.size
- 8;
2609 e
->rx_mini_pending
= s
->ethrxq
[pi
->first_qset
].rspq
.size
;
2610 e
->rx_jumbo_pending
= 0;
2611 e
->tx_pending
= s
->ethtxq
[pi
->first_qset
].q
.size
;
2614 static int set_sge_param(struct net_device
*dev
, struct ethtool_ringparam
*e
)
2617 const struct port_info
*pi
= netdev_priv(dev
);
2618 struct adapter
*adapter
= pi
->adapter
;
2619 struct sge
*s
= &adapter
->sge
;
2621 if (e
->rx_pending
> MAX_RX_BUFFERS
|| e
->rx_jumbo_pending
||
2622 e
->tx_pending
> MAX_TXQ_ENTRIES
||
2623 e
->rx_mini_pending
> MAX_RSPQ_ENTRIES
||
2624 e
->rx_mini_pending
< MIN_RSPQ_ENTRIES
||
2625 e
->rx_pending
< MIN_FL_ENTRIES
|| e
->tx_pending
< MIN_TXQ_ENTRIES
)
2628 if (adapter
->flags
& FULL_INIT_DONE
)
2631 for (i
= 0; i
< pi
->nqsets
; ++i
) {
2632 s
->ethtxq
[pi
->first_qset
+ i
].q
.size
= e
->tx_pending
;
2633 s
->ethrxq
[pi
->first_qset
+ i
].fl
.size
= e
->rx_pending
+ 8;
2634 s
->ethrxq
[pi
->first_qset
+ i
].rspq
.size
= e
->rx_mini_pending
;
2639 static int closest_timer(const struct sge
*s
, int time
)
2641 int i
, delta
, match
= 0, min_delta
= INT_MAX
;
2643 for (i
= 0; i
< ARRAY_SIZE(s
->timer_val
); i
++) {
2644 delta
= time
- s
->timer_val
[i
];
2647 if (delta
< min_delta
) {
2655 static int closest_thres(const struct sge
*s
, int thres
)
2657 int i
, delta
, match
= 0, min_delta
= INT_MAX
;
2659 for (i
= 0; i
< ARRAY_SIZE(s
->counter_val
); i
++) {
2660 delta
= thres
- s
->counter_val
[i
];
2663 if (delta
< min_delta
) {
2672 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2674 static unsigned int qtimer_val(const struct adapter
*adap
,
2675 const struct sge_rspq
*q
)
2677 unsigned int idx
= q
->intr_params
>> 1;
2679 return idx
< SGE_NTIMERS
? adap
->sge
.timer_val
[idx
] : 0;
2683 * set_rspq_intr_params - set a queue's interrupt holdoff parameters
2685 * @us: the hold-off time in us, or 0 to disable timer
2686 * @cnt: the hold-off packet count, or 0 to disable counter
2688 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2689 * one of the two needs to be enabled for the queue to generate interrupts.
2691 static int set_rspq_intr_params(struct sge_rspq
*q
,
2692 unsigned int us
, unsigned int cnt
)
2694 struct adapter
*adap
= q
->adap
;
2696 if ((us
| cnt
) == 0)
2703 new_idx
= closest_thres(&adap
->sge
, cnt
);
2704 if (q
->desc
&& q
->pktcnt_idx
!= new_idx
) {
2705 /* the queue has already been created, update it */
2706 v
= FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ
) |
2707 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH
) |
2708 FW_PARAMS_PARAM_YZ(q
->cntxt_id
);
2709 err
= t4_set_params(adap
, adap
->fn
, adap
->fn
, 0, 1, &v
,
2714 q
->pktcnt_idx
= new_idx
;
2717 us
= us
== 0 ? 6 : closest_timer(&adap
->sge
, us
);
2718 q
->intr_params
= QINTR_TIMER_IDX(us
) | (cnt
> 0 ? QINTR_CNT_EN
: 0);
2723 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2724 * @dev: the network device
2725 * @us: the hold-off time in us, or 0 to disable timer
2726 * @cnt: the hold-off packet count, or 0 to disable counter
2728 * Set the RX interrupt hold-off parameters for a network device.
2730 static int set_rx_intr_params(struct net_device
*dev
,
2731 unsigned int us
, unsigned int cnt
)
2734 struct port_info
*pi
= netdev_priv(dev
);
2735 struct adapter
*adap
= pi
->adapter
;
2736 struct sge_eth_rxq
*q
= &adap
->sge
.ethrxq
[pi
->first_qset
];
2738 for (i
= 0; i
< pi
->nqsets
; i
++, q
++) {
2739 err
= set_rspq_intr_params(&q
->rspq
, us
, cnt
);
2746 static int set_coalesce(struct net_device
*dev
, struct ethtool_coalesce
*c
)
2748 return set_rx_intr_params(dev
, c
->rx_coalesce_usecs
,
2749 c
->rx_max_coalesced_frames
);
2752 static int get_coalesce(struct net_device
*dev
, struct ethtool_coalesce
*c
)
2754 const struct port_info
*pi
= netdev_priv(dev
);
2755 const struct adapter
*adap
= pi
->adapter
;
2756 const struct sge_rspq
*rq
= &adap
->sge
.ethrxq
[pi
->first_qset
].rspq
;
2758 c
->rx_coalesce_usecs
= qtimer_val(adap
, rq
);
2759 c
->rx_max_coalesced_frames
= (rq
->intr_params
& QINTR_CNT_EN
) ?
2760 adap
->sge
.counter_val
[rq
->pktcnt_idx
] : 0;
2765 * eeprom_ptov - translate a physical EEPROM address to virtual
2766 * @phys_addr: the physical EEPROM address
2767 * @fn: the PCI function number
2768 * @sz: size of function-specific area
2770 * Translate a physical EEPROM address to virtual. The first 1K is
2771 * accessed through virtual addresses starting at 31K, the rest is
2772 * accessed through virtual addresses starting at 0.
2774 * The mapping is as follows:
2775 * [0..1K) -> [31K..32K)
2776 * [1K..1K+A) -> [31K-A..31K)
2777 * [1K+A..ES) -> [0..ES-A-1K)
2779 * where A = @fn * @sz, and ES = EEPROM size.
2781 static int eeprom_ptov(unsigned int phys_addr
, unsigned int fn
, unsigned int sz
)
2784 if (phys_addr
< 1024)
2785 return phys_addr
+ (31 << 10);
2786 if (phys_addr
< 1024 + fn
)
2787 return 31744 - fn
+ phys_addr
- 1024;
2788 if (phys_addr
< EEPROMSIZE
)
2789 return phys_addr
- 1024 - fn
;
2794 * The next two routines implement eeprom read/write from physical addresses.
2796 static int eeprom_rd_phys(struct adapter
*adap
, unsigned int phys_addr
, u32
*v
)
2798 int vaddr
= eeprom_ptov(phys_addr
, adap
->fn
, EEPROMPFSIZE
);
2801 vaddr
= pci_read_vpd(adap
->pdev
, vaddr
, sizeof(u32
), v
);
2802 return vaddr
< 0 ? vaddr
: 0;
2805 static int eeprom_wr_phys(struct adapter
*adap
, unsigned int phys_addr
, u32 v
)
2807 int vaddr
= eeprom_ptov(phys_addr
, adap
->fn
, EEPROMPFSIZE
);
2810 vaddr
= pci_write_vpd(adap
->pdev
, vaddr
, sizeof(u32
), &v
);
2811 return vaddr
< 0 ? vaddr
: 0;
2814 #define EEPROM_MAGIC 0x38E2F10C
2816 static int get_eeprom(struct net_device
*dev
, struct ethtool_eeprom
*e
,
2820 struct adapter
*adapter
= netdev2adap(dev
);
2822 u8
*buf
= kmalloc(EEPROMSIZE
, GFP_KERNEL
);
2826 e
->magic
= EEPROM_MAGIC
;
2827 for (i
= e
->offset
& ~3; !err
&& i
< e
->offset
+ e
->len
; i
+= 4)
2828 err
= eeprom_rd_phys(adapter
, i
, (u32
*)&buf
[i
]);
2831 memcpy(data
, buf
+ e
->offset
, e
->len
);
2836 static int set_eeprom(struct net_device
*dev
, struct ethtool_eeprom
*eeprom
,
2841 u32 aligned_offset
, aligned_len
, *p
;
2842 struct adapter
*adapter
= netdev2adap(dev
);
2844 if (eeprom
->magic
!= EEPROM_MAGIC
)
2847 aligned_offset
= eeprom
->offset
& ~3;
2848 aligned_len
= (eeprom
->len
+ (eeprom
->offset
& 3) + 3) & ~3;
2850 if (adapter
->fn
> 0) {
2851 u32 start
= 1024 + adapter
->fn
* EEPROMPFSIZE
;
2853 if (aligned_offset
< start
||
2854 aligned_offset
+ aligned_len
> start
+ EEPROMPFSIZE
)
2858 if (aligned_offset
!= eeprom
->offset
|| aligned_len
!= eeprom
->len
) {
2860 * RMW possibly needed for first or last words.
2862 buf
= kmalloc(aligned_len
, GFP_KERNEL
);
2865 err
= eeprom_rd_phys(adapter
, aligned_offset
, (u32
*)buf
);
2866 if (!err
&& aligned_len
> 4)
2867 err
= eeprom_rd_phys(adapter
,
2868 aligned_offset
+ aligned_len
- 4,
2869 (u32
*)&buf
[aligned_len
- 4]);
2872 memcpy(buf
+ (eeprom
->offset
& 3), data
, eeprom
->len
);
2876 err
= t4_seeprom_wp(adapter
, false);
2880 for (p
= (u32
*)buf
; !err
&& aligned_len
; aligned_len
-= 4, p
++) {
2881 err
= eeprom_wr_phys(adapter
, aligned_offset
, *p
);
2882 aligned_offset
+= 4;
2886 err
= t4_seeprom_wp(adapter
, true);
2893 static int set_flash(struct net_device
*netdev
, struct ethtool_flash
*ef
)
2896 const struct firmware
*fw
;
2897 struct adapter
*adap
= netdev2adap(netdev
);
2899 ef
->data
[sizeof(ef
->data
) - 1] = '\0';
2900 ret
= request_firmware(&fw
, ef
->data
, adap
->pdev_dev
);
2904 ret
= t4_load_fw(adap
, fw
->data
, fw
->size
);
2905 release_firmware(fw
);
2907 dev_info(adap
->pdev_dev
, "loaded firmware %s\n", ef
->data
);
2911 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2912 #define BCAST_CRC 0xa0ccc1a6
2914 static void get_wol(struct net_device
*dev
, struct ethtool_wolinfo
*wol
)
2916 wol
->supported
= WAKE_BCAST
| WAKE_MAGIC
;
2917 wol
->wolopts
= netdev2adap(dev
)->wol
;
2918 memset(&wol
->sopass
, 0, sizeof(wol
->sopass
));
2921 static int set_wol(struct net_device
*dev
, struct ethtool_wolinfo
*wol
)
2924 struct port_info
*pi
= netdev_priv(dev
);
2926 if (wol
->wolopts
& ~WOL_SUPPORTED
)
2928 t4_wol_magic_enable(pi
->adapter
, pi
->tx_chan
,
2929 (wol
->wolopts
& WAKE_MAGIC
) ? dev
->dev_addr
: NULL
);
2930 if (wol
->wolopts
& WAKE_BCAST
) {
2931 err
= t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 0xfe, ~0ULL,
2934 err
= t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 1,
2935 ~6ULL, ~0ULL, BCAST_CRC
, true);
2937 t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 0, 0, 0, 0, false);
2941 static int cxgb_set_features(struct net_device
*dev
, netdev_features_t features
)
2943 const struct port_info
*pi
= netdev_priv(dev
);
2944 netdev_features_t changed
= dev
->features
^ features
;
2947 if (!(changed
& NETIF_F_HW_VLAN_CTAG_RX
))
2950 err
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, -1,
2952 !!(features
& NETIF_F_HW_VLAN_CTAG_RX
), true);
2954 dev
->features
= features
^ NETIF_F_HW_VLAN_CTAG_RX
;
2958 static u32
get_rss_table_size(struct net_device
*dev
)
2960 const struct port_info
*pi
= netdev_priv(dev
);
2962 return pi
->rss_size
;
2965 static int get_rss_table(struct net_device
*dev
, u32
*p
, u8
*key
)
2967 const struct port_info
*pi
= netdev_priv(dev
);
2968 unsigned int n
= pi
->rss_size
;
2975 static int set_rss_table(struct net_device
*dev
, const u32
*p
, const u8
*key
)
2978 struct port_info
*pi
= netdev_priv(dev
);
2980 for (i
= 0; i
< pi
->rss_size
; i
++)
2982 if (pi
->adapter
->flags
& FULL_INIT_DONE
)
2983 return write_rss(pi
, pi
->rss
);
2987 static int get_rxnfc(struct net_device
*dev
, struct ethtool_rxnfc
*info
,
2990 const struct port_info
*pi
= netdev_priv(dev
);
2992 switch (info
->cmd
) {
2993 case ETHTOOL_GRXFH
: {
2994 unsigned int v
= pi
->rss_mode
;
2997 switch (info
->flow_type
) {
2999 if (v
& FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
)
3000 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
3001 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
3002 else if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
3003 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
3006 if ((v
& FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
) &&
3007 (v
& FW_RSS_VI_CONFIG_CMD_UDPEN
))
3008 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
3009 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
3010 else if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
3011 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
3014 case AH_ESP_V4_FLOW
:
3016 if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
3017 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
3020 if (v
& FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
)
3021 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
3022 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
3023 else if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
3024 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
3027 if ((v
& FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
) &&
3028 (v
& FW_RSS_VI_CONFIG_CMD_UDPEN
))
3029 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
3030 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
3031 else if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
3032 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
3035 case AH_ESP_V6_FLOW
:
3037 if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
3038 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
3043 case ETHTOOL_GRXRINGS
:
3044 info
->data
= pi
->nqsets
;
3050 static const struct ethtool_ops cxgb_ethtool_ops
= {
3051 .get_settings
= get_settings
,
3052 .set_settings
= set_settings
,
3053 .get_drvinfo
= get_drvinfo
,
3054 .get_msglevel
= get_msglevel
,
3055 .set_msglevel
= set_msglevel
,
3056 .get_ringparam
= get_sge_param
,
3057 .set_ringparam
= set_sge_param
,
3058 .get_coalesce
= get_coalesce
,
3059 .set_coalesce
= set_coalesce
,
3060 .get_eeprom_len
= get_eeprom_len
,
3061 .get_eeprom
= get_eeprom
,
3062 .set_eeprom
= set_eeprom
,
3063 .get_pauseparam
= get_pauseparam
,
3064 .set_pauseparam
= set_pauseparam
,
3065 .get_link
= ethtool_op_get_link
,
3066 .get_strings
= get_strings
,
3067 .set_phys_id
= identify_port
,
3068 .nway_reset
= restart_autoneg
,
3069 .get_sset_count
= get_sset_count
,
3070 .get_ethtool_stats
= get_stats
,
3071 .get_regs_len
= get_regs_len
,
3072 .get_regs
= get_regs
,
3075 .get_rxnfc
= get_rxnfc
,
3076 .get_rxfh_indir_size
= get_rss_table_size
,
3077 .get_rxfh
= get_rss_table
,
3078 .set_rxfh
= set_rss_table
,
3079 .flash_device
= set_flash
,
3085 static ssize_t
mem_read(struct file
*file
, char __user
*buf
, size_t count
,
3089 loff_t avail
= file_inode(file
)->i_size
;
3090 unsigned int mem
= (uintptr_t)file
->private_data
& 3;
3091 struct adapter
*adap
= file
->private_data
- mem
;
3099 if (count
> avail
- pos
)
3100 count
= avail
- pos
;
3102 data
= t4_alloc_mem(count
);
3106 spin_lock(&adap
->win0_lock
);
3107 ret
= t4_memory_rw(adap
, 0, mem
, pos
, count
, data
, T4_MEMORY_READ
);
3108 spin_unlock(&adap
->win0_lock
);
3113 ret
= copy_to_user(buf
, data
, count
);
3119 *ppos
= pos
+ count
;
3123 static const struct file_operations mem_debugfs_fops
= {
3124 .owner
= THIS_MODULE
,
3125 .open
= simple_open
,
3127 .llseek
= default_llseek
,
3130 static void add_debugfs_mem(struct adapter
*adap
, const char *name
,
3131 unsigned int idx
, unsigned int size_mb
)
3135 de
= debugfs_create_file(name
, S_IRUSR
, adap
->debugfs_root
,
3136 (void *)adap
+ idx
, &mem_debugfs_fops
);
3137 if (de
&& de
->d_inode
)
3138 de
->d_inode
->i_size
= size_mb
<< 20;
3141 static int setup_debugfs(struct adapter
*adap
)
3146 if (IS_ERR_OR_NULL(adap
->debugfs_root
))
3149 i
= t4_read_reg(adap
, MA_TARGET_MEM_ENABLE
);
3150 if (i
& EDRAM0_ENABLE
) {
3151 size
= t4_read_reg(adap
, MA_EDRAM0_BAR
);
3152 add_debugfs_mem(adap
, "edc0", MEM_EDC0
, EDRAM_SIZE_GET(size
));
3154 if (i
& EDRAM1_ENABLE
) {
3155 size
= t4_read_reg(adap
, MA_EDRAM1_BAR
);
3156 add_debugfs_mem(adap
, "edc1", MEM_EDC1
, EDRAM_SIZE_GET(size
));
3158 if (is_t4(adap
->params
.chip
)) {
3159 size
= t4_read_reg(adap
, MA_EXT_MEMORY_BAR
);
3160 if (i
& EXT_MEM_ENABLE
)
3161 add_debugfs_mem(adap
, "mc", MEM_MC
,
3162 EXT_MEM_SIZE_GET(size
));
3164 if (i
& EXT_MEM_ENABLE
) {
3165 size
= t4_read_reg(adap
, MA_EXT_MEMORY_BAR
);
3166 add_debugfs_mem(adap
, "mc0", MEM_MC0
,
3167 EXT_MEM_SIZE_GET(size
));
3169 if (i
& EXT_MEM1_ENABLE
) {
3170 size
= t4_read_reg(adap
, MA_EXT_MEMORY1_BAR
);
3171 add_debugfs_mem(adap
, "mc1", MEM_MC1
,
3172 EXT_MEM_SIZE_GET(size
));
3176 debugfs_create_file("l2t", S_IRUSR
, adap
->debugfs_root
, adap
,
3182 * upper-layer driver support
3186 * Allocate an active-open TID and set it to the supplied value.
3188 int cxgb4_alloc_atid(struct tid_info
*t
, void *data
)
3192 spin_lock_bh(&t
->atid_lock
);
3194 union aopen_entry
*p
= t
->afree
;
3196 atid
= (p
- t
->atid_tab
) + t
->atid_base
;
3201 spin_unlock_bh(&t
->atid_lock
);
3204 EXPORT_SYMBOL(cxgb4_alloc_atid
);
3207 * Release an active-open TID.
3209 void cxgb4_free_atid(struct tid_info
*t
, unsigned int atid
)
3211 union aopen_entry
*p
= &t
->atid_tab
[atid
- t
->atid_base
];
3213 spin_lock_bh(&t
->atid_lock
);
3217 spin_unlock_bh(&t
->atid_lock
);
3219 EXPORT_SYMBOL(cxgb4_free_atid
);
3222 * Allocate a server TID and set it to the supplied value.
3224 int cxgb4_alloc_stid(struct tid_info
*t
, int family
, void *data
)
3228 spin_lock_bh(&t
->stid_lock
);
3229 if (family
== PF_INET
) {
3230 stid
= find_first_zero_bit(t
->stid_bmap
, t
->nstids
);
3231 if (stid
< t
->nstids
)
3232 __set_bit(stid
, t
->stid_bmap
);
3236 stid
= bitmap_find_free_region(t
->stid_bmap
, t
->nstids
, 2);
3241 t
->stid_tab
[stid
].data
= data
;
3242 stid
+= t
->stid_base
;
3243 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3244 * This is equivalent to 4 TIDs. With CLIP enabled it
3247 if (family
== PF_INET
)
3250 t
->stids_in_use
+= 4;
3252 spin_unlock_bh(&t
->stid_lock
);
3255 EXPORT_SYMBOL(cxgb4_alloc_stid
);
3257 /* Allocate a server filter TID and set it to the supplied value.
3259 int cxgb4_alloc_sftid(struct tid_info
*t
, int family
, void *data
)
3263 spin_lock_bh(&t
->stid_lock
);
3264 if (family
== PF_INET
) {
3265 stid
= find_next_zero_bit(t
->stid_bmap
,
3266 t
->nstids
+ t
->nsftids
, t
->nstids
);
3267 if (stid
< (t
->nstids
+ t
->nsftids
))
3268 __set_bit(stid
, t
->stid_bmap
);
3275 t
->stid_tab
[stid
].data
= data
;
3277 stid
+= t
->sftid_base
;
3280 spin_unlock_bh(&t
->stid_lock
);
3283 EXPORT_SYMBOL(cxgb4_alloc_sftid
);
3285 /* Release a server TID.
3287 void cxgb4_free_stid(struct tid_info
*t
, unsigned int stid
, int family
)
3289 /* Is it a server filter TID? */
3290 if (t
->nsftids
&& (stid
>= t
->sftid_base
)) {
3291 stid
-= t
->sftid_base
;
3294 stid
-= t
->stid_base
;
3297 spin_lock_bh(&t
->stid_lock
);
3298 if (family
== PF_INET
)
3299 __clear_bit(stid
, t
->stid_bmap
);
3301 bitmap_release_region(t
->stid_bmap
, stid
, 2);
3302 t
->stid_tab
[stid
].data
= NULL
;
3303 if (family
== PF_INET
)
3306 t
->stids_in_use
-= 4;
3307 spin_unlock_bh(&t
->stid_lock
);
3309 EXPORT_SYMBOL(cxgb4_free_stid
);
3312 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3314 static void mk_tid_release(struct sk_buff
*skb
, unsigned int chan
,
3317 struct cpl_tid_release
*req
;
3319 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, chan
);
3320 req
= (struct cpl_tid_release
*)__skb_put(skb
, sizeof(*req
));
3321 INIT_TP_WR(req
, tid
);
3322 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE
, tid
));
3326 * Queue a TID release request and if necessary schedule a work queue to
3329 static void cxgb4_queue_tid_release(struct tid_info
*t
, unsigned int chan
,
3332 void **p
= &t
->tid_tab
[tid
];
3333 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
3335 spin_lock_bh(&adap
->tid_release_lock
);
3336 *p
= adap
->tid_release_head
;
3337 /* Low 2 bits encode the Tx channel number */
3338 adap
->tid_release_head
= (void **)((uintptr_t)p
| chan
);
3339 if (!adap
->tid_release_task_busy
) {
3340 adap
->tid_release_task_busy
= true;
3341 queue_work(adap
->workq
, &adap
->tid_release_task
);
3343 spin_unlock_bh(&adap
->tid_release_lock
);
3347 * Process the list of pending TID release requests.
3349 static void process_tid_release_list(struct work_struct
*work
)
3351 struct sk_buff
*skb
;
3352 struct adapter
*adap
;
3354 adap
= container_of(work
, struct adapter
, tid_release_task
);
3356 spin_lock_bh(&adap
->tid_release_lock
);
3357 while (adap
->tid_release_head
) {
3358 void **p
= adap
->tid_release_head
;
3359 unsigned int chan
= (uintptr_t)p
& 3;
3360 p
= (void *)p
- chan
;
3362 adap
->tid_release_head
= *p
;
3364 spin_unlock_bh(&adap
->tid_release_lock
);
3366 while (!(skb
= alloc_skb(sizeof(struct cpl_tid_release
),
3368 schedule_timeout_uninterruptible(1);
3370 mk_tid_release(skb
, chan
, p
- adap
->tids
.tid_tab
);
3371 t4_ofld_send(adap
, skb
);
3372 spin_lock_bh(&adap
->tid_release_lock
);
3374 adap
->tid_release_task_busy
= false;
3375 spin_unlock_bh(&adap
->tid_release_lock
);
3379 * Release a TID and inform HW. If we are unable to allocate the release
3380 * message we defer to a work queue.
3382 void cxgb4_remove_tid(struct tid_info
*t
, unsigned int chan
, unsigned int tid
)
3385 struct sk_buff
*skb
;
3386 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
3388 old
= t
->tid_tab
[tid
];
3389 skb
= alloc_skb(sizeof(struct cpl_tid_release
), GFP_ATOMIC
);
3391 t
->tid_tab
[tid
] = NULL
;
3392 mk_tid_release(skb
, chan
, tid
);
3393 t4_ofld_send(adap
, skb
);
3395 cxgb4_queue_tid_release(t
, chan
, tid
);
3397 atomic_dec(&t
->tids_in_use
);
3399 EXPORT_SYMBOL(cxgb4_remove_tid
);
3402 * Allocate and initialize the TID tables. Returns 0 on success.
3404 static int tid_init(struct tid_info
*t
)
3407 unsigned int stid_bmap_size
;
3408 unsigned int natids
= t
->natids
;
3409 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
3411 stid_bmap_size
= BITS_TO_LONGS(t
->nstids
+ t
->nsftids
);
3412 size
= t
->ntids
* sizeof(*t
->tid_tab
) +
3413 natids
* sizeof(*t
->atid_tab
) +
3414 t
->nstids
* sizeof(*t
->stid_tab
) +
3415 t
->nsftids
* sizeof(*t
->stid_tab
) +
3416 stid_bmap_size
* sizeof(long) +
3417 t
->nftids
* sizeof(*t
->ftid_tab
) +
3418 t
->nsftids
* sizeof(*t
->ftid_tab
);
3420 t
->tid_tab
= t4_alloc_mem(size
);
3424 t
->atid_tab
= (union aopen_entry
*)&t
->tid_tab
[t
->ntids
];
3425 t
->stid_tab
= (struct serv_entry
*)&t
->atid_tab
[natids
];
3426 t
->stid_bmap
= (unsigned long *)&t
->stid_tab
[t
->nstids
+ t
->nsftids
];
3427 t
->ftid_tab
= (struct filter_entry
*)&t
->stid_bmap
[stid_bmap_size
];
3428 spin_lock_init(&t
->stid_lock
);
3429 spin_lock_init(&t
->atid_lock
);
3431 t
->stids_in_use
= 0;
3433 t
->atids_in_use
= 0;
3434 atomic_set(&t
->tids_in_use
, 0);
3436 /* Setup the free list for atid_tab and clear the stid bitmap. */
3439 t
->atid_tab
[natids
- 1].next
= &t
->atid_tab
[natids
];
3440 t
->afree
= t
->atid_tab
;
3442 bitmap_zero(t
->stid_bmap
, t
->nstids
+ t
->nsftids
);
3443 /* Reserve stid 0 for T4/T5 adapters */
3444 if (!t
->stid_base
&&
3445 (is_t4(adap
->params
.chip
) || is_t5(adap
->params
.chip
)))
3446 __set_bit(0, t
->stid_bmap
);
3451 int cxgb4_clip_get(const struct net_device
*dev
,
3452 const struct in6_addr
*lip
)
3454 struct adapter
*adap
;
3455 struct fw_clip_cmd c
;
3457 adap
= netdev2adap(dev
);
3458 memset(&c
, 0, sizeof(c
));
3459 c
.op_to_write
= htonl(FW_CMD_OP(FW_CLIP_CMD
) |
3460 FW_CMD_REQUEST
| FW_CMD_WRITE
);
3461 c
.alloc_to_len16
= htonl(F_FW_CLIP_CMD_ALLOC
| FW_LEN16(c
));
3462 c
.ip_hi
= *(__be64
*)(lip
->s6_addr
);
3463 c
.ip_lo
= *(__be64
*)(lip
->s6_addr
+ 8);
3464 return t4_wr_mbox_meat(adap
, adap
->mbox
, &c
, sizeof(c
), &c
, false);
3466 EXPORT_SYMBOL(cxgb4_clip_get
);
3468 int cxgb4_clip_release(const struct net_device
*dev
,
3469 const struct in6_addr
*lip
)
3471 struct adapter
*adap
;
3472 struct fw_clip_cmd c
;
3474 adap
= netdev2adap(dev
);
3475 memset(&c
, 0, sizeof(c
));
3476 c
.op_to_write
= htonl(FW_CMD_OP(FW_CLIP_CMD
) |
3477 FW_CMD_REQUEST
| FW_CMD_READ
);
3478 c
.alloc_to_len16
= htonl(F_FW_CLIP_CMD_FREE
| FW_LEN16(c
));
3479 c
.ip_hi
= *(__be64
*)(lip
->s6_addr
);
3480 c
.ip_lo
= *(__be64
*)(lip
->s6_addr
+ 8);
3481 return t4_wr_mbox_meat(adap
, adap
->mbox
, &c
, sizeof(c
), &c
, false);
3483 EXPORT_SYMBOL(cxgb4_clip_release
);
3486 * cxgb4_create_server - create an IP server
3488 * @stid: the server TID
3489 * @sip: local IP address to bind server to
3490 * @sport: the server's TCP port
3491 * @queue: queue to direct messages from this server to
3493 * Create an IP server for the given port and address.
3494 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3496 int cxgb4_create_server(const struct net_device
*dev
, unsigned int stid
,
3497 __be32 sip
, __be16 sport
, __be16 vlan
,
3501 struct sk_buff
*skb
;
3502 struct adapter
*adap
;
3503 struct cpl_pass_open_req
*req
;
3506 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
3510 adap
= netdev2adap(dev
);
3511 req
= (struct cpl_pass_open_req
*)__skb_put(skb
, sizeof(*req
));
3513 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ
, stid
));
3514 req
->local_port
= sport
;
3515 req
->peer_port
= htons(0);
3516 req
->local_ip
= sip
;
3517 req
->peer_ip
= htonl(0);
3518 chan
= rxq_to_chan(&adap
->sge
, queue
);
3519 req
->opt0
= cpu_to_be64(TX_CHAN(chan
));
3520 req
->opt1
= cpu_to_be64(CONN_POLICY_ASK
|
3521 SYN_RSS_ENABLE
| SYN_RSS_QUEUE(queue
));
3522 ret
= t4_mgmt_tx(adap
, skb
);
3523 return net_xmit_eval(ret
);
3525 EXPORT_SYMBOL(cxgb4_create_server
);
3527 /* cxgb4_create_server6 - create an IPv6 server
3529 * @stid: the server TID
3530 * @sip: local IPv6 address to bind server to
3531 * @sport: the server's TCP port
3532 * @queue: queue to direct messages from this server to
3534 * Create an IPv6 server for the given port and address.
3535 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3537 int cxgb4_create_server6(const struct net_device
*dev
, unsigned int stid
,
3538 const struct in6_addr
*sip
, __be16 sport
,
3542 struct sk_buff
*skb
;
3543 struct adapter
*adap
;
3544 struct cpl_pass_open_req6
*req
;
3547 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
3551 adap
= netdev2adap(dev
);
3552 req
= (struct cpl_pass_open_req6
*)__skb_put(skb
, sizeof(*req
));
3554 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6
, stid
));
3555 req
->local_port
= sport
;
3556 req
->peer_port
= htons(0);
3557 req
->local_ip_hi
= *(__be64
*)(sip
->s6_addr
);
3558 req
->local_ip_lo
= *(__be64
*)(sip
->s6_addr
+ 8);
3559 req
->peer_ip_hi
= cpu_to_be64(0);
3560 req
->peer_ip_lo
= cpu_to_be64(0);
3561 chan
= rxq_to_chan(&adap
->sge
, queue
);
3562 req
->opt0
= cpu_to_be64(TX_CHAN(chan
));
3563 req
->opt1
= cpu_to_be64(CONN_POLICY_ASK
|
3564 SYN_RSS_ENABLE
| SYN_RSS_QUEUE(queue
));
3565 ret
= t4_mgmt_tx(adap
, skb
);
3566 return net_xmit_eval(ret
);
3568 EXPORT_SYMBOL(cxgb4_create_server6
);
3570 int cxgb4_remove_server(const struct net_device
*dev
, unsigned int stid
,
3571 unsigned int queue
, bool ipv6
)
3573 struct sk_buff
*skb
;
3574 struct adapter
*adap
;
3575 struct cpl_close_listsvr_req
*req
;
3578 adap
= netdev2adap(dev
);
3580 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
3584 req
= (struct cpl_close_listsvr_req
*)__skb_put(skb
, sizeof(*req
));
3586 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ
, stid
));
3587 req
->reply_ctrl
= htons(NO_REPLY(0) | (ipv6
? LISTSVR_IPV6(1) :
3588 LISTSVR_IPV6(0)) | QUEUENO(queue
));
3589 ret
= t4_mgmt_tx(adap
, skb
);
3590 return net_xmit_eval(ret
);
3592 EXPORT_SYMBOL(cxgb4_remove_server
);
3595 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3596 * @mtus: the HW MTU table
3597 * @mtu: the target MTU
3598 * @idx: index of selected entry in the MTU table
3600 * Returns the index and the value in the HW MTU table that is closest to
3601 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3602 * table, in which case that smallest available value is selected.
3604 unsigned int cxgb4_best_mtu(const unsigned short *mtus
, unsigned short mtu
,
3609 while (i
< NMTUS
- 1 && mtus
[i
+ 1] <= mtu
)
3615 EXPORT_SYMBOL(cxgb4_best_mtu
);
3618 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3619 * @mtus: the HW MTU table
3620 * @header_size: Header Size
3621 * @data_size_max: maximum Data Segment Size
3622 * @data_size_align: desired Data Segment Size Alignment (2^N)
3623 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3625 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3626 * MTU Table based solely on a Maximum MTU parameter, we break that
3627 * parameter up into a Header Size and Maximum Data Segment Size, and
3628 * provide a desired Data Segment Size Alignment. If we find an MTU in
3629 * the Hardware MTU Table which will result in a Data Segment Size with
3630 * the requested alignment _and_ that MTU isn't "too far" from the
3631 * closest MTU, then we'll return that rather than the closest MTU.
3633 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus
,
3634 unsigned short header_size
,
3635 unsigned short data_size_max
,
3636 unsigned short data_size_align
,
3637 unsigned int *mtu_idxp
)
3639 unsigned short max_mtu
= header_size
+ data_size_max
;
3640 unsigned short data_size_align_mask
= data_size_align
- 1;
3641 int mtu_idx
, aligned_mtu_idx
;
3643 /* Scan the MTU Table till we find an MTU which is larger than our
3644 * Maximum MTU or we reach the end of the table. Along the way,
3645 * record the last MTU found, if any, which will result in a Data
3646 * Segment Length matching the requested alignment.
3648 for (mtu_idx
= 0, aligned_mtu_idx
= -1; mtu_idx
< NMTUS
; mtu_idx
++) {
3649 unsigned short data_size
= mtus
[mtu_idx
] - header_size
;
3651 /* If this MTU minus the Header Size would result in a
3652 * Data Segment Size of the desired alignment, remember it.
3654 if ((data_size
& data_size_align_mask
) == 0)
3655 aligned_mtu_idx
= mtu_idx
;
3657 /* If we're not at the end of the Hardware MTU Table and the
3658 * next element is larger than our Maximum MTU, drop out of
3661 if (mtu_idx
+1 < NMTUS
&& mtus
[mtu_idx
+1] > max_mtu
)
3665 /* If we fell out of the loop because we ran to the end of the table,
3666 * then we just have to use the last [largest] entry.
3668 if (mtu_idx
== NMTUS
)
3671 /* If we found an MTU which resulted in the requested Data Segment
3672 * Length alignment and that's "not far" from the largest MTU which is
3673 * less than or equal to the maximum MTU, then use that.
3675 if (aligned_mtu_idx
>= 0 &&
3676 mtu_idx
- aligned_mtu_idx
<= 1)
3677 mtu_idx
= aligned_mtu_idx
;
3679 /* If the caller has passed in an MTU Index pointer, pass the
3680 * MTU Index back. Return the MTU value.
3683 *mtu_idxp
= mtu_idx
;
3684 return mtus
[mtu_idx
];
3686 EXPORT_SYMBOL(cxgb4_best_aligned_mtu
);
3689 * cxgb4_port_chan - get the HW channel of a port
3690 * @dev: the net device for the port
3692 * Return the HW Tx channel of the given port.
3694 unsigned int cxgb4_port_chan(const struct net_device
*dev
)
3696 return netdev2pinfo(dev
)->tx_chan
;
3698 EXPORT_SYMBOL(cxgb4_port_chan
);
3700 unsigned int cxgb4_dbfifo_count(const struct net_device
*dev
, int lpfifo
)
3702 struct adapter
*adap
= netdev2adap(dev
);
3703 u32 v1
, v2
, lp_count
, hp_count
;
3705 v1
= t4_read_reg(adap
, A_SGE_DBFIFO_STATUS
);
3706 v2
= t4_read_reg(adap
, SGE_DBFIFO_STATUS2
);
3707 if (is_t4(adap
->params
.chip
)) {
3708 lp_count
= G_LP_COUNT(v1
);
3709 hp_count
= G_HP_COUNT(v1
);
3711 lp_count
= G_LP_COUNT_T5(v1
);
3712 hp_count
= G_HP_COUNT_T5(v2
);
3714 return lpfifo
? lp_count
: hp_count
;
3716 EXPORT_SYMBOL(cxgb4_dbfifo_count
);
3719 * cxgb4_port_viid - get the VI id of a port
3720 * @dev: the net device for the port
3722 * Return the VI id of the given port.
3724 unsigned int cxgb4_port_viid(const struct net_device
*dev
)
3726 return netdev2pinfo(dev
)->viid
;
3728 EXPORT_SYMBOL(cxgb4_port_viid
);
3731 * cxgb4_port_idx - get the index of a port
3732 * @dev: the net device for the port
3734 * Return the index of the given port.
3736 unsigned int cxgb4_port_idx(const struct net_device
*dev
)
3738 return netdev2pinfo(dev
)->port_id
;
3740 EXPORT_SYMBOL(cxgb4_port_idx
);
3742 void cxgb4_get_tcp_stats(struct pci_dev
*pdev
, struct tp_tcp_stats
*v4
,
3743 struct tp_tcp_stats
*v6
)
3745 struct adapter
*adap
= pci_get_drvdata(pdev
);
3747 spin_lock(&adap
->stats_lock
);
3748 t4_tp_get_tcp_stats(adap
, v4
, v6
);
3749 spin_unlock(&adap
->stats_lock
);
3751 EXPORT_SYMBOL(cxgb4_get_tcp_stats
);
3753 void cxgb4_iscsi_init(struct net_device
*dev
, unsigned int tag_mask
,
3754 const unsigned int *pgsz_order
)
3756 struct adapter
*adap
= netdev2adap(dev
);
3758 t4_write_reg(adap
, ULP_RX_ISCSI_TAGMASK
, tag_mask
);
3759 t4_write_reg(adap
, ULP_RX_ISCSI_PSZ
, HPZ0(pgsz_order
[0]) |
3760 HPZ1(pgsz_order
[1]) | HPZ2(pgsz_order
[2]) |
3761 HPZ3(pgsz_order
[3]));
3763 EXPORT_SYMBOL(cxgb4_iscsi_init
);
3765 int cxgb4_flush_eq_cache(struct net_device
*dev
)
3767 struct adapter
*adap
= netdev2adap(dev
);
3770 ret
= t4_fwaddrspace_write(adap
, adap
->mbox
,
3771 0xe1000000 + A_SGE_CTXT_CMD
, 0x20000000);
3774 EXPORT_SYMBOL(cxgb4_flush_eq_cache
);
3776 static int read_eq_indices(struct adapter
*adap
, u16 qid
, u16
*pidx
, u16
*cidx
)
3778 u32 addr
= t4_read_reg(adap
, A_SGE_DBQ_CTXT_BADDR
) + 24 * qid
+ 8;
3782 spin_lock(&adap
->win0_lock
);
3783 ret
= t4_memory_rw(adap
, 0, MEM_EDC0
, addr
,
3784 sizeof(indices
), (__be32
*)&indices
,
3786 spin_unlock(&adap
->win0_lock
);
3788 *cidx
= (be64_to_cpu(indices
) >> 25) & 0xffff;
3789 *pidx
= (be64_to_cpu(indices
) >> 9) & 0xffff;
3794 int cxgb4_sync_txq_pidx(struct net_device
*dev
, u16 qid
, u16 pidx
,
3797 struct adapter
*adap
= netdev2adap(dev
);
3798 u16 hw_pidx
, hw_cidx
;
3801 ret
= read_eq_indices(adap
, qid
, &hw_pidx
, &hw_cidx
);
3805 if (pidx
!= hw_pidx
) {
3808 if (pidx
>= hw_pidx
)
3809 delta
= pidx
- hw_pidx
;
3811 delta
= size
- hw_pidx
+ pidx
;
3813 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
3814 QID(qid
) | PIDX(delta
));
3819 EXPORT_SYMBOL(cxgb4_sync_txq_pidx
);
3821 void cxgb4_disable_db_coalescing(struct net_device
*dev
)
3823 struct adapter
*adap
;
3825 adap
= netdev2adap(dev
);
3826 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_NOCOALESCE
,
3829 EXPORT_SYMBOL(cxgb4_disable_db_coalescing
);
3831 void cxgb4_enable_db_coalescing(struct net_device
*dev
)
3833 struct adapter
*adap
;
3835 adap
= netdev2adap(dev
);
3836 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_NOCOALESCE
, 0);
3838 EXPORT_SYMBOL(cxgb4_enable_db_coalescing
);
3840 int cxgb4_read_tpte(struct net_device
*dev
, u32 stag
, __be32
*tpte
)
3842 struct adapter
*adap
;
3843 u32 offset
, memtype
, memaddr
;
3844 u32 edc0_size
, edc1_size
, mc0_size
, mc1_size
;
3845 u32 edc0_end
, edc1_end
, mc0_end
, mc1_end
;
3848 adap
= netdev2adap(dev
);
3850 offset
= ((stag
>> 8) * 32) + adap
->vres
.stag
.start
;
3852 /* Figure out where the offset lands in the Memory Type/Address scheme.
3853 * This code assumes that the memory is laid out starting at offset 0
3854 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3855 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have
3856 * MC0, and some have both MC0 and MC1.
3858 edc0_size
= EDRAM_SIZE_GET(t4_read_reg(adap
, MA_EDRAM0_BAR
)) << 20;
3859 edc1_size
= EDRAM_SIZE_GET(t4_read_reg(adap
, MA_EDRAM1_BAR
)) << 20;
3860 mc0_size
= EXT_MEM_SIZE_GET(t4_read_reg(adap
, MA_EXT_MEMORY_BAR
)) << 20;
3862 edc0_end
= edc0_size
;
3863 edc1_end
= edc0_end
+ edc1_size
;
3864 mc0_end
= edc1_end
+ mc0_size
;
3866 if (offset
< edc0_end
) {
3869 } else if (offset
< edc1_end
) {
3871 memaddr
= offset
- edc0_end
;
3873 if (offset
< mc0_end
) {
3875 memaddr
= offset
- edc1_end
;
3876 } else if (is_t4(adap
->params
.chip
)) {
3877 /* T4 only has a single memory channel */
3880 mc1_size
= EXT_MEM_SIZE_GET(
3882 MA_EXT_MEMORY1_BAR
)) << 20;
3883 mc1_end
= mc0_end
+ mc1_size
;
3884 if (offset
< mc1_end
) {
3886 memaddr
= offset
- mc0_end
;
3888 /* offset beyond the end of any memory */
3894 spin_lock(&adap
->win0_lock
);
3895 ret
= t4_memory_rw(adap
, 0, memtype
, memaddr
, 32, tpte
, T4_MEMORY_READ
);
3896 spin_unlock(&adap
->win0_lock
);
3900 dev_err(adap
->pdev_dev
, "stag %#x, offset %#x out of range\n",
3904 EXPORT_SYMBOL(cxgb4_read_tpte
);
3906 u64
cxgb4_read_sge_timestamp(struct net_device
*dev
)
3909 struct adapter
*adap
;
3911 adap
= netdev2adap(dev
);
3912 lo
= t4_read_reg(adap
, SGE_TIMESTAMP_LO
);
3913 hi
= GET_TSVAL(t4_read_reg(adap
, SGE_TIMESTAMP_HI
));
3915 return ((u64
)hi
<< 32) | (u64
)lo
;
3917 EXPORT_SYMBOL(cxgb4_read_sge_timestamp
);
3919 static struct pci_driver cxgb4_driver
;
3921 static void check_neigh_update(struct neighbour
*neigh
)
3923 const struct device
*parent
;
3924 const struct net_device
*netdev
= neigh
->dev
;
3926 if (netdev
->priv_flags
& IFF_802_1Q_VLAN
)
3927 netdev
= vlan_dev_real_dev(netdev
);
3928 parent
= netdev
->dev
.parent
;
3929 if (parent
&& parent
->driver
== &cxgb4_driver
.driver
)
3930 t4_l2t_update(dev_get_drvdata(parent
), neigh
);
3933 static int netevent_cb(struct notifier_block
*nb
, unsigned long event
,
3937 case NETEVENT_NEIGH_UPDATE
:
3938 check_neigh_update(data
);
3940 case NETEVENT_REDIRECT
:
3947 static bool netevent_registered
;
3948 static struct notifier_block cxgb4_netevent_nb
= {
3949 .notifier_call
= netevent_cb
3952 static void drain_db_fifo(struct adapter
*adap
, int usecs
)
3954 u32 v1
, v2
, lp_count
, hp_count
;
3957 v1
= t4_read_reg(adap
, A_SGE_DBFIFO_STATUS
);
3958 v2
= t4_read_reg(adap
, SGE_DBFIFO_STATUS2
);
3959 if (is_t4(adap
->params
.chip
)) {
3960 lp_count
= G_LP_COUNT(v1
);
3961 hp_count
= G_HP_COUNT(v1
);
3963 lp_count
= G_LP_COUNT_T5(v1
);
3964 hp_count
= G_HP_COUNT_T5(v2
);
3967 if (lp_count
== 0 && hp_count
== 0)
3969 set_current_state(TASK_UNINTERRUPTIBLE
);
3970 schedule_timeout(usecs_to_jiffies(usecs
));
3974 static void disable_txq_db(struct sge_txq
*q
)
3976 unsigned long flags
;
3978 spin_lock_irqsave(&q
->db_lock
, flags
);
3980 spin_unlock_irqrestore(&q
->db_lock
, flags
);
3983 static void enable_txq_db(struct adapter
*adap
, struct sge_txq
*q
)
3985 spin_lock_irq(&q
->db_lock
);
3986 if (q
->db_pidx_inc
) {
3987 /* Make sure that all writes to the TX descriptors
3988 * are committed before we tell HW about them.
3991 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
3992 QID(q
->cntxt_id
) | PIDX(q
->db_pidx_inc
));
3996 spin_unlock_irq(&q
->db_lock
);
3999 static void disable_dbs(struct adapter
*adap
)
4003 for_each_ethrxq(&adap
->sge
, i
)
4004 disable_txq_db(&adap
->sge
.ethtxq
[i
].q
);
4005 for_each_ofldrxq(&adap
->sge
, i
)
4006 disable_txq_db(&adap
->sge
.ofldtxq
[i
].q
);
4007 for_each_port(adap
, i
)
4008 disable_txq_db(&adap
->sge
.ctrlq
[i
].q
);
4011 static void enable_dbs(struct adapter
*adap
)
4015 for_each_ethrxq(&adap
->sge
, i
)
4016 enable_txq_db(adap
, &adap
->sge
.ethtxq
[i
].q
);
4017 for_each_ofldrxq(&adap
->sge
, i
)
4018 enable_txq_db(adap
, &adap
->sge
.ofldtxq
[i
].q
);
4019 for_each_port(adap
, i
)
4020 enable_txq_db(adap
, &adap
->sge
.ctrlq
[i
].q
);
4023 static void notify_rdma_uld(struct adapter
*adap
, enum cxgb4_control cmd
)
4025 if (adap
->uld_handle
[CXGB4_ULD_RDMA
])
4026 ulds
[CXGB4_ULD_RDMA
].control(adap
->uld_handle
[CXGB4_ULD_RDMA
],
4030 static void process_db_full(struct work_struct
*work
)
4032 struct adapter
*adap
;
4034 adap
= container_of(work
, struct adapter
, db_full_task
);
4036 drain_db_fifo(adap
, dbfifo_drain_delay
);
4038 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_EMPTY
);
4039 t4_set_reg_field(adap
, SGE_INT_ENABLE3
,
4040 DBFIFO_HP_INT
| DBFIFO_LP_INT
,
4041 DBFIFO_HP_INT
| DBFIFO_LP_INT
);
4044 static void sync_txq_pidx(struct adapter
*adap
, struct sge_txq
*q
)
4046 u16 hw_pidx
, hw_cidx
;
4049 spin_lock_irq(&q
->db_lock
);
4050 ret
= read_eq_indices(adap
, (u16
)q
->cntxt_id
, &hw_pidx
, &hw_cidx
);
4053 if (q
->db_pidx
!= hw_pidx
) {
4056 if (q
->db_pidx
>= hw_pidx
)
4057 delta
= q
->db_pidx
- hw_pidx
;
4059 delta
= q
->size
- hw_pidx
+ q
->db_pidx
;
4061 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
4062 QID(q
->cntxt_id
) | PIDX(delta
));
4067 spin_unlock_irq(&q
->db_lock
);
4069 CH_WARN(adap
, "DB drop recovery failed.\n");
4071 static void recover_all_queues(struct adapter
*adap
)
4075 for_each_ethrxq(&adap
->sge
, i
)
4076 sync_txq_pidx(adap
, &adap
->sge
.ethtxq
[i
].q
);
4077 for_each_ofldrxq(&adap
->sge
, i
)
4078 sync_txq_pidx(adap
, &adap
->sge
.ofldtxq
[i
].q
);
4079 for_each_port(adap
, i
)
4080 sync_txq_pidx(adap
, &adap
->sge
.ctrlq
[i
].q
);
4083 static void process_db_drop(struct work_struct
*work
)
4085 struct adapter
*adap
;
4087 adap
= container_of(work
, struct adapter
, db_drop_task
);
4089 if (is_t4(adap
->params
.chip
)) {
4090 drain_db_fifo(adap
, dbfifo_drain_delay
);
4091 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_DROP
);
4092 drain_db_fifo(adap
, dbfifo_drain_delay
);
4093 recover_all_queues(adap
);
4094 drain_db_fifo(adap
, dbfifo_drain_delay
);
4096 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_EMPTY
);
4098 u32 dropped_db
= t4_read_reg(adap
, 0x010ac);
4099 u16 qid
= (dropped_db
>> 15) & 0x1ffff;
4100 u16 pidx_inc
= dropped_db
& 0x1fff;
4102 unsigned short udb_density
;
4103 unsigned long qpshift
;
4107 dev_warn(adap
->pdev_dev
,
4108 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
4110 (dropped_db
>> 14) & 1,
4111 (dropped_db
>> 13) & 1,
4114 drain_db_fifo(adap
, 1);
4116 s_qpp
= QUEUESPERPAGEPF1
* adap
->fn
;
4117 udb_density
= 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap
,
4118 SGE_EGRESS_QUEUES_PER_PAGE_PF
) >> s_qpp
);
4119 qpshift
= PAGE_SHIFT
- ilog2(udb_density
);
4120 udb
= qid
<< qpshift
;
4122 page
= udb
/ PAGE_SIZE
;
4123 udb
+= (qid
- (page
* udb_density
)) * 128;
4125 writel(PIDX(pidx_inc
), adap
->bar2
+ udb
+ 8);
4127 /* Re-enable BAR2 WC */
4128 t4_set_reg_field(adap
, 0x10b0, 1<<15, 1<<15);
4131 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_DROPPED_DB
, 0);
4134 void t4_db_full(struct adapter
*adap
)
4136 if (is_t4(adap
->params
.chip
)) {
4138 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_FULL
);
4139 t4_set_reg_field(adap
, SGE_INT_ENABLE3
,
4140 DBFIFO_HP_INT
| DBFIFO_LP_INT
, 0);
4141 queue_work(adap
->workq
, &adap
->db_full_task
);
4145 void t4_db_dropped(struct adapter
*adap
)
4147 if (is_t4(adap
->params
.chip
)) {
4149 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_FULL
);
4151 queue_work(adap
->workq
, &adap
->db_drop_task
);
4154 static void uld_attach(struct adapter
*adap
, unsigned int uld
)
4157 struct cxgb4_lld_info lli
;
4160 lli
.pdev
= adap
->pdev
;
4162 lli
.l2t
= adap
->l2t
;
4163 lli
.tids
= &adap
->tids
;
4164 lli
.ports
= adap
->port
;
4165 lli
.vr
= &adap
->vres
;
4166 lli
.mtus
= adap
->params
.mtus
;
4167 if (uld
== CXGB4_ULD_RDMA
) {
4168 lli
.rxq_ids
= adap
->sge
.rdma_rxq
;
4169 lli
.ciq_ids
= adap
->sge
.rdma_ciq
;
4170 lli
.nrxq
= adap
->sge
.rdmaqs
;
4171 lli
.nciq
= adap
->sge
.rdmaciqs
;
4172 } else if (uld
== CXGB4_ULD_ISCSI
) {
4173 lli
.rxq_ids
= adap
->sge
.ofld_rxq
;
4174 lli
.nrxq
= adap
->sge
.ofldqsets
;
4176 lli
.ntxq
= adap
->sge
.ofldqsets
;
4177 lli
.nchan
= adap
->params
.nports
;
4178 lli
.nports
= adap
->params
.nports
;
4179 lli
.wr_cred
= adap
->params
.ofldq_wr_cred
;
4180 lli
.adapter_type
= adap
->params
.chip
;
4181 lli
.iscsi_iolen
= MAXRXDATA_GET(t4_read_reg(adap
, TP_PARA_REG2
));
4182 lli
.cclk_ps
= 1000000000 / adap
->params
.vpd
.cclk
;
4183 lli
.udb_density
= 1 << QUEUESPERPAGEPF0_GET(
4184 t4_read_reg(adap
, SGE_EGRESS_QUEUES_PER_PAGE_PF
) >>
4186 lli
.ucq_density
= 1 << QUEUESPERPAGEPF0_GET(
4187 t4_read_reg(adap
, SGE_INGRESS_QUEUES_PER_PAGE_PF
) >>
4189 lli
.filt_mode
= adap
->params
.tp
.vlan_pri_map
;
4190 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
4191 for (i
= 0; i
< NCHAN
; i
++)
4193 lli
.gts_reg
= adap
->regs
+ MYPF_REG(SGE_PF_GTS
);
4194 lli
.db_reg
= adap
->regs
+ MYPF_REG(SGE_PF_KDOORBELL
);
4195 lli
.fw_vers
= adap
->params
.fw_vers
;
4196 lli
.dbfifo_int_thresh
= dbfifo_int_thresh
;
4197 lli
.sge_ingpadboundary
= adap
->sge
.fl_align
;
4198 lli
.sge_egrstatuspagesize
= adap
->sge
.stat_len
;
4199 lli
.sge_pktshift
= adap
->sge
.pktshift
;
4200 lli
.enable_fw_ofld_conn
= adap
->flags
& FW_OFLD_CONN
;
4201 lli
.max_ordird_qp
= adap
->params
.max_ordird_qp
;
4202 lli
.max_ird_adapter
= adap
->params
.max_ird_adapter
;
4203 lli
.ulptx_memwrite_dsgl
= adap
->params
.ulptx_memwrite_dsgl
;
4205 handle
= ulds
[uld
].add(&lli
);
4206 if (IS_ERR(handle
)) {
4207 dev_warn(adap
->pdev_dev
,
4208 "could not attach to the %s driver, error %ld\n",
4209 uld_str
[uld
], PTR_ERR(handle
));
4213 adap
->uld_handle
[uld
] = handle
;
4215 if (!netevent_registered
) {
4216 register_netevent_notifier(&cxgb4_netevent_nb
);
4217 netevent_registered
= true;
4220 if (adap
->flags
& FULL_INIT_DONE
)
4221 ulds
[uld
].state_change(handle
, CXGB4_STATE_UP
);
4224 static void attach_ulds(struct adapter
*adap
)
4228 spin_lock(&adap_rcu_lock
);
4229 list_add_tail_rcu(&adap
->rcu_node
, &adap_rcu_list
);
4230 spin_unlock(&adap_rcu_lock
);
4232 mutex_lock(&uld_mutex
);
4233 list_add_tail(&adap
->list_node
, &adapter_list
);
4234 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
4236 uld_attach(adap
, i
);
4237 mutex_unlock(&uld_mutex
);
4240 static void detach_ulds(struct adapter
*adap
)
4244 mutex_lock(&uld_mutex
);
4245 list_del(&adap
->list_node
);
4246 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
4247 if (adap
->uld_handle
[i
]) {
4248 ulds
[i
].state_change(adap
->uld_handle
[i
],
4249 CXGB4_STATE_DETACH
);
4250 adap
->uld_handle
[i
] = NULL
;
4252 if (netevent_registered
&& list_empty(&adapter_list
)) {
4253 unregister_netevent_notifier(&cxgb4_netevent_nb
);
4254 netevent_registered
= false;
4256 mutex_unlock(&uld_mutex
);
4258 spin_lock(&adap_rcu_lock
);
4259 list_del_rcu(&adap
->rcu_node
);
4260 spin_unlock(&adap_rcu_lock
);
4263 static void notify_ulds(struct adapter
*adap
, enum cxgb4_state new_state
)
4267 mutex_lock(&uld_mutex
);
4268 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
4269 if (adap
->uld_handle
[i
])
4270 ulds
[i
].state_change(adap
->uld_handle
[i
], new_state
);
4271 mutex_unlock(&uld_mutex
);
4275 * cxgb4_register_uld - register an upper-layer driver
4276 * @type: the ULD type
4277 * @p: the ULD methods
4279 * Registers an upper-layer driver with this driver and notifies the ULD
4280 * about any presently available devices that support its type. Returns
4281 * %-EBUSY if a ULD of the same type is already registered.
4283 int cxgb4_register_uld(enum cxgb4_uld type
, const struct cxgb4_uld_info
*p
)
4286 struct adapter
*adap
;
4288 if (type
>= CXGB4_ULD_MAX
)
4290 mutex_lock(&uld_mutex
);
4291 if (ulds
[type
].add
) {
4296 list_for_each_entry(adap
, &adapter_list
, list_node
)
4297 uld_attach(adap
, type
);
4298 out
: mutex_unlock(&uld_mutex
);
4301 EXPORT_SYMBOL(cxgb4_register_uld
);
4304 * cxgb4_unregister_uld - unregister an upper-layer driver
4305 * @type: the ULD type
4307 * Unregisters an existing upper-layer driver.
4309 int cxgb4_unregister_uld(enum cxgb4_uld type
)
4311 struct adapter
*adap
;
4313 if (type
>= CXGB4_ULD_MAX
)
4315 mutex_lock(&uld_mutex
);
4316 list_for_each_entry(adap
, &adapter_list
, list_node
)
4317 adap
->uld_handle
[type
] = NULL
;
4318 ulds
[type
].add
= NULL
;
4319 mutex_unlock(&uld_mutex
);
4322 EXPORT_SYMBOL(cxgb4_unregister_uld
);
4324 /* Check if netdev on which event is occured belongs to us or not. Return
4325 * success (true) if it belongs otherwise failure (false).
4326 * Called with rcu_read_lock() held.
4328 static bool cxgb4_netdev(const struct net_device
*netdev
)
4330 struct adapter
*adap
;
4333 list_for_each_entry_rcu(adap
, &adap_rcu_list
, rcu_node
)
4334 for (i
= 0; i
< MAX_NPORTS
; i
++)
4335 if (adap
->port
[i
] == netdev
)
4340 static int clip_add(struct net_device
*event_dev
, struct inet6_ifaddr
*ifa
,
4341 unsigned long event
)
4343 int ret
= NOTIFY_DONE
;
4346 if (cxgb4_netdev(event_dev
)) {
4349 ret
= cxgb4_clip_get(event_dev
,
4350 (const struct in6_addr
*)ifa
->addr
.s6_addr
);
4358 cxgb4_clip_release(event_dev
,
4359 (const struct in6_addr
*)ifa
->addr
.s6_addr
);
4370 static int cxgb4_inet6addr_handler(struct notifier_block
*this,
4371 unsigned long event
, void *data
)
4373 struct inet6_ifaddr
*ifa
= data
;
4374 struct net_device
*event_dev
;
4375 int ret
= NOTIFY_DONE
;
4376 struct bonding
*bond
= netdev_priv(ifa
->idev
->dev
);
4377 struct list_head
*iter
;
4378 struct slave
*slave
;
4379 struct pci_dev
*first_pdev
= NULL
;
4381 if (ifa
->idev
->dev
->priv_flags
& IFF_802_1Q_VLAN
) {
4382 event_dev
= vlan_dev_real_dev(ifa
->idev
->dev
);
4383 ret
= clip_add(event_dev
, ifa
, event
);
4384 } else if (ifa
->idev
->dev
->flags
& IFF_MASTER
) {
4385 /* It is possible that two different adapters are bonded in one
4386 * bond. We need to find such different adapters and add clip
4387 * in all of them only once.
4389 read_lock(&bond
->lock
);
4390 bond_for_each_slave(bond
, slave
, iter
) {
4392 ret
= clip_add(slave
->dev
, ifa
, event
);
4393 /* If clip_add is success then only initialize
4394 * first_pdev since it means it is our device
4396 if (ret
== NOTIFY_OK
)
4397 first_pdev
= to_pci_dev(
4398 slave
->dev
->dev
.parent
);
4399 } else if (first_pdev
!=
4400 to_pci_dev(slave
->dev
->dev
.parent
))
4401 ret
= clip_add(slave
->dev
, ifa
, event
);
4403 read_unlock(&bond
->lock
);
4405 ret
= clip_add(ifa
->idev
->dev
, ifa
, event
);
4410 static struct notifier_block cxgb4_inet6addr_notifier
= {
4411 .notifier_call
= cxgb4_inet6addr_handler
4414 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4415 * a physical device.
4416 * The physical device reference is needed to send the actul CLIP command.
4418 static int update_dev_clip(struct net_device
*root_dev
, struct net_device
*dev
)
4420 struct inet6_dev
*idev
= NULL
;
4421 struct inet6_ifaddr
*ifa
;
4424 idev
= __in6_dev_get(root_dev
);
4428 read_lock_bh(&idev
->lock
);
4429 list_for_each_entry(ifa
, &idev
->addr_list
, if_list
) {
4430 ret
= cxgb4_clip_get(dev
,
4431 (const struct in6_addr
*)ifa
->addr
.s6_addr
);
4435 read_unlock_bh(&idev
->lock
);
4440 static int update_root_dev_clip(struct net_device
*dev
)
4442 struct net_device
*root_dev
= NULL
;
4445 /* First populate the real net device's IPv6 addresses */
4446 ret
= update_dev_clip(dev
, dev
);
4450 /* Parse all bond and vlan devices layered on top of the physical dev */
4451 for (i
= 0; i
< VLAN_N_VID
; i
++) {
4452 root_dev
= __vlan_find_dev_deep_rcu(dev
, htons(ETH_P_8021Q
), i
);
4456 ret
= update_dev_clip(root_dev
, dev
);
4463 static void update_clip(const struct adapter
*adap
)
4466 struct net_device
*dev
;
4471 for (i
= 0; i
< MAX_NPORTS
; i
++) {
4472 dev
= adap
->port
[i
];
4476 ret
= update_root_dev_clip(dev
);
4485 * cxgb_up - enable the adapter
4486 * @adap: adapter being enabled
4488 * Called when the first port is enabled, this function performs the
4489 * actions necessary to make an adapter operational, such as completing
4490 * the initialization of HW modules, and enabling interrupts.
4492 * Must be called with the rtnl lock held.
4494 static int cxgb_up(struct adapter
*adap
)
4498 err
= setup_sge_queues(adap
);
4501 err
= setup_rss(adap
);
4505 if (adap
->flags
& USING_MSIX
) {
4506 name_msix_vecs(adap
);
4507 err
= request_irq(adap
->msix_info
[0].vec
, t4_nondata_intr
, 0,
4508 adap
->msix_info
[0].desc
, adap
);
4512 err
= request_msix_queue_irqs(adap
);
4514 free_irq(adap
->msix_info
[0].vec
, adap
);
4518 err
= request_irq(adap
->pdev
->irq
, t4_intr_handler(adap
),
4519 (adap
->flags
& USING_MSI
) ? 0 : IRQF_SHARED
,
4520 adap
->port
[0]->name
, adap
);
4526 t4_intr_enable(adap
);
4527 adap
->flags
|= FULL_INIT_DONE
;
4528 notify_ulds(adap
, CXGB4_STATE_UP
);
4533 dev_err(adap
->pdev_dev
, "request_irq failed, err %d\n", err
);
4535 t4_free_sge_resources(adap
);
4539 static void cxgb_down(struct adapter
*adapter
)
4541 t4_intr_disable(adapter
);
4542 cancel_work_sync(&adapter
->tid_release_task
);
4543 cancel_work_sync(&adapter
->db_full_task
);
4544 cancel_work_sync(&adapter
->db_drop_task
);
4545 adapter
->tid_release_task_busy
= false;
4546 adapter
->tid_release_head
= NULL
;
4548 if (adapter
->flags
& USING_MSIX
) {
4549 free_msix_queue_irqs(adapter
);
4550 free_irq(adapter
->msix_info
[0].vec
, adapter
);
4552 free_irq(adapter
->pdev
->irq
, adapter
);
4553 quiesce_rx(adapter
);
4554 t4_sge_stop(adapter
);
4555 t4_free_sge_resources(adapter
);
4556 adapter
->flags
&= ~FULL_INIT_DONE
;
4560 * net_device operations
4562 static int cxgb_open(struct net_device
*dev
)
4565 struct port_info
*pi
= netdev_priv(dev
);
4566 struct adapter
*adapter
= pi
->adapter
;
4568 netif_carrier_off(dev
);
4570 if (!(adapter
->flags
& FULL_INIT_DONE
)) {
4571 err
= cxgb_up(adapter
);
4576 err
= link_start(dev
);
4578 netif_tx_start_all_queues(dev
);
4582 static int cxgb_close(struct net_device
*dev
)
4584 struct port_info
*pi
= netdev_priv(dev
);
4585 struct adapter
*adapter
= pi
->adapter
;
4587 netif_tx_stop_all_queues(dev
);
4588 netif_carrier_off(dev
);
4589 return t4_enable_vi(adapter
, adapter
->fn
, pi
->viid
, false, false);
4592 /* Return an error number if the indicated filter isn't writable ...
4594 static int writable_filter(struct filter_entry
*f
)
4604 /* Delete the filter at the specified index (if valid). The checks for all
4605 * the common problems with doing this like the filter being locked, currently
4606 * pending in another operation, etc.
4608 static int delete_filter(struct adapter
*adapter
, unsigned int fidx
)
4610 struct filter_entry
*f
;
4613 if (fidx
>= adapter
->tids
.nftids
+ adapter
->tids
.nsftids
)
4616 f
= &adapter
->tids
.ftid_tab
[fidx
];
4617 ret
= writable_filter(f
);
4621 return del_filter_wr(adapter
, fidx
);
4626 int cxgb4_create_server_filter(const struct net_device
*dev
, unsigned int stid
,
4627 __be32 sip
, __be16 sport
, __be16 vlan
,
4628 unsigned int queue
, unsigned char port
, unsigned char mask
)
4631 struct filter_entry
*f
;
4632 struct adapter
*adap
;
4636 adap
= netdev2adap(dev
);
4638 /* Adjust stid to correct filter index */
4639 stid
-= adap
->tids
.sftid_base
;
4640 stid
+= adap
->tids
.nftids
;
4642 /* Check to make sure the filter requested is writable ...
4644 f
= &adap
->tids
.ftid_tab
[stid
];
4645 ret
= writable_filter(f
);
4649 /* Clear out any old resources being used by the filter before
4650 * we start constructing the new filter.
4653 clear_filter(adap
, f
);
4655 /* Clear out filter specifications */
4656 memset(&f
->fs
, 0, sizeof(struct ch_filter_specification
));
4657 f
->fs
.val
.lport
= cpu_to_be16(sport
);
4658 f
->fs
.mask
.lport
= ~0;
4660 if ((val
[0] | val
[1] | val
[2] | val
[3]) != 0) {
4661 for (i
= 0; i
< 4; i
++) {
4662 f
->fs
.val
.lip
[i
] = val
[i
];
4663 f
->fs
.mask
.lip
[i
] = ~0;
4665 if (adap
->params
.tp
.vlan_pri_map
& F_PORT
) {
4666 f
->fs
.val
.iport
= port
;
4667 f
->fs
.mask
.iport
= mask
;
4671 if (adap
->params
.tp
.vlan_pri_map
& F_PROTOCOL
) {
4672 f
->fs
.val
.proto
= IPPROTO_TCP
;
4673 f
->fs
.mask
.proto
= ~0;
4678 /* Mark filter as locked */
4682 ret
= set_filter_wr(adap
, stid
);
4684 clear_filter(adap
, f
);
4690 EXPORT_SYMBOL(cxgb4_create_server_filter
);
4692 int cxgb4_remove_server_filter(const struct net_device
*dev
, unsigned int stid
,
4693 unsigned int queue
, bool ipv6
)
4696 struct filter_entry
*f
;
4697 struct adapter
*adap
;
4699 adap
= netdev2adap(dev
);
4701 /* Adjust stid to correct filter index */
4702 stid
-= adap
->tids
.sftid_base
;
4703 stid
+= adap
->tids
.nftids
;
4705 f
= &adap
->tids
.ftid_tab
[stid
];
4706 /* Unlock the filter */
4709 ret
= delete_filter(adap
, stid
);
4715 EXPORT_SYMBOL(cxgb4_remove_server_filter
);
4717 static struct rtnl_link_stats64
*cxgb_get_stats(struct net_device
*dev
,
4718 struct rtnl_link_stats64
*ns
)
4720 struct port_stats stats
;
4721 struct port_info
*p
= netdev_priv(dev
);
4722 struct adapter
*adapter
= p
->adapter
;
4724 /* Block retrieving statistics during EEH error
4725 * recovery. Otherwise, the recovery might fail
4726 * and the PCI device will be removed permanently
4728 spin_lock(&adapter
->stats_lock
);
4729 if (!netif_device_present(dev
)) {
4730 spin_unlock(&adapter
->stats_lock
);
4733 t4_get_port_stats(adapter
, p
->tx_chan
, &stats
);
4734 spin_unlock(&adapter
->stats_lock
);
4736 ns
->tx_bytes
= stats
.tx_octets
;
4737 ns
->tx_packets
= stats
.tx_frames
;
4738 ns
->rx_bytes
= stats
.rx_octets
;
4739 ns
->rx_packets
= stats
.rx_frames
;
4740 ns
->multicast
= stats
.rx_mcast_frames
;
4742 /* detailed rx_errors */
4743 ns
->rx_length_errors
= stats
.rx_jabber
+ stats
.rx_too_long
+
4745 ns
->rx_over_errors
= 0;
4746 ns
->rx_crc_errors
= stats
.rx_fcs_err
;
4747 ns
->rx_frame_errors
= stats
.rx_symbol_err
;
4748 ns
->rx_fifo_errors
= stats
.rx_ovflow0
+ stats
.rx_ovflow1
+
4749 stats
.rx_ovflow2
+ stats
.rx_ovflow3
+
4750 stats
.rx_trunc0
+ stats
.rx_trunc1
+
4751 stats
.rx_trunc2
+ stats
.rx_trunc3
;
4752 ns
->rx_missed_errors
= 0;
4754 /* detailed tx_errors */
4755 ns
->tx_aborted_errors
= 0;
4756 ns
->tx_carrier_errors
= 0;
4757 ns
->tx_fifo_errors
= 0;
4758 ns
->tx_heartbeat_errors
= 0;
4759 ns
->tx_window_errors
= 0;
4761 ns
->tx_errors
= stats
.tx_error_frames
;
4762 ns
->rx_errors
= stats
.rx_symbol_err
+ stats
.rx_fcs_err
+
4763 ns
->rx_length_errors
+ stats
.rx_len_err
+ ns
->rx_fifo_errors
;
4767 static int cxgb_ioctl(struct net_device
*dev
, struct ifreq
*req
, int cmd
)
4770 int ret
= 0, prtad
, devad
;
4771 struct port_info
*pi
= netdev_priv(dev
);
4772 struct mii_ioctl_data
*data
= (struct mii_ioctl_data
*)&req
->ifr_data
;
4776 if (pi
->mdio_addr
< 0)
4778 data
->phy_id
= pi
->mdio_addr
;
4782 if (mdio_phy_id_is_c45(data
->phy_id
)) {
4783 prtad
= mdio_phy_id_prtad(data
->phy_id
);
4784 devad
= mdio_phy_id_devad(data
->phy_id
);
4785 } else if (data
->phy_id
< 32) {
4786 prtad
= data
->phy_id
;
4788 data
->reg_num
&= 0x1f;
4792 mbox
= pi
->adapter
->fn
;
4793 if (cmd
== SIOCGMIIREG
)
4794 ret
= t4_mdio_rd(pi
->adapter
, mbox
, prtad
, devad
,
4795 data
->reg_num
, &data
->val_out
);
4797 ret
= t4_mdio_wr(pi
->adapter
, mbox
, prtad
, devad
,
4798 data
->reg_num
, data
->val_in
);
4806 static void cxgb_set_rxmode(struct net_device
*dev
)
4808 /* unfortunately we can't return errors to the stack */
4809 set_rxmode(dev
, -1, false);
4812 static int cxgb_change_mtu(struct net_device
*dev
, int new_mtu
)
4815 struct port_info
*pi
= netdev_priv(dev
);
4817 if (new_mtu
< 81 || new_mtu
> MAX_MTU
) /* accommodate SACK */
4819 ret
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, new_mtu
, -1,
4826 static int cxgb_set_mac_addr(struct net_device
*dev
, void *p
)
4829 struct sockaddr
*addr
= p
;
4830 struct port_info
*pi
= netdev_priv(dev
);
4832 if (!is_valid_ether_addr(addr
->sa_data
))
4833 return -EADDRNOTAVAIL
;
4835 ret
= t4_change_mac(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
,
4836 pi
->xact_addr_filt
, addr
->sa_data
, true, true);
4840 memcpy(dev
->dev_addr
, addr
->sa_data
, dev
->addr_len
);
4841 pi
->xact_addr_filt
= ret
;
4845 #ifdef CONFIG_NET_POLL_CONTROLLER
4846 static void cxgb_netpoll(struct net_device
*dev
)
4848 struct port_info
*pi
= netdev_priv(dev
);
4849 struct adapter
*adap
= pi
->adapter
;
4851 if (adap
->flags
& USING_MSIX
) {
4853 struct sge_eth_rxq
*rx
= &adap
->sge
.ethrxq
[pi
->first_qset
];
4855 for (i
= pi
->nqsets
; i
; i
--, rx
++)
4856 t4_sge_intr_msix(0, &rx
->rspq
);
4858 t4_intr_handler(adap
)(0, adap
);
4862 static const struct net_device_ops cxgb4_netdev_ops
= {
4863 .ndo_open
= cxgb_open
,
4864 .ndo_stop
= cxgb_close
,
4865 .ndo_start_xmit
= t4_eth_xmit
,
4866 .ndo_select_queue
= cxgb_select_queue
,
4867 .ndo_get_stats64
= cxgb_get_stats
,
4868 .ndo_set_rx_mode
= cxgb_set_rxmode
,
4869 .ndo_set_mac_address
= cxgb_set_mac_addr
,
4870 .ndo_set_features
= cxgb_set_features
,
4871 .ndo_validate_addr
= eth_validate_addr
,
4872 .ndo_do_ioctl
= cxgb_ioctl
,
4873 .ndo_change_mtu
= cxgb_change_mtu
,
4874 #ifdef CONFIG_NET_POLL_CONTROLLER
4875 .ndo_poll_controller
= cxgb_netpoll
,
4879 void t4_fatal_err(struct adapter
*adap
)
4881 t4_set_reg_field(adap
, SGE_CONTROL
, GLOBALENABLE
, 0);
4882 t4_intr_disable(adap
);
4883 dev_alert(adap
->pdev_dev
, "encountered fatal error, adapter stopped\n");
4886 /* Return the specified PCI-E Configuration Space register from our Physical
4887 * Function. We try first via a Firmware LDST Command since we prefer to let
4888 * the firmware own all of these registers, but if that fails we go for it
4889 * directly ourselves.
4891 static u32
t4_read_pcie_cfg4(struct adapter
*adap
, int reg
)
4893 struct fw_ldst_cmd ldst_cmd
;
4897 /* Construct and send the Firmware LDST Command to retrieve the
4898 * specified PCI-E Configuration Space register.
4900 memset(&ldst_cmd
, 0, sizeof(ldst_cmd
));
4901 ldst_cmd
.op_to_addrspace
=
4902 htonl(FW_CMD_OP(FW_LDST_CMD
) |
4905 FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FUNC_PCIE
));
4906 ldst_cmd
.cycles_to_len16
= htonl(FW_LEN16(ldst_cmd
));
4907 ldst_cmd
.u
.pcie
.select_naccess
= FW_LDST_CMD_NACCESS(1);
4908 ldst_cmd
.u
.pcie
.ctrl_to_fn
=
4909 (FW_LDST_CMD_LC
| FW_LDST_CMD_FN(adap
->fn
));
4910 ldst_cmd
.u
.pcie
.r
= reg
;
4911 ret
= t4_wr_mbox(adap
, adap
->mbox
, &ldst_cmd
, sizeof(ldst_cmd
),
4914 /* If the LDST Command suucceeded, exctract the returned register
4915 * value. Otherwise read it directly ourself.
4918 val
= ntohl(ldst_cmd
.u
.pcie
.data
[0]);
4920 t4_hw_pci_read_cfg4(adap
, reg
, &val
);
4925 static void setup_memwin(struct adapter
*adap
)
4927 u32 mem_win0_base
, mem_win1_base
, mem_win2_base
, mem_win2_aperture
;
4929 if (is_t4(adap
->params
.chip
)) {
4932 /* Truncation intentional: we only read the bottom 32-bits of
4933 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor
4934 * mechanism to read BAR0 instead of using
4935 * pci_resource_start() because we could be operating from
4936 * within a Virtual Machine which is trapping our accesses to
4937 * our Configuration Space and we need to set up the PCI-E
4938 * Memory Window decoders with the actual addresses which will
4939 * be coming across the PCI-E link.
4941 bar0
= t4_read_pcie_cfg4(adap
, PCI_BASE_ADDRESS_0
);
4942 bar0
&= PCI_BASE_ADDRESS_MEM_MASK
;
4943 adap
->t4_bar0
= bar0
;
4945 mem_win0_base
= bar0
+ MEMWIN0_BASE
;
4946 mem_win1_base
= bar0
+ MEMWIN1_BASE
;
4947 mem_win2_base
= bar0
+ MEMWIN2_BASE
;
4948 mem_win2_aperture
= MEMWIN2_APERTURE
;
4950 /* For T5, only relative offset inside the PCIe BAR is passed */
4951 mem_win0_base
= MEMWIN0_BASE
;
4952 mem_win1_base
= MEMWIN1_BASE
;
4953 mem_win2_base
= MEMWIN2_BASE_T5
;
4954 mem_win2_aperture
= MEMWIN2_APERTURE_T5
;
4956 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 0),
4957 mem_win0_base
| BIR(0) |
4958 WINDOW(ilog2(MEMWIN0_APERTURE
) - 10));
4959 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 1),
4960 mem_win1_base
| BIR(0) |
4961 WINDOW(ilog2(MEMWIN1_APERTURE
) - 10));
4962 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 2),
4963 mem_win2_base
| BIR(0) |
4964 WINDOW(ilog2(mem_win2_aperture
) - 10));
4965 t4_read_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 2));
4968 static void setup_memwin_rdma(struct adapter
*adap
)
4970 if (adap
->vres
.ocq
.size
) {
4974 start
= t4_read_pcie_cfg4(adap
, PCI_BASE_ADDRESS_2
);
4975 start
&= PCI_BASE_ADDRESS_MEM_MASK
;
4976 start
+= OCQ_WIN_OFFSET(adap
->pdev
, &adap
->vres
);
4977 sz_kb
= roundup_pow_of_two(adap
->vres
.ocq
.size
) >> 10;
4979 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 3),
4980 start
| BIR(1) | WINDOW(ilog2(sz_kb
)));
4982 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET
, 3),
4983 adap
->vres
.ocq
.start
);
4985 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET
, 3));
4989 static int adap_init1(struct adapter
*adap
, struct fw_caps_config_cmd
*c
)
4994 /* get device capabilities */
4995 memset(c
, 0, sizeof(*c
));
4996 c
->op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4997 FW_CMD_REQUEST
| FW_CMD_READ
);
4998 c
->cfvalid_to_len16
= htonl(FW_LEN16(*c
));
4999 ret
= t4_wr_mbox(adap
, adap
->fn
, c
, sizeof(*c
), c
);
5003 /* select capabilities we'll be using */
5004 if (c
->niccaps
& htons(FW_CAPS_CONFIG_NIC_VM
)) {
5006 c
->niccaps
^= htons(FW_CAPS_CONFIG_NIC_VM
);
5008 c
->niccaps
= htons(FW_CAPS_CONFIG_NIC_VM
);
5009 } else if (vf_acls
) {
5010 dev_err(adap
->pdev_dev
, "virtualization ACLs not supported");
5013 c
->op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5014 FW_CMD_REQUEST
| FW_CMD_WRITE
);
5015 ret
= t4_wr_mbox(adap
, adap
->fn
, c
, sizeof(*c
), NULL
);
5019 ret
= t4_config_glbl_rss(adap
, adap
->fn
,
5020 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL
,
5021 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN
|
5022 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP
);
5026 ret
= t4_cfg_pfvf(adap
, adap
->fn
, adap
->fn
, 0, MAX_EGRQ
, 64, MAX_INGQ
,
5027 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF
, FW_CMD_CAP_PF
);
5033 /* tweak some settings */
5034 t4_write_reg(adap
, TP_SHIFT_CNT
, 0x64f8849);
5035 t4_write_reg(adap
, ULP_RX_TDDP_PSZ
, HPZ0(PAGE_SHIFT
- 12));
5036 t4_write_reg(adap
, TP_PIO_ADDR
, TP_INGRESS_CONFIG
);
5037 v
= t4_read_reg(adap
, TP_PIO_DATA
);
5038 t4_write_reg(adap
, TP_PIO_DATA
, v
& ~CSUM_HAS_PSEUDO_HDR
);
5040 /* first 4 Tx modulation queues point to consecutive Tx channels */
5041 adap
->params
.tp
.tx_modq_map
= 0xE4;
5042 t4_write_reg(adap
, A_TP_TX_MOD_QUEUE_REQ_MAP
,
5043 V_TX_MOD_QUEUE_REQ_MAP(adap
->params
.tp
.tx_modq_map
));
5045 /* associate each Tx modulation queue with consecutive Tx channels */
5047 t4_write_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
5048 &v
, 1, A_TP_TX_SCHED_HDR
);
5049 t4_write_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
5050 &v
, 1, A_TP_TX_SCHED_FIFO
);
5051 t4_write_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
5052 &v
, 1, A_TP_TX_SCHED_PCMD
);
5054 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
5055 if (is_offload(adap
)) {
5056 t4_write_reg(adap
, A_TP_TX_MOD_QUEUE_WEIGHT0
,
5057 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
5058 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
5059 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
5060 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT
));
5061 t4_write_reg(adap
, A_TP_TX_MOD_CHANNEL_WEIGHT
,
5062 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
5063 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
5064 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
5065 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT
));
5068 /* get basic stuff going */
5069 return t4_early_init(adap
, adap
->fn
);
5073 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
5075 #define MAX_ATIDS 8192U
5078 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5080 * If the firmware we're dealing with has Configuration File support, then
5081 * we use that to perform all configuration
5085 * Tweak configuration based on module parameters, etc. Most of these have
5086 * defaults assigned to them by Firmware Configuration Files (if we're using
5087 * them) but need to be explicitly set if we're using hard-coded
5088 * initialization. But even in the case of using Firmware Configuration
5089 * Files, we'd like to expose the ability to change these via module
5090 * parameters so these are essentially common tweaks/settings for
5091 * Configuration Files and hard-coded initialization ...
5093 static int adap_init0_tweaks(struct adapter
*adapter
)
5096 * Fix up various Host-Dependent Parameters like Page Size, Cache
5097 * Line Size, etc. The firmware default is for a 4KB Page Size and
5098 * 64B Cache Line Size ...
5100 t4_fixup_host_params(adapter
, PAGE_SIZE
, L1_CACHE_BYTES
);
5103 * Process module parameters which affect early initialization.
5105 if (rx_dma_offset
!= 2 && rx_dma_offset
!= 0) {
5106 dev_err(&adapter
->pdev
->dev
,
5107 "Ignoring illegal rx_dma_offset=%d, using 2\n",
5111 t4_set_reg_field(adapter
, SGE_CONTROL
,
5113 PKTSHIFT(rx_dma_offset
));
5116 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
5117 * adds the pseudo header itself.
5119 t4_tp_wr_bits_indirect(adapter
, TP_INGRESS_CONFIG
,
5120 CSUM_HAS_PSEUDO_HDR
, 0);
5126 * Attempt to initialize the adapter via a Firmware Configuration File.
5128 static int adap_init0_config(struct adapter
*adapter
, int reset
)
5130 struct fw_caps_config_cmd caps_cmd
;
5131 const struct firmware
*cf
;
5132 unsigned long mtype
= 0, maddr
= 0;
5133 u32 finiver
, finicsum
, cfcsum
;
5135 int config_issued
= 0;
5136 char *fw_config_file
, fw_config_file_path
[256];
5137 char *config_name
= NULL
;
5140 * Reset device if necessary.
5143 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
5144 PIORSTMODE
| PIORST
);
5150 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
5151 * then use that. Otherwise, use the configuration file stored
5152 * in the adapter flash ...
5154 switch (CHELSIO_CHIP_VERSION(adapter
->params
.chip
)) {
5156 fw_config_file
= FW4_CFNAME
;
5159 fw_config_file
= FW5_CFNAME
;
5162 dev_err(adapter
->pdev_dev
, "Device %d is not supported\n",
5163 adapter
->pdev
->device
);
5168 ret
= request_firmware(&cf
, fw_config_file
, adapter
->pdev_dev
);
5170 config_name
= "On FLASH";
5171 mtype
= FW_MEMTYPE_CF_FLASH
;
5172 maddr
= t4_flash_cfg_addr(adapter
);
5174 u32 params
[7], val
[7];
5176 sprintf(fw_config_file_path
,
5177 "/lib/firmware/%s", fw_config_file
);
5178 config_name
= fw_config_file_path
;
5180 if (cf
->size
>= FLASH_CFG_MAX_SIZE
)
5183 params
[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
5184 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF
));
5185 ret
= t4_query_params(adapter
, adapter
->mbox
,
5186 adapter
->fn
, 0, 1, params
, val
);
5189 * For t4_memory_rw() below addresses and
5190 * sizes have to be in terms of multiples of 4
5191 * bytes. So, if the Configuration File isn't
5192 * a multiple of 4 bytes in length we'll have
5193 * to write that out separately since we can't
5194 * guarantee that the bytes following the
5195 * residual byte in the buffer returned by
5196 * request_firmware() are zeroed out ...
5198 size_t resid
= cf
->size
& 0x3;
5199 size_t size
= cf
->size
& ~0x3;
5200 __be32
*data
= (__be32
*)cf
->data
;
5202 mtype
= FW_PARAMS_PARAM_Y_GET(val
[0]);
5203 maddr
= FW_PARAMS_PARAM_Z_GET(val
[0]) << 16;
5205 spin_lock(&adapter
->win0_lock
);
5206 ret
= t4_memory_rw(adapter
, 0, mtype
, maddr
,
5207 size
, data
, T4_MEMORY_WRITE
);
5208 if (ret
== 0 && resid
!= 0) {
5215 last
.word
= data
[size
>> 2];
5216 for (i
= resid
; i
< 4; i
++)
5218 ret
= t4_memory_rw(adapter
, 0, mtype
,
5223 spin_unlock(&adapter
->win0_lock
);
5227 release_firmware(cf
);
5233 * Issue a Capability Configuration command to the firmware to get it
5234 * to parse the Configuration File. We don't use t4_fw_config_file()
5235 * because we want the ability to modify various features after we've
5236 * processed the configuration file ...
5238 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
5239 caps_cmd
.op_to_write
=
5240 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5243 caps_cmd
.cfvalid_to_len16
=
5244 htonl(FW_CAPS_CONFIG_CMD_CFVALID
|
5245 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype
) |
5246 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr
>> 16) |
5247 FW_LEN16(caps_cmd
));
5248 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
5251 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
5252 * Configuration File in FLASH), our last gasp effort is to use the
5253 * Firmware Configuration File which is embedded in the firmware. A
5254 * very few early versions of the firmware didn't have one embedded
5255 * but we can ignore those.
5257 if (ret
== -ENOENT
) {
5258 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
5259 caps_cmd
.op_to_write
=
5260 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5263 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
5264 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
,
5265 sizeof(caps_cmd
), &caps_cmd
);
5266 config_name
= "Firmware Default";
5273 finiver
= ntohl(caps_cmd
.finiver
);
5274 finicsum
= ntohl(caps_cmd
.finicsum
);
5275 cfcsum
= ntohl(caps_cmd
.cfcsum
);
5276 if (finicsum
!= cfcsum
)
5277 dev_warn(adapter
->pdev_dev
, "Configuration File checksum "\
5278 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
5282 * And now tell the firmware to use the configuration we just loaded.
5284 caps_cmd
.op_to_write
=
5285 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5288 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
5289 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
5295 * Tweak configuration based on system architecture, module
5298 ret
= adap_init0_tweaks(adapter
);
5303 * And finally tell the firmware to initialize itself using the
5304 * parameters from the Configuration File.
5306 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
5311 * Return successfully and note that we're operating with parameters
5312 * not supplied by the driver, rather than from hard-wired
5313 * initialization constants burried in the driver.
5315 adapter
->flags
|= USING_SOFT_PARAMS
;
5316 dev_info(adapter
->pdev_dev
, "Successfully configured using Firmware "\
5317 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5318 config_name
, finiver
, cfcsum
);
5322 * Something bad happened. Return the error ... (If the "error"
5323 * is that there's no Configuration File on the adapter we don't
5324 * want to issue a warning since this is fairly common.)
5327 if (config_issued
&& ret
!= -ENOENT
)
5328 dev_warn(adapter
->pdev_dev
, "\"%s\" configuration file error %d\n",
5334 * Attempt to initialize the adapter via hard-coded, driver supplied
5337 static int adap_init0_no_config(struct adapter
*adapter
, int reset
)
5339 struct sge
*s
= &adapter
->sge
;
5340 struct fw_caps_config_cmd caps_cmd
;
5345 * Reset device if necessary
5348 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
5349 PIORSTMODE
| PIORST
);
5355 * Get device capabilities and select which we'll be using.
5357 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
5358 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5359 FW_CMD_REQUEST
| FW_CMD_READ
);
5360 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
5361 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
5366 if (caps_cmd
.niccaps
& htons(FW_CAPS_CONFIG_NIC_VM
)) {
5368 caps_cmd
.niccaps
^= htons(FW_CAPS_CONFIG_NIC_VM
);
5370 caps_cmd
.niccaps
= htons(FW_CAPS_CONFIG_NIC_VM
);
5371 } else if (vf_acls
) {
5372 dev_err(adapter
->pdev_dev
, "virtualization ACLs not supported");
5375 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5376 FW_CMD_REQUEST
| FW_CMD_WRITE
);
5377 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
5383 * Tweak configuration based on system architecture, module
5386 ret
= adap_init0_tweaks(adapter
);
5391 * Select RSS Global Mode we want to use. We use "Basic Virtual"
5392 * mode which maps each Virtual Interface to its own section of
5393 * the RSS Table and we turn on all map and hash enables ...
5395 adapter
->flags
|= RSS_TNLALLLOOKUP
;
5396 ret
= t4_config_glbl_rss(adapter
, adapter
->mbox
,
5397 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL
,
5398 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN
|
5399 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ
|
5400 ((adapter
->flags
& RSS_TNLALLLOOKUP
) ?
5401 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP
: 0));
5406 * Set up our own fundamental resource provisioning ...
5408 ret
= t4_cfg_pfvf(adapter
, adapter
->mbox
, adapter
->fn
, 0,
5409 PFRES_NEQ
, PFRES_NETHCTRL
,
5410 PFRES_NIQFLINT
, PFRES_NIQ
,
5411 PFRES_TC
, PFRES_NVI
,
5412 FW_PFVF_CMD_CMASK_MASK
,
5413 pfvfres_pmask(adapter
, adapter
->fn
, 0),
5415 PFRES_R_CAPS
, PFRES_WX_CAPS
);
5420 * Perform low level SGE initialization. We need to do this before we
5421 * send the firmware the INITIALIZE command because that will cause
5422 * any other PF Drivers which are waiting for the Master
5423 * Initialization to proceed forward.
5425 for (i
= 0; i
< SGE_NTIMERS
- 1; i
++)
5426 s
->timer_val
[i
] = min(intr_holdoff
[i
], MAX_SGE_TIMERVAL
);
5427 s
->timer_val
[SGE_NTIMERS
- 1] = MAX_SGE_TIMERVAL
;
5428 s
->counter_val
[0] = 1;
5429 for (i
= 1; i
< SGE_NCOUNTERS
; i
++)
5430 s
->counter_val
[i
] = min(intr_cnt
[i
- 1],
5431 THRESHOLD_0_GET(THRESHOLD_0_MASK
));
5432 t4_sge_init(adapter
);
5434 #ifdef CONFIG_PCI_IOV
5436 * Provision resource limits for Virtual Functions. We currently
5437 * grant them all the same static resource limits except for the Port
5438 * Access Rights Mask which we're assigning based on the PF. All of
5439 * the static provisioning stuff for both the PF and VF really needs
5440 * to be managed in a persistent manner for each device which the
5441 * firmware controls.
5446 for (pf
= 0; pf
< ARRAY_SIZE(num_vf
); pf
++) {
5447 if (num_vf
[pf
] <= 0)
5450 /* VF numbering starts at 1! */
5451 for (vf
= 1; vf
<= num_vf
[pf
]; vf
++) {
5452 ret
= t4_cfg_pfvf(adapter
, adapter
->mbox
,
5454 VFRES_NEQ
, VFRES_NETHCTRL
,
5455 VFRES_NIQFLINT
, VFRES_NIQ
,
5456 VFRES_TC
, VFRES_NVI
,
5457 FW_PFVF_CMD_CMASK_MASK
,
5461 VFRES_R_CAPS
, VFRES_WX_CAPS
);
5463 dev_warn(adapter
->pdev_dev
,
5465 "provision pf/vf=%d/%d; "
5466 "err=%d\n", pf
, vf
, ret
);
5473 * Set up the default filter mode. Later we'll want to implement this
5474 * via a firmware command, etc. ... This needs to be done before the
5475 * firmare initialization command ... If the selected set of fields
5476 * isn't equal to the default value, we'll need to make sure that the
5477 * field selections will fit in the 36-bit budget.
5479 if (tp_vlan_pri_map
!= TP_VLAN_PRI_MAP_DEFAULT
) {
5482 for (j
= TP_VLAN_PRI_MAP_FIRST
; j
<= TP_VLAN_PRI_MAP_LAST
; j
++)
5483 switch (tp_vlan_pri_map
& (1 << j
)) {
5485 /* compressed filter field not enabled */
5505 case ETHERTYPE_MASK
:
5511 case MPSHITTYPE_MASK
:
5514 case FRAGMENTATION_MASK
:
5520 dev_err(adapter
->pdev_dev
,
5521 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5522 " using %#x\n", tp_vlan_pri_map
, bits
,
5523 TP_VLAN_PRI_MAP_DEFAULT
);
5524 tp_vlan_pri_map
= TP_VLAN_PRI_MAP_DEFAULT
;
5527 v
= tp_vlan_pri_map
;
5528 t4_write_indirect(adapter
, TP_PIO_ADDR
, TP_PIO_DATA
,
5529 &v
, 1, TP_VLAN_PRI_MAP
);
5532 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5533 * to support any of the compressed filter fields above. Newer
5534 * versions of the firmware do this automatically but it doesn't hurt
5535 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5536 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5537 * since the firmware automatically turns this on and off when we have
5538 * a non-zero number of filters active (since it does have a
5539 * performance impact).
5541 if (tp_vlan_pri_map
)
5542 t4_set_reg_field(adapter
, TP_GLOBAL_CONFIG
,
5543 FIVETUPLELOOKUP_MASK
,
5544 FIVETUPLELOOKUP_MASK
);
5547 * Tweak some settings.
5549 t4_write_reg(adapter
, TP_SHIFT_CNT
, SYNSHIFTMAX(6) |
5550 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5551 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5552 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5555 * Get basic stuff going by issuing the Firmware Initialize command.
5556 * Note that this _must_ be after all PFVF commands ...
5558 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
5563 * Return successfully!
5565 dev_info(adapter
->pdev_dev
, "Successfully configured using built-in "\
5566 "driver parameters\n");
5570 * Something bad happened. Return the error ...
5576 static struct fw_info fw_info_array
[] = {
5579 .fs_name
= FW4_CFNAME
,
5580 .fw_mod_name
= FW4_FNAME
,
5582 .chip
= FW_HDR_CHIP_T4
,
5583 .fw_ver
= __cpu_to_be32(FW_VERSION(T4
)),
5584 .intfver_nic
= FW_INTFVER(T4
, NIC
),
5585 .intfver_vnic
= FW_INTFVER(T4
, VNIC
),
5586 .intfver_ri
= FW_INTFVER(T4
, RI
),
5587 .intfver_iscsi
= FW_INTFVER(T4
, ISCSI
),
5588 .intfver_fcoe
= FW_INTFVER(T4
, FCOE
),
5592 .fs_name
= FW5_CFNAME
,
5593 .fw_mod_name
= FW5_FNAME
,
5595 .chip
= FW_HDR_CHIP_T5
,
5596 .fw_ver
= __cpu_to_be32(FW_VERSION(T5
)),
5597 .intfver_nic
= FW_INTFVER(T5
, NIC
),
5598 .intfver_vnic
= FW_INTFVER(T5
, VNIC
),
5599 .intfver_ri
= FW_INTFVER(T5
, RI
),
5600 .intfver_iscsi
= FW_INTFVER(T5
, ISCSI
),
5601 .intfver_fcoe
= FW_INTFVER(T5
, FCOE
),
5606 static struct fw_info
*find_fw_info(int chip
)
5610 for (i
= 0; i
< ARRAY_SIZE(fw_info_array
); i
++) {
5611 if (fw_info_array
[i
].chip
== chip
)
5612 return &fw_info_array
[i
];
5618 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5620 static int adap_init0(struct adapter
*adap
)
5624 enum dev_state state
;
5625 u32 params
[7], val
[7];
5626 struct fw_caps_config_cmd caps_cmd
;
5630 * Contact FW, advertising Master capability (and potentially forcing
5631 * ourselves as the Master PF if our module parameter force_init is
5634 ret
= t4_fw_hello(adap
, adap
->mbox
, adap
->fn
,
5635 force_init
? MASTER_MUST
: MASTER_MAY
,
5638 dev_err(adap
->pdev_dev
, "could not connect to FW, error %d\n",
5642 if (ret
== adap
->mbox
)
5643 adap
->flags
|= MASTER_PF
;
5644 if (force_init
&& state
== DEV_STATE_INIT
)
5645 state
= DEV_STATE_UNINIT
;
5648 * If we're the Master PF Driver and the device is uninitialized,
5649 * then let's consider upgrading the firmware ... (We always want
5650 * to check the firmware version number in order to A. get it for
5651 * later reporting and B. to warn if the currently loaded firmware
5652 * is excessively mismatched relative to the driver.)
5654 t4_get_fw_version(adap
, &adap
->params
.fw_vers
);
5655 t4_get_tp_version(adap
, &adap
->params
.tp_vers
);
5656 if ((adap
->flags
& MASTER_PF
) && state
!= DEV_STATE_INIT
) {
5657 struct fw_info
*fw_info
;
5658 struct fw_hdr
*card_fw
;
5659 const struct firmware
*fw
;
5660 const u8
*fw_data
= NULL
;
5661 unsigned int fw_size
= 0;
5663 /* This is the firmware whose headers the driver was compiled
5666 fw_info
= find_fw_info(CHELSIO_CHIP_VERSION(adap
->params
.chip
));
5667 if (fw_info
== NULL
) {
5668 dev_err(adap
->pdev_dev
,
5669 "unable to get firmware info for chip %d.\n",
5670 CHELSIO_CHIP_VERSION(adap
->params
.chip
));
5674 /* allocate memory to read the header of the firmware on the
5677 card_fw
= t4_alloc_mem(sizeof(*card_fw
));
5679 /* Get FW from from /lib/firmware/ */
5680 ret
= request_firmware(&fw
, fw_info
->fw_mod_name
,
5683 dev_err(adap
->pdev_dev
,
5684 "unable to load firmware image %s, error %d\n",
5685 fw_info
->fw_mod_name
, ret
);
5691 /* upgrade FW logic */
5692 ret
= t4_prep_fw(adap
, fw_info
, fw_data
, fw_size
, card_fw
,
5697 release_firmware(fw
);
5698 t4_free_mem(card_fw
);
5705 * Grab VPD parameters. This should be done after we establish a
5706 * connection to the firmware since some of the VPD parameters
5707 * (notably the Core Clock frequency) are retrieved via requests to
5708 * the firmware. On the other hand, we need these fairly early on
5709 * so we do this right after getting ahold of the firmware.
5711 ret
= get_vpd_params(adap
, &adap
->params
.vpd
);
5716 * Find out what ports are available to us. Note that we need to do
5717 * this before calling adap_init0_no_config() since it needs nports
5721 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
5722 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC
);
5723 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 1, &v
, &port_vec
);
5727 adap
->params
.nports
= hweight32(port_vec
);
5728 adap
->params
.portvec
= port_vec
;
5731 * If the firmware is initialized already (and we're not forcing a
5732 * master initialization), note that we're living with existing
5733 * adapter parameters. Otherwise, it's time to try initializing the
5736 if (state
== DEV_STATE_INIT
) {
5737 dev_info(adap
->pdev_dev
, "Coming up as %s: "\
5738 "Adapter already initialized\n",
5739 adap
->flags
& MASTER_PF
? "MASTER" : "SLAVE");
5740 adap
->flags
|= USING_SOFT_PARAMS
;
5742 dev_info(adap
->pdev_dev
, "Coming up as MASTER: "\
5743 "Initializing adapter\n");
5746 * If the firmware doesn't support Configuration
5747 * Files warn user and exit,
5750 dev_warn(adap
->pdev_dev
, "Firmware doesn't support "
5751 "configuration file.\n");
5753 ret
= adap_init0_no_config(adap
, reset
);
5756 * Find out whether we're dealing with a version of
5757 * the firmware which has configuration file support.
5759 params
[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
5760 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF
));
5761 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 1,
5765 * If the firmware doesn't support Configuration
5766 * Files, use the old Driver-based, hard-wired
5767 * initialization. Otherwise, try using the
5768 * Configuration File support and fall back to the
5769 * Driver-based initialization if there's no
5770 * Configuration File found.
5773 ret
= adap_init0_no_config(adap
, reset
);
5776 * The firmware provides us with a memory
5777 * buffer where we can load a Configuration
5778 * File from the host if we want to override
5779 * the Configuration File in flash.
5782 ret
= adap_init0_config(adap
, reset
);
5783 if (ret
== -ENOENT
) {
5784 dev_info(adap
->pdev_dev
,
5785 "No Configuration File present "
5786 "on adapter. Using hard-wired "
5787 "configuration parameters.\n");
5788 ret
= adap_init0_no_config(adap
, reset
);
5793 dev_err(adap
->pdev_dev
,
5794 "could not initialize adapter, error %d\n",
5801 * If we're living with non-hard-coded parameters (either from a
5802 * Firmware Configuration File or values programmed by a different PF
5803 * Driver), give the SGE code a chance to pull in anything that it
5804 * needs ... Note that this must be called after we retrieve our VPD
5805 * parameters in order to know how to convert core ticks to seconds.
5807 if (adap
->flags
& USING_SOFT_PARAMS
) {
5808 ret
= t4_sge_init(adap
);
5813 if (is_bypass_device(adap
->pdev
->device
))
5814 adap
->params
.bypass
= 1;
5817 * Grab some of our basic fundamental operating parameters.
5819 #define FW_PARAM_DEV(param) \
5820 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5821 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5823 #define FW_PARAM_PFVF(param) \
5824 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5825 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
5826 FW_PARAMS_PARAM_Y(0) | \
5827 FW_PARAMS_PARAM_Z(0)
5829 params
[0] = FW_PARAM_PFVF(EQ_START
);
5830 params
[1] = FW_PARAM_PFVF(L2T_START
);
5831 params
[2] = FW_PARAM_PFVF(L2T_END
);
5832 params
[3] = FW_PARAM_PFVF(FILTER_START
);
5833 params
[4] = FW_PARAM_PFVF(FILTER_END
);
5834 params
[5] = FW_PARAM_PFVF(IQFLINT_START
);
5835 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6, params
, val
);
5838 adap
->sge
.egr_start
= val
[0];
5839 adap
->l2t_start
= val
[1];
5840 adap
->l2t_end
= val
[2];
5841 adap
->tids
.ftid_base
= val
[3];
5842 adap
->tids
.nftids
= val
[4] - val
[3] + 1;
5843 adap
->sge
.ingr_start
= val
[5];
5845 /* query params related to active filter region */
5846 params
[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START
);
5847 params
[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END
);
5848 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 2, params
, val
);
5849 /* If Active filter size is set we enable establishing
5850 * offload connection through firmware work request
5852 if ((val
[0] != val
[1]) && (ret
>= 0)) {
5853 adap
->flags
|= FW_OFLD_CONN
;
5854 adap
->tids
.aftid_base
= val
[0];
5855 adap
->tids
.aftid_end
= val
[1];
5858 /* If we're running on newer firmware, let it know that we're
5859 * prepared to deal with encapsulated CPL messages. Older
5860 * firmware won't understand this and we'll just get
5861 * unencapsulated messages ...
5863 params
[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP
);
5865 (void) t4_set_params(adap
, adap
->mbox
, adap
->fn
, 0, 1, params
, val
);
5868 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5869 * capability. Earlier versions of the firmware didn't have the
5870 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5871 * permission to use ULPTX MEMWRITE DSGL.
5873 if (is_t4(adap
->params
.chip
)) {
5874 adap
->params
.ulptx_memwrite_dsgl
= false;
5876 params
[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL
);
5877 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0,
5879 adap
->params
.ulptx_memwrite_dsgl
= (ret
== 0 && val
[0] != 0);
5883 * Get device capabilities so we can determine what resources we need
5886 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
5887 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5888 FW_CMD_REQUEST
| FW_CMD_READ
);
5889 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
5890 ret
= t4_wr_mbox(adap
, adap
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
5895 if (caps_cmd
.ofldcaps
) {
5896 /* query offload-related parameters */
5897 params
[0] = FW_PARAM_DEV(NTID
);
5898 params
[1] = FW_PARAM_PFVF(SERVER_START
);
5899 params
[2] = FW_PARAM_PFVF(SERVER_END
);
5900 params
[3] = FW_PARAM_PFVF(TDDP_START
);
5901 params
[4] = FW_PARAM_PFVF(TDDP_END
);
5902 params
[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ
);
5903 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6,
5907 adap
->tids
.ntids
= val
[0];
5908 adap
->tids
.natids
= min(adap
->tids
.ntids
/ 2, MAX_ATIDS
);
5909 adap
->tids
.stid_base
= val
[1];
5910 adap
->tids
.nstids
= val
[2] - val
[1] + 1;
5912 * Setup server filter region. Divide the availble filter
5913 * region into two parts. Regular filters get 1/3rd and server
5914 * filters get 2/3rd part. This is only enabled if workarond
5916 * 1. For regular filters.
5917 * 2. Server filter: This are special filters which are used
5918 * to redirect SYN packets to offload queue.
5920 if (adap
->flags
& FW_OFLD_CONN
&& !is_bypass(adap
)) {
5921 adap
->tids
.sftid_base
= adap
->tids
.ftid_base
+
5922 DIV_ROUND_UP(adap
->tids
.nftids
, 3);
5923 adap
->tids
.nsftids
= adap
->tids
.nftids
-
5924 DIV_ROUND_UP(adap
->tids
.nftids
, 3);
5925 adap
->tids
.nftids
= adap
->tids
.sftid_base
-
5926 adap
->tids
.ftid_base
;
5928 adap
->vres
.ddp
.start
= val
[3];
5929 adap
->vres
.ddp
.size
= val
[4] - val
[3] + 1;
5930 adap
->params
.ofldq_wr_cred
= val
[5];
5932 adap
->params
.offload
= 1;
5934 if (caps_cmd
.rdmacaps
) {
5935 params
[0] = FW_PARAM_PFVF(STAG_START
);
5936 params
[1] = FW_PARAM_PFVF(STAG_END
);
5937 params
[2] = FW_PARAM_PFVF(RQ_START
);
5938 params
[3] = FW_PARAM_PFVF(RQ_END
);
5939 params
[4] = FW_PARAM_PFVF(PBL_START
);
5940 params
[5] = FW_PARAM_PFVF(PBL_END
);
5941 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6,
5945 adap
->vres
.stag
.start
= val
[0];
5946 adap
->vres
.stag
.size
= val
[1] - val
[0] + 1;
5947 adap
->vres
.rq
.start
= val
[2];
5948 adap
->vres
.rq
.size
= val
[3] - val
[2] + 1;
5949 adap
->vres
.pbl
.start
= val
[4];
5950 adap
->vres
.pbl
.size
= val
[5] - val
[4] + 1;
5952 params
[0] = FW_PARAM_PFVF(SQRQ_START
);
5953 params
[1] = FW_PARAM_PFVF(SQRQ_END
);
5954 params
[2] = FW_PARAM_PFVF(CQ_START
);
5955 params
[3] = FW_PARAM_PFVF(CQ_END
);
5956 params
[4] = FW_PARAM_PFVF(OCQ_START
);
5957 params
[5] = FW_PARAM_PFVF(OCQ_END
);
5958 ret
= t4_query_params(adap
, 0, 0, 0, 6, params
, val
);
5961 adap
->vres
.qp
.start
= val
[0];
5962 adap
->vres
.qp
.size
= val
[1] - val
[0] + 1;
5963 adap
->vres
.cq
.start
= val
[2];
5964 adap
->vres
.cq
.size
= val
[3] - val
[2] + 1;
5965 adap
->vres
.ocq
.start
= val
[4];
5966 adap
->vres
.ocq
.size
= val
[5] - val
[4] + 1;
5968 params
[0] = FW_PARAM_DEV(MAXORDIRD_QP
);
5969 params
[1] = FW_PARAM_DEV(MAXIRD_ADAPTER
);
5970 ret
= t4_query_params(adap
, 0, 0, 0, 2, params
, val
);
5972 adap
->params
.max_ordird_qp
= 8;
5973 adap
->params
.max_ird_adapter
= 32 * adap
->tids
.ntids
;
5976 adap
->params
.max_ordird_qp
= val
[0];
5977 adap
->params
.max_ird_adapter
= val
[1];
5979 dev_info(adap
->pdev_dev
,
5980 "max_ordird_qp %d max_ird_adapter %d\n",
5981 adap
->params
.max_ordird_qp
,
5982 adap
->params
.max_ird_adapter
);
5984 if (caps_cmd
.iscsicaps
) {
5985 params
[0] = FW_PARAM_PFVF(ISCSI_START
);
5986 params
[1] = FW_PARAM_PFVF(ISCSI_END
);
5987 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 2,
5991 adap
->vres
.iscsi
.start
= val
[0];
5992 adap
->vres
.iscsi
.size
= val
[1] - val
[0] + 1;
5994 #undef FW_PARAM_PFVF
5997 /* The MTU/MSS Table is initialized by now, so load their values. If
5998 * we're initializing the adapter, then we'll make any modifications
5999 * we want to the MTU/MSS Table and also initialize the congestion
6002 t4_read_mtu_tbl(adap
, adap
->params
.mtus
, NULL
);
6003 if (state
!= DEV_STATE_INIT
) {
6006 /* The default MTU Table contains values 1492 and 1500.
6007 * However, for TCP, it's better to have two values which are
6008 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
6009 * This allows us to have a TCP Data Payload which is a
6010 * multiple of 8 regardless of what combination of TCP Options
6011 * are in use (always a multiple of 4 bytes) which is
6012 * important for performance reasons. For instance, if no
6013 * options are in use, then we have a 20-byte IP header and a
6014 * 20-byte TCP header. In this case, a 1500-byte MSS would
6015 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
6016 * which is not a multiple of 8. So using an MSS of 1488 in
6017 * this case results in a TCP Data Payload of 1448 bytes which
6018 * is a multiple of 8. On the other hand, if 12-byte TCP Time
6019 * Stamps have been negotiated, then an MTU of 1500 bytes
6020 * results in a TCP Data Payload of 1448 bytes which, as
6021 * above, is a multiple of 8 bytes ...
6023 for (i
= 0; i
< NMTUS
; i
++)
6024 if (adap
->params
.mtus
[i
] == 1492) {
6025 adap
->params
.mtus
[i
] = 1488;
6029 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
6030 adap
->params
.b_wnd
);
6032 t4_init_tp_params(adap
);
6033 adap
->flags
|= FW_OK
;
6037 * Something bad happened. If a command timed out or failed with EIO
6038 * FW does not operate within its spec or something catastrophic
6039 * happened to HW/FW, stop issuing commands.
6042 if (ret
!= -ETIMEDOUT
&& ret
!= -EIO
)
6043 t4_fw_bye(adap
, adap
->mbox
);
6049 static pci_ers_result_t
eeh_err_detected(struct pci_dev
*pdev
,
6050 pci_channel_state_t state
)
6053 struct adapter
*adap
= pci_get_drvdata(pdev
);
6059 adap
->flags
&= ~FW_OK
;
6060 notify_ulds(adap
, CXGB4_STATE_START_RECOVERY
);
6061 spin_lock(&adap
->stats_lock
);
6062 for_each_port(adap
, i
) {
6063 struct net_device
*dev
= adap
->port
[i
];
6065 netif_device_detach(dev
);
6066 netif_carrier_off(dev
);
6068 spin_unlock(&adap
->stats_lock
);
6069 if (adap
->flags
& FULL_INIT_DONE
)
6072 if ((adap
->flags
& DEV_ENABLED
)) {
6073 pci_disable_device(pdev
);
6074 adap
->flags
&= ~DEV_ENABLED
;
6076 out
: return state
== pci_channel_io_perm_failure
?
6077 PCI_ERS_RESULT_DISCONNECT
: PCI_ERS_RESULT_NEED_RESET
;
6080 static pci_ers_result_t
eeh_slot_reset(struct pci_dev
*pdev
)
6083 struct fw_caps_config_cmd c
;
6084 struct adapter
*adap
= pci_get_drvdata(pdev
);
6087 pci_restore_state(pdev
);
6088 pci_save_state(pdev
);
6089 return PCI_ERS_RESULT_RECOVERED
;
6092 if (!(adap
->flags
& DEV_ENABLED
)) {
6093 if (pci_enable_device(pdev
)) {
6094 dev_err(&pdev
->dev
, "Cannot reenable PCI "
6095 "device after reset\n");
6096 return PCI_ERS_RESULT_DISCONNECT
;
6098 adap
->flags
|= DEV_ENABLED
;
6101 pci_set_master(pdev
);
6102 pci_restore_state(pdev
);
6103 pci_save_state(pdev
);
6104 pci_cleanup_aer_uncorrect_error_status(pdev
);
6106 if (t4_wait_dev_ready(adap
) < 0)
6107 return PCI_ERS_RESULT_DISCONNECT
;
6108 if (t4_fw_hello(adap
, adap
->fn
, adap
->fn
, MASTER_MUST
, NULL
) < 0)
6109 return PCI_ERS_RESULT_DISCONNECT
;
6110 adap
->flags
|= FW_OK
;
6111 if (adap_init1(adap
, &c
))
6112 return PCI_ERS_RESULT_DISCONNECT
;
6114 for_each_port(adap
, i
) {
6115 struct port_info
*p
= adap2pinfo(adap
, i
);
6117 ret
= t4_alloc_vi(adap
, adap
->fn
, p
->tx_chan
, adap
->fn
, 0, 1,
6120 return PCI_ERS_RESULT_DISCONNECT
;
6122 p
->xact_addr_filt
= -1;
6125 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
6126 adap
->params
.b_wnd
);
6129 return PCI_ERS_RESULT_DISCONNECT
;
6130 return PCI_ERS_RESULT_RECOVERED
;
6133 static void eeh_resume(struct pci_dev
*pdev
)
6136 struct adapter
*adap
= pci_get_drvdata(pdev
);
6142 for_each_port(adap
, i
) {
6143 struct net_device
*dev
= adap
->port
[i
];
6145 if (netif_running(dev
)) {
6147 cxgb_set_rxmode(dev
);
6149 netif_device_attach(dev
);
6154 static const struct pci_error_handlers cxgb4_eeh
= {
6155 .error_detected
= eeh_err_detected
,
6156 .slot_reset
= eeh_slot_reset
,
6157 .resume
= eeh_resume
,
6160 static inline bool is_x_10g_port(const struct link_config
*lc
)
6162 return (lc
->supported
& FW_PORT_CAP_SPEED_10G
) != 0 ||
6163 (lc
->supported
& FW_PORT_CAP_SPEED_40G
) != 0;
6166 static inline void init_rspq(struct adapter
*adap
, struct sge_rspq
*q
,
6167 unsigned int us
, unsigned int cnt
,
6168 unsigned int size
, unsigned int iqe_size
)
6171 set_rspq_intr_params(q
, us
, cnt
);
6172 q
->iqe_len
= iqe_size
;
6177 * Perform default configuration of DMA queues depending on the number and type
6178 * of ports we found and the number of available CPUs. Most settings can be
6179 * modified by the admin prior to actual use.
6181 static void cfg_queues(struct adapter
*adap
)
6183 struct sge
*s
= &adap
->sge
;
6184 int i
, n10g
= 0, qidx
= 0;
6185 #ifndef CONFIG_CHELSIO_T4_DCB
6190 for_each_port(adap
, i
)
6191 n10g
+= is_x_10g_port(&adap2pinfo(adap
, i
)->link_cfg
);
6192 #ifdef CONFIG_CHELSIO_T4_DCB
6193 /* For Data Center Bridging support we need to be able to support up
6194 * to 8 Traffic Priorities; each of which will be assigned to its
6195 * own TX Queue in order to prevent Head-Of-Line Blocking.
6197 if (adap
->params
.nports
* 8 > MAX_ETH_QSETS
) {
6198 dev_err(adap
->pdev_dev
, "MAX_ETH_QSETS=%d < %d!\n",
6199 MAX_ETH_QSETS
, adap
->params
.nports
* 8);
6203 for_each_port(adap
, i
) {
6204 struct port_info
*pi
= adap2pinfo(adap
, i
);
6206 pi
->first_qset
= qidx
;
6210 #else /* !CONFIG_CHELSIO_T4_DCB */
6212 * We default to 1 queue per non-10G port and up to # of cores queues
6216 q10g
= (MAX_ETH_QSETS
- (adap
->params
.nports
- n10g
)) / n10g
;
6217 if (q10g
> netif_get_num_default_rss_queues())
6218 q10g
= netif_get_num_default_rss_queues();
6220 for_each_port(adap
, i
) {
6221 struct port_info
*pi
= adap2pinfo(adap
, i
);
6223 pi
->first_qset
= qidx
;
6224 pi
->nqsets
= is_x_10g_port(&pi
->link_cfg
) ? q10g
: 1;
6227 #endif /* !CONFIG_CHELSIO_T4_DCB */
6230 s
->max_ethqsets
= qidx
; /* MSI-X may lower it later */
6232 if (is_offload(adap
)) {
6234 * For offload we use 1 queue/channel if all ports are up to 1G,
6235 * otherwise we divide all available queues amongst the channels
6236 * capped by the number of available cores.
6239 i
= min_t(int, ARRAY_SIZE(s
->ofldrxq
),
6241 s
->ofldqsets
= roundup(i
, adap
->params
.nports
);
6243 s
->ofldqsets
= adap
->params
.nports
;
6244 /* For RDMA one Rx queue per channel suffices */
6245 s
->rdmaqs
= adap
->params
.nports
;
6246 s
->rdmaciqs
= adap
->params
.nports
;
6249 for (i
= 0; i
< ARRAY_SIZE(s
->ethrxq
); i
++) {
6250 struct sge_eth_rxq
*r
= &s
->ethrxq
[i
];
6252 init_rspq(adap
, &r
->rspq
, 5, 10, 1024, 64);
6256 for (i
= 0; i
< ARRAY_SIZE(s
->ethtxq
); i
++)
6257 s
->ethtxq
[i
].q
.size
= 1024;
6259 for (i
= 0; i
< ARRAY_SIZE(s
->ctrlq
); i
++)
6260 s
->ctrlq
[i
].q
.size
= 512;
6262 for (i
= 0; i
< ARRAY_SIZE(s
->ofldtxq
); i
++)
6263 s
->ofldtxq
[i
].q
.size
= 1024;
6265 for (i
= 0; i
< ARRAY_SIZE(s
->ofldrxq
); i
++) {
6266 struct sge_ofld_rxq
*r
= &s
->ofldrxq
[i
];
6268 init_rspq(adap
, &r
->rspq
, 5, 1, 1024, 64);
6269 r
->rspq
.uld
= CXGB4_ULD_ISCSI
;
6273 for (i
= 0; i
< ARRAY_SIZE(s
->rdmarxq
); i
++) {
6274 struct sge_ofld_rxq
*r
= &s
->rdmarxq
[i
];
6276 init_rspq(adap
, &r
->rspq
, 5, 1, 511, 64);
6277 r
->rspq
.uld
= CXGB4_ULD_RDMA
;
6281 ciq_size
= 64 + adap
->vres
.cq
.size
+ adap
->tids
.nftids
;
6282 if (ciq_size
> SGE_MAX_IQ_SIZE
) {
6283 CH_WARN(adap
, "CIQ size too small for available IQs\n");
6284 ciq_size
= SGE_MAX_IQ_SIZE
;
6287 for (i
= 0; i
< ARRAY_SIZE(s
->rdmaciq
); i
++) {
6288 struct sge_ofld_rxq
*r
= &s
->rdmaciq
[i
];
6290 init_rspq(adap
, &r
->rspq
, 5, 1, ciq_size
, 64);
6291 r
->rspq
.uld
= CXGB4_ULD_RDMA
;
6294 init_rspq(adap
, &s
->fw_evtq
, 0, 1, 1024, 64);
6295 init_rspq(adap
, &s
->intrq
, 0, 1, 2 * MAX_INGQ
, 64);
6299 * Reduce the number of Ethernet queues across all ports to at most n.
6300 * n provides at least one queue per port.
6302 static void reduce_ethqs(struct adapter
*adap
, int n
)
6305 struct port_info
*pi
;
6307 while (n
< adap
->sge
.ethqsets
)
6308 for_each_port(adap
, i
) {
6309 pi
= adap2pinfo(adap
, i
);
6310 if (pi
->nqsets
> 1) {
6312 adap
->sge
.ethqsets
--;
6313 if (adap
->sge
.ethqsets
<= n
)
6319 for_each_port(adap
, i
) {
6320 pi
= adap2pinfo(adap
, i
);
6326 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
6327 #define EXTRA_VECS 2
6329 static int enable_msix(struct adapter
*adap
)
6333 struct sge
*s
= &adap
->sge
;
6334 unsigned int nchan
= adap
->params
.nports
;
6335 struct msix_entry entries
[MAX_INGQ
+ 1];
6337 for (i
= 0; i
< ARRAY_SIZE(entries
); ++i
)
6338 entries
[i
].entry
= i
;
6340 want
= s
->max_ethqsets
+ EXTRA_VECS
;
6341 if (is_offload(adap
)) {
6342 want
+= s
->rdmaqs
+ s
->rdmaciqs
+ s
->ofldqsets
;
6343 /* need nchan for each possible ULD */
6344 ofld_need
= 3 * nchan
;
6346 #ifdef CONFIG_CHELSIO_T4_DCB
6347 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
6350 need
= 8 * adap
->params
.nports
+ EXTRA_VECS
+ ofld_need
;
6352 need
= adap
->params
.nports
+ EXTRA_VECS
+ ofld_need
;
6354 want
= pci_enable_msix_range(adap
->pdev
, entries
, need
, want
);
6359 * Distribute available vectors to the various queue groups.
6360 * Every group gets its minimum requirement and NIC gets top
6361 * priority for leftovers.
6363 i
= want
- EXTRA_VECS
- ofld_need
;
6364 if (i
< s
->max_ethqsets
) {
6365 s
->max_ethqsets
= i
;
6366 if (i
< s
->ethqsets
)
6367 reduce_ethqs(adap
, i
);
6369 if (is_offload(adap
)) {
6370 i
= want
- EXTRA_VECS
- s
->max_ethqsets
;
6371 i
-= ofld_need
- nchan
;
6372 s
->ofldqsets
= (i
/ nchan
) * nchan
; /* round down */
6374 for (i
= 0; i
< want
; ++i
)
6375 adap
->msix_info
[i
].vec
= entries
[i
].vector
;
6382 static int init_rss(struct adapter
*adap
)
6386 for_each_port(adap
, i
) {
6387 struct port_info
*pi
= adap2pinfo(adap
, i
);
6389 pi
->rss
= kcalloc(pi
->rss_size
, sizeof(u16
), GFP_KERNEL
);
6392 for (j
= 0; j
< pi
->rss_size
; j
++)
6393 pi
->rss
[j
] = ethtool_rxfh_indir_default(j
, pi
->nqsets
);
6398 static void print_port_info(const struct net_device
*dev
)
6402 const char *spd
= "";
6403 const struct port_info
*pi
= netdev_priv(dev
);
6404 const struct adapter
*adap
= pi
->adapter
;
6406 if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_2_5GB
)
6408 else if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_5_0GB
)
6410 else if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_8_0GB
)
6413 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_100M
)
6414 bufp
+= sprintf(bufp
, "100/");
6415 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_1G
)
6416 bufp
+= sprintf(bufp
, "1000/");
6417 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_10G
)
6418 bufp
+= sprintf(bufp
, "10G/");
6419 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_40G
)
6420 bufp
+= sprintf(bufp
, "40G/");
6423 sprintf(bufp
, "BASE-%s", t4_get_port_type_description(pi
->port_type
));
6425 netdev_info(dev
, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
6426 adap
->params
.vpd
.id
,
6427 CHELSIO_CHIP_RELEASE(adap
->params
.chip
), buf
,
6428 is_offload(adap
) ? "R" : "", adap
->params
.pci
.width
, spd
,
6429 (adap
->flags
& USING_MSIX
) ? " MSI-X" :
6430 (adap
->flags
& USING_MSI
) ? " MSI" : "");
6431 netdev_info(dev
, "S/N: %s, P/N: %s\n",
6432 adap
->params
.vpd
.sn
, adap
->params
.vpd
.pn
);
6435 static void enable_pcie_relaxed_ordering(struct pci_dev
*dev
)
6437 pcie_capability_set_word(dev
, PCI_EXP_DEVCTL
, PCI_EXP_DEVCTL_RELAX_EN
);
6441 * Free the following resources:
6442 * - memory used for tables
6445 * - resources FW is holding for us
6447 static void free_some_resources(struct adapter
*adapter
)
6451 t4_free_mem(adapter
->l2t
);
6452 t4_free_mem(adapter
->tids
.tid_tab
);
6453 disable_msi(adapter
);
6455 for_each_port(adapter
, i
)
6456 if (adapter
->port
[i
]) {
6457 kfree(adap2pinfo(adapter
, i
)->rss
);
6458 free_netdev(adapter
->port
[i
]);
6460 if (adapter
->flags
& FW_OK
)
6461 t4_fw_bye(adapter
, adapter
->fn
);
6464 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
6465 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
6466 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
6467 #define SEGMENT_SIZE 128
6469 static int init_one(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
6471 int func
, i
, err
, s_qpp
, qpp
, num_seg
;
6472 struct port_info
*pi
;
6473 bool highdma
= false;
6474 struct adapter
*adapter
= NULL
;
6476 printk_once(KERN_INFO
"%s - version %s\n", DRV_DESC
, DRV_VERSION
);
6478 err
= pci_request_regions(pdev
, KBUILD_MODNAME
);
6480 /* Just info, some other driver may have claimed the device. */
6481 dev_info(&pdev
->dev
, "cannot obtain PCI resources\n");
6485 err
= pci_enable_device(pdev
);
6487 dev_err(&pdev
->dev
, "cannot enable PCI device\n");
6488 goto out_release_regions
;
6491 if (!pci_set_dma_mask(pdev
, DMA_BIT_MASK(64))) {
6493 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
6495 dev_err(&pdev
->dev
, "unable to obtain 64-bit DMA for "
6496 "coherent allocations\n");
6497 goto out_disable_device
;
6500 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
6502 dev_err(&pdev
->dev
, "no usable DMA configuration\n");
6503 goto out_disable_device
;
6507 pci_enable_pcie_error_reporting(pdev
);
6508 enable_pcie_relaxed_ordering(pdev
);
6509 pci_set_master(pdev
);
6510 pci_save_state(pdev
);
6512 adapter
= kzalloc(sizeof(*adapter
), GFP_KERNEL
);
6515 goto out_disable_device
;
6518 adapter
->workq
= create_singlethread_workqueue("cxgb4");
6519 if (!adapter
->workq
) {
6521 goto out_free_adapter
;
6524 /* PCI device has been enabled */
6525 adapter
->flags
|= DEV_ENABLED
;
6527 adapter
->regs
= pci_ioremap_bar(pdev
, 0);
6528 if (!adapter
->regs
) {
6529 dev_err(&pdev
->dev
, "cannot map device registers\n");
6531 goto out_free_adapter
;
6534 /* We control everything through one PF */
6535 func
= SOURCEPF_GET(readl(adapter
->regs
+ PL_WHOAMI
));
6536 if (func
!= ent
->driver_data
) {
6537 pci_save_state(pdev
); /* to restore SR-IOV later */
6541 adapter
->pdev
= pdev
;
6542 adapter
->pdev_dev
= &pdev
->dev
;
6543 adapter
->mbox
= func
;
6545 adapter
->msg_enable
= dflt_msg_enable
;
6546 memset(adapter
->chan_map
, 0xff, sizeof(adapter
->chan_map
));
6548 spin_lock_init(&adapter
->stats_lock
);
6549 spin_lock_init(&adapter
->tid_release_lock
);
6551 INIT_WORK(&adapter
->tid_release_task
, process_tid_release_list
);
6552 INIT_WORK(&adapter
->db_full_task
, process_db_full
);
6553 INIT_WORK(&adapter
->db_drop_task
, process_db_drop
);
6555 err
= t4_prep_adapter(adapter
);
6557 goto out_unmap_bar0
;
6559 if (!is_t4(adapter
->params
.chip
)) {
6560 s_qpp
= QUEUESPERPAGEPF1
* adapter
->fn
;
6561 qpp
= 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter
,
6562 SGE_EGRESS_QUEUES_PER_PAGE_PF
) >> s_qpp
);
6563 num_seg
= PAGE_SIZE
/ SEGMENT_SIZE
;
6565 /* Each segment size is 128B. Write coalescing is enabled only
6566 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6567 * queue is less no of segments that can be accommodated in
6570 if (qpp
> num_seg
) {
6572 "Incorrect number of egress queues per page\n");
6574 goto out_unmap_bar0
;
6576 adapter
->bar2
= ioremap_wc(pci_resource_start(pdev
, 2),
6577 pci_resource_len(pdev
, 2));
6578 if (!adapter
->bar2
) {
6579 dev_err(&pdev
->dev
, "cannot map device bar2 region\n");
6581 goto out_unmap_bar0
;
6585 setup_memwin(adapter
);
6586 err
= adap_init0(adapter
);
6587 setup_memwin_rdma(adapter
);
6591 for_each_port(adapter
, i
) {
6592 struct net_device
*netdev
;
6594 netdev
= alloc_etherdev_mq(sizeof(struct port_info
),
6601 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
6603 adapter
->port
[i
] = netdev
;
6604 pi
= netdev_priv(netdev
);
6605 pi
->adapter
= adapter
;
6606 pi
->xact_addr_filt
= -1;
6608 netdev
->irq
= pdev
->irq
;
6610 netdev
->hw_features
= NETIF_F_SG
| TSO_FLAGS
|
6611 NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
6612 NETIF_F_RXCSUM
| NETIF_F_RXHASH
|
6613 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
;
6615 netdev
->hw_features
|= NETIF_F_HIGHDMA
;
6616 netdev
->features
|= netdev
->hw_features
;
6617 netdev
->vlan_features
= netdev
->features
& VLAN_FEAT
;
6619 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
6621 netdev
->netdev_ops
= &cxgb4_netdev_ops
;
6622 #ifdef CONFIG_CHELSIO_T4_DCB
6623 netdev
->dcbnl_ops
= &cxgb4_dcb_ops
;
6624 cxgb4_dcb_state_init(netdev
);
6626 netdev
->ethtool_ops
= &cxgb_ethtool_ops
;
6629 pci_set_drvdata(pdev
, adapter
);
6631 if (adapter
->flags
& FW_OK
) {
6632 err
= t4_port_init(adapter
, func
, func
, 0);
6638 * Configure queues and allocate tables now, they can be needed as
6639 * soon as the first register_netdev completes.
6641 cfg_queues(adapter
);
6643 adapter
->l2t
= t4_init_l2t();
6644 if (!adapter
->l2t
) {
6645 /* We tolerate a lack of L2T, giving up some functionality */
6646 dev_warn(&pdev
->dev
, "could not allocate L2T, continuing\n");
6647 adapter
->params
.offload
= 0;
6650 if (is_offload(adapter
) && tid_init(&adapter
->tids
) < 0) {
6651 dev_warn(&pdev
->dev
, "could not allocate TID table, "
6653 adapter
->params
.offload
= 0;
6656 /* See what interrupts we'll be using */
6657 if (msi
> 1 && enable_msix(adapter
) == 0)
6658 adapter
->flags
|= USING_MSIX
;
6659 else if (msi
> 0 && pci_enable_msi(pdev
) == 0)
6660 adapter
->flags
|= USING_MSI
;
6662 err
= init_rss(adapter
);
6667 * The card is now ready to go. If any errors occur during device
6668 * registration we do not fail the whole card but rather proceed only
6669 * with the ports we manage to register successfully. However we must
6670 * register at least one net device.
6672 for_each_port(adapter
, i
) {
6673 pi
= adap2pinfo(adapter
, i
);
6674 netif_set_real_num_tx_queues(adapter
->port
[i
], pi
->nqsets
);
6675 netif_set_real_num_rx_queues(adapter
->port
[i
], pi
->nqsets
);
6677 err
= register_netdev(adapter
->port
[i
]);
6680 adapter
->chan_map
[pi
->tx_chan
] = i
;
6681 print_port_info(adapter
->port
[i
]);
6684 dev_err(&pdev
->dev
, "could not register any net devices\n");
6688 dev_warn(&pdev
->dev
, "only %d net devices registered\n", i
);
6692 if (cxgb4_debugfs_root
) {
6693 adapter
->debugfs_root
= debugfs_create_dir(pci_name(pdev
),
6694 cxgb4_debugfs_root
);
6695 setup_debugfs(adapter
);
6698 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6699 pdev
->needs_freset
= 1;
6701 if (is_offload(adapter
))
6702 attach_ulds(adapter
);
6705 #ifdef CONFIG_PCI_IOV
6706 if (func
< ARRAY_SIZE(num_vf
) && num_vf
[func
] > 0)
6707 if (pci_enable_sriov(pdev
, num_vf
[func
]) == 0)
6708 dev_info(&pdev
->dev
,
6709 "instantiated %u virtual functions\n",
6715 free_some_resources(adapter
);
6717 if (!is_t4(adapter
->params
.chip
))
6718 iounmap(adapter
->bar2
);
6720 iounmap(adapter
->regs
);
6723 destroy_workqueue(adapter
->workq
);
6727 pci_disable_pcie_error_reporting(pdev
);
6728 pci_disable_device(pdev
);
6729 out_release_regions
:
6730 pci_release_regions(pdev
);
6734 static void remove_one(struct pci_dev
*pdev
)
6736 struct adapter
*adapter
= pci_get_drvdata(pdev
);
6738 #ifdef CONFIG_PCI_IOV
6739 pci_disable_sriov(pdev
);
6746 /* Tear down per-adapter Work Queue first since it can contain
6747 * references to our adapter data structure.
6749 destroy_workqueue(adapter
->workq
);
6751 if (is_offload(adapter
))
6752 detach_ulds(adapter
);
6754 for_each_port(adapter
, i
)
6755 if (adapter
->port
[i
]->reg_state
== NETREG_REGISTERED
)
6756 unregister_netdev(adapter
->port
[i
]);
6758 debugfs_remove_recursive(adapter
->debugfs_root
);
6760 /* If we allocated filters, free up state associated with any
6763 if (adapter
->tids
.ftid_tab
) {
6764 struct filter_entry
*f
= &adapter
->tids
.ftid_tab
[0];
6765 for (i
= 0; i
< (adapter
->tids
.nftids
+
6766 adapter
->tids
.nsftids
); i
++, f
++)
6768 clear_filter(adapter
, f
);
6771 if (adapter
->flags
& FULL_INIT_DONE
)
6774 free_some_resources(adapter
);
6775 iounmap(adapter
->regs
);
6776 if (!is_t4(adapter
->params
.chip
))
6777 iounmap(adapter
->bar2
);
6778 pci_disable_pcie_error_reporting(pdev
);
6779 if ((adapter
->flags
& DEV_ENABLED
)) {
6780 pci_disable_device(pdev
);
6781 adapter
->flags
&= ~DEV_ENABLED
;
6783 pci_release_regions(pdev
);
6787 pci_release_regions(pdev
);
6790 static struct pci_driver cxgb4_driver
= {
6791 .name
= KBUILD_MODNAME
,
6792 .id_table
= cxgb4_pci_tbl
,
6794 .remove
= remove_one
,
6795 .shutdown
= remove_one
,
6796 .err_handler
= &cxgb4_eeh
,
6799 static int __init
cxgb4_init_module(void)
6803 /* Debugfs support is optional, just warn if this fails */
6804 cxgb4_debugfs_root
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
6805 if (!cxgb4_debugfs_root
)
6806 pr_warn("could not create debugfs entry, continuing\n");
6808 ret
= pci_register_driver(&cxgb4_driver
);
6810 debugfs_remove(cxgb4_debugfs_root
);
6812 register_inet6addr_notifier(&cxgb4_inet6addr_notifier
);
6817 static void __exit
cxgb4_cleanup_module(void)
6819 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier
);
6820 pci_unregister_driver(&cxgb4_driver
);
6821 debugfs_remove(cxgb4_debugfs_root
); /* NULL ok */
6824 module_init(cxgb4_init_module
);
6825 module_exit(cxgb4_cleanup_module
);