2 * Copyright (c) 2006-2008 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/list.h>
34 #include <linux/slab.h>
35 #include <net/neighbour.h>
36 #include <linux/notifier.h>
37 #include <linux/atomic.h>
38 #include <linux/proc_fs.h>
39 #include <linux/if_vlan.h>
40 #include <net/netevent.h>
41 #include <linux/highmem.h>
42 #include <linux/vmalloc.h>
43 #include <linux/export.h>
47 #include "cxgb3_ioctl.h"
48 #include "cxgb3_ctl_defs.h"
49 #include "cxgb3_defs.h"
51 #include "firmware_exports.h"
52 #include "cxgb3_offload.h"
54 static LIST_HEAD(client_list
);
55 static LIST_HEAD(ofld_dev_list
);
56 static DEFINE_MUTEX(cxgb3_db_lock
);
58 static DEFINE_RWLOCK(adapter_list_lock
);
59 static LIST_HEAD(adapter_list
);
61 static const unsigned int MAX_ATIDS
= 64 * 1024;
62 static const unsigned int ATID_BASE
= 0x10000;
64 static void cxgb_neigh_update(struct neighbour
*neigh
);
65 static void cxgb_redirect(struct dst_entry
*old
, struct neighbour
*old_neigh
,
66 struct dst_entry
*new, struct neighbour
*new_neigh
,
69 static inline int offload_activated(struct t3cdev
*tdev
)
71 const struct adapter
*adapter
= tdev2adap(tdev
);
73 return test_bit(OFFLOAD_DEVMAP_BIT
, &adapter
->open_device_map
);
77 * cxgb3_register_client - register an offload client
80 * Add the client to the client list,
81 * and call backs the client for each activated offload device
83 void cxgb3_register_client(struct cxgb3_client
*client
)
87 mutex_lock(&cxgb3_db_lock
);
88 list_add_tail(&client
->client_list
, &client_list
);
91 list_for_each_entry(tdev
, &ofld_dev_list
, ofld_dev_list
) {
92 if (offload_activated(tdev
))
96 mutex_unlock(&cxgb3_db_lock
);
99 EXPORT_SYMBOL(cxgb3_register_client
);
102 * cxgb3_unregister_client - unregister an offload client
103 * @client: the client
105 * Remove the client to the client list,
106 * and call backs the client for each activated offload device.
108 void cxgb3_unregister_client(struct cxgb3_client
*client
)
112 mutex_lock(&cxgb3_db_lock
);
113 list_del(&client
->client_list
);
115 if (client
->remove
) {
116 list_for_each_entry(tdev
, &ofld_dev_list
, ofld_dev_list
) {
117 if (offload_activated(tdev
))
118 client
->remove(tdev
);
121 mutex_unlock(&cxgb3_db_lock
);
124 EXPORT_SYMBOL(cxgb3_unregister_client
);
127 * cxgb3_add_clients - activate registered clients for an offload device
128 * @tdev: the offload device
130 * Call backs all registered clients once a offload device is activated
132 void cxgb3_add_clients(struct t3cdev
*tdev
)
134 struct cxgb3_client
*client
;
136 mutex_lock(&cxgb3_db_lock
);
137 list_for_each_entry(client
, &client_list
, client_list
) {
141 mutex_unlock(&cxgb3_db_lock
);
145 * cxgb3_remove_clients - deactivates registered clients
146 * for an offload device
147 * @tdev: the offload device
149 * Call backs all registered clients once a offload device is deactivated
151 void cxgb3_remove_clients(struct t3cdev
*tdev
)
153 struct cxgb3_client
*client
;
155 mutex_lock(&cxgb3_db_lock
);
156 list_for_each_entry(client
, &client_list
, client_list
) {
158 client
->remove(tdev
);
160 mutex_unlock(&cxgb3_db_lock
);
163 void cxgb3_event_notify(struct t3cdev
*tdev
, u32 event
, u32 port
)
165 struct cxgb3_client
*client
;
167 mutex_lock(&cxgb3_db_lock
);
168 list_for_each_entry(client
, &client_list
, client_list
) {
169 if (client
->event_handler
)
170 client
->event_handler(tdev
, event
, port
);
172 mutex_unlock(&cxgb3_db_lock
);
175 static struct net_device
*get_iff_from_mac(struct adapter
*adapter
,
176 const unsigned char *mac
,
181 for_each_port(adapter
, i
) {
182 struct net_device
*dev
= adapter
->port
[i
];
184 if (!memcmp(dev
->dev_addr
, mac
, ETH_ALEN
)) {
186 if (vlan
&& vlan
!= VLAN_VID_MASK
) {
187 dev
= __vlan_find_dev_deep(dev
, vlan
);
188 } else if (netif_is_bond_slave(dev
)) {
189 struct net_device
*upper_dev
;
192 netdev_master_upper_dev_get_rcu(dev
)))
202 static int cxgb_ulp_iscsi_ctl(struct adapter
*adapter
, unsigned int req
,
207 unsigned int val
= 0;
208 struct ulp_iscsi_info
*uiip
= data
;
211 case ULP_ISCSI_GET_PARAMS
:
212 uiip
->pdev
= adapter
->pdev
;
213 uiip
->llimit
= t3_read_reg(adapter
, A_ULPRX_ISCSI_LLIMIT
);
214 uiip
->ulimit
= t3_read_reg(adapter
, A_ULPRX_ISCSI_ULIMIT
);
215 uiip
->tagmask
= t3_read_reg(adapter
, A_ULPRX_ISCSI_TAGMASK
);
217 val
= t3_read_reg(adapter
, A_ULPRX_ISCSI_PSZ
);
218 for (i
= 0; i
< 4; i
++, val
>>= 8)
219 uiip
->pgsz_factor
[i
] = val
& 0xFF;
221 val
= t3_read_reg(adapter
, A_TP_PARA_REG7
);
223 uiip
->max_rxsz
= min((val
>> S_PMMAXXFERLEN0
)&M_PMMAXXFERLEN0
,
224 (val
>> S_PMMAXXFERLEN1
)&M_PMMAXXFERLEN1
);
226 * On tx, the iscsi pdu has to be <= tx page size and has to
227 * fit into the Tx PM FIFO.
229 val
= min(adapter
->params
.tp
.tx_pg_size
,
230 t3_read_reg(adapter
, A_PM1_TX_CFG
) >> 17);
231 uiip
->max_txsz
= min(val
, uiip
->max_txsz
);
233 /* set MaxRxData to 16224 */
234 val
= t3_read_reg(adapter
, A_TP_PARA_REG2
);
235 if ((val
>> S_MAXRXDATA
) != 0x3f60) {
236 val
&= (M_RXCOALESCESIZE
<< S_RXCOALESCESIZE
);
237 val
|= V_MAXRXDATA(0x3f60);
239 "%s, iscsi set MaxRxData to 16224 (0x%x).\n",
241 t3_write_reg(adapter
, A_TP_PARA_REG2
, val
);
245 * on rx, the iscsi pdu has to be < rx page size and the
246 * the max rx data length programmed in TP
248 val
= min(adapter
->params
.tp
.rx_pg_size
,
249 ((t3_read_reg(adapter
, A_TP_PARA_REG2
)) >>
250 S_MAXRXDATA
) & M_MAXRXDATA
);
251 uiip
->max_rxsz
= min(val
, uiip
->max_rxsz
);
253 case ULP_ISCSI_SET_PARAMS
:
254 t3_write_reg(adapter
, A_ULPRX_ISCSI_TAGMASK
, uiip
->tagmask
);
255 /* program the ddp page sizes */
256 for (i
= 0; i
< 4; i
++)
257 val
|= (uiip
->pgsz_factor
[i
] & 0xF) << (8 * i
);
258 if (val
&& (val
!= t3_read_reg(adapter
, A_ULPRX_ISCSI_PSZ
))) {
260 "%s, setting iscsi pgsz 0x%x, %u,%u,%u,%u.\n",
261 adapter
->name
, val
, uiip
->pgsz_factor
[0],
262 uiip
->pgsz_factor
[1], uiip
->pgsz_factor
[2],
263 uiip
->pgsz_factor
[3]);
264 t3_write_reg(adapter
, A_ULPRX_ISCSI_PSZ
, val
);
273 /* Response queue used for RDMA events. */
274 #define ASYNC_NOTIF_RSPQ 0
276 static int cxgb_rdma_ctl(struct adapter
*adapter
, unsigned int req
, void *data
)
281 case RDMA_GET_PARAMS
: {
282 struct rdma_info
*rdma
= data
;
283 struct pci_dev
*pdev
= adapter
->pdev
;
285 rdma
->udbell_physbase
= pci_resource_start(pdev
, 2);
286 rdma
->udbell_len
= pci_resource_len(pdev
, 2);
288 t3_read_reg(adapter
, A_ULPTX_TPT_LLIMIT
);
289 rdma
->tpt_top
= t3_read_reg(adapter
, A_ULPTX_TPT_ULIMIT
);
291 t3_read_reg(adapter
, A_ULPTX_PBL_LLIMIT
);
292 rdma
->pbl_top
= t3_read_reg(adapter
, A_ULPTX_PBL_ULIMIT
);
293 rdma
->rqt_base
= t3_read_reg(adapter
, A_ULPRX_RQ_LLIMIT
);
294 rdma
->rqt_top
= t3_read_reg(adapter
, A_ULPRX_RQ_ULIMIT
);
295 rdma
->kdb_addr
= adapter
->regs
+ A_SG_KDOORBELL
;
301 struct rdma_cq_op
*rdma
= data
;
303 /* may be called in any context */
304 spin_lock_irqsave(&adapter
->sge
.reg_lock
, flags
);
305 ret
= t3_sge_cqcntxt_op(adapter
, rdma
->id
, rdma
->op
,
307 spin_unlock_irqrestore(&adapter
->sge
.reg_lock
, flags
);
311 struct ch_mem_range
*t
= data
;
314 if ((t
->addr
& 7) || (t
->len
& 7))
316 if (t
->mem_id
== MEM_CM
)
318 else if (t
->mem_id
== MEM_PMRX
)
319 mem
= &adapter
->pmrx
;
320 else if (t
->mem_id
== MEM_PMTX
)
321 mem
= &adapter
->pmtx
;
326 t3_mc7_bd_read(mem
, t
->addr
/ 8, t
->len
/ 8,
333 struct rdma_cq_setup
*rdma
= data
;
335 spin_lock_irq(&adapter
->sge
.reg_lock
);
337 t3_sge_init_cqcntxt(adapter
, rdma
->id
,
338 rdma
->base_addr
, rdma
->size
,
340 rdma
->ovfl_mode
, rdma
->credits
,
342 spin_unlock_irq(&adapter
->sge
.reg_lock
);
345 case RDMA_CQ_DISABLE
:
346 spin_lock_irq(&adapter
->sge
.reg_lock
);
347 ret
= t3_sge_disable_cqcntxt(adapter
, *(unsigned int *)data
);
348 spin_unlock_irq(&adapter
->sge
.reg_lock
);
350 case RDMA_CTRL_QP_SETUP
:{
351 struct rdma_ctrlqp_setup
*rdma
= data
;
353 spin_lock_irq(&adapter
->sge
.reg_lock
);
354 ret
= t3_sge_init_ecntxt(adapter
, FW_RI_SGEEC_START
, 0,
357 rdma
->base_addr
, rdma
->size
,
358 FW_RI_TID_START
, 1, 0);
359 spin_unlock_irq(&adapter
->sge
.reg_lock
);
363 spin_lock(&adapter
->stats_lock
);
364 t3_tp_get_mib_stats(adapter
, (struct tp_mib_stats
*)data
);
365 spin_unlock(&adapter
->stats_lock
);
374 static int cxgb_offload_ctl(struct t3cdev
*tdev
, unsigned int req
, void *data
)
376 struct adapter
*adapter
= tdev2adap(tdev
);
377 struct tid_range
*tid
;
379 struct iff_mac
*iffmacp
;
380 struct ddp_params
*ddpp
;
381 struct adap_ports
*ports
;
382 struct ofld_page_info
*rx_page_info
;
383 struct tp_params
*tp
= &adapter
->params
.tp
;
387 case GET_MAX_OUTSTANDING_WR
:
388 *(unsigned int *)data
= FW_WR_NUM
;
391 *(unsigned int *)data
= WR_FLITS
;
393 case GET_TX_MAX_CHUNK
:
394 *(unsigned int *)data
= 1 << 20; /* 1MB */
398 tid
->num
= t3_mc5_size(&adapter
->mc5
) -
399 adapter
->params
.mc5
.nroutes
-
400 adapter
->params
.mc5
.nfilters
- adapter
->params
.mc5
.nservers
;
405 tid
->num
= adapter
->params
.mc5
.nservers
;
406 tid
->base
= t3_mc5_size(&adapter
->mc5
) - tid
->num
-
407 adapter
->params
.mc5
.nfilters
- adapter
->params
.mc5
.nroutes
;
409 case GET_L2T_CAPACITY
:
410 *(unsigned int *)data
= 2048;
415 mtup
->mtus
= adapter
->params
.mtus
;
417 case GET_IFF_FROM_MAC
:
419 iffmacp
->dev
= get_iff_from_mac(adapter
, iffmacp
->mac_addr
,
425 ddpp
->llimit
= t3_read_reg(adapter
, A_ULPRX_TDDP_LLIMIT
);
426 ddpp
->ulimit
= t3_read_reg(adapter
, A_ULPRX_TDDP_ULIMIT
);
427 ddpp
->tag_mask
= t3_read_reg(adapter
, A_ULPRX_TDDP_TAGMASK
);
431 ports
->nports
= adapter
->params
.nports
;
432 for_each_port(adapter
, i
)
433 ports
->lldevs
[i
] = adapter
->port
[i
];
435 case ULP_ISCSI_GET_PARAMS
:
436 case ULP_ISCSI_SET_PARAMS
:
437 if (!offload_running(adapter
))
439 return cxgb_ulp_iscsi_ctl(adapter
, req
, data
);
440 case RDMA_GET_PARAMS
:
443 case RDMA_CQ_DISABLE
:
444 case RDMA_CTRL_QP_SETUP
:
447 if (!offload_running(adapter
))
449 return cxgb_rdma_ctl(adapter
, req
, data
);
450 case GET_RX_PAGE_INFO
:
452 rx_page_info
->page_size
= tp
->rx_pg_size
;
453 rx_page_info
->num
= tp
->rx_num_pgs
;
455 case GET_ISCSI_IPV4ADDR
: {
456 struct iscsi_ipv4addr
*p
= data
;
457 struct port_info
*pi
= netdev_priv(p
->dev
);
458 p
->ipv4addr
= pi
->iscsi_ipv4addr
;
461 case GET_EMBEDDED_INFO
: {
462 struct ch_embedded_info
*e
= data
;
464 spin_lock(&adapter
->stats_lock
);
465 t3_get_fw_version(adapter
, &e
->fw_vers
);
466 t3_get_tp_version(adapter
, &e
->tp_vers
);
467 spin_unlock(&adapter
->stats_lock
);
477 * Dummy handler for Rx offload packets in case we get an offload packet before
478 * proper processing is setup. This complains and drops the packet as it isn't
479 * normal to get offload packets at this stage.
481 static int rx_offload_blackhole(struct t3cdev
*dev
, struct sk_buff
**skbs
,
485 dev_kfree_skb_any(skbs
[n
]);
489 static void dummy_neigh_update(struct t3cdev
*dev
, struct neighbour
*neigh
)
493 void cxgb3_set_dummy_ops(struct t3cdev
*dev
)
495 dev
->recv
= rx_offload_blackhole
;
496 dev
->neigh_update
= dummy_neigh_update
;
500 * Free an active-open TID.
502 void *cxgb3_free_atid(struct t3cdev
*tdev
, int atid
)
504 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
505 union active_open_entry
*p
= atid2entry(t
, atid
);
506 void *ctx
= p
->t3c_tid
.ctx
;
508 spin_lock_bh(&t
->atid_lock
);
512 spin_unlock_bh(&t
->atid_lock
);
517 EXPORT_SYMBOL(cxgb3_free_atid
);
520 * Free a server TID and return it to the free pool.
522 void cxgb3_free_stid(struct t3cdev
*tdev
, int stid
)
524 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
525 union listen_entry
*p
= stid2entry(t
, stid
);
527 spin_lock_bh(&t
->stid_lock
);
531 spin_unlock_bh(&t
->stid_lock
);
534 EXPORT_SYMBOL(cxgb3_free_stid
);
536 void cxgb3_insert_tid(struct t3cdev
*tdev
, struct cxgb3_client
*client
,
537 void *ctx
, unsigned int tid
)
539 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
541 t
->tid_tab
[tid
].client
= client
;
542 t
->tid_tab
[tid
].ctx
= ctx
;
543 atomic_inc(&t
->tids_in_use
);
546 EXPORT_SYMBOL(cxgb3_insert_tid
);
549 * Populate a TID_RELEASE WR. The skb must be already propely sized.
551 static inline void mk_tid_release(struct sk_buff
*skb
, unsigned int tid
)
553 struct cpl_tid_release
*req
;
555 skb
->priority
= CPL_PRIORITY_SETUP
;
556 req
= (struct cpl_tid_release
*)__skb_put(skb
, sizeof(*req
));
557 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
558 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE
, tid
));
561 static void t3_process_tid_release_list(struct work_struct
*work
)
563 struct t3c_data
*td
= container_of(work
, struct t3c_data
,
566 struct t3cdev
*tdev
= td
->dev
;
569 spin_lock_bh(&td
->tid_release_lock
);
570 while (td
->tid_release_list
) {
571 struct t3c_tid_entry
*p
= td
->tid_release_list
;
573 td
->tid_release_list
= p
->ctx
;
574 spin_unlock_bh(&td
->tid_release_lock
);
576 skb
= alloc_skb(sizeof(struct cpl_tid_release
),
579 skb
= td
->nofail_skb
;
581 spin_lock_bh(&td
->tid_release_lock
);
582 p
->ctx
= (void *)td
->tid_release_list
;
583 td
->tid_release_list
= p
;
586 mk_tid_release(skb
, p
- td
->tid_maps
.tid_tab
);
587 cxgb3_ofld_send(tdev
, skb
);
589 if (skb
== td
->nofail_skb
)
591 alloc_skb(sizeof(struct cpl_tid_release
),
593 spin_lock_bh(&td
->tid_release_lock
);
595 td
->release_list_incomplete
= (td
->tid_release_list
== NULL
) ? 0 : 1;
596 spin_unlock_bh(&td
->tid_release_lock
);
600 alloc_skb(sizeof(struct cpl_tid_release
),
604 /* use ctx as a next pointer in the tid release list */
605 void cxgb3_queue_tid_release(struct t3cdev
*tdev
, unsigned int tid
)
607 struct t3c_data
*td
= T3C_DATA(tdev
);
608 struct t3c_tid_entry
*p
= &td
->tid_maps
.tid_tab
[tid
];
610 spin_lock_bh(&td
->tid_release_lock
);
611 p
->ctx
= (void *)td
->tid_release_list
;
613 td
->tid_release_list
= p
;
614 if (!p
->ctx
|| td
->release_list_incomplete
)
615 schedule_work(&td
->tid_release_task
);
616 spin_unlock_bh(&td
->tid_release_lock
);
619 EXPORT_SYMBOL(cxgb3_queue_tid_release
);
622 * Remove a tid from the TID table. A client may defer processing its last
623 * CPL message if it is locked at the time it arrives, and while the message
624 * sits in the client's backlog the TID may be reused for another connection.
625 * To handle this we atomically switch the TID association if it still points
626 * to the original client context.
628 void cxgb3_remove_tid(struct t3cdev
*tdev
, void *ctx
, unsigned int tid
)
630 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
632 BUG_ON(tid
>= t
->ntids
);
633 if (tdev
->type
== T3A
)
634 (void)cmpxchg(&t
->tid_tab
[tid
].ctx
, ctx
, NULL
);
638 skb
= alloc_skb(sizeof(struct cpl_tid_release
), GFP_ATOMIC
);
640 mk_tid_release(skb
, tid
);
641 cxgb3_ofld_send(tdev
, skb
);
642 t
->tid_tab
[tid
].ctx
= NULL
;
644 cxgb3_queue_tid_release(tdev
, tid
);
646 atomic_dec(&t
->tids_in_use
);
649 EXPORT_SYMBOL(cxgb3_remove_tid
);
651 int cxgb3_alloc_atid(struct t3cdev
*tdev
, struct cxgb3_client
*client
,
655 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
657 spin_lock_bh(&t
->atid_lock
);
659 t
->atids_in_use
+ atomic_read(&t
->tids_in_use
) + MC5_MIN_TIDS
<=
661 union active_open_entry
*p
= t
->afree
;
663 atid
= (p
- t
->atid_tab
) + t
->atid_base
;
665 p
->t3c_tid
.ctx
= ctx
;
666 p
->t3c_tid
.client
= client
;
669 spin_unlock_bh(&t
->atid_lock
);
673 EXPORT_SYMBOL(cxgb3_alloc_atid
);
675 int cxgb3_alloc_stid(struct t3cdev
*tdev
, struct cxgb3_client
*client
,
679 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
681 spin_lock_bh(&t
->stid_lock
);
683 union listen_entry
*p
= t
->sfree
;
685 stid
= (p
- t
->stid_tab
) + t
->stid_base
;
687 p
->t3c_tid
.ctx
= ctx
;
688 p
->t3c_tid
.client
= client
;
691 spin_unlock_bh(&t
->stid_lock
);
695 EXPORT_SYMBOL(cxgb3_alloc_stid
);
697 /* Get the t3cdev associated with a net_device */
698 struct t3cdev
*dev2t3cdev(struct net_device
*dev
)
700 const struct port_info
*pi
= netdev_priv(dev
);
702 return (struct t3cdev
*)pi
->adapter
;
705 EXPORT_SYMBOL(dev2t3cdev
);
707 static int do_smt_write_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
709 struct cpl_smt_write_rpl
*rpl
= cplhdr(skb
);
711 if (rpl
->status
!= CPL_ERR_NONE
)
713 "Unexpected SMT_WRITE_RPL status %u for entry %u\n",
714 rpl
->status
, GET_TID(rpl
));
716 return CPL_RET_BUF_DONE
;
719 static int do_l2t_write_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
721 struct cpl_l2t_write_rpl
*rpl
= cplhdr(skb
);
723 if (rpl
->status
!= CPL_ERR_NONE
)
725 "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
726 rpl
->status
, GET_TID(rpl
));
728 return CPL_RET_BUF_DONE
;
731 static int do_rte_write_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
733 struct cpl_rte_write_rpl
*rpl
= cplhdr(skb
);
735 if (rpl
->status
!= CPL_ERR_NONE
)
737 "Unexpected RTE_WRITE_RPL status %u for entry %u\n",
738 rpl
->status
, GET_TID(rpl
));
740 return CPL_RET_BUF_DONE
;
743 static int do_act_open_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
745 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
746 unsigned int atid
= G_TID(ntohl(rpl
->atid
));
747 struct t3c_tid_entry
*t3c_tid
;
749 t3c_tid
= lookup_atid(&(T3C_DATA(dev
))->tid_maps
, atid
);
750 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
&&
751 t3c_tid
->client
->handlers
&&
752 t3c_tid
->client
->handlers
[CPL_ACT_OPEN_RPL
]) {
753 return t3c_tid
->client
->handlers
[CPL_ACT_OPEN_RPL
] (dev
, skb
,
757 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
758 dev
->name
, CPL_ACT_OPEN_RPL
);
759 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
763 static int do_stid_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
765 union opcode_tid
*p
= cplhdr(skb
);
766 unsigned int stid
= G_TID(ntohl(p
->opcode_tid
));
767 struct t3c_tid_entry
*t3c_tid
;
769 t3c_tid
= lookup_stid(&(T3C_DATA(dev
))->tid_maps
, stid
);
770 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
771 t3c_tid
->client
->handlers
[p
->opcode
]) {
772 return t3c_tid
->client
->handlers
[p
->opcode
] (dev
, skb
,
775 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
776 dev
->name
, p
->opcode
);
777 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
781 static int do_hwtid_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
783 union opcode_tid
*p
= cplhdr(skb
);
784 unsigned int hwtid
= G_TID(ntohl(p
->opcode_tid
));
785 struct t3c_tid_entry
*t3c_tid
;
787 t3c_tid
= lookup_tid(&(T3C_DATA(dev
))->tid_maps
, hwtid
);
788 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
789 t3c_tid
->client
->handlers
[p
->opcode
]) {
790 return t3c_tid
->client
->handlers
[p
->opcode
]
791 (dev
, skb
, t3c_tid
->ctx
);
793 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
794 dev
->name
, p
->opcode
);
795 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
799 static int do_cr(struct t3cdev
*dev
, struct sk_buff
*skb
)
801 struct cpl_pass_accept_req
*req
= cplhdr(skb
);
802 unsigned int stid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
803 struct tid_info
*t
= &(T3C_DATA(dev
))->tid_maps
;
804 struct t3c_tid_entry
*t3c_tid
;
805 unsigned int tid
= GET_TID(req
);
807 if (unlikely(tid
>= t
->ntids
)) {
808 printk("%s: passive open TID %u too large\n",
810 t3_fatal_err(tdev2adap(dev
));
811 return CPL_RET_BUF_DONE
;
814 t3c_tid
= lookup_stid(t
, stid
);
815 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
816 t3c_tid
->client
->handlers
[CPL_PASS_ACCEPT_REQ
]) {
817 return t3c_tid
->client
->handlers
[CPL_PASS_ACCEPT_REQ
]
818 (dev
, skb
, t3c_tid
->ctx
);
820 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
821 dev
->name
, CPL_PASS_ACCEPT_REQ
);
822 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
827 * Returns an sk_buff for a reply CPL message of size len. If the input
828 * sk_buff has no other users it is trimmed and reused, otherwise a new buffer
829 * is allocated. The input skb must be of size at least len. Note that this
830 * operation does not destroy the original skb data even if it decides to reuse
833 static struct sk_buff
*cxgb3_get_cpl_reply_skb(struct sk_buff
*skb
, size_t len
,
836 if (likely(!skb_cloned(skb
))) {
837 BUG_ON(skb
->len
< len
);
838 __skb_trim(skb
, len
);
841 skb
= alloc_skb(len
, gfp
);
848 static int do_abort_req_rss(struct t3cdev
*dev
, struct sk_buff
*skb
)
850 union opcode_tid
*p
= cplhdr(skb
);
851 unsigned int hwtid
= G_TID(ntohl(p
->opcode_tid
));
852 struct t3c_tid_entry
*t3c_tid
;
854 t3c_tid
= lookup_tid(&(T3C_DATA(dev
))->tid_maps
, hwtid
);
855 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
856 t3c_tid
->client
->handlers
[p
->opcode
]) {
857 return t3c_tid
->client
->handlers
[p
->opcode
]
858 (dev
, skb
, t3c_tid
->ctx
);
860 struct cpl_abort_req_rss
*req
= cplhdr(skb
);
861 struct cpl_abort_rpl
*rpl
;
862 struct sk_buff
*reply_skb
;
863 unsigned int tid
= GET_TID(req
);
864 u8 cmd
= req
->status
;
866 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
867 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
)
870 reply_skb
= cxgb3_get_cpl_reply_skb(skb
,
876 printk("do_abort_req_rss: couldn't get skb!\n");
879 reply_skb
->priority
= CPL_PRIORITY_DATA
;
880 __skb_put(reply_skb
, sizeof(struct cpl_abort_rpl
));
881 rpl
= cplhdr(reply_skb
);
883 htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL
));
884 rpl
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
885 OPCODE_TID(rpl
) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL
, tid
));
887 cxgb3_ofld_send(dev
, reply_skb
);
889 return CPL_RET_BUF_DONE
;
893 static int do_act_establish(struct t3cdev
*dev
, struct sk_buff
*skb
)
895 struct cpl_act_establish
*req
= cplhdr(skb
);
896 unsigned int atid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
897 struct tid_info
*t
= &(T3C_DATA(dev
))->tid_maps
;
898 struct t3c_tid_entry
*t3c_tid
;
899 unsigned int tid
= GET_TID(req
);
901 if (unlikely(tid
>= t
->ntids
)) {
902 printk("%s: active establish TID %u too large\n",
904 t3_fatal_err(tdev2adap(dev
));
905 return CPL_RET_BUF_DONE
;
908 t3c_tid
= lookup_atid(t
, atid
);
909 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
910 t3c_tid
->client
->handlers
[CPL_ACT_ESTABLISH
]) {
911 return t3c_tid
->client
->handlers
[CPL_ACT_ESTABLISH
]
912 (dev
, skb
, t3c_tid
->ctx
);
914 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
915 dev
->name
, CPL_ACT_ESTABLISH
);
916 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
920 static int do_trace(struct t3cdev
*dev
, struct sk_buff
*skb
)
922 struct cpl_trace_pkt
*p
= cplhdr(skb
);
924 skb
->protocol
= htons(0xffff);
925 skb
->dev
= dev
->lldev
;
926 skb_pull(skb
, sizeof(*p
));
927 skb_reset_mac_header(skb
);
928 netif_receive_skb(skb
);
933 * That skb would better have come from process_responses() where we abuse
934 * ->priority and ->csum to carry our data. NB: if we get to per-arch
935 * ->csum, the things might get really interesting here.
938 static inline u32
get_hwtid(struct sk_buff
*skb
)
940 return ntohl((__force __be32
)skb
->priority
) >> 8 & 0xfffff;
943 static inline u32
get_opcode(struct sk_buff
*skb
)
945 return G_OPCODE(ntohl((__force __be32
)skb
->csum
));
948 static int do_term(struct t3cdev
*dev
, struct sk_buff
*skb
)
950 unsigned int hwtid
= get_hwtid(skb
);
951 unsigned int opcode
= get_opcode(skb
);
952 struct t3c_tid_entry
*t3c_tid
;
954 t3c_tid
= lookup_tid(&(T3C_DATA(dev
))->tid_maps
, hwtid
);
955 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
956 t3c_tid
->client
->handlers
[opcode
]) {
957 return t3c_tid
->client
->handlers
[opcode
] (dev
, skb
,
960 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
962 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
966 static int nb_callback(struct notifier_block
*self
, unsigned long event
,
970 case (NETEVENT_NEIGH_UPDATE
):{
971 cxgb_neigh_update((struct neighbour
*)ctx
);
974 case (NETEVENT_REDIRECT
):{
975 struct netevent_redirect
*nr
= ctx
;
976 cxgb_redirect(nr
->old
, nr
->old_neigh
,
977 nr
->new, nr
->new_neigh
,
979 cxgb_neigh_update(nr
->new_neigh
);
988 static struct notifier_block nb
= {
989 .notifier_call
= nb_callback
993 * Process a received packet with an unknown/unexpected CPL opcode.
995 static int do_bad_cpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
997 printk(KERN_ERR
"%s: received bad CPL command 0x%x\n", dev
->name
,
999 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
1003 * Handlers for each CPL opcode
1005 static cpl_handler_func cpl_handlers
[NUM_CPL_CMDS
];
1008 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied
1009 * to unregister an existing handler.
1011 void t3_register_cpl_handler(unsigned int opcode
, cpl_handler_func h
)
1013 if (opcode
< NUM_CPL_CMDS
)
1014 cpl_handlers
[opcode
] = h
? h
: do_bad_cpl
;
1016 printk(KERN_ERR
"T3C: handler registration for "
1017 "opcode %x failed\n", opcode
);
1020 EXPORT_SYMBOL(t3_register_cpl_handler
);
1023 * T3CDEV's receive method.
1025 static int process_rx(struct t3cdev
*dev
, struct sk_buff
**skbs
, int n
)
1028 struct sk_buff
*skb
= *skbs
++;
1029 unsigned int opcode
= get_opcode(skb
);
1030 int ret
= cpl_handlers
[opcode
] (dev
, skb
);
1033 if (ret
& CPL_RET_UNKNOWN_TID
) {
1034 union opcode_tid
*p
= cplhdr(skb
);
1036 printk(KERN_ERR
"%s: CPL message (opcode %u) had "
1037 "unknown TID %u\n", dev
->name
, opcode
,
1038 G_TID(ntohl(p
->opcode_tid
)));
1041 if (ret
& CPL_RET_BUF_DONE
)
1048 * Sends an sk_buff to a T3C driver after dealing with any active network taps.
1050 int cxgb3_ofld_send(struct t3cdev
*dev
, struct sk_buff
*skb
)
1055 r
= dev
->send(dev
, skb
);
1060 EXPORT_SYMBOL(cxgb3_ofld_send
);
1062 static int is_offloading(struct net_device
*dev
)
1064 struct adapter
*adapter
;
1067 read_lock_bh(&adapter_list_lock
);
1068 list_for_each_entry(adapter
, &adapter_list
, adapter_list
) {
1069 for_each_port(adapter
, i
) {
1070 if (dev
== adapter
->port
[i
]) {
1071 read_unlock_bh(&adapter_list_lock
);
1076 read_unlock_bh(&adapter_list_lock
);
1080 static void cxgb_neigh_update(struct neighbour
*neigh
)
1082 struct net_device
*dev
;
1087 if (dev
&& (is_offloading(dev
))) {
1088 struct t3cdev
*tdev
= dev2t3cdev(dev
);
1091 t3_l2t_update(tdev
, neigh
);
1095 static void set_l2t_ix(struct t3cdev
*tdev
, u32 tid
, struct l2t_entry
*e
)
1097 struct sk_buff
*skb
;
1098 struct cpl_set_tcb_field
*req
;
1100 skb
= alloc_skb(sizeof(*req
), GFP_ATOMIC
);
1102 printk(KERN_ERR
"%s: cannot allocate skb!\n", __func__
);
1105 skb
->priority
= CPL_PRIORITY_CONTROL
;
1106 req
= (struct cpl_set_tcb_field
*)skb_put(skb
, sizeof(*req
));
1107 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
1108 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1111 req
->word
= htons(W_TCB_L2T_IX
);
1112 req
->mask
= cpu_to_be64(V_TCB_L2T_IX(M_TCB_L2T_IX
));
1113 req
->val
= cpu_to_be64(V_TCB_L2T_IX(e
->idx
));
1114 tdev
->send(tdev
, skb
);
1117 static void cxgb_redirect(struct dst_entry
*old
, struct neighbour
*old_neigh
,
1118 struct dst_entry
*new, struct neighbour
*new_neigh
,
1121 struct net_device
*olddev
, *newdev
;
1122 struct tid_info
*ti
;
1123 struct t3cdev
*tdev
;
1126 struct l2t_entry
*e
;
1127 struct t3c_tid_entry
*te
;
1129 olddev
= old_neigh
->dev
;
1130 newdev
= new_neigh
->dev
;
1132 if (!is_offloading(olddev
))
1134 if (!is_offloading(newdev
)) {
1135 printk(KERN_WARNING
"%s: Redirect to non-offload "
1136 "device ignored.\n", __func__
);
1139 tdev
= dev2t3cdev(olddev
);
1141 if (tdev
!= dev2t3cdev(newdev
)) {
1142 printk(KERN_WARNING
"%s: Redirect to different "
1143 "offload device ignored.\n", __func__
);
1147 /* Add new L2T entry */
1148 e
= t3_l2t_get(tdev
, new, newdev
, daddr
);
1150 printk(KERN_ERR
"%s: couldn't allocate new l2t entry!\n",
1155 /* Walk tid table and notify clients of dst change. */
1156 ti
= &(T3C_DATA(tdev
))->tid_maps
;
1157 for (tid
= 0; tid
< ti
->ntids
; tid
++) {
1158 te
= lookup_tid(ti
, tid
);
1160 if (te
&& te
->ctx
&& te
->client
&& te
->client
->redirect
) {
1161 update_tcb
= te
->client
->redirect(te
->ctx
, old
, new, e
);
1164 l2t_hold(L2DATA(tdev
), e
);
1166 set_l2t_ix(tdev
, tid
, e
);
1170 l2t_release(tdev
, e
);
1174 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1175 * The allocated memory is cleared.
1177 void *cxgb_alloc_mem(unsigned long size
)
1179 void *p
= kzalloc(size
, GFP_KERNEL
);
1187 * Free memory allocated through t3_alloc_mem().
1189 void cxgb_free_mem(void *addr
)
1191 if (is_vmalloc_addr(addr
))
1198 * Allocate and initialize the TID tables. Returns 0 on success.
1200 static int init_tid_tabs(struct tid_info
*t
, unsigned int ntids
,
1201 unsigned int natids
, unsigned int nstids
,
1202 unsigned int atid_base
, unsigned int stid_base
)
1204 unsigned long size
= ntids
* sizeof(*t
->tid_tab
) +
1205 natids
* sizeof(*t
->atid_tab
) + nstids
* sizeof(*t
->stid_tab
);
1207 t
->tid_tab
= cxgb_alloc_mem(size
);
1211 t
->stid_tab
= (union listen_entry
*)&t
->tid_tab
[ntids
];
1212 t
->atid_tab
= (union active_open_entry
*)&t
->stid_tab
[nstids
];
1215 t
->stid_base
= stid_base
;
1218 t
->atid_base
= atid_base
;
1220 t
->stids_in_use
= t
->atids_in_use
= 0;
1221 atomic_set(&t
->tids_in_use
, 0);
1222 spin_lock_init(&t
->stid_lock
);
1223 spin_lock_init(&t
->atid_lock
);
1226 * Setup the free lists for stid_tab and atid_tab.
1230 t
->stid_tab
[nstids
- 1].next
= &t
->stid_tab
[nstids
];
1231 t
->sfree
= t
->stid_tab
;
1235 t
->atid_tab
[natids
- 1].next
= &t
->atid_tab
[natids
];
1236 t
->afree
= t
->atid_tab
;
1241 static void free_tid_maps(struct tid_info
*t
)
1243 cxgb_free_mem(t
->tid_tab
);
1246 static inline void add_adapter(struct adapter
*adap
)
1248 write_lock_bh(&adapter_list_lock
);
1249 list_add_tail(&adap
->adapter_list
, &adapter_list
);
1250 write_unlock_bh(&adapter_list_lock
);
1253 static inline void remove_adapter(struct adapter
*adap
)
1255 write_lock_bh(&adapter_list_lock
);
1256 list_del(&adap
->adapter_list
);
1257 write_unlock_bh(&adapter_list_lock
);
1260 int cxgb3_offload_activate(struct adapter
*adapter
)
1262 struct t3cdev
*dev
= &adapter
->tdev
;
1265 struct tid_range stid_range
, tid_range
;
1266 struct mtutab mtutab
;
1267 unsigned int l2t_capacity
;
1269 t
= kzalloc(sizeof(*t
), GFP_KERNEL
);
1274 if (dev
->ctl(dev
, GET_TX_MAX_CHUNK
, &t
->tx_max_chunk
) < 0 ||
1275 dev
->ctl(dev
, GET_MAX_OUTSTANDING_WR
, &t
->max_wrs
) < 0 ||
1276 dev
->ctl(dev
, GET_L2T_CAPACITY
, &l2t_capacity
) < 0 ||
1277 dev
->ctl(dev
, GET_MTUS
, &mtutab
) < 0 ||
1278 dev
->ctl(dev
, GET_TID_RANGE
, &tid_range
) < 0 ||
1279 dev
->ctl(dev
, GET_STID_RANGE
, &stid_range
) < 0)
1283 RCU_INIT_POINTER(dev
->l2opt
, t3_init_l2t(l2t_capacity
));
1287 natids
= min(tid_range
.num
/ 2, MAX_ATIDS
);
1288 err
= init_tid_tabs(&t
->tid_maps
, tid_range
.num
, natids
,
1289 stid_range
.num
, ATID_BASE
, stid_range
.base
);
1293 t
->mtus
= mtutab
.mtus
;
1294 t
->nmtus
= mtutab
.size
;
1296 INIT_WORK(&t
->tid_release_task
, t3_process_tid_release_list
);
1297 spin_lock_init(&t
->tid_release_lock
);
1298 INIT_LIST_HEAD(&t
->list_node
);
1302 dev
->recv
= process_rx
;
1303 dev
->neigh_update
= t3_l2t_update
;
1305 /* Register netevent handler once */
1306 if (list_empty(&adapter_list
))
1307 register_netevent_notifier(&nb
);
1309 t
->nofail_skb
= alloc_skb(sizeof(struct cpl_tid_release
), GFP_KERNEL
);
1310 t
->release_list_incomplete
= 0;
1312 add_adapter(adapter
);
1316 t3_free_l2t(L2DATA(dev
));
1317 RCU_INIT_POINTER(dev
->l2opt
, NULL
);
1323 static void clean_l2_data(struct rcu_head
*head
)
1325 struct l2t_data
*d
= container_of(head
, struct l2t_data
, rcu_head
);
1330 void cxgb3_offload_deactivate(struct adapter
*adapter
)
1332 struct t3cdev
*tdev
= &adapter
->tdev
;
1333 struct t3c_data
*t
= T3C_DATA(tdev
);
1336 remove_adapter(adapter
);
1337 if (list_empty(&adapter_list
))
1338 unregister_netevent_notifier(&nb
);
1340 free_tid_maps(&t
->tid_maps
);
1341 T3C_DATA(tdev
) = NULL
;
1345 RCU_INIT_POINTER(tdev
->l2opt
, NULL
);
1346 call_rcu(&d
->rcu_head
, clean_l2_data
);
1348 kfree_skb(t
->nofail_skb
);
1352 static inline void register_tdev(struct t3cdev
*tdev
)
1356 mutex_lock(&cxgb3_db_lock
);
1357 snprintf(tdev
->name
, sizeof(tdev
->name
), "ofld_dev%d", unit
++);
1358 list_add_tail(&tdev
->ofld_dev_list
, &ofld_dev_list
);
1359 mutex_unlock(&cxgb3_db_lock
);
1362 static inline void unregister_tdev(struct t3cdev
*tdev
)
1364 mutex_lock(&cxgb3_db_lock
);
1365 list_del(&tdev
->ofld_dev_list
);
1366 mutex_unlock(&cxgb3_db_lock
);
1369 static inline int adap2type(struct adapter
*adapter
)
1373 switch (adapter
->params
.rev
) {
1388 void cxgb3_adapter_ofld(struct adapter
*adapter
)
1390 struct t3cdev
*tdev
= &adapter
->tdev
;
1392 INIT_LIST_HEAD(&tdev
->ofld_dev_list
);
1394 cxgb3_set_dummy_ops(tdev
);
1395 tdev
->send
= t3_offload_tx
;
1396 tdev
->ctl
= cxgb_offload_ctl
;
1397 tdev
->type
= adap2type(adapter
);
1399 register_tdev(tdev
);
1402 void cxgb3_adapter_unofld(struct adapter
*adapter
)
1404 struct t3cdev
*tdev
= &adapter
->tdev
;
1407 tdev
->neigh_update
= NULL
;
1409 unregister_tdev(tdev
);
1412 void __init
cxgb3_offload_init(void)
1416 for (i
= 0; i
< NUM_CPL_CMDS
; ++i
)
1417 cpl_handlers
[i
] = do_bad_cpl
;
1419 t3_register_cpl_handler(CPL_SMT_WRITE_RPL
, do_smt_write_rpl
);
1420 t3_register_cpl_handler(CPL_L2T_WRITE_RPL
, do_l2t_write_rpl
);
1421 t3_register_cpl_handler(CPL_RTE_WRITE_RPL
, do_rte_write_rpl
);
1422 t3_register_cpl_handler(CPL_PASS_OPEN_RPL
, do_stid_rpl
);
1423 t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL
, do_stid_rpl
);
1424 t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ
, do_cr
);
1425 t3_register_cpl_handler(CPL_PASS_ESTABLISH
, do_hwtid_rpl
);
1426 t3_register_cpl_handler(CPL_ABORT_RPL_RSS
, do_hwtid_rpl
);
1427 t3_register_cpl_handler(CPL_ABORT_RPL
, do_hwtid_rpl
);
1428 t3_register_cpl_handler(CPL_RX_URG_NOTIFY
, do_hwtid_rpl
);
1429 t3_register_cpl_handler(CPL_RX_DATA
, do_hwtid_rpl
);
1430 t3_register_cpl_handler(CPL_TX_DATA_ACK
, do_hwtid_rpl
);
1431 t3_register_cpl_handler(CPL_TX_DMA_ACK
, do_hwtid_rpl
);
1432 t3_register_cpl_handler(CPL_ACT_OPEN_RPL
, do_act_open_rpl
);
1433 t3_register_cpl_handler(CPL_PEER_CLOSE
, do_hwtid_rpl
);
1434 t3_register_cpl_handler(CPL_CLOSE_CON_RPL
, do_hwtid_rpl
);
1435 t3_register_cpl_handler(CPL_ABORT_REQ_RSS
, do_abort_req_rss
);
1436 t3_register_cpl_handler(CPL_ACT_ESTABLISH
, do_act_establish
);
1437 t3_register_cpl_handler(CPL_SET_TCB_RPL
, do_hwtid_rpl
);
1438 t3_register_cpl_handler(CPL_GET_TCB_RPL
, do_hwtid_rpl
);
1439 t3_register_cpl_handler(CPL_RDMA_TERMINATE
, do_term
);
1440 t3_register_cpl_handler(CPL_RDMA_EC_STATUS
, do_hwtid_rpl
);
1441 t3_register_cpl_handler(CPL_TRACE_PKT
, do_trace
);
1442 t3_register_cpl_handler(CPL_RX_DATA_DDP
, do_hwtid_rpl
);
1443 t3_register_cpl_handler(CPL_RX_DDP_COMPLETE
, do_hwtid_rpl
);
1444 t3_register_cpl_handler(CPL_ISCSI_HDR
, do_hwtid_rpl
);