2 * cxgb4i.c: Chelsio T4 iSCSI driver.
4 * Copyright (c) 2010-2015 Chelsio Communications, Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
10 * Written by: Karen Xie (kxie@chelsio.com)
11 * Rakesh Ranjan (rranjan@chelsio.com)
14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/moduleparam.h>
19 #include <scsi/scsi_host.h>
22 #include <linux/netdevice.h>
23 #include <net/addrconf.h>
28 #include "cxgb4_uld.h"
34 static unsigned int dbg_level
;
36 #include "../libcxgbi.h"
38 #define DRV_MODULE_NAME "cxgb4i"
39 #define DRV_MODULE_DESC "Chelsio T4/T5 iSCSI Driver"
40 #define DRV_MODULE_VERSION "0.9.5-ko"
41 #define DRV_MODULE_RELDATE "Apr. 2015"
43 static char version
[] =
44 DRV_MODULE_DESC
" " DRV_MODULE_NAME
45 " v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
47 MODULE_AUTHOR("Chelsio Communications, Inc.");
48 MODULE_DESCRIPTION(DRV_MODULE_DESC
);
49 MODULE_VERSION(DRV_MODULE_VERSION
);
50 MODULE_LICENSE("GPL");
52 module_param(dbg_level
, uint
, 0644);
53 MODULE_PARM_DESC(dbg_level
, "Debug flag (default=0)");
55 #define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024)
56 static int cxgb4i_rcv_win
= -1;
57 module_param(cxgb4i_rcv_win
, int, 0644);
58 MODULE_PARM_DESC(cxgb4i_rcv_win
, "TCP reveive window in bytes");
60 #define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024)
61 static int cxgb4i_snd_win
= -1;
62 module_param(cxgb4i_snd_win
, int, 0644);
63 MODULE_PARM_DESC(cxgb4i_snd_win
, "TCP send window in bytes");
65 static int cxgb4i_rx_credit_thres
= 10 * 1024;
66 module_param(cxgb4i_rx_credit_thres
, int, 0644);
67 MODULE_PARM_DESC(cxgb4i_rx_credit_thres
,
68 "RX credits return threshold in bytes (default=10KB)");
70 static unsigned int cxgb4i_max_connect
= (8 * 1024);
71 module_param(cxgb4i_max_connect
, uint
, 0644);
72 MODULE_PARM_DESC(cxgb4i_max_connect
, "Maximum number of connections");
74 static unsigned short cxgb4i_sport_base
= 20000;
75 module_param(cxgb4i_sport_base
, ushort
, 0644);
76 MODULE_PARM_DESC(cxgb4i_sport_base
, "Starting port number (default 20000)");
78 typedef void (*cxgb4i_cplhandler_func
)(struct cxgbi_device
*, struct sk_buff
*);
80 static void *t4_uld_add(const struct cxgb4_lld_info
*);
81 static int t4_uld_rx_handler(void *, const __be64
*, const struct pkt_gl
*);
82 static int t4_uld_state_change(void *, enum cxgb4_state state
);
83 static inline int send_tx_flowc_wr(struct cxgbi_sock
*);
85 static const struct cxgb4_uld_info cxgb4i_uld_info
= {
86 .name
= DRV_MODULE_NAME
,
87 .nrxq
= MAX_ULD_QSETS
,
91 .rx_handler
= t4_uld_rx_handler
,
92 .state_change
= t4_uld_state_change
,
95 static struct scsi_host_template cxgb4i_host_template
= {
96 .module
= THIS_MODULE
,
97 .name
= DRV_MODULE_NAME
,
98 .proc_name
= DRV_MODULE_NAME
,
99 .can_queue
= CXGB4I_SCSI_HOST_QDEPTH
,
100 .queuecommand
= iscsi_queuecommand
,
101 .change_queue_depth
= scsi_change_queue_depth
,
102 .sg_tablesize
= SG_ALL
,
103 .max_sectors
= 0xFFFF,
104 .cmd_per_lun
= ISCSI_DEF_CMD_PER_LUN
,
105 .eh_abort_handler
= iscsi_eh_abort
,
106 .eh_device_reset_handler
= iscsi_eh_device_reset
,
107 .eh_target_reset_handler
= iscsi_eh_recover_target
,
108 .target_alloc
= iscsi_target_alloc
,
109 .use_clustering
= DISABLE_CLUSTERING
,
111 .track_queue_depth
= 1,
114 static struct iscsi_transport cxgb4i_iscsi_transport
= {
115 .owner
= THIS_MODULE
,
116 .name
= DRV_MODULE_NAME
,
117 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
| CAP_HDRDGST
|
118 CAP_DATADGST
| CAP_DIGEST_OFFLOAD
|
119 CAP_PADDING_OFFLOAD
| CAP_TEXT_NEGO
,
120 .attr_is_visible
= cxgbi_attr_is_visible
,
121 .get_host_param
= cxgbi_get_host_param
,
122 .set_host_param
= cxgbi_set_host_param
,
123 /* session management */
124 .create_session
= cxgbi_create_session
,
125 .destroy_session
= cxgbi_destroy_session
,
126 .get_session_param
= iscsi_session_get_param
,
127 /* connection management */
128 .create_conn
= cxgbi_create_conn
,
129 .bind_conn
= cxgbi_bind_conn
,
130 .destroy_conn
= iscsi_tcp_conn_teardown
,
131 .start_conn
= iscsi_conn_start
,
132 .stop_conn
= iscsi_conn_stop
,
133 .get_conn_param
= iscsi_conn_get_param
,
134 .set_param
= cxgbi_set_conn_param
,
135 .get_stats
= cxgbi_get_conn_stats
,
136 /* pdu xmit req from user space */
137 .send_pdu
= iscsi_conn_send_pdu
,
139 .init_task
= iscsi_tcp_task_init
,
140 .xmit_task
= iscsi_tcp_task_xmit
,
141 .cleanup_task
= cxgbi_cleanup_task
,
143 .alloc_pdu
= cxgbi_conn_alloc_pdu
,
144 .init_pdu
= cxgbi_conn_init_pdu
,
145 .xmit_pdu
= cxgbi_conn_xmit_pdu
,
146 .parse_pdu_itt
= cxgbi_parse_pdu_itt
,
147 /* TCP connect/disconnect */
148 .get_ep_param
= cxgbi_get_ep_param
,
149 .ep_connect
= cxgbi_ep_connect
,
150 .ep_poll
= cxgbi_ep_poll
,
151 .ep_disconnect
= cxgbi_ep_disconnect
,
152 /* Error recovery timeout call */
153 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
156 static struct scsi_transport_template
*cxgb4i_stt
;
159 * CPL (Chelsio Protocol Language) defines a message passing interface between
160 * the host driver and Chelsio asic.
161 * The section below implments CPLs that related to iscsi tcp connection
162 * open/close/abort and data send/receive.
165 #define RCV_BUFSIZ_MASK 0x3FFU
166 #define MAX_IMM_TX_PKT_LEN 256
168 static int push_tx_frames(struct cxgbi_sock
*, int);
171 * is_ofld_imm - check whether a packet can be sent as immediate data
174 * Returns true if a packet can be sent as an offload WR with immediate
175 * data. We currently use the same limit as for Ethernet packets.
177 static inline bool is_ofld_imm(const struct sk_buff
*skb
)
181 if (likely(cxgbi_skcb_test_flag(skb
, SKCBF_TX_NEED_HDR
)))
182 len
+= sizeof(struct fw_ofld_tx_data_wr
);
184 return len
<= MAX_IMM_TX_PKT_LEN
;
187 static void send_act_open_req(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
190 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
191 int t4
= is_t4(lldi
->adapter_type
);
192 int wscale
= cxgbi_sock_compute_wscale(csk
->mss_idx
);
193 unsigned long long opt0
;
195 unsigned int qid_atid
= ((unsigned int)csk
->atid
) |
196 (((unsigned int)csk
->rss_qid
) << 14);
198 opt0
= KEEP_ALIVE_F
|
199 WND_SCALE_V(wscale
) |
200 MSS_IDX_V(csk
->mss_idx
) |
201 L2T_IDX_V(((struct l2t_entry
*)csk
->l2t
)->idx
) |
202 TX_CHAN_V(csk
->tx_chan
) |
203 SMAC_SEL_V(csk
->smac_idx
) |
204 ULP_MODE_V(ULP_MODE_ISCSI
) |
205 RCV_BUFSIZ_V(csk
->rcv_win
>> 10);
207 opt2
= RX_CHANNEL_V(0) |
209 RSS_QUEUE_V(csk
->rss_qid
);
211 if (is_t4(lldi
->adapter_type
)) {
212 struct cpl_act_open_req
*req
=
213 (struct cpl_act_open_req
*)skb
->head
;
216 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
,
218 req
->local_port
= csk
->saddr
.sin_port
;
219 req
->peer_port
= csk
->daddr
.sin_port
;
220 req
->local_ip
= csk
->saddr
.sin_addr
.s_addr
;
221 req
->peer_ip
= csk
->daddr
.sin_addr
.s_addr
;
222 req
->opt0
= cpu_to_be64(opt0
);
223 req
->params
= cpu_to_be32(cxgb4_select_ntuple(
224 csk
->cdev
->ports
[csk
->port_id
],
226 opt2
|= RX_FC_VALID_F
;
227 req
->opt2
= cpu_to_be32(opt2
);
229 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
230 "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
231 csk
, &req
->local_ip
, ntohs(req
->local_port
),
232 &req
->peer_ip
, ntohs(req
->peer_port
),
233 csk
->atid
, csk
->rss_qid
);
235 struct cpl_t5_act_open_req
*req
=
236 (struct cpl_t5_act_open_req
*)skb
->head
;
237 u32 isn
= (prandom_u32() & ~7UL) - 1;
240 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
,
242 req
->local_port
= csk
->saddr
.sin_port
;
243 req
->peer_port
= csk
->daddr
.sin_port
;
244 req
->local_ip
= csk
->saddr
.sin_addr
.s_addr
;
245 req
->peer_ip
= csk
->daddr
.sin_addr
.s_addr
;
246 req
->opt0
= cpu_to_be64(opt0
);
247 req
->params
= cpu_to_be64(FILTER_TUPLE_V(
249 csk
->cdev
->ports
[csk
->port_id
],
251 req
->rsvd
= cpu_to_be32(isn
);
252 opt2
|= T5_ISS_VALID
;
253 opt2
|= T5_OPT_2_VALID_F
;
255 req
->opt2
= cpu_to_be32(opt2
);
257 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
258 "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
259 csk
, &req
->local_ip
, ntohs(req
->local_port
),
260 &req
->peer_ip
, ntohs(req
->peer_port
),
261 csk
->atid
, csk
->rss_qid
);
264 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->port_id
);
266 pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n",
267 (&csk
->saddr
), (&csk
->daddr
), t4
? 4 : 5, csk
,
268 csk
->state
, csk
->flags
, csk
->atid
, csk
->rss_qid
);
270 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
273 #if IS_ENABLED(CONFIG_IPV6)
274 static void send_act_open_req6(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
277 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
278 int t4
= is_t4(lldi
->adapter_type
);
279 int wscale
= cxgbi_sock_compute_wscale(csk
->mss_idx
);
280 unsigned long long opt0
;
282 unsigned int qid_atid
= ((unsigned int)csk
->atid
) |
283 (((unsigned int)csk
->rss_qid
) << 14);
285 opt0
= KEEP_ALIVE_F
|
286 WND_SCALE_V(wscale
) |
287 MSS_IDX_V(csk
->mss_idx
) |
288 L2T_IDX_V(((struct l2t_entry
*)csk
->l2t
)->idx
) |
289 TX_CHAN_V(csk
->tx_chan
) |
290 SMAC_SEL_V(csk
->smac_idx
) |
291 ULP_MODE_V(ULP_MODE_ISCSI
) |
292 RCV_BUFSIZ_V(csk
->rcv_win
>> 10);
294 opt2
= RX_CHANNEL_V(0) |
297 RSS_QUEUE_V(csk
->rss_qid
);
300 struct cpl_act_open_req6
*req
=
301 (struct cpl_act_open_req6
*)skb
->head
;
304 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6
,
306 req
->local_port
= csk
->saddr6
.sin6_port
;
307 req
->peer_port
= csk
->daddr6
.sin6_port
;
309 req
->local_ip_hi
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
);
310 req
->local_ip_lo
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
+
312 req
->peer_ip_hi
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
);
313 req
->peer_ip_lo
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
+
316 req
->opt0
= cpu_to_be64(opt0
);
318 opt2
|= RX_FC_VALID_F
;
319 req
->opt2
= cpu_to_be32(opt2
);
321 req
->params
= cpu_to_be32(cxgb4_select_ntuple(
322 csk
->cdev
->ports
[csk
->port_id
],
325 struct cpl_t5_act_open_req6
*req
=
326 (struct cpl_t5_act_open_req6
*)skb
->head
;
329 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6
,
331 req
->local_port
= csk
->saddr6
.sin6_port
;
332 req
->peer_port
= csk
->daddr6
.sin6_port
;
333 req
->local_ip_hi
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
);
334 req
->local_ip_lo
= *(__be64
*)(csk
->saddr6
.sin6_addr
.s6_addr
+
336 req
->peer_ip_hi
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
);
337 req
->peer_ip_lo
= *(__be64
*)(csk
->daddr6
.sin6_addr
.s6_addr
+
339 req
->opt0
= cpu_to_be64(opt0
);
341 opt2
|= T5_OPT_2_VALID_F
;
342 req
->opt2
= cpu_to_be32(opt2
);
344 req
->params
= cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple(
345 csk
->cdev
->ports
[csk
->port_id
],
349 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->port_id
);
351 pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n",
352 t4
? 4 : 5, csk
, csk
->state
, csk
->flags
, csk
->atid
,
353 &csk
->saddr6
.sin6_addr
, ntohs(csk
->saddr
.sin_port
),
354 &csk
->daddr6
.sin6_addr
, ntohs(csk
->daddr
.sin_port
),
357 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
361 static void send_close_req(struct cxgbi_sock
*csk
)
363 struct sk_buff
*skb
= csk
->cpl_close
;
364 struct cpl_close_con_req
*req
= (struct cpl_close_con_req
*)skb
->head
;
365 unsigned int tid
= csk
->tid
;
367 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
368 "csk 0x%p,%u,0x%lx, tid %u.\n",
369 csk
, csk
->state
, csk
->flags
, csk
->tid
);
370 csk
->cpl_close
= NULL
;
371 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
372 INIT_TP_WR(req
, tid
);
373 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
, tid
));
376 cxgbi_sock_skb_entail(csk
, skb
);
377 if (csk
->state
>= CTP_ESTABLISHED
)
378 push_tx_frames(csk
, 1);
381 static void abort_arp_failure(void *handle
, struct sk_buff
*skb
)
383 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)handle
;
384 struct cpl_abort_req
*req
;
386 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
387 "csk 0x%p,%u,0x%lx, tid %u, abort.\n",
388 csk
, csk
->state
, csk
->flags
, csk
->tid
);
389 req
= (struct cpl_abort_req
*)skb
->data
;
390 req
->cmd
= CPL_ABORT_NO_RST
;
391 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
394 static void send_abort_req(struct cxgbi_sock
*csk
)
396 struct cpl_abort_req
*req
;
397 struct sk_buff
*skb
= csk
->cpl_abort_req
;
399 if (unlikely(csk
->state
== CTP_ABORTING
) || !skb
|| !csk
->cdev
)
402 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
)) {
403 send_tx_flowc_wr(csk
);
404 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
407 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
408 cxgbi_sock_set_flag(csk
, CTPF_ABORT_RPL_PENDING
);
409 cxgbi_sock_purge_write_queue(csk
);
411 csk
->cpl_abort_req
= NULL
;
412 req
= (struct cpl_abort_req
*)skb
->head
;
413 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
414 req
->cmd
= CPL_ABORT_SEND_RST
;
415 t4_set_arp_err_handler(skb
, csk
, abort_arp_failure
);
416 INIT_TP_WR(req
, csk
->tid
);
417 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ
, csk
->tid
));
418 req
->rsvd0
= htonl(csk
->snd_nxt
);
419 req
->rsvd1
= !cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
);
421 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
422 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
423 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->snd_nxt
,
426 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
429 static void send_abort_rpl(struct cxgbi_sock
*csk
, int rst_status
)
431 struct sk_buff
*skb
= csk
->cpl_abort_rpl
;
432 struct cpl_abort_rpl
*rpl
= (struct cpl_abort_rpl
*)skb
->head
;
434 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
435 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
436 csk
, csk
->state
, csk
->flags
, csk
->tid
, rst_status
);
438 csk
->cpl_abort_rpl
= NULL
;
439 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
440 INIT_TP_WR(rpl
, csk
->tid
);
441 OPCODE_TID(rpl
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL
, csk
->tid
));
442 rpl
->cmd
= rst_status
;
443 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
447 * CPL connection rx data ack: host ->
448 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
451 static u32
send_rx_credits(struct cxgbi_sock
*csk
, u32 credits
)
454 struct cpl_rx_data_ack
*req
;
456 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
457 "csk 0x%p,%u,0x%lx,%u, credit %u.\n",
458 csk
, csk
->state
, csk
->flags
, csk
->tid
, credits
);
460 skb
= alloc_wr(sizeof(*req
), 0, GFP_ATOMIC
);
462 pr_info("csk 0x%p, credit %u, OOM.\n", csk
, credits
);
465 req
= (struct cpl_rx_data_ack
*)skb
->head
;
467 set_wr_txq(skb
, CPL_PRIORITY_ACK
, csk
->port_id
);
468 INIT_TP_WR(req
, csk
->tid
);
469 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK
,
471 req
->credit_dack
= cpu_to_be32(RX_CREDITS_V(credits
)
473 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
478 * sgl_len - calculates the size of an SGL of the given capacity
479 * @n: the number of SGL entries
480 * Calculates the number of flits needed for a scatter/gather list that
481 * can hold the given number of entries.
483 static inline unsigned int sgl_len(unsigned int n
)
486 return (3 * n
) / 2 + (n
& 1) + 2;
490 * calc_tx_flits_ofld - calculate # of flits for an offload packet
493 * Returns the number of flits needed for the given offload packet.
494 * These packets are already fully constructed and no additional headers
497 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff
*skb
)
499 unsigned int flits
, cnt
;
501 if (is_ofld_imm(skb
))
502 return DIV_ROUND_UP(skb
->len
, 8);
503 flits
= skb_transport_offset(skb
) / 8;
504 cnt
= skb_shinfo(skb
)->nr_frags
;
505 if (skb_tail_pointer(skb
) != skb_transport_header(skb
))
507 return flits
+ sgl_len(cnt
);
510 #define FLOWC_WR_NPARAMS_MIN 9
511 static inline int tx_flowc_wr_credits(int *nparamsp
, int *flowclenp
)
513 int nparams
, flowclen16
, flowclen
;
515 nparams
= FLOWC_WR_NPARAMS_MIN
;
516 flowclen
= offsetof(struct fw_flowc_wr
, mnemval
[nparams
]);
517 flowclen16
= DIV_ROUND_UP(flowclen
, 16);
518 flowclen
= flowclen16
* 16;
520 * Return the number of 16-byte credits used by the FlowC request.
521 * Pass back the nparams and actual FlowC length if requested.
526 *flowclenp
= flowclen
;
531 static inline int send_tx_flowc_wr(struct cxgbi_sock
*csk
)
534 struct fw_flowc_wr
*flowc
;
535 int nparams
, flowclen16
, flowclen
;
537 flowclen16
= tx_flowc_wr_credits(&nparams
, &flowclen
);
538 skb
= alloc_wr(flowclen
, 0, GFP_ATOMIC
);
539 flowc
= (struct fw_flowc_wr
*)skb
->head
;
540 flowc
->op_to_nparams
=
541 htonl(FW_WR_OP_V(FW_FLOWC_WR
) | FW_FLOWC_WR_NPARAMS_V(nparams
));
542 flowc
->flowid_len16
=
543 htonl(FW_WR_LEN16_V(flowclen16
) | FW_WR_FLOWID_V(csk
->tid
));
544 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
545 flowc
->mnemval
[0].val
= htonl(csk
->cdev
->pfvf
);
546 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
547 flowc
->mnemval
[1].val
= htonl(csk
->tx_chan
);
548 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
549 flowc
->mnemval
[2].val
= htonl(csk
->tx_chan
);
550 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
551 flowc
->mnemval
[3].val
= htonl(csk
->rss_qid
);
552 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SNDNXT
;
553 flowc
->mnemval
[4].val
= htonl(csk
->snd_nxt
);
554 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_RCVNXT
;
555 flowc
->mnemval
[5].val
= htonl(csk
->rcv_nxt
);
556 flowc
->mnemval
[6].mnemonic
= FW_FLOWC_MNEM_SNDBUF
;
557 flowc
->mnemval
[6].val
= htonl(csk
->snd_win
);
558 flowc
->mnemval
[7].mnemonic
= FW_FLOWC_MNEM_MSS
;
559 flowc
->mnemval
[7].val
= htonl(csk
->advmss
);
560 flowc
->mnemval
[8].mnemonic
= 0;
561 flowc
->mnemval
[8].val
= 0;
562 flowc
->mnemval
[8].mnemonic
= FW_FLOWC_MNEM_TXDATAPLEN_MAX
;
563 flowc
->mnemval
[8].val
= 16384;
565 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
567 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
568 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n",
569 csk
, csk
->tid
, 0, csk
->tx_chan
, csk
->rss_qid
,
570 csk
->snd_nxt
, csk
->rcv_nxt
, csk
->snd_win
,
573 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
578 static inline void make_tx_data_wr(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
579 int dlen
, int len
, u32 credits
, int compl)
581 struct fw_ofld_tx_data_wr
*req
;
582 unsigned int submode
= cxgbi_skcb_ulp_mode(skb
) & 3;
583 unsigned int wr_ulp_mode
= 0, val
;
584 bool imm
= is_ofld_imm(skb
);
586 req
= (struct fw_ofld_tx_data_wr
*)__skb_push(skb
, sizeof(*req
));
589 req
->op_to_immdlen
= htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR
) |
591 FW_WR_IMMDLEN_V(dlen
));
592 req
->flowid_len16
= htonl(FW_WR_FLOWID_V(csk
->tid
) |
593 FW_WR_LEN16_V(credits
));
596 cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR
) |
600 cpu_to_be32(FW_WR_FLOWID_V(csk
->tid
) |
601 FW_WR_LEN16_V(credits
));
604 wr_ulp_mode
= FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI
) |
605 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode
);
606 val
= skb_peek(&csk
->write_queue
) ? 0 : 1;
607 req
->tunnel_to_proxy
= htonl(wr_ulp_mode
|
608 FW_OFLD_TX_DATA_WR_SHOVE_V(val
));
609 req
->plen
= htonl(len
);
610 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
))
611 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
614 static void arp_failure_skb_discard(void *handle
, struct sk_buff
*skb
)
619 static int push_tx_frames(struct cxgbi_sock
*csk
, int req_completion
)
624 if (unlikely(csk
->state
< CTP_ESTABLISHED
||
625 csk
->state
== CTP_CLOSE_WAIT_1
|| csk
->state
>= CTP_ABORTING
)) {
626 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
|
627 1 << CXGBI_DBG_PDU_TX
,
628 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
629 csk
, csk
->state
, csk
->flags
, csk
->tid
);
633 while (csk
->wr_cred
&& (skb
= skb_peek(&csk
->write_queue
)) != NULL
) {
636 unsigned int credits_needed
;
639 skb_reset_transport_header(skb
);
640 if (is_ofld_imm(skb
))
641 credits_needed
= DIV_ROUND_UP(dlen
, 16);
643 credits_needed
= DIV_ROUND_UP(
644 8 * calc_tx_flits_ofld(skb
),
647 if (likely(cxgbi_skcb_test_flag(skb
, SKCBF_TX_NEED_HDR
)))
648 credits_needed
+= DIV_ROUND_UP(
649 sizeof(struct fw_ofld_tx_data_wr
),
653 * Assumes the initial credits is large enough to support
654 * fw_flowc_wr plus largest possible first payload
656 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
)) {
657 flowclen16
= send_tx_flowc_wr(csk
);
658 csk
->wr_cred
-= flowclen16
;
659 csk
->wr_una_cred
+= flowclen16
;
660 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
663 if (csk
->wr_cred
< credits_needed
) {
664 log_debug(1 << CXGBI_DBG_PDU_TX
,
665 "csk 0x%p, skb %u/%u, wr %d < %u.\n",
666 csk
, skb
->len
, skb
->data_len
,
667 credits_needed
, csk
->wr_cred
);
670 __skb_unlink(skb
, &csk
->write_queue
);
671 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
672 skb
->csum
= credits_needed
+ flowclen16
;
673 csk
->wr_cred
-= credits_needed
;
674 csk
->wr_una_cred
+= credits_needed
;
675 cxgbi_sock_enqueue_wr(csk
, skb
);
677 log_debug(1 << CXGBI_DBG_PDU_TX
,
678 "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
679 csk
, skb
->len
, skb
->data_len
, credits_needed
,
680 csk
->wr_cred
, csk
->wr_una_cred
);
682 if (likely(cxgbi_skcb_test_flag(skb
, SKCBF_TX_NEED_HDR
))) {
683 len
+= cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb
));
684 make_tx_data_wr(csk
, skb
, dlen
, len
, credits_needed
,
687 cxgbi_skcb_clear_flag(skb
, SKCBF_TX_NEED_HDR
);
689 total_size
+= skb
->truesize
;
690 t4_set_arp_err_handler(skb
, csk
, arp_failure_skb_discard
);
692 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
693 "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
694 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
, len
);
696 cxgb4_l2t_send(csk
->cdev
->ports
[csk
->port_id
], skb
, csk
->l2t
);
701 static inline void free_atid(struct cxgbi_sock
*csk
)
703 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
705 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
)) {
706 cxgb4_free_atid(lldi
->tids
, csk
->atid
);
707 cxgbi_sock_clear_flag(csk
, CTPF_HAS_ATID
);
712 static void do_act_establish(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
714 struct cxgbi_sock
*csk
;
715 struct cpl_act_establish
*req
= (struct cpl_act_establish
*)skb
->data
;
716 unsigned short tcp_opt
= ntohs(req
->tcp_opt
);
717 unsigned int tid
= GET_TID(req
);
718 unsigned int atid
= TID_TID_G(ntohl(req
->tos_atid
));
719 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
720 struct tid_info
*t
= lldi
->tids
;
721 u32 rcv_isn
= be32_to_cpu(req
->rcv_isn
);
723 csk
= lookup_atid(t
, atid
);
724 if (unlikely(!csk
)) {
725 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid
, cdev
);
729 if (csk
->atid
!= atid
) {
730 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n",
731 atid
, csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->atid
);
735 pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
736 (&csk
->saddr
), (&csk
->daddr
),
737 atid
, tid
, csk
, csk
->state
, csk
->flags
, rcv_isn
);
739 module_put(THIS_MODULE
);
743 cxgb4_insert_tid(lldi
->tids
, csk
, tid
);
744 cxgbi_sock_set_flag(csk
, CTPF_HAS_TID
);
748 spin_lock_bh(&csk
->lock
);
749 if (unlikely(csk
->state
!= CTP_ACTIVE_OPEN
))
750 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
751 csk
, csk
->state
, csk
->flags
, csk
->tid
);
753 if (csk
->retry_timer
.function
) {
754 del_timer(&csk
->retry_timer
);
755 csk
->retry_timer
.function
= NULL
;
758 csk
->copied_seq
= csk
->rcv_wup
= csk
->rcv_nxt
= rcv_isn
;
760 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
763 if (csk
->rcv_win
> (RCV_BUFSIZ_MASK
<< 10))
764 csk
->rcv_wup
-= csk
->rcv_win
- (RCV_BUFSIZ_MASK
<< 10);
766 csk
->advmss
= lldi
->mtus
[TCPOPT_MSS_G(tcp_opt
)] - 40;
767 if (TCPOPT_TSTAMP_G(tcp_opt
))
769 if (csk
->advmss
< 128)
772 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
773 "csk 0x%p, mss_idx %u, advmss %u.\n",
774 csk
, TCPOPT_MSS_G(tcp_opt
), csk
->advmss
);
776 cxgbi_sock_established(csk
, ntohl(req
->snd_isn
), ntohs(req
->tcp_opt
));
778 if (unlikely(cxgbi_sock_flag(csk
, CTPF_ACTIVE_CLOSE_NEEDED
)))
781 if (skb_queue_len(&csk
->write_queue
))
782 push_tx_frames(csk
, 0);
783 cxgbi_conn_tx_open(csk
);
785 spin_unlock_bh(&csk
->lock
);
791 static int act_open_rpl_status_to_errno(int status
)
794 case CPL_ERR_CONN_RESET
:
795 return -ECONNREFUSED
;
796 case CPL_ERR_ARP_MISS
:
797 return -EHOSTUNREACH
;
798 case CPL_ERR_CONN_TIMEDOUT
:
800 case CPL_ERR_TCAM_FULL
:
802 case CPL_ERR_CONN_EXIST
:
809 static void csk_act_open_retry_timer(unsigned long data
)
811 struct sk_buff
*skb
= NULL
;
812 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)data
;
813 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(csk
->cdev
);
814 void (*send_act_open_func
)(struct cxgbi_sock
*, struct sk_buff
*,
816 int t4
= is_t4(lldi
->adapter_type
), size
, size6
;
818 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
819 "csk 0x%p,%u,0x%lx,%u.\n",
820 csk
, csk
->state
, csk
->flags
, csk
->tid
);
823 spin_lock_bh(&csk
->lock
);
826 size
= sizeof(struct cpl_act_open_req
);
827 size6
= sizeof(struct cpl_act_open_req6
);
829 size
= sizeof(struct cpl_t5_act_open_req
);
830 size6
= sizeof(struct cpl_t5_act_open_req6
);
833 if (csk
->csk_family
== AF_INET
) {
834 send_act_open_func
= send_act_open_req
;
835 skb
= alloc_wr(size
, 0, GFP_ATOMIC
);
836 #if IS_ENABLED(CONFIG_IPV6)
838 send_act_open_func
= send_act_open_req6
;
839 skb
= alloc_wr(size6
, 0, GFP_ATOMIC
);
844 cxgbi_sock_fail_act_open(csk
, -ENOMEM
);
846 skb
->sk
= (struct sock
*)csk
;
847 t4_set_arp_err_handler(skb
, csk
,
848 cxgbi_sock_act_open_req_arp_failure
);
849 send_act_open_func(csk
, skb
, csk
->l2t
);
852 spin_unlock_bh(&csk
->lock
);
857 static inline bool is_neg_adv(unsigned int status
)
859 return status
== CPL_ERR_RTX_NEG_ADVICE
||
860 status
== CPL_ERR_KEEPALV_NEG_ADVICE
||
861 status
== CPL_ERR_PERSIST_NEG_ADVICE
;
864 static void do_act_open_rpl(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
866 struct cxgbi_sock
*csk
;
867 struct cpl_act_open_rpl
*rpl
= (struct cpl_act_open_rpl
*)skb
->data
;
868 unsigned int tid
= GET_TID(rpl
);
870 TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl
->atid_status
)));
871 unsigned int status
= AOPEN_STATUS_G(be32_to_cpu(rpl
->atid_status
));
872 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
873 struct tid_info
*t
= lldi
->tids
;
875 csk
= lookup_atid(t
, atid
);
876 if (unlikely(!csk
)) {
877 pr_err("NO matching conn. atid %u, tid %u.\n", atid
, tid
);
881 pr_info_ipaddr("tid %u/%u, status %u.\n"
882 "csk 0x%p,%u,0x%lx. ", (&csk
->saddr
), (&csk
->daddr
),
883 atid
, tid
, status
, csk
, csk
->state
, csk
->flags
);
885 if (is_neg_adv(status
))
888 module_put(THIS_MODULE
);
890 if (status
&& status
!= CPL_ERR_TCAM_FULL
&&
891 status
!= CPL_ERR_CONN_EXIST
&&
892 status
!= CPL_ERR_ARP_MISS
)
893 cxgb4_remove_tid(lldi
->tids
, csk
->port_id
, GET_TID(rpl
));
896 spin_lock_bh(&csk
->lock
);
898 if (status
== CPL_ERR_CONN_EXIST
&&
899 csk
->retry_timer
.function
!= csk_act_open_retry_timer
) {
900 csk
->retry_timer
.function
= csk_act_open_retry_timer
;
901 mod_timer(&csk
->retry_timer
, jiffies
+ HZ
/ 2);
903 cxgbi_sock_fail_act_open(csk
,
904 act_open_rpl_status_to_errno(status
));
906 spin_unlock_bh(&csk
->lock
);
912 static void do_peer_close(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
914 struct cxgbi_sock
*csk
;
915 struct cpl_peer_close
*req
= (struct cpl_peer_close
*)skb
->data
;
916 unsigned int tid
= GET_TID(req
);
917 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
918 struct tid_info
*t
= lldi
->tids
;
920 csk
= lookup_tid(t
, tid
);
921 if (unlikely(!csk
)) {
922 pr_err("can't find connection for tid %u.\n", tid
);
925 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
926 (&csk
->saddr
), (&csk
->daddr
),
927 csk
, csk
->state
, csk
->flags
, csk
->tid
);
928 cxgbi_sock_rcv_peer_close(csk
);
933 static void do_close_con_rpl(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
935 struct cxgbi_sock
*csk
;
936 struct cpl_close_con_rpl
*rpl
= (struct cpl_close_con_rpl
*)skb
->data
;
937 unsigned int tid
= GET_TID(rpl
);
938 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
939 struct tid_info
*t
= lldi
->tids
;
941 csk
= lookup_tid(t
, tid
);
942 if (unlikely(!csk
)) {
943 pr_err("can't find connection for tid %u.\n", tid
);
946 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
947 (&csk
->saddr
), (&csk
->daddr
),
948 csk
, csk
->state
, csk
->flags
, csk
->tid
);
949 cxgbi_sock_rcv_close_conn_rpl(csk
, ntohl(rpl
->snd_nxt
));
954 static int abort_status_to_errno(struct cxgbi_sock
*csk
, int abort_reason
,
957 switch (abort_reason
) {
958 case CPL_ERR_BAD_SYN
: /* fall through */
959 case CPL_ERR_CONN_RESET
:
960 return csk
->state
> CTP_ESTABLISHED
?
961 -EPIPE
: -ECONNRESET
;
962 case CPL_ERR_XMIT_TIMEDOUT
:
963 case CPL_ERR_PERSIST_TIMEDOUT
:
964 case CPL_ERR_FINWAIT2_TIMEDOUT
:
965 case CPL_ERR_KEEPALIVE_TIMEDOUT
:
972 static void do_abort_req_rss(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
974 struct cxgbi_sock
*csk
;
975 struct cpl_abort_req_rss
*req
= (struct cpl_abort_req_rss
*)skb
->data
;
976 unsigned int tid
= GET_TID(req
);
977 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
978 struct tid_info
*t
= lldi
->tids
;
979 int rst_status
= CPL_ABORT_NO_RST
;
981 csk
= lookup_tid(t
, tid
);
982 if (unlikely(!csk
)) {
983 pr_err("can't find connection for tid %u.\n", tid
);
987 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
988 (&csk
->saddr
), (&csk
->daddr
),
989 csk
, csk
->state
, csk
->flags
, csk
->tid
, req
->status
);
991 if (is_neg_adv(req
->status
))
995 spin_lock_bh(&csk
->lock
);
997 cxgbi_sock_clear_flag(csk
, CTPF_ABORT_REQ_RCVD
);
999 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
)) {
1000 send_tx_flowc_wr(csk
);
1001 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
1004 cxgbi_sock_set_flag(csk
, CTPF_ABORT_REQ_RCVD
);
1005 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
1007 send_abort_rpl(csk
, rst_status
);
1009 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
)) {
1010 csk
->err
= abort_status_to_errno(csk
, req
->status
, &rst_status
);
1011 cxgbi_sock_closed(csk
);
1014 spin_unlock_bh(&csk
->lock
);
1015 cxgbi_sock_put(csk
);
1020 static void do_abort_rpl_rss(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1022 struct cxgbi_sock
*csk
;
1023 struct cpl_abort_rpl_rss
*rpl
= (struct cpl_abort_rpl_rss
*)skb
->data
;
1024 unsigned int tid
= GET_TID(rpl
);
1025 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1026 struct tid_info
*t
= lldi
->tids
;
1028 csk
= lookup_tid(t
, tid
);
1033 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
1034 (&csk
->saddr
), (&csk
->daddr
), csk
,
1035 csk
->state
, csk
->flags
, csk
->tid
, rpl
->status
);
1037 if (rpl
->status
== CPL_ERR_ABORT_FAILED
)
1040 cxgbi_sock_rcv_abort_rpl(csk
);
1045 static void do_rx_data(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1047 struct cxgbi_sock
*csk
;
1048 struct cpl_rx_data
*cpl
= (struct cpl_rx_data
*)skb
->data
;
1049 unsigned int tid
= GET_TID(cpl
);
1050 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1051 struct tid_info
*t
= lldi
->tids
;
1053 csk
= lookup_tid(t
, tid
);
1055 pr_err("can't find connection for tid %u.\n", tid
);
1057 /* not expecting this, reset the connection. */
1058 pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk
, tid
);
1059 spin_lock_bh(&csk
->lock
);
1060 send_abort_req(csk
);
1061 spin_unlock_bh(&csk
->lock
);
1066 static void do_rx_iscsi_hdr(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1068 struct cxgbi_sock
*csk
;
1069 struct cpl_iscsi_hdr
*cpl
= (struct cpl_iscsi_hdr
*)skb
->data
;
1070 unsigned short pdu_len_ddp
= be16_to_cpu(cpl
->pdu_len_ddp
);
1071 unsigned int tid
= GET_TID(cpl
);
1072 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1073 struct tid_info
*t
= lldi
->tids
;
1075 csk
= lookup_tid(t
, tid
);
1076 if (unlikely(!csk
)) {
1077 pr_err("can't find conn. for tid %u.\n", tid
);
1081 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1082 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
1083 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
, skb
->len
,
1086 spin_lock_bh(&csk
->lock
);
1088 if (unlikely(csk
->state
>= CTP_PASSIVE_CLOSE
)) {
1089 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1090 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1091 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1092 if (csk
->state
!= CTP_ABORTING
)
1098 cxgbi_skcb_tcp_seq(skb
) = ntohl(cpl
->seq
);
1099 cxgbi_skcb_flags(skb
) = 0;
1101 skb_reset_transport_header(skb
);
1102 __skb_pull(skb
, sizeof(*cpl
));
1103 __pskb_trim(skb
, ntohs(cpl
->len
));
1105 if (!csk
->skb_ulp_lhdr
) {
1107 unsigned int hlen
, dlen
, plen
;
1109 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1110 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n",
1111 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
);
1112 csk
->skb_ulp_lhdr
= skb
;
1113 cxgbi_skcb_set_flag(skb
, SKCBF_RX_HDR
);
1115 if (cxgbi_skcb_tcp_seq(skb
) != csk
->rcv_nxt
) {
1116 pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n",
1117 csk
->tid
, cxgbi_skcb_tcp_seq(skb
),
1123 hlen
= ntohs(cpl
->len
);
1124 dlen
= ntohl(*(unsigned int *)(bhs
+ 4)) & 0xFFFFFF;
1126 plen
= ISCSI_PDU_LEN_G(pdu_len_ddp
);
1127 if (is_t4(lldi
->adapter_type
))
1130 if ((hlen
+ dlen
) != plen
) {
1131 pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len "
1132 "mismatch %u != %u + %u, seq 0x%x.\n",
1133 csk
->tid
, plen
, hlen
, dlen
,
1134 cxgbi_skcb_tcp_seq(skb
));
1138 cxgbi_skcb_rx_pdulen(skb
) = (hlen
+ dlen
+ 3) & (~0x3);
1140 cxgbi_skcb_rx_pdulen(skb
) += csk
->dcrc_len
;
1141 csk
->rcv_nxt
+= cxgbi_skcb_rx_pdulen(skb
);
1143 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1144 "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n",
1145 csk
, skb
, *bhs
, hlen
, dlen
,
1146 ntohl(*((unsigned int *)(bhs
+ 16))),
1147 ntohl(*((unsigned int *)(bhs
+ 24))));
1150 struct sk_buff
*lskb
= csk
->skb_ulp_lhdr
;
1152 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_DATA
);
1153 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1154 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
1155 csk
, csk
->state
, csk
->flags
, skb
, lskb
);
1158 __skb_queue_tail(&csk
->receive_queue
, skb
);
1159 spin_unlock_bh(&csk
->lock
);
1163 send_abort_req(csk
);
1165 spin_unlock_bh(&csk
->lock
);
1170 static void do_rx_data_ddp(struct cxgbi_device
*cdev
,
1171 struct sk_buff
*skb
)
1173 struct cxgbi_sock
*csk
;
1174 struct sk_buff
*lskb
;
1175 struct cpl_rx_data_ddp
*rpl
= (struct cpl_rx_data_ddp
*)skb
->data
;
1176 unsigned int tid
= GET_TID(rpl
);
1177 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1178 struct tid_info
*t
= lldi
->tids
;
1179 unsigned int status
= ntohl(rpl
->ddpvld
);
1181 csk
= lookup_tid(t
, tid
);
1182 if (unlikely(!csk
)) {
1183 pr_err("can't find connection for tid %u.\n", tid
);
1187 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
1188 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
1189 csk
, csk
->state
, csk
->flags
, skb
, status
, csk
->skb_ulp_lhdr
);
1191 spin_lock_bh(&csk
->lock
);
1193 if (unlikely(csk
->state
>= CTP_PASSIVE_CLOSE
)) {
1194 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1195 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1196 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1197 if (csk
->state
!= CTP_ABORTING
)
1203 if (!csk
->skb_ulp_lhdr
) {
1204 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk
->tid
);
1208 lskb
= csk
->skb_ulp_lhdr
;
1209 csk
->skb_ulp_lhdr
= NULL
;
1211 cxgbi_skcb_rx_ddigest(lskb
) = ntohl(rpl
->ulp_crc
);
1213 if (ntohs(rpl
->len
) != cxgbi_skcb_rx_pdulen(lskb
))
1214 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n",
1215 csk
->tid
, ntohs(rpl
->len
), cxgbi_skcb_rx_pdulen(lskb
));
1217 if (status
& (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT
)) {
1218 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
1219 csk
, lskb
, status
, cxgbi_skcb_flags(lskb
));
1220 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_HCRC_ERR
);
1222 if (status
& (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT
)) {
1223 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
1224 csk
, lskb
, status
, cxgbi_skcb_flags(lskb
));
1225 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_DCRC_ERR
);
1227 if (status
& (1 << CPL_RX_DDP_STATUS_PAD_SHIFT
)) {
1228 log_debug(1 << CXGBI_DBG_PDU_RX
,
1229 "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n",
1231 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_PAD_ERR
);
1233 if ((status
& (1 << CPL_RX_DDP_STATUS_DDP_SHIFT
)) &&
1234 !cxgbi_skcb_test_flag(lskb
, SKCBF_RX_DATA
)) {
1235 log_debug(1 << CXGBI_DBG_PDU_RX
,
1236 "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n",
1238 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_DATA_DDPD
);
1240 log_debug(1 << CXGBI_DBG_PDU_RX
,
1241 "csk 0x%p, lskb 0x%p, f 0x%lx.\n",
1242 csk
, lskb
, cxgbi_skcb_flags(lskb
));
1244 cxgbi_skcb_set_flag(lskb
, SKCBF_RX_STATUS
);
1245 cxgbi_conn_pdu_ready(csk
);
1246 spin_unlock_bh(&csk
->lock
);
1250 send_abort_req(csk
);
1252 spin_unlock_bh(&csk
->lock
);
1257 static void do_fw4_ack(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1259 struct cxgbi_sock
*csk
;
1260 struct cpl_fw4_ack
*rpl
= (struct cpl_fw4_ack
*)skb
->data
;
1261 unsigned int tid
= GET_TID(rpl
);
1262 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1263 struct tid_info
*t
= lldi
->tids
;
1265 csk
= lookup_tid(t
, tid
);
1267 pr_err("can't find connection for tid %u.\n", tid
);
1269 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1270 "csk 0x%p,%u,0x%lx,%u.\n",
1271 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1272 cxgbi_sock_rcv_wr_ack(csk
, rpl
->credits
, ntohl(rpl
->snd_una
),
1278 static void do_set_tcb_rpl(struct cxgbi_device
*cdev
, struct sk_buff
*skb
)
1280 struct cpl_set_tcb_rpl
*rpl
= (struct cpl_set_tcb_rpl
*)skb
->data
;
1281 unsigned int tid
= GET_TID(rpl
);
1282 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1283 struct tid_info
*t
= lldi
->tids
;
1284 struct cxgbi_sock
*csk
;
1286 csk
= lookup_tid(t
, tid
);
1288 pr_err("can't find conn. for tid %u.\n", tid
);
1290 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1291 "csk 0x%p,%u,%lx,%u, status 0x%x.\n",
1292 csk
, csk
->state
, csk
->flags
, csk
->tid
, rpl
->status
);
1294 if (rpl
->status
!= CPL_ERR_NONE
)
1295 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n",
1296 csk
, tid
, rpl
->status
);
1301 static int alloc_cpls(struct cxgbi_sock
*csk
)
1303 csk
->cpl_close
= alloc_wr(sizeof(struct cpl_close_con_req
),
1305 if (!csk
->cpl_close
)
1308 csk
->cpl_abort_req
= alloc_wr(sizeof(struct cpl_abort_req
),
1310 if (!csk
->cpl_abort_req
)
1313 csk
->cpl_abort_rpl
= alloc_wr(sizeof(struct cpl_abort_rpl
),
1315 if (!csk
->cpl_abort_rpl
)
1320 cxgbi_sock_free_cpl_skbs(csk
);
1324 static inline void l2t_put(struct cxgbi_sock
*csk
)
1327 cxgb4_l2t_release(csk
->l2t
);
1329 cxgbi_sock_put(csk
);
1333 static void release_offload_resources(struct cxgbi_sock
*csk
)
1335 struct cxgb4_lld_info
*lldi
;
1336 #if IS_ENABLED(CONFIG_IPV6)
1337 struct net_device
*ndev
= csk
->cdev
->ports
[csk
->port_id
];
1340 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1341 "csk 0x%p,%u,0x%lx,%u.\n",
1342 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1344 cxgbi_sock_free_cpl_skbs(csk
);
1345 if (csk
->wr_cred
!= csk
->wr_max_cred
) {
1346 cxgbi_sock_purge_wr_queue(csk
);
1347 cxgbi_sock_reset_wr_list(csk
);
1351 #if IS_ENABLED(CONFIG_IPV6)
1352 if (csk
->csk_family
== AF_INET6
)
1353 cxgb4_clip_release(ndev
,
1354 (const u32
*)&csk
->saddr6
.sin6_addr
, 1);
1357 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
))
1359 else if (cxgbi_sock_flag(csk
, CTPF_HAS_TID
)) {
1360 lldi
= cxgbi_cdev_priv(csk
->cdev
);
1361 cxgb4_remove_tid(lldi
->tids
, 0, csk
->tid
);
1362 cxgbi_sock_clear_flag(csk
, CTPF_HAS_TID
);
1363 cxgbi_sock_put(csk
);
1369 static int init_act_open(struct cxgbi_sock
*csk
)
1371 struct cxgbi_device
*cdev
= csk
->cdev
;
1372 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1373 struct net_device
*ndev
= cdev
->ports
[csk
->port_id
];
1374 struct sk_buff
*skb
= NULL
;
1375 struct neighbour
*n
= NULL
;
1378 unsigned int size
, size6
;
1379 int t4
= is_t4(lldi
->adapter_type
);
1380 unsigned int linkspeed
;
1381 unsigned int rcv_winf
, snd_winf
;
1383 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1384 "csk 0x%p,%u,0x%lx,%u.\n",
1385 csk
, csk
->state
, csk
->flags
, csk
->tid
);
1387 if (csk
->csk_family
== AF_INET
)
1388 daddr
= &csk
->daddr
.sin_addr
.s_addr
;
1389 #if IS_ENABLED(CONFIG_IPV6)
1390 else if (csk
->csk_family
== AF_INET6
)
1391 daddr
= &csk
->daddr6
.sin6_addr
;
1394 pr_err("address family 0x%x not supported\n", csk
->csk_family
);
1398 n
= dst_neigh_lookup(csk
->dst
, daddr
);
1401 pr_err("%s, can't get neighbour of csk->dst.\n", ndev
->name
);
1405 csk
->atid
= cxgb4_alloc_atid(lldi
->tids
, csk
);
1406 if (csk
->atid
< 0) {
1407 pr_err("%s, NO atid available.\n", ndev
->name
);
1410 cxgbi_sock_set_flag(csk
, CTPF_HAS_ATID
);
1411 cxgbi_sock_get(csk
);
1413 csk
->l2t
= cxgb4_l2t_get(lldi
->l2t
, n
, ndev
, 0);
1415 pr_err("%s, cannot alloc l2t.\n", ndev
->name
);
1416 goto rel_resource_without_clip
;
1418 cxgbi_sock_get(csk
);
1420 #if IS_ENABLED(CONFIG_IPV6)
1421 if (csk
->csk_family
== AF_INET6
)
1422 cxgb4_clip_get(ndev
, (const u32
*)&csk
->saddr6
.sin6_addr
, 1);
1426 size
= sizeof(struct cpl_act_open_req
);
1427 size6
= sizeof(struct cpl_act_open_req6
);
1429 size
= sizeof(struct cpl_t5_act_open_req
);
1430 size6
= sizeof(struct cpl_t5_act_open_req6
);
1433 if (csk
->csk_family
== AF_INET
)
1434 skb
= alloc_wr(size
, 0, GFP_NOIO
);
1435 #if IS_ENABLED(CONFIG_IPV6)
1437 skb
= alloc_wr(size6
, 0, GFP_NOIO
);
1442 skb
->sk
= (struct sock
*)csk
;
1443 t4_set_arp_err_handler(skb
, csk
, cxgbi_sock_act_open_req_arp_failure
);
1446 csk
->mtu
= dst_mtu(csk
->dst
);
1447 cxgb4_best_mtu(lldi
->mtus
, csk
->mtu
, &csk
->mss_idx
);
1448 csk
->tx_chan
= cxgb4_port_chan(ndev
);
1449 /* SMT two entries per row */
1450 csk
->smac_idx
= ((cxgb4_port_viid(ndev
) & 0x7F)) << 1;
1451 step
= lldi
->ntxq
/ lldi
->nchan
;
1452 csk
->txq_idx
= cxgb4_port_idx(ndev
) * step
;
1453 step
= lldi
->nrxq
/ lldi
->nchan
;
1454 csk
->rss_qid
= lldi
->rxq_ids
[cxgb4_port_idx(ndev
) * step
];
1455 linkspeed
= ((struct port_info
*)netdev_priv(ndev
))->link_cfg
.speed
;
1456 csk
->snd_win
= cxgb4i_snd_win
;
1457 csk
->rcv_win
= cxgb4i_rcv_win
;
1458 if (cxgb4i_rcv_win
<= 0) {
1459 csk
->rcv_win
= CXGB4I_DEFAULT_10G_RCV_WIN
;
1460 rcv_winf
= linkspeed
/ SPEED_10000
;
1462 csk
->rcv_win
*= rcv_winf
;
1464 if (cxgb4i_snd_win
<= 0) {
1465 csk
->snd_win
= CXGB4I_DEFAULT_10G_SND_WIN
;
1466 snd_winf
= linkspeed
/ SPEED_10000
;
1468 csk
->snd_win
*= snd_winf
;
1470 csk
->wr_cred
= lldi
->wr_cred
-
1471 DIV_ROUND_UP(sizeof(struct cpl_abort_req
), 16);
1472 csk
->wr_max_cred
= csk
->wr_cred
;
1473 csk
->wr_una_cred
= 0;
1474 cxgbi_sock_reset_wr_list(csk
);
1477 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n",
1478 (&csk
->saddr
), (&csk
->daddr
), csk
, csk
->state
,
1479 csk
->flags
, csk
->tx_chan
, csk
->txq_idx
, csk
->rss_qid
,
1480 csk
->mtu
, csk
->mss_idx
, csk
->smac_idx
);
1482 /* must wait for either a act_open_rpl or act_open_establish */
1483 try_module_get(THIS_MODULE
);
1484 cxgbi_sock_set_state(csk
, CTP_ACTIVE_OPEN
);
1485 if (csk
->csk_family
== AF_INET
)
1486 send_act_open_req(csk
, skb
, csk
->l2t
);
1487 #if IS_ENABLED(CONFIG_IPV6)
1489 send_act_open_req6(csk
, skb
, csk
->l2t
);
1496 #if IS_ENABLED(CONFIG_IPV6)
1497 if (csk
->csk_family
== AF_INET6
)
1498 cxgb4_clip_release(ndev
,
1499 (const u32
*)&csk
->saddr6
.sin6_addr
, 1);
1501 rel_resource_without_clip
:
1509 static cxgb4i_cplhandler_func cxgb4i_cplhandlers
[NUM_CPL_CMDS
] = {
1510 [CPL_ACT_ESTABLISH
] = do_act_establish
,
1511 [CPL_ACT_OPEN_RPL
] = do_act_open_rpl
,
1512 [CPL_PEER_CLOSE
] = do_peer_close
,
1513 [CPL_ABORT_REQ_RSS
] = do_abort_req_rss
,
1514 [CPL_ABORT_RPL_RSS
] = do_abort_rpl_rss
,
1515 [CPL_CLOSE_CON_RPL
] = do_close_con_rpl
,
1516 [CPL_FW4_ACK
] = do_fw4_ack
,
1517 [CPL_ISCSI_HDR
] = do_rx_iscsi_hdr
,
1518 [CPL_ISCSI_DATA
] = do_rx_iscsi_hdr
,
1519 [CPL_SET_TCB_RPL
] = do_set_tcb_rpl
,
1520 [CPL_RX_DATA_DDP
] = do_rx_data_ddp
,
1521 [CPL_RX_ISCSI_DDP
] = do_rx_data_ddp
,
1522 [CPL_RX_DATA
] = do_rx_data
,
1525 static int cxgb4i_ofld_init(struct cxgbi_device
*cdev
)
1529 if (cxgb4i_max_connect
> CXGB4I_MAX_CONN
)
1530 cxgb4i_max_connect
= CXGB4I_MAX_CONN
;
1532 rc
= cxgbi_device_portmap_create(cdev
, cxgb4i_sport_base
,
1533 cxgb4i_max_connect
);
1537 cdev
->csk_release_offload_resources
= release_offload_resources
;
1538 cdev
->csk_push_tx_frames
= push_tx_frames
;
1539 cdev
->csk_send_abort_req
= send_abort_req
;
1540 cdev
->csk_send_close_req
= send_close_req
;
1541 cdev
->csk_send_rx_credits
= send_rx_credits
;
1542 cdev
->csk_alloc_cpls
= alloc_cpls
;
1543 cdev
->csk_init_act_open
= init_act_open
;
1545 pr_info("cdev 0x%p, offload up, added.\n", cdev
);
1550 ulp_mem_io_set_hdr(struct cxgbi_device
*cdev
,
1551 struct ulp_mem_io
*req
,
1552 unsigned int wr_len
, unsigned int dlen
,
1553 unsigned int pm_addr
,
1556 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1557 struct ulptx_idata
*idata
= (struct ulptx_idata
*)(req
+ 1);
1559 INIT_ULPTX_WR(req
, wr_len
, 0, tid
);
1560 req
->wr
.wr_hi
= htonl(FW_WR_OP_V(FW_ULPTX_WR
) |
1562 req
->cmd
= htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE
) |
1563 ULP_MEMIO_ORDER_V(is_t4(lldi
->adapter_type
)) |
1564 T5_ULP_MEMIO_IMM_V(!is_t4(lldi
->adapter_type
)));
1565 req
->dlen
= htonl(ULP_MEMIO_DATA_LEN_V(dlen
>> 5));
1566 req
->lock_addr
= htonl(ULP_MEMIO_ADDR_V(pm_addr
>> 5));
1567 req
->len16
= htonl(DIV_ROUND_UP(wr_len
- sizeof(req
->wr
), 16));
1569 idata
->cmd_more
= htonl(ULPTX_CMD_V(ULP_TX_SC_IMM
));
1570 idata
->len
= htonl(dlen
);
1573 static struct sk_buff
*
1574 ddp_ppod_init_idata(struct cxgbi_device
*cdev
,
1575 struct cxgbi_ppm
*ppm
,
1576 unsigned int idx
, unsigned int npods
,
1579 unsigned int pm_addr
= (idx
<< PPOD_SIZE_SHIFT
) + ppm
->llimit
;
1580 unsigned int dlen
= npods
<< PPOD_SIZE_SHIFT
;
1581 unsigned int wr_len
= roundup(sizeof(struct ulp_mem_io
) +
1582 sizeof(struct ulptx_idata
) + dlen
, 16);
1583 struct sk_buff
*skb
= alloc_wr(wr_len
, 0, GFP_ATOMIC
);
1586 pr_err("%s: %s idx %u, npods %u, OOM.\n",
1587 __func__
, ppm
->ndev
->name
, idx
, npods
);
1591 ulp_mem_io_set_hdr(cdev
, (struct ulp_mem_io
*)skb
->head
, wr_len
, dlen
,
1597 static int ddp_ppod_write_idata(struct cxgbi_ppm
*ppm
, struct cxgbi_sock
*csk
,
1598 struct cxgbi_task_tag_info
*ttinfo
,
1599 unsigned int idx
, unsigned int npods
,
1600 struct scatterlist
**sg_pp
,
1601 unsigned int *sg_off
)
1603 struct cxgbi_device
*cdev
= csk
->cdev
;
1604 struct sk_buff
*skb
= ddp_ppod_init_idata(cdev
, ppm
, idx
, npods
,
1606 struct ulp_mem_io
*req
;
1607 struct ulptx_idata
*idata
;
1608 struct cxgbi_pagepod
*ppod
;
1614 req
= (struct ulp_mem_io
*)skb
->head
;
1615 idata
= (struct ulptx_idata
*)(req
+ 1);
1616 ppod
= (struct cxgbi_pagepod
*)(idata
+ 1);
1618 for (i
= 0; i
< npods
; i
++, ppod
++)
1619 cxgbi_ddp_set_one_ppod(ppod
, ttinfo
, sg_pp
, sg_off
);
1621 cxgbi_skcb_set_flag(skb
, SKCBF_TX_MEM_WRITE
);
1622 cxgbi_skcb_set_flag(skb
, SKCBF_TX_FLAG_COMPL
);
1623 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->port_id
);
1625 spin_lock_bh(&csk
->lock
);
1626 cxgbi_sock_skb_entail(csk
, skb
);
1627 spin_unlock_bh(&csk
->lock
);
1632 static int ddp_set_map(struct cxgbi_ppm
*ppm
, struct cxgbi_sock
*csk
,
1633 struct cxgbi_task_tag_info
*ttinfo
)
1635 unsigned int pidx
= ttinfo
->idx
;
1636 unsigned int npods
= ttinfo
->npods
;
1637 unsigned int i
, cnt
;
1639 struct scatterlist
*sg
= ttinfo
->sgl
;
1640 unsigned int offset
= 0;
1642 ttinfo
->cid
= csk
->port_id
;
1644 for (i
= 0; i
< npods
; i
+= cnt
, pidx
+= cnt
) {
1647 if (cnt
> ULPMEM_IDATA_MAX_NPPODS
)
1648 cnt
= ULPMEM_IDATA_MAX_NPPODS
;
1649 err
= ddp_ppod_write_idata(ppm
, csk
, ttinfo
, pidx
, cnt
,
1658 static int ddp_setup_conn_pgidx(struct cxgbi_sock
*csk
, unsigned int tid
,
1659 int pg_idx
, bool reply
)
1661 struct sk_buff
*skb
;
1662 struct cpl_set_tcb_field
*req
;
1664 if (!pg_idx
|| pg_idx
>= DDP_PGIDX_MAX
)
1667 skb
= alloc_wr(sizeof(*req
), 0, GFP_KERNEL
);
1671 /* set up ulp page size */
1672 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1673 INIT_TP_WR(req
, csk
->tid
);
1674 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1675 req
->reply_ctrl
= htons(NO_REPLY_V(reply
) | QUEUENO_V(csk
->rss_qid
));
1676 req
->word_cookie
= htons(0);
1677 req
->mask
= cpu_to_be64(0x3 << 8);
1678 req
->val
= cpu_to_be64(pg_idx
<< 8);
1679 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->port_id
);
1681 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1682 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk
, csk
->tid
, pg_idx
);
1684 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
1688 static int ddp_setup_conn_digest(struct cxgbi_sock
*csk
, unsigned int tid
,
1689 int hcrc
, int dcrc
, int reply
)
1691 struct sk_buff
*skb
;
1692 struct cpl_set_tcb_field
*req
;
1697 skb
= alloc_wr(sizeof(*req
), 0, GFP_KERNEL
);
1701 csk
->hcrc_len
= (hcrc
? 4 : 0);
1702 csk
->dcrc_len
= (dcrc
? 4 : 0);
1703 /* set up ulp submode */
1704 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1705 INIT_TP_WR(req
, tid
);
1706 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1707 req
->reply_ctrl
= htons(NO_REPLY_V(reply
) | QUEUENO_V(csk
->rss_qid
));
1708 req
->word_cookie
= htons(0);
1709 req
->mask
= cpu_to_be64(0x3 << 4);
1710 req
->val
= cpu_to_be64(((hcrc
? ULP_CRC_HEADER
: 0) |
1711 (dcrc
? ULP_CRC_DATA
: 0)) << 4);
1712 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->port_id
);
1714 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1715 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk
, csk
->tid
, hcrc
, dcrc
);
1717 cxgb4_ofld_send(csk
->cdev
->ports
[csk
->port_id
], skb
);
1721 static struct cxgbi_ppm
*cdev2ppm(struct cxgbi_device
*cdev
)
1723 return (struct cxgbi_ppm
*)(*((struct cxgb4_lld_info
*)
1724 (cxgbi_cdev_priv(cdev
)))->iscsi_ppm
);
1727 static int cxgb4i_ddp_init(struct cxgbi_device
*cdev
)
1729 struct cxgb4_lld_info
*lldi
= cxgbi_cdev_priv(cdev
);
1730 struct net_device
*ndev
= cdev
->ports
[0];
1731 struct cxgbi_tag_format tformat
;
1735 if (!lldi
->vr
->iscsi
.size
) {
1736 pr_warn("%s, iscsi NOT enabled, check config!\n", ndev
->name
);
1740 cdev
->flags
|= CXGBI_FLAG_USE_PPOD_OFLDQ
;
1741 ppmax
= lldi
->vr
->iscsi
.size
>> PPOD_SIZE_SHIFT
;
1743 memset(&tformat
, 0, sizeof(struct cxgbi_tag_format
));
1744 for (i
= 0; i
< 4; i
++)
1745 tformat
.pgsz_order
[i
] = (lldi
->iscsi_pgsz_order
>> (i
<< 3))
1747 cxgbi_tagmask_check(lldi
->iscsi_tagmask
, &tformat
);
1749 cxgbi_ddp_ppm_setup(lldi
->iscsi_ppm
, cdev
, &tformat
, ppmax
,
1750 lldi
->iscsi_llimit
, lldi
->vr
->iscsi
.start
, 2);
1752 cdev
->csk_ddp_setup_digest
= ddp_setup_conn_digest
;
1753 cdev
->csk_ddp_setup_pgidx
= ddp_setup_conn_pgidx
;
1754 cdev
->csk_ddp_set_map
= ddp_set_map
;
1755 cdev
->tx_max_size
= min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD
,
1756 lldi
->iscsi_iolen
- ISCSI_PDU_NONPAYLOAD_LEN
);
1757 cdev
->rx_max_size
= min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD
,
1758 lldi
->iscsi_iolen
- ISCSI_PDU_NONPAYLOAD_LEN
);
1759 cdev
->cdev2ppm
= cdev2ppm
;
1764 static void *t4_uld_add(const struct cxgb4_lld_info
*lldi
)
1766 struct cxgbi_device
*cdev
;
1767 struct port_info
*pi
;
1770 cdev
= cxgbi_device_register(sizeof(*lldi
), lldi
->nports
);
1772 pr_info("t4 device 0x%p, register failed.\n", lldi
);
1775 pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n",
1776 cdev
, lldi
->adapter_type
, lldi
->nports
,
1777 lldi
->ports
[0]->name
, lldi
->nchan
, lldi
->ntxq
,
1778 lldi
->nrxq
, lldi
->wr_cred
);
1779 for (i
= 0; i
< lldi
->nrxq
; i
++)
1780 log_debug(1 << CXGBI_DBG_DEV
,
1781 "t4 0x%p, rxq id #%d: %u.\n",
1782 cdev
, i
, lldi
->rxq_ids
[i
]);
1784 memcpy(cxgbi_cdev_priv(cdev
), lldi
, sizeof(*lldi
));
1785 cdev
->flags
= CXGBI_FLAG_DEV_T4
;
1786 cdev
->pdev
= lldi
->pdev
;
1787 cdev
->ports
= lldi
->ports
;
1788 cdev
->nports
= lldi
->nports
;
1789 cdev
->mtus
= lldi
->mtus
;
1790 cdev
->nmtus
= NMTUS
;
1791 cdev
->rx_credit_thres
= cxgb4i_rx_credit_thres
;
1792 cdev
->skb_tx_rsvd
= CXGB4I_TX_HEADER_LEN
;
1793 cdev
->skb_rx_extra
= sizeof(struct cpl_iscsi_hdr
);
1794 cdev
->itp
= &cxgb4i_iscsi_transport
;
1796 cdev
->pfvf
= FW_VIID_PFN_G(cxgb4_port_viid(lldi
->ports
[0]))
1798 pr_info("cdev 0x%p,%s, pfvf %u.\n",
1799 cdev
, lldi
->ports
[0]->name
, cdev
->pfvf
);
1801 rc
= cxgb4i_ddp_init(cdev
);
1803 pr_info("t4 0x%p ddp init failed.\n", cdev
);
1806 rc
= cxgb4i_ofld_init(cdev
);
1808 pr_info("t4 0x%p ofld init failed.\n", cdev
);
1812 rc
= cxgbi_hbas_add(cdev
, CXGB4I_MAX_LUN
, CXGBI_MAX_CONN
,
1813 &cxgb4i_host_template
, cxgb4i_stt
);
1817 for (i
= 0; i
< cdev
->nports
; i
++) {
1818 pi
= netdev_priv(lldi
->ports
[i
]);
1819 cdev
->hbas
[i
]->port_id
= pi
->port_id
;
1824 cxgbi_device_unregister(cdev
);
1825 return ERR_PTR(-ENOMEM
);
1828 #define RX_PULL_LEN 128
1829 static int t4_uld_rx_handler(void *handle
, const __be64
*rsp
,
1830 const struct pkt_gl
*pgl
)
1832 const struct cpl_act_establish
*rpl
;
1833 struct sk_buff
*skb
;
1835 struct cxgbi_device
*cdev
= handle
;
1838 unsigned int len
= 64 - sizeof(struct rsp_ctrl
) - 8;
1840 skb
= alloc_wr(len
, 0, GFP_ATOMIC
);
1843 skb_copy_to_linear_data(skb
, &rsp
[1], len
);
1845 if (unlikely(*(u8
*)rsp
!= *(u8
*)pgl
->va
)) {
1846 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
1847 pgl
->va
, be64_to_cpu(*rsp
),
1848 be64_to_cpu(*(u64
*)pgl
->va
),
1852 skb
= cxgb4_pktgl_to_skb(pgl
, RX_PULL_LEN
, RX_PULL_LEN
);
1857 rpl
= (struct cpl_act_establish
*)skb
->data
;
1858 opc
= rpl
->ot
.opcode
;
1859 log_debug(1 << CXGBI_DBG_TOE
,
1860 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
1861 cdev
, opc
, rpl
->ot
.opcode_tid
, ntohl(rpl
->ot
.opcode_tid
), skb
);
1862 if (cxgb4i_cplhandlers
[opc
])
1863 cxgb4i_cplhandlers
[opc
](cdev
, skb
);
1865 pr_err("No handler for opcode 0x%x.\n", opc
);
1870 log_debug(1 << CXGBI_DBG_TOE
, "OOM bailing out.\n");
1874 static int t4_uld_state_change(void *handle
, enum cxgb4_state state
)
1876 struct cxgbi_device
*cdev
= handle
;
1879 case CXGB4_STATE_UP
:
1880 pr_info("cdev 0x%p, UP.\n", cdev
);
1882 case CXGB4_STATE_START_RECOVERY
:
1883 pr_info("cdev 0x%p, RECOVERY.\n", cdev
);
1884 /* close all connections */
1886 case CXGB4_STATE_DOWN
:
1887 pr_info("cdev 0x%p, DOWN.\n", cdev
);
1889 case CXGB4_STATE_DETACH
:
1890 pr_info("cdev 0x%p, DETACH.\n", cdev
);
1891 cxgbi_device_unregister(cdev
);
1894 pr_info("cdev 0x%p, unknown state %d.\n", cdev
, state
);
1900 static int __init
cxgb4i_init_module(void)
1904 printk(KERN_INFO
"%s", version
);
1906 rc
= cxgbi_iscsi_init(&cxgb4i_iscsi_transport
, &cxgb4i_stt
);
1909 cxgb4_register_uld(CXGB4_ULD_ISCSI
, &cxgb4i_uld_info
);
1914 static void __exit
cxgb4i_exit_module(void)
1916 cxgb4_unregister_uld(CXGB4_ULD_ISCSI
);
1917 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4
);
1918 cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport
, &cxgb4i_stt
);
1921 module_init(cxgb4i_init_module
);
1922 module_exit(cxgb4i_exit_module
);