]>
Commit | Line | Data |
---|---|---|
7b36b6e0 | 1 | /* |
2 | * cxgb4i.c: Chelsio T4 iSCSI driver. | |
3 | * | |
1149a5ed | 4 | * Copyright (c) 2010-2015 Chelsio Communications, Inc. |
7b36b6e0 | 5 | * |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License as published by | |
8 | * the Free Software Foundation. | |
9 | * | |
10 | * Written by: Karen Xie (kxie@chelsio.com) | |
11 | * Rakesh Ranjan (rranjan@chelsio.com) | |
12 | */ | |
13 | ||
14 | #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ | |
15 | ||
28466e0f | 16 | #include <linux/kernel.h> |
7b36b6e0 | 17 | #include <linux/module.h> |
18 | #include <linux/moduleparam.h> | |
19 | #include <scsi/scsi_host.h> | |
20 | #include <net/tcp.h> | |
21 | #include <net/dst.h> | |
22 | #include <linux/netdevice.h> | |
759a0cc5 | 23 | #include <net/addrconf.h> |
7b36b6e0 | 24 | |
3bd3e8bf | 25 | #include "t4_regs.h" |
7b36b6e0 | 26 | #include "t4_msg.h" |
27 | #include "cxgb4.h" | |
28 | #include "cxgb4_uld.h" | |
29 | #include "t4fw_api.h" | |
30 | #include "l2t.h" | |
31 | #include "cxgb4i.h" | |
211a84e3 | 32 | #include "clip_tbl.h" |
7b36b6e0 | 33 | |
34 | static unsigned int dbg_level; | |
35 | ||
36 | #include "../libcxgbi.h" | |
37 | ||
b5a5fe4e VP |
38 | #ifdef CONFIG_CHELSIO_T4_DCB |
39 | #include <net/dcbevent.h> | |
40 | #include "cxgb4_dcb.h" | |
41 | #endif | |
42 | ||
7b36b6e0 | 43 | #define DRV_MODULE_NAME "cxgb4i" |
cb22bdc8 | 44 | #define DRV_MODULE_DESC "Chelsio T4-T6 iSCSI Driver" |
0ea5bf3d KX |
45 | #define DRV_MODULE_VERSION "0.9.5-ko" |
46 | #define DRV_MODULE_RELDATE "Apr. 2015" | |
7b36b6e0 | 47 | |
48 | static char version[] = | |
49 | DRV_MODULE_DESC " " DRV_MODULE_NAME | |
0ea5bf3d | 50 | " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; |
7b36b6e0 | 51 | |
52 | MODULE_AUTHOR("Chelsio Communications, Inc."); | |
53 | MODULE_DESCRIPTION(DRV_MODULE_DESC); | |
54 | MODULE_VERSION(DRV_MODULE_VERSION); | |
55 | MODULE_LICENSE("GPL"); | |
56 | ||
57 | module_param(dbg_level, uint, 0644); | |
58 | MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)"); | |
59 | ||
81daf10c KX |
60 | #define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024) |
61 | static int cxgb4i_rcv_win = -1; | |
7b36b6e0 | 62 | module_param(cxgb4i_rcv_win, int, 0644); |
21112e13 | 63 | MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP receive window in bytes"); |
7b36b6e0 | 64 | |
81daf10c KX |
65 | #define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024) |
66 | static int cxgb4i_snd_win = -1; | |
7b36b6e0 | 67 | module_param(cxgb4i_snd_win, int, 0644); |
68 | MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes"); | |
69 | ||
70 | static int cxgb4i_rx_credit_thres = 10 * 1024; | |
71 | module_param(cxgb4i_rx_credit_thres, int, 0644); | |
72 | MODULE_PARM_DESC(cxgb4i_rx_credit_thres, | |
73 | "RX credits return threshold in bytes (default=10KB)"); | |
74 | ||
75 | static unsigned int cxgb4i_max_connect = (8 * 1024); | |
76 | module_param(cxgb4i_max_connect, uint, 0644); | |
77 | MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections"); | |
78 | ||
79 | static unsigned short cxgb4i_sport_base = 20000; | |
80 | module_param(cxgb4i_sport_base, ushort, 0644); | |
81 | MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)"); | |
82 | ||
83 | typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *); | |
84 | ||
85 | static void *t4_uld_add(const struct cxgb4_lld_info *); | |
86 | static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *); | |
87 | static int t4_uld_state_change(void *, enum cxgb4_state state); | |
64bfead8 | 88 | static inline int send_tx_flowc_wr(struct cxgbi_sock *); |
7b36b6e0 | 89 | |
90 | static const struct cxgb4_uld_info cxgb4i_uld_info = { | |
91 | .name = DRV_MODULE_NAME, | |
0fbc81b3 | 92 | .nrxq = MAX_ULD_QSETS, |
ab677ff4 | 93 | .ntxq = MAX_ULD_QSETS, |
0fbc81b3 HS |
94 | .rxq_size = 1024, |
95 | .lro = false, | |
7b36b6e0 | 96 | .add = t4_uld_add, |
97 | .rx_handler = t4_uld_rx_handler, | |
98 | .state_change = t4_uld_state_change, | |
99 | }; | |
100 | ||
101 | static struct scsi_host_template cxgb4i_host_template = { | |
102 | .module = THIS_MODULE, | |
103 | .name = DRV_MODULE_NAME, | |
104 | .proc_name = DRV_MODULE_NAME, | |
105 | .can_queue = CXGB4I_SCSI_HOST_QDEPTH, | |
106 | .queuecommand = iscsi_queuecommand, | |
db5ed4df | 107 | .change_queue_depth = scsi_change_queue_depth, |
7b36b6e0 | 108 | .sg_tablesize = SG_ALL, |
109 | .max_sectors = 0xFFFF, | |
110 | .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, | |
b6a05c82 | 111 | .eh_timed_out = iscsi_eh_cmd_timed_out, |
7b36b6e0 | 112 | .eh_abort_handler = iscsi_eh_abort, |
113 | .eh_device_reset_handler = iscsi_eh_device_reset, | |
114 | .eh_target_reset_handler = iscsi_eh_recover_target, | |
115 | .target_alloc = iscsi_target_alloc, | |
4af14d11 | 116 | .dma_boundary = PAGE_SIZE - 1, |
7b36b6e0 | 117 | .this_id = -1, |
c40ecc12 | 118 | .track_queue_depth = 1, |
7b36b6e0 | 119 | }; |
120 | ||
121 | static struct iscsi_transport cxgb4i_iscsi_transport = { | |
122 | .owner = THIS_MODULE, | |
123 | .name = DRV_MODULE_NAME, | |
124 | .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST | | |
125 | CAP_DATADGST | CAP_DIGEST_OFFLOAD | | |
fdafd4df | 126 | CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, |
3128c6c7 | 127 | .attr_is_visible = cxgbi_attr_is_visible, |
7b36b6e0 | 128 | .get_host_param = cxgbi_get_host_param, |
129 | .set_host_param = cxgbi_set_host_param, | |
130 | /* session management */ | |
131 | .create_session = cxgbi_create_session, | |
132 | .destroy_session = cxgbi_destroy_session, | |
133 | .get_session_param = iscsi_session_get_param, | |
134 | /* connection management */ | |
135 | .create_conn = cxgbi_create_conn, | |
136 | .bind_conn = cxgbi_bind_conn, | |
137 | .destroy_conn = iscsi_tcp_conn_teardown, | |
138 | .start_conn = iscsi_conn_start, | |
139 | .stop_conn = iscsi_conn_stop, | |
c71b9b66 | 140 | .get_conn_param = iscsi_conn_get_param, |
7b36b6e0 | 141 | .set_param = cxgbi_set_conn_param, |
142 | .get_stats = cxgbi_get_conn_stats, | |
143 | /* pdu xmit req from user space */ | |
144 | .send_pdu = iscsi_conn_send_pdu, | |
145 | /* task */ | |
146 | .init_task = iscsi_tcp_task_init, | |
147 | .xmit_task = iscsi_tcp_task_xmit, | |
148 | .cleanup_task = cxgbi_cleanup_task, | |
149 | /* pdu */ | |
150 | .alloc_pdu = cxgbi_conn_alloc_pdu, | |
151 | .init_pdu = cxgbi_conn_init_pdu, | |
152 | .xmit_pdu = cxgbi_conn_xmit_pdu, | |
153 | .parse_pdu_itt = cxgbi_parse_pdu_itt, | |
154 | /* TCP connect/disconnect */ | |
c71b9b66 | 155 | .get_ep_param = cxgbi_get_ep_param, |
7b36b6e0 | 156 | .ep_connect = cxgbi_ep_connect, |
157 | .ep_poll = cxgbi_ep_poll, | |
158 | .ep_disconnect = cxgbi_ep_disconnect, | |
159 | /* Error recovery timeout call */ | |
160 | .session_recovery_timedout = iscsi_session_recovery_timedout, | |
161 | }; | |
162 | ||
b5a5fe4e VP |
163 | #ifdef CONFIG_CHELSIO_T4_DCB |
164 | static int | |
165 | cxgb4_dcb_change_notify(struct notifier_block *, unsigned long, void *); | |
166 | ||
167 | static struct notifier_block cxgb4_dcb_change = { | |
168 | .notifier_call = cxgb4_dcb_change_notify, | |
169 | }; | |
170 | #endif | |
171 | ||
7b36b6e0 | 172 | static struct scsi_transport_template *cxgb4i_stt; |
173 | ||
174 | /* | |
175 | * CPL (Chelsio Protocol Language) defines a message passing interface between | |
176 | * the host driver and Chelsio asic. | |
177 | * The section below implments CPLs that related to iscsi tcp connection | |
178 | * open/close/abort and data send/receive. | |
179 | */ | |
759a0cc5 | 180 | |
7b36b6e0 | 181 | #define RCV_BUFSIZ_MASK 0x3FFU |
d3074cce | 182 | #define MAX_IMM_TX_PKT_LEN 256 |
7b36b6e0 | 183 | |
7b36b6e0 | 184 | static int push_tx_frames(struct cxgbi_sock *, int); |
185 | ||
186 | /* | |
187 | * is_ofld_imm - check whether a packet can be sent as immediate data | |
188 | * @skb: the packet | |
189 | * | |
190 | * Returns true if a packet can be sent as an offload WR with immediate | |
191 | * data. We currently use the same limit as for Ethernet packets. | |
192 | */ | |
84944d8c | 193 | static inline bool is_ofld_imm(const struct sk_buff *skb) |
7b36b6e0 | 194 | { |
84944d8c KX |
195 | int len = skb->len; |
196 | ||
197 | if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) | |
198 | len += sizeof(struct fw_ofld_tx_data_wr); | |
199 | ||
e33c2482 VP |
200 | if (likely(cxgbi_skcb_test_flag((struct sk_buff *)skb, SKCBF_TX_ISO))) |
201 | len += sizeof(struct cpl_tx_data_iso); | |
202 | ||
203 | return (len <= MAX_IMM_OFLD_TX_DATA_WR_LEN); | |
7b36b6e0 | 204 | } |
205 | ||
206 | static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, | |
207 | struct l2t_entry *e) | |
208 | { | |
3bd3e8bf | 209 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); |
7b36b6e0 | 210 | int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); |
211 | unsigned long long opt0; | |
212 | unsigned int opt2; | |
213 | unsigned int qid_atid = ((unsigned int)csk->atid) | | |
214 | (((unsigned int)csk->rss_qid) << 14); | |
215 | ||
d7990b0c AB |
216 | opt0 = KEEP_ALIVE_F | |
217 | WND_SCALE_V(wscale) | | |
218 | MSS_IDX_V(csk->mss_idx) | | |
219 | L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | | |
220 | TX_CHAN_V(csk->tx_chan) | | |
221 | SMAC_SEL_V(csk->smac_idx) | | |
222 | ULP_MODE_V(ULP_MODE_ISCSI) | | |
81daf10c | 223 | RCV_BUFSIZ_V(csk->rcv_win >> 10); |
1eb88a5d | 224 | |
d7990b0c AB |
225 | opt2 = RX_CHANNEL_V(0) | |
226 | RSS_QUEUE_VALID_F | | |
d7990b0c | 227 | RSS_QUEUE_V(csk->rss_qid); |
7b36b6e0 | 228 | |
3bd3e8bf KX |
229 | if (is_t4(lldi->adapter_type)) { |
230 | struct cpl_act_open_req *req = | |
231 | (struct cpl_act_open_req *)skb->head; | |
232 | ||
3bd3e8bf KX |
233 | INIT_TP_WR(req, 0); |
234 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, | |
7b36b6e0 | 235 | qid_atid)); |
3bd3e8bf KX |
236 | req->local_port = csk->saddr.sin_port; |
237 | req->peer_port = csk->daddr.sin_port; | |
238 | req->local_ip = csk->saddr.sin_addr.s_addr; | |
239 | req->peer_ip = csk->daddr.sin_addr.s_addr; | |
240 | req->opt0 = cpu_to_be64(opt0); | |
ac0245ff KX |
241 | req->params = cpu_to_be32(cxgb4_select_ntuple( |
242 | csk->cdev->ports[csk->port_id], | |
243 | csk->l2t)); | |
d7990b0c | 244 | opt2 |= RX_FC_VALID_F; |
3bd3e8bf | 245 | req->opt2 = cpu_to_be32(opt2); |
7b36b6e0 | 246 | |
3bd3e8bf KX |
247 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
248 | "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", | |
249 | csk, &req->local_ip, ntohs(req->local_port), | |
250 | &req->peer_ip, ntohs(req->peer_port), | |
251 | csk->atid, csk->rss_qid); | |
586be7cb | 252 | } else if (is_t5(lldi->adapter_type)) { |
3bd3e8bf KX |
253 | struct cpl_t5_act_open_req *req = |
254 | (struct cpl_t5_act_open_req *)skb->head; | |
1eb88a5d | 255 | u32 isn = (prandom_u32() & ~7UL) - 1; |
3bd3e8bf | 256 | |
3bd3e8bf KX |
257 | INIT_TP_WR(req, 0); |
258 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, | |
259 | qid_atid)); | |
260 | req->local_port = csk->saddr.sin_port; | |
261 | req->peer_port = csk->daddr.sin_port; | |
262 | req->local_ip = csk->saddr.sin_addr.s_addr; | |
263 | req->peer_ip = csk->daddr.sin_addr.s_addr; | |
264 | req->opt0 = cpu_to_be64(opt0); | |
d7990b0c | 265 | req->params = cpu_to_be64(FILTER_TUPLE_V( |
ac0245ff KX |
266 | cxgb4_select_ntuple( |
267 | csk->cdev->ports[csk->port_id], | |
268 | csk->l2t))); | |
1eb88a5d KX |
269 | req->rsvd = cpu_to_be32(isn); |
270 | opt2 |= T5_ISS_VALID; | |
271 | opt2 |= T5_OPT_2_VALID_F; | |
272 | ||
3bd3e8bf KX |
273 | req->opt2 = cpu_to_be32(opt2); |
274 | ||
275 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
276 | "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", | |
277 | csk, &req->local_ip, ntohs(req->local_port), | |
278 | &req->peer_ip, ntohs(req->peer_port), | |
279 | csk->atid, csk->rss_qid); | |
586be7cb VP |
280 | } else { |
281 | struct cpl_t6_act_open_req *req = | |
282 | (struct cpl_t6_act_open_req *)skb->head; | |
283 | u32 isn = (prandom_u32() & ~7UL) - 1; | |
284 | ||
285 | INIT_TP_WR(req, 0); | |
286 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, | |
287 | qid_atid)); | |
288 | req->local_port = csk->saddr.sin_port; | |
289 | req->peer_port = csk->daddr.sin_port; | |
290 | req->local_ip = csk->saddr.sin_addr.s_addr; | |
291 | req->peer_ip = csk->daddr.sin_addr.s_addr; | |
292 | req->opt0 = cpu_to_be64(opt0); | |
293 | req->params = cpu_to_be64(FILTER_TUPLE_V( | |
294 | cxgb4_select_ntuple( | |
295 | csk->cdev->ports[csk->port_id], | |
296 | csk->l2t))); | |
297 | req->rsvd = cpu_to_be32(isn); | |
298 | ||
299 | opt2 |= T5_ISS_VALID; | |
300 | opt2 |= RX_FC_DISABLE_F; | |
301 | opt2 |= T5_OPT_2_VALID_F; | |
302 | ||
303 | req->opt2 = cpu_to_be32(opt2); | |
304 | req->rsvd2 = cpu_to_be32(0); | |
305 | req->opt3 = cpu_to_be32(0); | |
306 | ||
307 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
308 | "csk t6 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", | |
309 | csk, &req->local_ip, ntohs(req->local_port), | |
310 | &req->peer_ip, ntohs(req->peer_port), | |
311 | csk->atid, csk->rss_qid); | |
3bd3e8bf KX |
312 | } |
313 | ||
314 | set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); | |
759a0cc5 AB |
315 | |
316 | pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n", | |
586be7cb VP |
317 | (&csk->saddr), (&csk->daddr), |
318 | CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, | |
759a0cc5 AB |
319 | csk->state, csk->flags, csk->atid, csk->rss_qid); |
320 | ||
321 | cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); | |
322 | } | |
323 | ||
f42bb57c | 324 | #if IS_ENABLED(CONFIG_IPV6) |
759a0cc5 AB |
325 | static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb, |
326 | struct l2t_entry *e) | |
327 | { | |
328 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); | |
759a0cc5 AB |
329 | int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); |
330 | unsigned long long opt0; | |
331 | unsigned int opt2; | |
332 | unsigned int qid_atid = ((unsigned int)csk->atid) | | |
333 | (((unsigned int)csk->rss_qid) << 14); | |
334 | ||
d7990b0c AB |
335 | opt0 = KEEP_ALIVE_F | |
336 | WND_SCALE_V(wscale) | | |
337 | MSS_IDX_V(csk->mss_idx) | | |
338 | L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | | |
339 | TX_CHAN_V(csk->tx_chan) | | |
340 | SMAC_SEL_V(csk->smac_idx) | | |
341 | ULP_MODE_V(ULP_MODE_ISCSI) | | |
81daf10c | 342 | RCV_BUFSIZ_V(csk->rcv_win >> 10); |
759a0cc5 | 343 | |
d7990b0c AB |
344 | opt2 = RX_CHANNEL_V(0) | |
345 | RSS_QUEUE_VALID_F | | |
d7990b0c | 346 | RSS_QUEUE_V(csk->rss_qid); |
759a0cc5 | 347 | |
586be7cb | 348 | if (is_t4(lldi->adapter_type)) { |
759a0cc5 AB |
349 | struct cpl_act_open_req6 *req = |
350 | (struct cpl_act_open_req6 *)skb->head; | |
351 | ||
352 | INIT_TP_WR(req, 0); | |
353 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, | |
354 | qid_atid)); | |
355 | req->local_port = csk->saddr6.sin6_port; | |
356 | req->peer_port = csk->daddr6.sin6_port; | |
357 | ||
358 | req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); | |
359 | req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + | |
360 | 8); | |
361 | req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); | |
362 | req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + | |
363 | 8); | |
364 | ||
365 | req->opt0 = cpu_to_be64(opt0); | |
366 | ||
d7990b0c | 367 | opt2 |= RX_FC_VALID_F; |
759a0cc5 AB |
368 | req->opt2 = cpu_to_be32(opt2); |
369 | ||
370 | req->params = cpu_to_be32(cxgb4_select_ntuple( | |
371 | csk->cdev->ports[csk->port_id], | |
372 | csk->l2t)); | |
586be7cb | 373 | } else if (is_t5(lldi->adapter_type)) { |
759a0cc5 AB |
374 | struct cpl_t5_act_open_req6 *req = |
375 | (struct cpl_t5_act_open_req6 *)skb->head; | |
376 | ||
377 | INIT_TP_WR(req, 0); | |
378 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, | |
379 | qid_atid)); | |
380 | req->local_port = csk->saddr6.sin6_port; | |
381 | req->peer_port = csk->daddr6.sin6_port; | |
382 | req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); | |
383 | req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + | |
384 | 8); | |
385 | req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); | |
386 | req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + | |
387 | 8); | |
388 | req->opt0 = cpu_to_be64(opt0); | |
389 | ||
d7990b0c | 390 | opt2 |= T5_OPT_2_VALID_F; |
759a0cc5 AB |
391 | req->opt2 = cpu_to_be32(opt2); |
392 | ||
d7990b0c | 393 | req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( |
759a0cc5 AB |
394 | csk->cdev->ports[csk->port_id], |
395 | csk->l2t))); | |
586be7cb VP |
396 | } else { |
397 | struct cpl_t6_act_open_req6 *req = | |
398 | (struct cpl_t6_act_open_req6 *)skb->head; | |
399 | ||
400 | INIT_TP_WR(req, 0); | |
401 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, | |
402 | qid_atid)); | |
403 | req->local_port = csk->saddr6.sin6_port; | |
404 | req->peer_port = csk->daddr6.sin6_port; | |
405 | req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); | |
406 | req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + | |
407 | 8); | |
408 | req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); | |
409 | req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + | |
410 | 8); | |
411 | req->opt0 = cpu_to_be64(opt0); | |
412 | ||
413 | opt2 |= RX_FC_DISABLE_F; | |
414 | opt2 |= T5_OPT_2_VALID_F; | |
415 | ||
416 | req->opt2 = cpu_to_be32(opt2); | |
417 | ||
418 | req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( | |
419 | csk->cdev->ports[csk->port_id], | |
420 | csk->l2t))); | |
421 | ||
422 | req->rsvd2 = cpu_to_be32(0); | |
423 | req->opt3 = cpu_to_be32(0); | |
759a0cc5 AB |
424 | } |
425 | ||
426 | set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); | |
427 | ||
428 | pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n", | |
586be7cb VP |
429 | CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, csk->state, |
430 | csk->flags, csk->atid, | |
759a0cc5 AB |
431 | &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port), |
432 | &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port), | |
433 | csk->rss_qid); | |
434 | ||
7b36b6e0 | 435 | cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); |
436 | } | |
f42bb57c | 437 | #endif |
7b36b6e0 | 438 | |
439 | static void send_close_req(struct cxgbi_sock *csk) | |
440 | { | |
441 | struct sk_buff *skb = csk->cpl_close; | |
442 | struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; | |
443 | unsigned int tid = csk->tid; | |
444 | ||
445 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
446 | "csk 0x%p,%u,0x%lx, tid %u.\n", | |
447 | csk, csk->state, csk->flags, csk->tid); | |
448 | csk->cpl_close = NULL; | |
449 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); | |
450 | INIT_TP_WR(req, tid); | |
451 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); | |
452 | req->rsvd = 0; | |
453 | ||
454 | cxgbi_sock_skb_entail(csk, skb); | |
455 | if (csk->state >= CTP_ESTABLISHED) | |
456 | push_tx_frames(csk, 1); | |
457 | } | |
458 | ||
459 | static void abort_arp_failure(void *handle, struct sk_buff *skb) | |
460 | { | |
461 | struct cxgbi_sock *csk = (struct cxgbi_sock *)handle; | |
462 | struct cpl_abort_req *req; | |
463 | ||
464 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
465 | "csk 0x%p,%u,0x%lx, tid %u, abort.\n", | |
466 | csk, csk->state, csk->flags, csk->tid); | |
467 | req = (struct cpl_abort_req *)skb->data; | |
468 | req->cmd = CPL_ABORT_NO_RST; | |
469 | cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); | |
470 | } | |
471 | ||
472 | static void send_abort_req(struct cxgbi_sock *csk) | |
473 | { | |
474 | struct cpl_abort_req *req; | |
475 | struct sk_buff *skb = csk->cpl_abort_req; | |
476 | ||
477 | if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev) | |
478 | return; | |
64bfead8 KX |
479 | |
480 | if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { | |
481 | send_tx_flowc_wr(csk); | |
482 | cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); | |
483 | } | |
484 | ||
7b36b6e0 | 485 | cxgbi_sock_set_state(csk, CTP_ABORTING); |
486 | cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); | |
487 | cxgbi_sock_purge_write_queue(csk); | |
488 | ||
489 | csk->cpl_abort_req = NULL; | |
490 | req = (struct cpl_abort_req *)skb->head; | |
2126bc5e | 491 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); |
7b36b6e0 | 492 | req->cmd = CPL_ABORT_SEND_RST; |
493 | t4_set_arp_err_handler(skb, csk, abort_arp_failure); | |
494 | INIT_TP_WR(req, csk->tid); | |
495 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); | |
496 | req->rsvd0 = htonl(csk->snd_nxt); | |
497 | req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); | |
498 | ||
499 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
500 | "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", | |
501 | csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, | |
502 | req->rsvd1); | |
503 | ||
504 | cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); | |
505 | } | |
506 | ||
507 | static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) | |
508 | { | |
509 | struct sk_buff *skb = csk->cpl_abort_rpl; | |
510 | struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; | |
511 | ||
512 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
513 | "csk 0x%p,%u,0x%lx,%u, status %d.\n", | |
514 | csk, csk->state, csk->flags, csk->tid, rst_status); | |
515 | ||
516 | csk->cpl_abort_rpl = NULL; | |
2126bc5e | 517 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); |
7b36b6e0 | 518 | INIT_TP_WR(rpl, csk->tid); |
519 | OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); | |
520 | rpl->cmd = rst_status; | |
521 | cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); | |
522 | } | |
523 | ||
524 | /* | |
525 | * CPL connection rx data ack: host -> | |
526 | * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of | |
527 | * credits sent. | |
528 | */ | |
529 | static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) | |
530 | { | |
531 | struct sk_buff *skb; | |
532 | struct cpl_rx_data_ack *req; | |
533 | ||
534 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
535 | "csk 0x%p,%u,0x%lx,%u, credit %u.\n", | |
536 | csk, csk->state, csk->flags, csk->tid, credits); | |
537 | ||
24d3f95a | 538 | skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); |
7b36b6e0 | 539 | if (!skb) { |
540 | pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); | |
541 | return 0; | |
542 | } | |
543 | req = (struct cpl_rx_data_ack *)skb->head; | |
544 | ||
545 | set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); | |
546 | INIT_TP_WR(req, csk->tid); | |
547 | OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, | |
548 | csk->tid)); | |
d7990b0c AB |
549 | req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) |
550 | | RX_FORCE_ACK_F); | |
7b36b6e0 | 551 | cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); |
552 | return credits; | |
553 | } | |
554 | ||
555 | /* | |
556 | * sgl_len - calculates the size of an SGL of the given capacity | |
557 | * @n: the number of SGL entries | |
558 | * Calculates the number of flits needed for a scatter/gather list that | |
559 | * can hold the given number of entries. | |
560 | */ | |
561 | static inline unsigned int sgl_len(unsigned int n) | |
562 | { | |
563 | n--; | |
564 | return (3 * n) / 2 + (n & 1) + 2; | |
565 | } | |
566 | ||
567 | /* | |
568 | * calc_tx_flits_ofld - calculate # of flits for an offload packet | |
569 | * @skb: the packet | |
570 | * | |
571 | * Returns the number of flits needed for the given offload packet. | |
572 | * These packets are already fully constructed and no additional headers | |
573 | * will be added. | |
574 | */ | |
575 | static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) | |
576 | { | |
577 | unsigned int flits, cnt; | |
578 | ||
579 | if (is_ofld_imm(skb)) | |
580 | return DIV_ROUND_UP(skb->len, 8); | |
581 | flits = skb_transport_offset(skb) / 8; | |
582 | cnt = skb_shinfo(skb)->nr_frags; | |
499e2e6f | 583 | if (skb_tail_pointer(skb) != skb_transport_header(skb)) |
7b36b6e0 | 584 | cnt++; |
585 | return flits + sgl_len(cnt); | |
586 | } | |
587 | ||
64bfead8 KX |
588 | #define FLOWC_WR_NPARAMS_MIN 9 |
589 | static inline int tx_flowc_wr_credits(int *nparamsp, int *flowclenp) | |
590 | { | |
591 | int nparams, flowclen16, flowclen; | |
592 | ||
593 | nparams = FLOWC_WR_NPARAMS_MIN; | |
b5a5fe4e VP |
594 | #ifdef CONFIG_CHELSIO_T4_DCB |
595 | nparams++; | |
596 | #endif | |
64bfead8 KX |
597 | flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); |
598 | flowclen16 = DIV_ROUND_UP(flowclen, 16); | |
599 | flowclen = flowclen16 * 16; | |
600 | /* | |
601 | * Return the number of 16-byte credits used by the FlowC request. | |
602 | * Pass back the nparams and actual FlowC length if requested. | |
603 | */ | |
604 | if (nparamsp) | |
605 | *nparamsp = nparams; | |
606 | if (flowclenp) | |
607 | *flowclenp = flowclen; | |
608 | ||
609 | return flowclen16; | |
610 | } | |
611 | ||
612 | static inline int send_tx_flowc_wr(struct cxgbi_sock *csk) | |
7b36b6e0 | 613 | { |
614 | struct sk_buff *skb; | |
615 | struct fw_flowc_wr *flowc; | |
64bfead8 | 616 | int nparams, flowclen16, flowclen; |
7b36b6e0 | 617 | |
b5a5fe4e VP |
618 | #ifdef CONFIG_CHELSIO_T4_DCB |
619 | u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan; | |
620 | #endif | |
64bfead8 | 621 | flowclen16 = tx_flowc_wr_credits(&nparams, &flowclen); |
24d3f95a | 622 | skb = alloc_wr(flowclen, 0, GFP_ATOMIC); |
7b36b6e0 | 623 | flowc = (struct fw_flowc_wr *)skb->head; |
624 | flowc->op_to_nparams = | |
64bfead8 | 625 | htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(nparams)); |
7b36b6e0 | 626 | flowc->flowid_len16 = |
64bfead8 | 627 | htonl(FW_WR_LEN16_V(flowclen16) | FW_WR_FLOWID_V(csk->tid)); |
7b36b6e0 | 628 | flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; |
e27d6169 | 629 | flowc->mnemval[0].val = htonl(csk->cdev->pfvf); |
7b36b6e0 | 630 | flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; |
631 | flowc->mnemval[1].val = htonl(csk->tx_chan); | |
632 | flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; | |
633 | flowc->mnemval[2].val = htonl(csk->tx_chan); | |
634 | flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; | |
635 | flowc->mnemval[3].val = htonl(csk->rss_qid); | |
636 | flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; | |
637 | flowc->mnemval[4].val = htonl(csk->snd_nxt); | |
638 | flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; | |
639 | flowc->mnemval[5].val = htonl(csk->rcv_nxt); | |
640 | flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; | |
81daf10c | 641 | flowc->mnemval[6].val = htonl(csk->snd_win); |
7b36b6e0 | 642 | flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; |
643 | flowc->mnemval[7].val = htonl(csk->advmss); | |
644 | flowc->mnemval[8].mnemonic = 0; | |
645 | flowc->mnemval[8].val = 0; | |
64bfead8 | 646 | flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; |
e33c2482 VP |
647 | if (csk->cdev->skb_iso_txhdr) |
648 | flowc->mnemval[8].val = cpu_to_be32(CXGBI_MAX_ISO_DATA_IN_SKB); | |
649 | else | |
650 | flowc->mnemval[8].val = cpu_to_be32(16128); | |
b5a5fe4e VP |
651 | #ifdef CONFIG_CHELSIO_T4_DCB |
652 | flowc->mnemval[9].mnemonic = FW_FLOWC_MNEM_DCBPRIO; | |
653 | if (vlan == CPL_L2T_VLAN_NONE) { | |
654 | pr_warn_ratelimited("csk %u without VLAN Tag on DCB Link\n", | |
655 | csk->tid); | |
656 | flowc->mnemval[9].val = cpu_to_be32(0); | |
657 | } else { | |
658 | flowc->mnemval[9].val = cpu_to_be32((vlan & VLAN_PRIO_MASK) >> | |
659 | VLAN_PRIO_SHIFT); | |
660 | } | |
661 | #endif | |
64bfead8 | 662 | |
2126bc5e | 663 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); |
7b36b6e0 | 664 | |
665 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
666 | "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n", | |
667 | csk, csk->tid, 0, csk->tx_chan, csk->rss_qid, | |
81daf10c | 668 | csk->snd_nxt, csk->rcv_nxt, csk->snd_win, |
7b36b6e0 | 669 | csk->advmss); |
670 | ||
671 | cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); | |
64bfead8 KX |
672 | |
673 | return flowclen16; | |
7b36b6e0 | 674 | } |
675 | ||
e33c2482 VP |
676 | static void |
677 | cxgb4i_make_tx_iso_cpl(struct sk_buff *skb, struct cpl_tx_data_iso *cpl) | |
7b36b6e0 | 678 | { |
e33c2482 VP |
679 | struct cxgbi_iso_info *info = (struct cxgbi_iso_info *)skb->head; |
680 | u32 imm_en = !!(info->flags & CXGBI_ISO_INFO_IMM_ENABLE); | |
681 | u32 fslice = !!(info->flags & CXGBI_ISO_INFO_FSLICE); | |
682 | u32 lslice = !!(info->flags & CXGBI_ISO_INFO_LSLICE); | |
683 | u32 pdu_type = (info->op == ISCSI_OP_SCSI_CMD) ? 0 : 1; | |
684 | u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3; | |
685 | ||
686 | cpl->op_to_scsi = cpu_to_be32(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | | |
687 | CPL_TX_DATA_ISO_FIRST_V(fslice) | | |
688 | CPL_TX_DATA_ISO_LAST_V(lslice) | | |
689 | CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | | |
690 | CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | | |
691 | CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | | |
692 | CPL_TX_DATA_ISO_IMMEDIATE_V(imm_en) | | |
693 | CPL_TX_DATA_ISO_SCSI_V(pdu_type)); | |
694 | ||
695 | cpl->ahs_len = info->ahs; | |
696 | cpl->mpdu = cpu_to_be16(DIV_ROUND_UP(info->mpdu, 4)); | |
697 | cpl->burst_size = cpu_to_be32(info->burst_size); | |
698 | cpl->len = cpu_to_be32(info->len); | |
699 | cpl->reserved2_seglen_offset = | |
700 | cpu_to_be32(CPL_TX_DATA_ISO_SEGLEN_OFFSET_V(info->segment_offset)); | |
701 | cpl->datasn_offset = cpu_to_be32(info->datasn_offset); | |
702 | cpl->buffer_offset = cpu_to_be32(info->buffer_offset); | |
703 | cpl->reserved3 = cpu_to_be32(0); | |
704 | log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, | |
705 | "iso: flags 0x%x, op %u, ahs %u, num_pdu %u, mpdu %u, " | |
706 | "burst_size %u, iso_len %u\n", | |
707 | info->flags, info->op, info->ahs, info->num_pdu, | |
708 | info->mpdu, info->burst_size << 2, info->len); | |
709 | } | |
710 | ||
711 | static void | |
712 | cxgb4i_make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, int dlen, | |
713 | int len, u32 credits, int compl) | |
714 | { | |
715 | struct cxgbi_device *cdev = csk->cdev; | |
716 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
7b36b6e0 | 717 | struct fw_ofld_tx_data_wr *req; |
e33c2482 VP |
718 | struct cpl_tx_data_iso *cpl; |
719 | u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3; | |
720 | u32 wr_ulp_mode = 0; | |
721 | u32 hdr_size = sizeof(*req); | |
722 | u32 opcode = FW_OFLD_TX_DATA_WR; | |
723 | u32 immlen = 0; | |
724 | u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : | |
725 | T6_TX_FORCE_F; | |
726 | ||
727 | if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) { | |
728 | hdr_size += sizeof(struct cpl_tx_data_iso); | |
729 | opcode = FW_ISCSI_TX_DATA_WR; | |
730 | immlen += sizeof(struct cpl_tx_data_iso); | |
731 | submode |= 8; | |
7b36b6e0 | 732 | } |
e33c2482 VP |
733 | |
734 | if (is_ofld_imm(skb)) | |
735 | immlen += dlen; | |
736 | ||
737 | req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, hdr_size); | |
738 | req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | | |
739 | FW_WR_COMPL_V(compl) | | |
740 | FW_WR_IMMDLEN_V(immlen)); | |
741 | req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | | |
742 | FW_WR_LEN16_V(credits)); | |
743 | req->plen = cpu_to_be32(len); | |
744 | cpl = (struct cpl_tx_data_iso *)(req + 1); | |
745 | ||
746 | if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) | |
747 | cxgb4i_make_tx_iso_cpl(skb, cpl); | |
748 | ||
7b36b6e0 | 749 | if (submode) |
e2ac9628 | 750 | wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) | |
e33c2482 VP |
751 | FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); |
752 | ||
753 | req->tunnel_to_proxy = cpu_to_be32(wr_ulp_mode | force | | |
754 | FW_OFLD_TX_DATA_WR_SHOVE_V(1U)); | |
755 | ||
7b36b6e0 | 756 | if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) |
757 | cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); | |
758 | } | |
759 | ||
760 | static void arp_failure_skb_discard(void *handle, struct sk_buff *skb) | |
761 | { | |
762 | kfree_skb(skb); | |
763 | } | |
764 | ||
765 | static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) | |
766 | { | |
767 | int total_size = 0; | |
768 | struct sk_buff *skb; | |
769 | ||
770 | if (unlikely(csk->state < CTP_ESTABLISHED || | |
771 | csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { | |
772 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK | | |
e33c2482 VP |
773 | 1 << CXGBI_DBG_PDU_TX, |
774 | "csk 0x%p,%u,0x%lx,%u, in closing state.\n", | |
775 | csk, csk->state, csk->flags, csk->tid); | |
7b36b6e0 | 776 | return 0; |
777 | } | |
778 | ||
e33c2482 VP |
779 | while (csk->wr_cred && ((skb = skb_peek(&csk->write_queue)) != NULL)) { |
780 | struct cxgbi_iso_info *iso_cpl; | |
781 | u32 dlen = skb->len; | |
782 | u32 len = skb->len; | |
783 | u32 iso_cpl_len = 0; | |
784 | u32 flowclen16 = 0; | |
785 | u32 credits_needed; | |
786 | u32 num_pdu = 1, hdr_len; | |
787 | ||
788 | if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) | |
789 | iso_cpl_len = sizeof(struct cpl_tx_data_iso); | |
7b36b6e0 | 790 | |
7b36b6e0 | 791 | if (is_ofld_imm(skb)) |
e33c2482 | 792 | credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); |
7b36b6e0 | 793 | else |
e33c2482 VP |
794 | credits_needed = |
795 | DIV_ROUND_UP((8 * calc_tx_flits_ofld(skb)) + | |
796 | iso_cpl_len, 16); | |
84944d8c KX |
797 | |
798 | if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) | |
e33c2482 VP |
799 | credits_needed += |
800 | DIV_ROUND_UP(sizeof(struct fw_ofld_tx_data_wr), 16); | |
7b36b6e0 | 801 | |
64bfead8 KX |
802 | /* |
803 | * Assumes the initial credits is large enough to support | |
804 | * fw_flowc_wr plus largest possible first payload | |
805 | */ | |
806 | if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { | |
807 | flowclen16 = send_tx_flowc_wr(csk); | |
808 | csk->wr_cred -= flowclen16; | |
809 | csk->wr_una_cred += flowclen16; | |
810 | cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); | |
811 | } | |
812 | ||
7b36b6e0 | 813 | if (csk->wr_cred < credits_needed) { |
814 | log_debug(1 << CXGBI_DBG_PDU_TX, | |
e33c2482 VP |
815 | "csk 0x%p, skb %u/%u, wr %d < %u.\n", |
816 | csk, skb->len, skb->data_len, | |
817 | credits_needed, csk->wr_cred); | |
818 | ||
819 | csk->no_tx_credits++; | |
7b36b6e0 | 820 | break; |
821 | } | |
e33c2482 VP |
822 | |
823 | csk->no_tx_credits = 0; | |
824 | ||
7b36b6e0 | 825 | __skb_unlink(skb, &csk->write_queue); |
2126bc5e | 826 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); |
e33c2482 | 827 | skb->csum = (__force __wsum)(credits_needed + flowclen16); |
7b36b6e0 | 828 | csk->wr_cred -= credits_needed; |
829 | csk->wr_una_cred += credits_needed; | |
830 | cxgbi_sock_enqueue_wr(csk, skb); | |
831 | ||
832 | log_debug(1 << CXGBI_DBG_PDU_TX, | |
833 | "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", | |
834 | csk, skb->len, skb->data_len, credits_needed, | |
835 | csk->wr_cred, csk->wr_una_cred); | |
836 | ||
e33c2482 VP |
837 | if (!req_completion && |
838 | ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || | |
839 | after(csk->write_seq, (csk->snd_una + csk->snd_win / 2)))) | |
840 | req_completion = 1; | |
841 | ||
7b36b6e0 | 842 | if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { |
e33c2482 VP |
843 | u32 ulp_mode = cxgbi_skcb_tx_ulp_mode(skb); |
844 | ||
845 | if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) { | |
846 | iso_cpl = (struct cxgbi_iso_info *)skb->head; | |
847 | num_pdu = iso_cpl->num_pdu; | |
848 | hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb); | |
849 | len += (cxgbi_ulp_extra_len(ulp_mode) * num_pdu) + | |
850 | (hdr_len * (num_pdu - 1)); | |
851 | } else { | |
852 | len += cxgbi_ulp_extra_len(ulp_mode); | |
853 | } | |
854 | ||
855 | cxgb4i_make_tx_data_wr(csk, skb, dlen, len, | |
856 | credits_needed, req_completion); | |
7b36b6e0 | 857 | csk->snd_nxt += len; |
858 | cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); | |
fe9b34bf VP |
859 | } else if (cxgbi_skcb_test_flag(skb, SKCBF_TX_FLAG_COMPL) && |
860 | (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { | |
861 | struct cpl_close_con_req *req = | |
862 | (struct cpl_close_con_req *)skb->data; | |
e33c2482 VP |
863 | |
864 | req->wr.wr_hi |= cpu_to_be32(FW_WR_COMPL_F); | |
7b36b6e0 | 865 | } |
e33c2482 | 866 | |
7b36b6e0 | 867 | total_size += skb->truesize; |
868 | t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard); | |
869 | ||
870 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, | |
e33c2482 VP |
871 | "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n", |
872 | csk, csk->state, csk->flags, csk->tid, skb, len); | |
7b36b6e0 | 873 | cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); |
874 | } | |
875 | return total_size; | |
876 | } | |
877 | ||
878 | static inline void free_atid(struct cxgbi_sock *csk) | |
879 | { | |
880 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); | |
881 | ||
882 | if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { | |
883 | cxgb4_free_atid(lldi->tids, csk->atid); | |
884 | cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); | |
885 | cxgbi_sock_put(csk); | |
886 | } | |
887 | } | |
888 | ||
889 | static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb) | |
890 | { | |
891 | struct cxgbi_sock *csk; | |
892 | struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data; | |
893 | unsigned short tcp_opt = ntohs(req->tcp_opt); | |
894 | unsigned int tid = GET_TID(req); | |
6c53e938 | 895 | unsigned int atid = TID_TID_G(ntohl(req->tos_atid)); |
7b36b6e0 | 896 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); |
897 | struct tid_info *t = lldi->tids; | |
898 | u32 rcv_isn = be32_to_cpu(req->rcv_isn); | |
899 | ||
900 | csk = lookup_atid(t, atid); | |
901 | if (unlikely(!csk)) { | |
902 | pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev); | |
903 | goto rel_skb; | |
904 | } | |
905 | ||
e27d6169 | 906 | if (csk->atid != atid) { |
907 | pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n", | |
908 | atid, csk, csk->state, csk->flags, csk->tid, csk->atid); | |
909 | goto rel_skb; | |
910 | } | |
911 | ||
759a0cc5 AB |
912 | pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", |
913 | (&csk->saddr), (&csk->daddr), | |
914 | atid, tid, csk, csk->state, csk->flags, rcv_isn); | |
915 | ||
1fe1fdb0 | 916 | module_put(cdev->owner); |
7b36b6e0 | 917 | |
918 | cxgbi_sock_get(csk); | |
919 | csk->tid = tid; | |
1dec4cec | 920 | cxgb4_insert_tid(lldi->tids, csk, tid, csk->csk_family); |
7b36b6e0 | 921 | cxgbi_sock_set_flag(csk, CTPF_HAS_TID); |
922 | ||
923 | free_atid(csk); | |
924 | ||
925 | spin_lock_bh(&csk->lock); | |
926 | if (unlikely(csk->state != CTP_ACTIVE_OPEN)) | |
927 | pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", | |
928 | csk, csk->state, csk->flags, csk->tid); | |
929 | ||
930 | if (csk->retry_timer.function) { | |
931 | del_timer(&csk->retry_timer); | |
932 | csk->retry_timer.function = NULL; | |
933 | } | |
934 | ||
935 | csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; | |
936 | /* | |
937 | * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't | |
938 | * pass through opt0. | |
939 | */ | |
81daf10c KX |
940 | if (csk->rcv_win > (RCV_BUFSIZ_MASK << 10)) |
941 | csk->rcv_wup -= csk->rcv_win - (RCV_BUFSIZ_MASK << 10); | |
7b36b6e0 | 942 | |
6c53e938 HS |
943 | csk->advmss = lldi->mtus[TCPOPT_MSS_G(tcp_opt)] - 40; |
944 | if (TCPOPT_TSTAMP_G(tcp_opt)) | |
7b36b6e0 | 945 | csk->advmss -= 12; |
946 | if (csk->advmss < 128) | |
947 | csk->advmss = 128; | |
948 | ||
949 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
950 | "csk 0x%p, mss_idx %u, advmss %u.\n", | |
6c53e938 | 951 | csk, TCPOPT_MSS_G(tcp_opt), csk->advmss); |
7b36b6e0 | 952 | |
953 | cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); | |
954 | ||
955 | if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) | |
956 | send_abort_req(csk); | |
957 | else { | |
958 | if (skb_queue_len(&csk->write_queue)) | |
959 | push_tx_frames(csk, 0); | |
960 | cxgbi_conn_tx_open(csk); | |
961 | } | |
962 | spin_unlock_bh(&csk->lock); | |
963 | ||
964 | rel_skb: | |
965 | __kfree_skb(skb); | |
966 | } | |
967 | ||
968 | static int act_open_rpl_status_to_errno(int status) | |
969 | { | |
970 | switch (status) { | |
971 | case CPL_ERR_CONN_RESET: | |
972 | return -ECONNREFUSED; | |
973 | case CPL_ERR_ARP_MISS: | |
974 | return -EHOSTUNREACH; | |
975 | case CPL_ERR_CONN_TIMEDOUT: | |
976 | return -ETIMEDOUT; | |
977 | case CPL_ERR_TCAM_FULL: | |
978 | return -ENOMEM; | |
979 | case CPL_ERR_CONN_EXIST: | |
980 | return -EADDRINUSE; | |
981 | default: | |
982 | return -EIO; | |
983 | } | |
984 | } | |
985 | ||
cd07f958 | 986 | static void csk_act_open_retry_timer(struct timer_list *t) |
7b36b6e0 | 987 | { |
001586a7 | 988 | struct sk_buff *skb = NULL; |
cd07f958 | 989 | struct cxgbi_sock *csk = from_timer(csk, t, retry_timer); |
3bd3e8bf | 990 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); |
759a0cc5 AB |
991 | void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *, |
992 | struct l2t_entry *); | |
993 | int t4 = is_t4(lldi->adapter_type), size, size6; | |
7b36b6e0 | 994 | |
995 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
996 | "csk 0x%p,%u,0x%lx,%u.\n", | |
997 | csk, csk->state, csk->flags, csk->tid); | |
998 | ||
999 | cxgbi_sock_get(csk); | |
1000 | spin_lock_bh(&csk->lock); | |
759a0cc5 AB |
1001 | |
1002 | if (t4) { | |
1003 | size = sizeof(struct cpl_act_open_req); | |
1004 | size6 = sizeof(struct cpl_act_open_req6); | |
1005 | } else { | |
1006 | size = sizeof(struct cpl_t5_act_open_req); | |
1007 | size6 = sizeof(struct cpl_t5_act_open_req6); | |
1008 | } | |
1009 | ||
1010 | if (csk->csk_family == AF_INET) { | |
1011 | send_act_open_func = send_act_open_req; | |
1012 | skb = alloc_wr(size, 0, GFP_ATOMIC); | |
f42bb57c | 1013 | #if IS_ENABLED(CONFIG_IPV6) |
759a0cc5 AB |
1014 | } else { |
1015 | send_act_open_func = send_act_open_req6; | |
1016 | skb = alloc_wr(size6, 0, GFP_ATOMIC); | |
f42bb57c | 1017 | #endif |
759a0cc5 AB |
1018 | } |
1019 | ||
7b36b6e0 | 1020 | if (!skb) |
1021 | cxgbi_sock_fail_act_open(csk, -ENOMEM); | |
1022 | else { | |
1023 | skb->sk = (struct sock *)csk; | |
1024 | t4_set_arp_err_handler(skb, csk, | |
759a0cc5 AB |
1025 | cxgbi_sock_act_open_req_arp_failure); |
1026 | send_act_open_func(csk, skb, csk->l2t); | |
7b36b6e0 | 1027 | } |
759a0cc5 | 1028 | |
7b36b6e0 | 1029 | spin_unlock_bh(&csk->lock); |
1030 | cxgbi_sock_put(csk); | |
759a0cc5 | 1031 | |
7b36b6e0 | 1032 | } |
1033 | ||
928567ad KX |
1034 | static inline bool is_neg_adv(unsigned int status) |
1035 | { | |
1036 | return status == CPL_ERR_RTX_NEG_ADVICE || | |
1037 | status == CPL_ERR_KEEPALV_NEG_ADVICE || | |
1038 | status == CPL_ERR_PERSIST_NEG_ADVICE; | |
1039 | } | |
1040 | ||
7b36b6e0 | 1041 | static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) |
1042 | { | |
1043 | struct cxgbi_sock *csk; | |
1044 | struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data; | |
1045 | unsigned int tid = GET_TID(rpl); | |
1046 | unsigned int atid = | |
6c53e938 HS |
1047 | TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl->atid_status))); |
1048 | unsigned int status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status)); | |
7b36b6e0 | 1049 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); |
1050 | struct tid_info *t = lldi->tids; | |
1051 | ||
1052 | csk = lookup_atid(t, atid); | |
1053 | if (unlikely(!csk)) { | |
1054 | pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid); | |
1055 | goto rel_skb; | |
1056 | } | |
1057 | ||
759a0cc5 AB |
1058 | pr_info_ipaddr("tid %u/%u, status %u.\n" |
1059 | "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr), | |
1060 | atid, tid, status, csk, csk->state, csk->flags); | |
7b36b6e0 | 1061 | |
928567ad | 1062 | if (is_neg_adv(status)) |
150cca7c KX |
1063 | goto rel_skb; |
1064 | ||
1fe1fdb0 | 1065 | module_put(cdev->owner); |
ee7255ad | 1066 | |
7b36b6e0 | 1067 | if (status && status != CPL_ERR_TCAM_FULL && |
1068 | status != CPL_ERR_CONN_EXIST && | |
1069 | status != CPL_ERR_ARP_MISS) | |
1dec4cec GG |
1070 | cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl), |
1071 | csk->csk_family); | |
7b36b6e0 | 1072 | |
1073 | cxgbi_sock_get(csk); | |
1074 | spin_lock_bh(&csk->lock); | |
1075 | ||
1076 | if (status == CPL_ERR_CONN_EXIST && | |
841b86f3 KC |
1077 | csk->retry_timer.function != csk_act_open_retry_timer) { |
1078 | csk->retry_timer.function = csk_act_open_retry_timer; | |
7b36b6e0 | 1079 | mod_timer(&csk->retry_timer, jiffies + HZ / 2); |
1080 | } else | |
1081 | cxgbi_sock_fail_act_open(csk, | |
1082 | act_open_rpl_status_to_errno(status)); | |
1083 | ||
1084 | spin_unlock_bh(&csk->lock); | |
1085 | cxgbi_sock_put(csk); | |
1086 | rel_skb: | |
1087 | __kfree_skb(skb); | |
1088 | } | |
1089 | ||
1090 | static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb) | |
1091 | { | |
1092 | struct cxgbi_sock *csk; | |
1093 | struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data; | |
1094 | unsigned int tid = GET_TID(req); | |
1095 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1096 | struct tid_info *t = lldi->tids; | |
1097 | ||
1098 | csk = lookup_tid(t, tid); | |
1099 | if (unlikely(!csk)) { | |
1100 | pr_err("can't find connection for tid %u.\n", tid); | |
1101 | goto rel_skb; | |
1102 | } | |
759a0cc5 AB |
1103 | pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", |
1104 | (&csk->saddr), (&csk->daddr), | |
1105 | csk, csk->state, csk->flags, csk->tid); | |
7b36b6e0 | 1106 | cxgbi_sock_rcv_peer_close(csk); |
1107 | rel_skb: | |
1108 | __kfree_skb(skb); | |
1109 | } | |
1110 | ||
1111 | static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) | |
1112 | { | |
1113 | struct cxgbi_sock *csk; | |
1114 | struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data; | |
1115 | unsigned int tid = GET_TID(rpl); | |
1116 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1117 | struct tid_info *t = lldi->tids; | |
1118 | ||
1119 | csk = lookup_tid(t, tid); | |
1120 | if (unlikely(!csk)) { | |
1121 | pr_err("can't find connection for tid %u.\n", tid); | |
1122 | goto rel_skb; | |
1123 | } | |
759a0cc5 AB |
1124 | pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", |
1125 | (&csk->saddr), (&csk->daddr), | |
1126 | csk, csk->state, csk->flags, csk->tid); | |
7b36b6e0 | 1127 | cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); |
1128 | rel_skb: | |
1129 | __kfree_skb(skb); | |
1130 | } | |
1131 | ||
1132 | static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, | |
1133 | int *need_rst) | |
1134 | { | |
1135 | switch (abort_reason) { | |
df561f66 | 1136 | case CPL_ERR_BAD_SYN: |
7b36b6e0 | 1137 | case CPL_ERR_CONN_RESET: |
1138 | return csk->state > CTP_ESTABLISHED ? | |
1139 | -EPIPE : -ECONNRESET; | |
1140 | case CPL_ERR_XMIT_TIMEDOUT: | |
1141 | case CPL_ERR_PERSIST_TIMEDOUT: | |
1142 | case CPL_ERR_FINWAIT2_TIMEDOUT: | |
1143 | case CPL_ERR_KEEPALIVE_TIMEDOUT: | |
1144 | return -ETIMEDOUT; | |
1145 | default: | |
1146 | return -EIO; | |
1147 | } | |
1148 | } | |
1149 | ||
1150 | static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb) | |
1151 | { | |
1152 | struct cxgbi_sock *csk; | |
1153 | struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data; | |
1154 | unsigned int tid = GET_TID(req); | |
1155 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1156 | struct tid_info *t = lldi->tids; | |
1157 | int rst_status = CPL_ABORT_NO_RST; | |
1158 | ||
1159 | csk = lookup_tid(t, tid); | |
1160 | if (unlikely(!csk)) { | |
1161 | pr_err("can't find connection for tid %u.\n", tid); | |
1162 | goto rel_skb; | |
1163 | } | |
1164 | ||
759a0cc5 AB |
1165 | pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", |
1166 | (&csk->saddr), (&csk->daddr), | |
1167 | csk, csk->state, csk->flags, csk->tid, req->status); | |
7b36b6e0 | 1168 | |
928567ad | 1169 | if (is_neg_adv(req->status)) |
7b36b6e0 | 1170 | goto rel_skb; |
1171 | ||
1172 | cxgbi_sock_get(csk); | |
1173 | spin_lock_bh(&csk->lock); | |
1174 | ||
7b07bf24 AB |
1175 | cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); |
1176 | ||
1177 | if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { | |
1178 | send_tx_flowc_wr(csk); | |
1179 | cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); | |
7b36b6e0 | 1180 | } |
1181 | ||
7b07bf24 AB |
1182 | cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); |
1183 | cxgbi_sock_set_state(csk, CTP_ABORTING); | |
1184 | ||
7b36b6e0 | 1185 | send_abort_rpl(csk, rst_status); |
1186 | ||
1187 | if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { | |
1188 | csk->err = abort_status_to_errno(csk, req->status, &rst_status); | |
1189 | cxgbi_sock_closed(csk); | |
1190 | } | |
7b07bf24 | 1191 | |
7b36b6e0 | 1192 | spin_unlock_bh(&csk->lock); |
1193 | cxgbi_sock_put(csk); | |
1194 | rel_skb: | |
1195 | __kfree_skb(skb); | |
1196 | } | |
1197 | ||
1198 | static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb) | |
1199 | { | |
1200 | struct cxgbi_sock *csk; | |
1201 | struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data; | |
1202 | unsigned int tid = GET_TID(rpl); | |
1203 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1204 | struct tid_info *t = lldi->tids; | |
1205 | ||
1206 | csk = lookup_tid(t, tid); | |
1207 | if (!csk) | |
1208 | goto rel_skb; | |
1209 | ||
f8e25f97 XW |
1210 | pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", |
1211 | (&csk->saddr), (&csk->daddr), csk, | |
1212 | csk->state, csk->flags, csk->tid, rpl->status); | |
7b36b6e0 | 1213 | |
1214 | if (rpl->status == CPL_ERR_ABORT_FAILED) | |
1215 | goto rel_skb; | |
1216 | ||
1217 | cxgbi_sock_rcv_abort_rpl(csk); | |
1218 | rel_skb: | |
1219 | __kfree_skb(skb); | |
1220 | } | |
1221 | ||
f7bcd2e1 KX |
1222 | static void do_rx_data(struct cxgbi_device *cdev, struct sk_buff *skb) |
1223 | { | |
1224 | struct cxgbi_sock *csk; | |
1225 | struct cpl_rx_data *cpl = (struct cpl_rx_data *)skb->data; | |
1226 | unsigned int tid = GET_TID(cpl); | |
1227 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1228 | struct tid_info *t = lldi->tids; | |
1229 | ||
1230 | csk = lookup_tid(t, tid); | |
1231 | if (!csk) { | |
1232 | pr_err("can't find connection for tid %u.\n", tid); | |
1233 | } else { | |
1234 | /* not expecting this, reset the connection. */ | |
1235 | pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk, tid); | |
1236 | spin_lock_bh(&csk->lock); | |
1237 | send_abort_req(csk); | |
1238 | spin_unlock_bh(&csk->lock); | |
1239 | } | |
1240 | __kfree_skb(skb); | |
1241 | } | |
1242 | ||
7b36b6e0 | 1243 | static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb) |
1244 | { | |
1245 | struct cxgbi_sock *csk; | |
1246 | struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; | |
1247 | unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); | |
1248 | unsigned int tid = GET_TID(cpl); | |
1249 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1250 | struct tid_info *t = lldi->tids; | |
7b36b6e0 | 1251 | |
1252 | csk = lookup_tid(t, tid); | |
1253 | if (unlikely(!csk)) { | |
1254 | pr_err("can't find conn. for tid %u.\n", tid); | |
1255 | goto rel_skb; | |
1256 | } | |
1257 | ||
1258 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1259 | "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", | |
1260 | csk, csk->state, csk->flags, csk->tid, skb, skb->len, | |
1261 | pdu_len_ddp); | |
1262 | ||
1263 | spin_lock_bh(&csk->lock); | |
1264 | ||
1265 | if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { | |
1266 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1267 | "csk 0x%p,%u,0x%lx,%u, bad state.\n", | |
1268 | csk, csk->state, csk->flags, csk->tid); | |
1269 | if (csk->state != CTP_ABORTING) | |
1270 | goto abort_conn; | |
1271 | else | |
1272 | goto discard; | |
1273 | } | |
1274 | ||
1275 | cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq); | |
e27d6169 | 1276 | cxgbi_skcb_flags(skb) = 0; |
1277 | ||
7b36b6e0 | 1278 | skb_reset_transport_header(skb); |
1279 | __skb_pull(skb, sizeof(*cpl)); | |
1280 | __pskb_trim(skb, ntohs(cpl->len)); | |
1281 | ||
1282 | if (!csk->skb_ulp_lhdr) { | |
1283 | unsigned char *bhs; | |
3bd3e8bf | 1284 | unsigned int hlen, dlen, plen; |
7b36b6e0 | 1285 | |
1286 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1287 | "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n", | |
1288 | csk, csk->state, csk->flags, csk->tid, skb); | |
1289 | csk->skb_ulp_lhdr = skb; | |
e27d6169 | 1290 | cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); |
7b36b6e0 | 1291 | |
26af1a36 VP |
1292 | if ((CHELSIO_CHIP_VERSION(lldi->adapter_type) <= CHELSIO_T5) && |
1293 | (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt)) { | |
7b36b6e0 | 1294 | pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n", |
e27d6169 | 1295 | csk->tid, cxgbi_skcb_tcp_seq(skb), |
7b36b6e0 | 1296 | csk->rcv_nxt); |
1297 | goto abort_conn; | |
1298 | } | |
1299 | ||
e27d6169 | 1300 | bhs = skb->data; |
7b36b6e0 | 1301 | hlen = ntohs(cpl->len); |
1302 | dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF; | |
1303 | ||
bdc590b9 | 1304 | plen = ISCSI_PDU_LEN_G(pdu_len_ddp); |
3bd3e8bf KX |
1305 | if (is_t4(lldi->adapter_type)) |
1306 | plen -= 40; | |
1307 | ||
1308 | if ((hlen + dlen) != plen) { | |
7b36b6e0 | 1309 | pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len " |
1310 | "mismatch %u != %u + %u, seq 0x%x.\n", | |
3bd3e8bf KX |
1311 | csk->tid, plen, hlen, dlen, |
1312 | cxgbi_skcb_tcp_seq(skb)); | |
7b36b6e0 | 1313 | goto abort_conn; |
1314 | } | |
1315 | ||
1316 | cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3); | |
1317 | if (dlen) | |
1318 | cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len; | |
1319 | csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb); | |
1320 | ||
1321 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1322 | "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n", | |
1323 | csk, skb, *bhs, hlen, dlen, | |
1324 | ntohl(*((unsigned int *)(bhs + 16))), | |
1325 | ntohl(*((unsigned int *)(bhs + 24)))); | |
1326 | ||
1327 | } else { | |
e27d6169 | 1328 | struct sk_buff *lskb = csk->skb_ulp_lhdr; |
7b36b6e0 | 1329 | |
e27d6169 | 1330 | cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); |
7b36b6e0 | 1331 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, |
1332 | "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", | |
1333 | csk, csk->state, csk->flags, skb, lskb); | |
1334 | } | |
1335 | ||
1336 | __skb_queue_tail(&csk->receive_queue, skb); | |
1337 | spin_unlock_bh(&csk->lock); | |
1338 | return; | |
1339 | ||
1340 | abort_conn: | |
1341 | send_abort_req(csk); | |
1342 | discard: | |
1343 | spin_unlock_bh(&csk->lock); | |
1344 | rel_skb: | |
1345 | __kfree_skb(skb); | |
1346 | } | |
1347 | ||
44830d8f VP |
1348 | static void do_rx_iscsi_data(struct cxgbi_device *cdev, struct sk_buff *skb) |
1349 | { | |
1350 | struct cxgbi_sock *csk; | |
1351 | struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; | |
1352 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1353 | struct tid_info *t = lldi->tids; | |
1354 | struct sk_buff *lskb; | |
1355 | u32 tid = GET_TID(cpl); | |
1356 | u16 pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); | |
1357 | ||
1358 | csk = lookup_tid(t, tid); | |
1359 | if (unlikely(!csk)) { | |
1360 | pr_err("can't find conn. for tid %u.\n", tid); | |
1361 | goto rel_skb; | |
1362 | } | |
1363 | ||
1364 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1365 | "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", | |
1366 | csk, csk->state, csk->flags, csk->tid, skb, | |
1367 | skb->len, pdu_len_ddp); | |
1368 | ||
1369 | spin_lock_bh(&csk->lock); | |
1370 | ||
1371 | if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { | |
1372 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1373 | "csk 0x%p,%u,0x%lx,%u, bad state.\n", | |
1374 | csk, csk->state, csk->flags, csk->tid); | |
1375 | ||
1376 | if (csk->state != CTP_ABORTING) | |
1377 | goto abort_conn; | |
1378 | else | |
1379 | goto discard; | |
1380 | } | |
1381 | ||
1382 | cxgbi_skcb_tcp_seq(skb) = be32_to_cpu(cpl->seq); | |
1383 | cxgbi_skcb_flags(skb) = 0; | |
1384 | ||
1385 | skb_reset_transport_header(skb); | |
1386 | __skb_pull(skb, sizeof(*cpl)); | |
1387 | __pskb_trim(skb, ntohs(cpl->len)); | |
1388 | ||
1389 | if (!csk->skb_ulp_lhdr) | |
1390 | csk->skb_ulp_lhdr = skb; | |
1391 | ||
1392 | lskb = csk->skb_ulp_lhdr; | |
1393 | cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); | |
1394 | ||
1395 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1396 | "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", | |
1397 | csk, csk->state, csk->flags, skb, lskb); | |
1398 | ||
1399 | __skb_queue_tail(&csk->receive_queue, skb); | |
1400 | spin_unlock_bh(&csk->lock); | |
1401 | return; | |
1402 | ||
1403 | abort_conn: | |
1404 | send_abort_req(csk); | |
1405 | discard: | |
1406 | spin_unlock_bh(&csk->lock); | |
1407 | rel_skb: | |
1408 | __kfree_skb(skb); | |
1409 | } | |
1410 | ||
1411 | static void | |
1412 | cxgb4i_process_ddpvld(struct cxgbi_sock *csk, | |
1413 | struct sk_buff *skb, u32 ddpvld) | |
1414 | { | |
1415 | if (ddpvld & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) { | |
1416 | pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n", | |
1417 | csk, skb, ddpvld, cxgbi_skcb_flags(skb)); | |
1418 | cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR); | |
1419 | } | |
1420 | ||
1421 | if (ddpvld & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) { | |
1422 | pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n", | |
1423 | csk, skb, ddpvld, cxgbi_skcb_flags(skb)); | |
1424 | cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR); | |
1425 | } | |
1426 | ||
1427 | if (ddpvld & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) { | |
1428 | log_debug(1 << CXGBI_DBG_PDU_RX, | |
1429 | "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n", | |
1430 | csk, skb, ddpvld); | |
1431 | cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR); | |
1432 | } | |
1433 | ||
1434 | if ((ddpvld & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) && | |
1435 | !cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) { | |
1436 | log_debug(1 << CXGBI_DBG_PDU_RX, | |
1437 | "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n", | |
1438 | csk, skb, ddpvld); | |
1439 | cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD); | |
1440 | } | |
1441 | } | |
1442 | ||
7b36b6e0 | 1443 | static void do_rx_data_ddp(struct cxgbi_device *cdev, |
1444 | struct sk_buff *skb) | |
1445 | { | |
1446 | struct cxgbi_sock *csk; | |
1447 | struct sk_buff *lskb; | |
1448 | struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data; | |
1449 | unsigned int tid = GET_TID(rpl); | |
1450 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1451 | struct tid_info *t = lldi->tids; | |
44830d8f | 1452 | u32 ddpvld = be32_to_cpu(rpl->ddpvld); |
7b36b6e0 | 1453 | |
1454 | csk = lookup_tid(t, tid); | |
1455 | if (unlikely(!csk)) { | |
1456 | pr_err("can't find connection for tid %u.\n", tid); | |
1457 | goto rel_skb; | |
1458 | } | |
1459 | ||
1460 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1461 | "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n", | |
44830d8f | 1462 | csk, csk->state, csk->flags, skb, ddpvld, csk->skb_ulp_lhdr); |
7b36b6e0 | 1463 | |
1464 | spin_lock_bh(&csk->lock); | |
1465 | ||
1466 | if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { | |
1467 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1468 | "csk 0x%p,%u,0x%lx,%u, bad state.\n", | |
1469 | csk, csk->state, csk->flags, csk->tid); | |
1470 | if (csk->state != CTP_ABORTING) | |
1471 | goto abort_conn; | |
1472 | else | |
1473 | goto discard; | |
1474 | } | |
1475 | ||
1476 | if (!csk->skb_ulp_lhdr) { | |
1477 | pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid); | |
1478 | goto abort_conn; | |
1479 | } | |
1480 | ||
1481 | lskb = csk->skb_ulp_lhdr; | |
1482 | csk->skb_ulp_lhdr = NULL; | |
1483 | ||
7b36b6e0 | 1484 | cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc); |
1485 | ||
1486 | if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb)) | |
1487 | pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n", | |
1488 | csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb)); | |
1489 | ||
44830d8f VP |
1490 | cxgb4i_process_ddpvld(csk, lskb, ddpvld); |
1491 | ||
7b36b6e0 | 1492 | log_debug(1 << CXGBI_DBG_PDU_RX, |
1493 | "csk 0x%p, lskb 0x%p, f 0x%lx.\n", | |
1494 | csk, lskb, cxgbi_skcb_flags(lskb)); | |
1495 | ||
e27d6169 | 1496 | cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS); |
7b36b6e0 | 1497 | cxgbi_conn_pdu_ready(csk); |
1498 | spin_unlock_bh(&csk->lock); | |
1499 | goto rel_skb; | |
1500 | ||
1501 | abort_conn: | |
1502 | send_abort_req(csk); | |
1503 | discard: | |
1504 | spin_unlock_bh(&csk->lock); | |
1505 | rel_skb: | |
1506 | __kfree_skb(skb); | |
1507 | } | |
1508 | ||
44830d8f VP |
1509 | static void |
1510 | do_rx_iscsi_cmp(struct cxgbi_device *cdev, struct sk_buff *skb) | |
1511 | { | |
1512 | struct cxgbi_sock *csk; | |
1513 | struct cpl_rx_iscsi_cmp *rpl = (struct cpl_rx_iscsi_cmp *)skb->data; | |
1514 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1515 | struct tid_info *t = lldi->tids; | |
1516 | struct sk_buff *data_skb = NULL; | |
1517 | u32 tid = GET_TID(rpl); | |
1518 | u32 ddpvld = be32_to_cpu(rpl->ddpvld); | |
1519 | u32 seq = be32_to_cpu(rpl->seq); | |
1520 | u16 pdu_len_ddp = be16_to_cpu(rpl->pdu_len_ddp); | |
1521 | ||
1522 | csk = lookup_tid(t, tid); | |
1523 | if (unlikely(!csk)) { | |
1524 | pr_err("can't find connection for tid %u.\n", tid); | |
1525 | goto rel_skb; | |
1526 | } | |
1527 | ||
1528 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, | |
1529 | "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p, len %u, " | |
1530 | "pdu_len_ddp %u, status %u.\n", | |
1531 | csk, csk->state, csk->flags, skb, ddpvld, csk->skb_ulp_lhdr, | |
1532 | ntohs(rpl->len), pdu_len_ddp, rpl->status); | |
1533 | ||
1534 | spin_lock_bh(&csk->lock); | |
1535 | ||
1536 | if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { | |
1537 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1538 | "csk 0x%p,%u,0x%lx,%u, bad state.\n", | |
1539 | csk, csk->state, csk->flags, csk->tid); | |
1540 | ||
1541 | if (csk->state != CTP_ABORTING) | |
1542 | goto abort_conn; | |
1543 | else | |
1544 | goto discard; | |
1545 | } | |
1546 | ||
1547 | cxgbi_skcb_tcp_seq(skb) = seq; | |
1548 | cxgbi_skcb_flags(skb) = 0; | |
1549 | cxgbi_skcb_rx_pdulen(skb) = 0; | |
1550 | ||
1551 | skb_reset_transport_header(skb); | |
1552 | __skb_pull(skb, sizeof(*rpl)); | |
1553 | __pskb_trim(skb, be16_to_cpu(rpl->len)); | |
1554 | ||
1555 | csk->rcv_nxt = seq + pdu_len_ddp; | |
1556 | ||
1557 | if (csk->skb_ulp_lhdr) { | |
1558 | data_skb = skb_peek(&csk->receive_queue); | |
1559 | if (!data_skb || | |
1560 | !cxgbi_skcb_test_flag(data_skb, SKCBF_RX_DATA)) { | |
1561 | pr_err("Error! freelist data not found 0x%p, tid %u\n", | |
1562 | data_skb, tid); | |
1563 | ||
1564 | goto abort_conn; | |
1565 | } | |
1566 | __skb_unlink(data_skb, &csk->receive_queue); | |
1567 | ||
1568 | cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA); | |
1569 | ||
1570 | __skb_queue_tail(&csk->receive_queue, skb); | |
1571 | __skb_queue_tail(&csk->receive_queue, data_skb); | |
1572 | } else { | |
1573 | __skb_queue_tail(&csk->receive_queue, skb); | |
1574 | } | |
1575 | ||
1576 | csk->skb_ulp_lhdr = NULL; | |
1577 | ||
1578 | cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); | |
1579 | cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS); | |
1580 | cxgbi_skcb_set_flag(skb, SKCBF_RX_ISCSI_COMPL); | |
1581 | cxgbi_skcb_rx_ddigest(skb) = be32_to_cpu(rpl->ulp_crc); | |
1582 | ||
1583 | cxgb4i_process_ddpvld(csk, skb, ddpvld); | |
1584 | ||
1585 | log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, skb 0x%p, f 0x%lx.\n", | |
1586 | csk, skb, cxgbi_skcb_flags(skb)); | |
1587 | ||
1588 | cxgbi_conn_pdu_ready(csk); | |
1589 | spin_unlock_bh(&csk->lock); | |
1590 | ||
1591 | return; | |
1592 | ||
1593 | abort_conn: | |
1594 | send_abort_req(csk); | |
1595 | discard: | |
1596 | spin_unlock_bh(&csk->lock); | |
1597 | rel_skb: | |
1598 | __kfree_skb(skb); | |
1599 | } | |
1600 | ||
7b36b6e0 | 1601 | static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb) |
1602 | { | |
1603 | struct cxgbi_sock *csk; | |
1604 | struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data; | |
1605 | unsigned int tid = GET_TID(rpl); | |
1606 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1607 | struct tid_info *t = lldi->tids; | |
1608 | ||
1609 | csk = lookup_tid(t, tid); | |
1610 | if (unlikely(!csk)) | |
1611 | pr_err("can't find connection for tid %u.\n", tid); | |
1612 | else { | |
1613 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1614 | "csk 0x%p,%u,0x%lx,%u.\n", | |
1615 | csk, csk->state, csk->flags, csk->tid); | |
1616 | cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una), | |
1617 | rpl->seq_vld); | |
1618 | } | |
1619 | __kfree_skb(skb); | |
1620 | } | |
1621 | ||
1622 | static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) | |
1623 | { | |
1624 | struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; | |
1625 | unsigned int tid = GET_TID(rpl); | |
1626 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1627 | struct tid_info *t = lldi->tids; | |
1628 | struct cxgbi_sock *csk; | |
1629 | ||
1630 | csk = lookup_tid(t, tid); | |
9e8f1c79 | 1631 | if (!csk) { |
7b36b6e0 | 1632 | pr_err("can't find conn. for tid %u.\n", tid); |
9e8f1c79 VP |
1633 | return; |
1634 | } | |
7b36b6e0 | 1635 | |
1636 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1637 | "csk 0x%p,%u,%lx,%u, status 0x%x.\n", | |
1638 | csk, csk->state, csk->flags, csk->tid, rpl->status); | |
1639 | ||
9e8f1c79 | 1640 | if (rpl->status != CPL_ERR_NONE) { |
7b36b6e0 | 1641 | pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n", |
1642 | csk, tid, rpl->status); | |
9e8f1c79 VP |
1643 | csk->err = -EINVAL; |
1644 | } | |
1645 | ||
1646 | complete(&csk->cmpl); | |
7b36b6e0 | 1647 | |
1648 | __kfree_skb(skb); | |
1649 | } | |
1650 | ||
1651 | static int alloc_cpls(struct cxgbi_sock *csk) | |
1652 | { | |
24d3f95a | 1653 | csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), |
1654 | 0, GFP_KERNEL); | |
7b36b6e0 | 1655 | if (!csk->cpl_close) |
1656 | return -ENOMEM; | |
1657 | ||
24d3f95a | 1658 | csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), |
1659 | 0, GFP_KERNEL); | |
7b36b6e0 | 1660 | if (!csk->cpl_abort_req) |
1661 | goto free_cpls; | |
1662 | ||
24d3f95a | 1663 | csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), |
1664 | 0, GFP_KERNEL); | |
7b36b6e0 | 1665 | if (!csk->cpl_abort_rpl) |
1666 | goto free_cpls; | |
1667 | return 0; | |
1668 | ||
1669 | free_cpls: | |
1670 | cxgbi_sock_free_cpl_skbs(csk); | |
1671 | return -ENOMEM; | |
1672 | } | |
1673 | ||
1674 | static inline void l2t_put(struct cxgbi_sock *csk) | |
1675 | { | |
1676 | if (csk->l2t) { | |
1677 | cxgb4_l2t_release(csk->l2t); | |
1678 | csk->l2t = NULL; | |
1679 | cxgbi_sock_put(csk); | |
1680 | } | |
1681 | } | |
1682 | ||
1683 | static void release_offload_resources(struct cxgbi_sock *csk) | |
1684 | { | |
1685 | struct cxgb4_lld_info *lldi; | |
211a84e3 AB |
1686 | #if IS_ENABLED(CONFIG_IPV6) |
1687 | struct net_device *ndev = csk->cdev->ports[csk->port_id]; | |
1688 | #endif | |
7b36b6e0 | 1689 | |
1690 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
1691 | "csk 0x%p,%u,0x%lx,%u.\n", | |
1692 | csk, csk->state, csk->flags, csk->tid); | |
1693 | ||
1694 | cxgbi_sock_free_cpl_skbs(csk); | |
9b3a081f | 1695 | cxgbi_sock_purge_write_queue(csk); |
7b36b6e0 | 1696 | if (csk->wr_cred != csk->wr_max_cred) { |
1697 | cxgbi_sock_purge_wr_queue(csk); | |
1698 | cxgbi_sock_reset_wr_list(csk); | |
1699 | } | |
1700 | ||
1701 | l2t_put(csk); | |
211a84e3 AB |
1702 | #if IS_ENABLED(CONFIG_IPV6) |
1703 | if (csk->csk_family == AF_INET6) | |
1704 | cxgb4_clip_release(ndev, | |
1705 | (const u32 *)&csk->saddr6.sin6_addr, 1); | |
1706 | #endif | |
1707 | ||
7b36b6e0 | 1708 | if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) |
1709 | free_atid(csk); | |
1710 | else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { | |
1711 | lldi = cxgbi_cdev_priv(csk->cdev); | |
1dec4cec GG |
1712 | cxgb4_remove_tid(lldi->tids, 0, csk->tid, |
1713 | csk->csk_family); | |
7b36b6e0 | 1714 | cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); |
1715 | cxgbi_sock_put(csk); | |
1716 | } | |
1717 | csk->dst = NULL; | |
7b36b6e0 | 1718 | } |
1719 | ||
b5a5fe4e VP |
1720 | #ifdef CONFIG_CHELSIO_T4_DCB |
1721 | static inline u8 get_iscsi_dcb_state(struct net_device *ndev) | |
1722 | { | |
1723 | return ndev->dcbnl_ops->getstate(ndev); | |
1724 | } | |
1725 | ||
1726 | static int select_priority(int pri_mask) | |
1727 | { | |
1728 | if (!pri_mask) | |
1729 | return 0; | |
1730 | return (ffs(pri_mask) - 1); | |
1731 | } | |
1732 | ||
1733 | static u8 get_iscsi_dcb_priority(struct net_device *ndev) | |
1734 | { | |
1735 | int rv; | |
1736 | u8 caps; | |
1737 | ||
1738 | struct dcb_app iscsi_dcb_app = { | |
1739 | .protocol = 3260 | |
1740 | }; | |
1741 | ||
1742 | rv = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps); | |
1743 | if (rv) | |
1744 | return 0; | |
1745 | ||
1746 | if (caps & DCB_CAP_DCBX_VER_IEEE) { | |
152e30fc | 1747 | iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM; |
b5a5fe4e | 1748 | rv = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); |
152e30fc VP |
1749 | if (!rv) { |
1750 | iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY; | |
1751 | rv = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app); | |
1752 | } | |
b5a5fe4e VP |
1753 | } else if (caps & DCB_CAP_DCBX_VER_CEE) { |
1754 | iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM; | |
1755 | rv = dcb_getapp(ndev, &iscsi_dcb_app); | |
1756 | } | |
1757 | ||
1758 | log_debug(1 << CXGBI_DBG_ISCSI, | |
1759 | "iSCSI priority is set to %u\n", select_priority(rv)); | |
1760 | return select_priority(rv); | |
1761 | } | |
1762 | #endif | |
1763 | ||
7b36b6e0 | 1764 | static int init_act_open(struct cxgbi_sock *csk) |
1765 | { | |
1766 | struct cxgbi_device *cdev = csk->cdev; | |
1767 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1768 | struct net_device *ndev = cdev->ports[csk->port_id]; | |
7b36b6e0 | 1769 | struct sk_buff *skb = NULL; |
759a0cc5 AB |
1770 | struct neighbour *n = NULL; |
1771 | void *daddr; | |
7b36b6e0 | 1772 | unsigned int step; |
6c9e2770 | 1773 | unsigned int rxq_idx; |
759a0cc5 | 1774 | unsigned int size, size6; |
81daf10c KX |
1775 | unsigned int linkspeed; |
1776 | unsigned int rcv_winf, snd_winf; | |
b5a5fe4e VP |
1777 | #ifdef CONFIG_CHELSIO_T4_DCB |
1778 | u8 priority = 0; | |
1779 | #endif | |
7b36b6e0 | 1780 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
1781 | "csk 0x%p,%u,0x%lx,%u.\n", | |
1782 | csk, csk->state, csk->flags, csk->tid); | |
1783 | ||
759a0cc5 AB |
1784 | if (csk->csk_family == AF_INET) |
1785 | daddr = &csk->daddr.sin_addr.s_addr; | |
e81fbf6c AB |
1786 | #if IS_ENABLED(CONFIG_IPV6) |
1787 | else if (csk->csk_family == AF_INET6) | |
759a0cc5 | 1788 | daddr = &csk->daddr6.sin6_addr; |
e81fbf6c AB |
1789 | #endif |
1790 | else { | |
1791 | pr_err("address family 0x%x not supported\n", csk->csk_family); | |
1792 | goto rel_resource; | |
1793 | } | |
759a0cc5 AB |
1794 | |
1795 | n = dst_neigh_lookup(csk->dst, daddr); | |
1796 | ||
1797 | if (!n) { | |
1798 | pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name); | |
1799 | goto rel_resource; | |
1800 | } | |
1801 | ||
71eb2ac5 VP |
1802 | if (!(n->nud_state & NUD_VALID)) |
1803 | neigh_event_send(n, NULL); | |
1804 | ||
7b36b6e0 | 1805 | csk->atid = cxgb4_alloc_atid(lldi->tids, csk); |
1806 | if (csk->atid < 0) { | |
1807 | pr_err("%s, NO atid available.\n", ndev->name); | |
338be072 | 1808 | goto rel_resource_without_clip; |
7b36b6e0 | 1809 | } |
1810 | cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); | |
1811 | cxgbi_sock_get(csk); | |
1812 | ||
b5a5fe4e VP |
1813 | #ifdef CONFIG_CHELSIO_T4_DCB |
1814 | if (get_iscsi_dcb_state(ndev)) | |
1815 | priority = get_iscsi_dcb_priority(ndev); | |
1816 | ||
1817 | csk->dcb_priority = priority; | |
1818 | csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, priority); | |
1819 | #else | |
51e059bd | 1820 | csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0); |
b5a5fe4e | 1821 | #endif |
7b36b6e0 | 1822 | if (!csk->l2t) { |
1823 | pr_err("%s, cannot alloc l2t.\n", ndev->name); | |
211a84e3 | 1824 | goto rel_resource_without_clip; |
7b36b6e0 | 1825 | } |
1826 | cxgbi_sock_get(csk); | |
1827 | ||
211a84e3 AB |
1828 | #if IS_ENABLED(CONFIG_IPV6) |
1829 | if (csk->csk_family == AF_INET6) | |
1830 | cxgb4_clip_get(ndev, (const u32 *)&csk->saddr6.sin6_addr, 1); | |
1831 | #endif | |
1832 | ||
586be7cb | 1833 | if (is_t4(lldi->adapter_type)) { |
759a0cc5 AB |
1834 | size = sizeof(struct cpl_act_open_req); |
1835 | size6 = sizeof(struct cpl_act_open_req6); | |
586be7cb | 1836 | } else if (is_t5(lldi->adapter_type)) { |
759a0cc5 AB |
1837 | size = sizeof(struct cpl_t5_act_open_req); |
1838 | size6 = sizeof(struct cpl_t5_act_open_req6); | |
586be7cb VP |
1839 | } else { |
1840 | size = sizeof(struct cpl_t6_act_open_req); | |
1841 | size6 = sizeof(struct cpl_t6_act_open_req6); | |
759a0cc5 AB |
1842 | } |
1843 | ||
1844 | if (csk->csk_family == AF_INET) | |
1845 | skb = alloc_wr(size, 0, GFP_NOIO); | |
f42bb57c | 1846 | #if IS_ENABLED(CONFIG_IPV6) |
759a0cc5 AB |
1847 | else |
1848 | skb = alloc_wr(size6, 0, GFP_NOIO); | |
f42bb57c | 1849 | #endif |
759a0cc5 | 1850 | |
7b36b6e0 | 1851 | if (!skb) |
1852 | goto rel_resource; | |
1853 | skb->sk = (struct sock *)csk; | |
1854 | t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure); | |
1855 | ||
1856 | if (!csk->mtu) | |
1857 | csk->mtu = dst_mtu(csk->dst); | |
1858 | cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx); | |
1859 | csk->tx_chan = cxgb4_port_chan(ndev); | |
02d805dc | 1860 | csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; |
7b36b6e0 | 1861 | step = lldi->ntxq / lldi->nchan; |
1862 | csk->txq_idx = cxgb4_port_idx(ndev) * step; | |
1863 | step = lldi->nrxq / lldi->nchan; | |
6c9e2770 VP |
1864 | rxq_idx = (cxgb4_port_idx(ndev) * step) + (cdev->rxq_idx_cntr % step); |
1865 | cdev->rxq_idx_cntr++; | |
1866 | csk->rss_qid = lldi->rxq_ids[rxq_idx]; | |
81daf10c KX |
1867 | linkspeed = ((struct port_info *)netdev_priv(ndev))->link_cfg.speed; |
1868 | csk->snd_win = cxgb4i_snd_win; | |
1869 | csk->rcv_win = cxgb4i_rcv_win; | |
1870 | if (cxgb4i_rcv_win <= 0) { | |
1871 | csk->rcv_win = CXGB4I_DEFAULT_10G_RCV_WIN; | |
1872 | rcv_winf = linkspeed / SPEED_10000; | |
1873 | if (rcv_winf) | |
1874 | csk->rcv_win *= rcv_winf; | |
1875 | } | |
1876 | if (cxgb4i_snd_win <= 0) { | |
1877 | csk->snd_win = CXGB4I_DEFAULT_10G_SND_WIN; | |
1878 | snd_winf = linkspeed / SPEED_10000; | |
1879 | if (snd_winf) | |
1880 | csk->snd_win *= snd_winf; | |
1881 | } | |
759a0cc5 AB |
1882 | csk->wr_cred = lldi->wr_cred - |
1883 | DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16); | |
1884 | csk->wr_max_cred = csk->wr_cred; | |
7b36b6e0 | 1885 | csk->wr_una_cred = 0; |
1886 | cxgbi_sock_reset_wr_list(csk); | |
1887 | csk->err = 0; | |
7b36b6e0 | 1888 | |
759a0cc5 AB |
1889 | pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n", |
1890 | (&csk->saddr), (&csk->daddr), csk, csk->state, | |
1891 | csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid, | |
1892 | csk->mtu, csk->mss_idx, csk->smac_idx); | |
1893 | ||
1894 | /* must wait for either a act_open_rpl or act_open_establish */ | |
1fe1fdb0 VP |
1895 | if (!try_module_get(cdev->owner)) { |
1896 | pr_err("%s, try_module_get failed.\n", ndev->name); | |
1897 | goto rel_resource; | |
1898 | } | |
1899 | ||
7b36b6e0 | 1900 | cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); |
759a0cc5 AB |
1901 | if (csk->csk_family == AF_INET) |
1902 | send_act_open_req(csk, skb, csk->l2t); | |
f42bb57c | 1903 | #if IS_ENABLED(CONFIG_IPV6) |
759a0cc5 AB |
1904 | else |
1905 | send_act_open_req6(csk, skb, csk->l2t); | |
f42bb57c | 1906 | #endif |
c4737377 | 1907 | neigh_release(n); |
759a0cc5 | 1908 | |
7b36b6e0 | 1909 | return 0; |
1910 | ||
1911 | rel_resource: | |
211a84e3 AB |
1912 | #if IS_ENABLED(CONFIG_IPV6) |
1913 | if (csk->csk_family == AF_INET6) | |
1914 | cxgb4_clip_release(ndev, | |
1915 | (const u32 *)&csk->saddr6.sin6_addr, 1); | |
1916 | #endif | |
1917 | rel_resource_without_clip: | |
c4737377 DM |
1918 | if (n) |
1919 | neigh_release(n); | |
7b36b6e0 | 1920 | if (skb) |
1921 | __kfree_skb(skb); | |
1922 | return -EINVAL; | |
1923 | } | |
1924 | ||
4665bdd5 | 1925 | static cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = { |
7b36b6e0 | 1926 | [CPL_ACT_ESTABLISH] = do_act_establish, |
1927 | [CPL_ACT_OPEN_RPL] = do_act_open_rpl, | |
1928 | [CPL_PEER_CLOSE] = do_peer_close, | |
1929 | [CPL_ABORT_REQ_RSS] = do_abort_req_rss, | |
1930 | [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss, | |
1931 | [CPL_CLOSE_CON_RPL] = do_close_con_rpl, | |
1932 | [CPL_FW4_ACK] = do_fw4_ack, | |
1933 | [CPL_ISCSI_HDR] = do_rx_iscsi_hdr, | |
44830d8f | 1934 | [CPL_ISCSI_DATA] = do_rx_iscsi_data, |
7b36b6e0 | 1935 | [CPL_SET_TCB_RPL] = do_set_tcb_rpl, |
1936 | [CPL_RX_DATA_DDP] = do_rx_data_ddp, | |
3bd3e8bf | 1937 | [CPL_RX_ISCSI_DDP] = do_rx_data_ddp, |
44830d8f | 1938 | [CPL_RX_ISCSI_CMP] = do_rx_iscsi_cmp, |
f7bcd2e1 | 1939 | [CPL_RX_DATA] = do_rx_data, |
7b36b6e0 | 1940 | }; |
1941 | ||
4665bdd5 | 1942 | static int cxgb4i_ofld_init(struct cxgbi_device *cdev) |
7b36b6e0 | 1943 | { |
1944 | int rc; | |
1945 | ||
1946 | if (cxgb4i_max_connect > CXGB4I_MAX_CONN) | |
1947 | cxgb4i_max_connect = CXGB4I_MAX_CONN; | |
1948 | ||
1949 | rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base, | |
1950 | cxgb4i_max_connect); | |
1951 | if (rc < 0) | |
1952 | return rc; | |
1953 | ||
1954 | cdev->csk_release_offload_resources = release_offload_resources; | |
1955 | cdev->csk_push_tx_frames = push_tx_frames; | |
1956 | cdev->csk_send_abort_req = send_abort_req; | |
1957 | cdev->csk_send_close_req = send_close_req; | |
1958 | cdev->csk_send_rx_credits = send_rx_credits; | |
1959 | cdev->csk_alloc_cpls = alloc_cpls; | |
1960 | cdev->csk_init_act_open = init_act_open; | |
1961 | ||
1962 | pr_info("cdev 0x%p, offload up, added.\n", cdev); | |
1963 | return 0; | |
1964 | } | |
1965 | ||
71f7a00b VP |
1966 | static inline void |
1967 | ulp_mem_io_set_hdr(struct cxgbi_device *cdev, | |
1968 | struct ulp_mem_io *req, | |
1969 | unsigned int wr_len, unsigned int dlen, | |
1970 | unsigned int pm_addr, | |
1971 | int tid) | |
1972 | { | |
1973 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); | |
1974 | struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1); | |
1975 | ||
1976 | INIT_ULPTX_WR(req, wr_len, 0, tid); | |
1977 | req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | | |
1978 | FW_WR_ATOMIC_V(0)); | |
1979 | req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | | |
1980 | ULP_MEMIO_ORDER_V(is_t4(lldi->adapter_type)) | | |
1981 | T5_ULP_MEMIO_IMM_V(!is_t4(lldi->adapter_type))); | |
1982 | req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); | |
1983 | req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); | |
1984 | req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); | |
1985 | ||
1986 | idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); | |
1987 | idata->len = htonl(dlen); | |
1988 | } | |
1989 | ||
1990 | static struct sk_buff * | |
1991 | ddp_ppod_init_idata(struct cxgbi_device *cdev, | |
1992 | struct cxgbi_ppm *ppm, | |
1993 | unsigned int idx, unsigned int npods, | |
1994 | unsigned int tid) | |
1995 | { | |
1996 | unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; | |
1997 | unsigned int dlen = npods << PPOD_SIZE_SHIFT; | |
1998 | unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + | |
1999 | sizeof(struct ulptx_idata) + dlen, 16); | |
2000 | struct sk_buff *skb = alloc_wr(wr_len, 0, GFP_ATOMIC); | |
2001 | ||
2002 | if (!skb) { | |
2003 | pr_err("%s: %s idx %u, npods %u, OOM.\n", | |
2004 | __func__, ppm->ndev->name, idx, npods); | |
2005 | return NULL; | |
2006 | } | |
2007 | ||
2008 | ulp_mem_io_set_hdr(cdev, (struct ulp_mem_io *)skb->head, wr_len, dlen, | |
2009 | pm_addr, tid); | |
2010 | ||
2011 | return skb; | |
2012 | } | |
2013 | ||
2014 | static int ddp_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, | |
2015 | struct cxgbi_task_tag_info *ttinfo, | |
2016 | unsigned int idx, unsigned int npods, | |
2017 | struct scatterlist **sg_pp, | |
2018 | unsigned int *sg_off) | |
2019 | { | |
2020 | struct cxgbi_device *cdev = csk->cdev; | |
2021 | struct sk_buff *skb = ddp_ppod_init_idata(cdev, ppm, idx, npods, | |
2022 | csk->tid); | |
2023 | struct ulp_mem_io *req; | |
2024 | struct ulptx_idata *idata; | |
2025 | struct cxgbi_pagepod *ppod; | |
2026 | int i; | |
2027 | ||
2028 | if (!skb) | |
2029 | return -ENOMEM; | |
2030 | ||
2031 | req = (struct ulp_mem_io *)skb->head; | |
2032 | idata = (struct ulptx_idata *)(req + 1); | |
2033 | ppod = (struct cxgbi_pagepod *)(idata + 1); | |
2034 | ||
2035 | for (i = 0; i < npods; i++, ppod++) | |
2036 | cxgbi_ddp_set_one_ppod(ppod, ttinfo, sg_pp, sg_off); | |
2037 | ||
2038 | cxgbi_skcb_set_flag(skb, SKCBF_TX_MEM_WRITE); | |
2039 | cxgbi_skcb_set_flag(skb, SKCBF_TX_FLAG_COMPL); | |
2040 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); | |
2041 | ||
2042 | spin_lock_bh(&csk->lock); | |
2043 | cxgbi_sock_skb_entail(csk, skb); | |
2044 | spin_unlock_bh(&csk->lock); | |
2045 | ||
2046 | return 0; | |
2047 | } | |
2048 | ||
2049 | static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, | |
2050 | struct cxgbi_task_tag_info *ttinfo) | |
2051 | { | |
2052 | unsigned int pidx = ttinfo->idx; | |
2053 | unsigned int npods = ttinfo->npods; | |
2054 | unsigned int i, cnt; | |
2055 | int err = 0; | |
2056 | struct scatterlist *sg = ttinfo->sgl; | |
2057 | unsigned int offset = 0; | |
2058 | ||
2059 | ttinfo->cid = csk->port_id; | |
2060 | ||
2061 | for (i = 0; i < npods; i += cnt, pidx += cnt) { | |
2062 | cnt = npods - i; | |
2063 | ||
2064 | if (cnt > ULPMEM_IDATA_MAX_NPPODS) | |
2065 | cnt = ULPMEM_IDATA_MAX_NPPODS; | |
2066 | err = ddp_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt, | |
2067 | &sg, &offset); | |
2068 | if (err < 0) | |
2069 | break; | |
2070 | } | |
2071 | ||
2072 | return err; | |
2073 | } | |
2074 | ||
7b36b6e0 | 2075 | static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid, |
9e8f1c79 | 2076 | int pg_idx) |
7b36b6e0 | 2077 | { |
2078 | struct sk_buff *skb; | |
2079 | struct cpl_set_tcb_field *req; | |
7b36b6e0 | 2080 | |
e27d6169 | 2081 | if (!pg_idx || pg_idx >= DDP_PGIDX_MAX) |
7b36b6e0 | 2082 | return 0; |
2083 | ||
24d3f95a | 2084 | skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); |
7b36b6e0 | 2085 | if (!skb) |
2086 | return -ENOMEM; | |
2087 | ||
e27d6169 | 2088 | /* set up ulp page size */ |
7b36b6e0 | 2089 | req = (struct cpl_set_tcb_field *)skb->head; |
2090 | INIT_TP_WR(req, csk->tid); | |
2091 | OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); | |
9e8f1c79 | 2092 | req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); |
e27d6169 | 2093 | req->word_cookie = htons(0); |
2094 | req->mask = cpu_to_be64(0x3 << 8); | |
2095 | req->val = cpu_to_be64(pg_idx << 8); | |
7b36b6e0 | 2096 | set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); |
2097 | ||
2098 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
2099 | "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx); | |
2100 | ||
9e8f1c79 | 2101 | reinit_completion(&csk->cmpl); |
7b36b6e0 | 2102 | cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); |
9e8f1c79 VP |
2103 | wait_for_completion(&csk->cmpl); |
2104 | ||
2105 | return csk->err; | |
7b36b6e0 | 2106 | } |
2107 | ||
2108 | static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, | |
9e8f1c79 | 2109 | int hcrc, int dcrc) |
7b36b6e0 | 2110 | { |
2111 | struct sk_buff *skb; | |
2112 | struct cpl_set_tcb_field *req; | |
7b36b6e0 | 2113 | |
e27d6169 | 2114 | if (!hcrc && !dcrc) |
2115 | return 0; | |
7b36b6e0 | 2116 | |
24d3f95a | 2117 | skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); |
7b36b6e0 | 2118 | if (!skb) |
2119 | return -ENOMEM; | |
2120 | ||
2121 | csk->hcrc_len = (hcrc ? 4 : 0); | |
2122 | csk->dcrc_len = (dcrc ? 4 : 0); | |
e27d6169 | 2123 | /* set up ulp submode */ |
7b36b6e0 | 2124 | req = (struct cpl_set_tcb_field *)skb->head; |
2125 | INIT_TP_WR(req, tid); | |
2126 | OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); | |
9e8f1c79 | 2127 | req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid)); |
e27d6169 | 2128 | req->word_cookie = htons(0); |
2129 | req->mask = cpu_to_be64(0x3 << 4); | |
2130 | req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) | | |
2131 | (dcrc ? ULP_CRC_DATA : 0)) << 4); | |
7b36b6e0 | 2132 | set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); |
2133 | ||
2134 | log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, | |
2135 | "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc); | |
2136 | ||
9e8f1c79 | 2137 | reinit_completion(&csk->cmpl); |
7b36b6e0 | 2138 | cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); |
9e8f1c79 VP |
2139 | wait_for_completion(&csk->cmpl); |
2140 | ||
2141 | return csk->err; | |
7b36b6e0 | 2142 | } |
2143 | ||
71f7a00b VP |
2144 | static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev) |
2145 | { | |
2146 | return (struct cxgbi_ppm *)(*((struct cxgb4_lld_info *) | |
2147 | (cxgbi_cdev_priv(cdev)))->iscsi_ppm); | |
2148 | } | |
2149 | ||
7b36b6e0 | 2150 | static int cxgb4i_ddp_init(struct cxgbi_device *cdev) |
2151 | { | |
71f7a00b VP |
2152 | struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); |
2153 | struct net_device *ndev = cdev->ports[0]; | |
2154 | struct cxgbi_tag_format tformat; | |
a248384e | 2155 | int i, err; |
71f7a00b VP |
2156 | |
2157 | if (!lldi->vr->iscsi.size) { | |
2158 | pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name); | |
2159 | return -EACCES; | |
2160 | } | |
2161 | ||
2162 | cdev->flags |= CXGBI_FLAG_USE_PPOD_OFLDQ; | |
71f7a00b VP |
2163 | |
2164 | memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); | |
2165 | for (i = 0; i < 4; i++) | |
2166 | tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3)) | |
2167 | & 0xF; | |
2168 | cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat); | |
2169 | ||
a248384e VP |
2170 | pr_info("iscsi_edram.start 0x%x iscsi_edram.size 0x%x", |
2171 | lldi->vr->ppod_edram.start, lldi->vr->ppod_edram.size); | |
2172 | ||
2173 | err = cxgbi_ddp_ppm_setup(lldi->iscsi_ppm, cdev, &tformat, | |
2174 | lldi->vr->iscsi.size, lldi->iscsi_llimit, | |
2175 | lldi->vr->iscsi.start, 2, | |
2176 | lldi->vr->ppod_edram.start, | |
2177 | lldi->vr->ppod_edram.size); | |
2178 | ||
2179 | if (err < 0) | |
2180 | return err; | |
71f7a00b | 2181 | |
7b36b6e0 | 2182 | cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; |
2183 | cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; | |
71f7a00b VP |
2184 | cdev->csk_ddp_set_map = ddp_set_map; |
2185 | cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, | |
2186 | lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN); | |
2187 | cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, | |
2188 | lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN); | |
2189 | cdev->cdev2ppm = cdev2ppm; | |
2190 | ||
7b36b6e0 | 2191 | return 0; |
2192 | } | |
2193 | ||
e33c2482 VP |
2194 | static bool is_memfree(struct adapter *adap) |
2195 | { | |
2196 | u32 io; | |
2197 | ||
2198 | io = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); | |
2199 | if (is_t5(adap->params.chip)) { | |
2200 | if ((io & EXT_MEM0_ENABLE_F) || (io & EXT_MEM1_ENABLE_F)) | |
2201 | return false; | |
2202 | } else if (io & EXT_MEM_ENABLE_F) { | |
2203 | return false; | |
2204 | } | |
2205 | ||
2206 | return true; | |
2207 | } | |
2208 | ||
7b36b6e0 | 2209 | static void *t4_uld_add(const struct cxgb4_lld_info *lldi) |
2210 | { | |
2211 | struct cxgbi_device *cdev; | |
2212 | struct port_info *pi; | |
e33c2482 VP |
2213 | struct net_device *ndev; |
2214 | struct adapter *adap; | |
2215 | struct tid_info *t; | |
2216 | u32 max_cmds = CXGB4I_SCSI_HOST_QDEPTH; | |
2217 | u32 max_conn = CXGBI_MAX_CONN; | |
7b36b6e0 | 2218 | int i, rc; |
2219 | ||
2220 | cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports); | |
2221 | if (!cdev) { | |
2222 | pr_info("t4 device 0x%p, register failed.\n", lldi); | |
2223 | return NULL; | |
2224 | } | |
2225 | pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n", | |
2226 | cdev, lldi->adapter_type, lldi->nports, | |
2227 | lldi->ports[0]->name, lldi->nchan, lldi->ntxq, | |
2228 | lldi->nrxq, lldi->wr_cred); | |
2229 | for (i = 0; i < lldi->nrxq; i++) | |
2230 | log_debug(1 << CXGBI_DBG_DEV, | |
2231 | "t4 0x%p, rxq id #%d: %u.\n", | |
2232 | cdev, i, lldi->rxq_ids[i]); | |
2233 | ||
2234 | memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi)); | |
2235 | cdev->flags = CXGBI_FLAG_DEV_T4; | |
2236 | cdev->pdev = lldi->pdev; | |
2237 | cdev->ports = lldi->ports; | |
2238 | cdev->nports = lldi->nports; | |
2239 | cdev->mtus = lldi->mtus; | |
2240 | cdev->nmtus = NMTUS; | |
586be7cb VP |
2241 | cdev->rx_credit_thres = (CHELSIO_CHIP_VERSION(lldi->adapter_type) <= |
2242 | CHELSIO_T5) ? cxgb4i_rx_credit_thres : 0; | |
7b36b6e0 | 2243 | cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN; |
2244 | cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr); | |
2245 | cdev->itp = &cxgb4i_iscsi_transport; | |
1fe1fdb0 | 2246 | cdev->owner = THIS_MODULE; |
7b36b6e0 | 2247 | |
b184487c | 2248 | cdev->pfvf = FW_PFVF_CMD_PFN_V(lldi->pf); |
e27d6169 | 2249 | pr_info("cdev 0x%p,%s, pfvf %u.\n", |
2250 | cdev, lldi->ports[0]->name, cdev->pfvf); | |
2251 | ||
7b36b6e0 | 2252 | rc = cxgb4i_ddp_init(cdev); |
2253 | if (rc) { | |
a248384e | 2254 | pr_info("t4 0x%p ddp init failed %d.\n", cdev, rc); |
7b36b6e0 | 2255 | goto err_out; |
2256 | } | |
e33c2482 VP |
2257 | |
2258 | ndev = cdev->ports[0]; | |
2259 | adap = netdev2adap(ndev); | |
2260 | if (adap) { | |
2261 | t = &adap->tids; | |
2262 | if (t->ntids <= CXGBI_MAX_CONN) | |
2263 | max_conn = t->ntids; | |
2264 | ||
2265 | if (is_memfree(adap)) { | |
2266 | cdev->flags |= CXGBI_FLAG_DEV_ISO_OFF; | |
2267 | max_cmds = CXGB4I_SCSI_HOST_QDEPTH >> 2; | |
2268 | ||
2269 | pr_info("%s: 0x%p, tid %u, SO adapter.\n", | |
2270 | ndev->name, cdev, t->ntids); | |
2271 | } | |
2272 | } else { | |
2273 | pr_info("%s, 0x%p, NO adapter struct.\n", ndev->name, cdev); | |
2274 | } | |
2275 | ||
2276 | /* ISO is enabled in T5/T6 firmware version >= 1.13.43.0 */ | |
2277 | if (!is_t4(lldi->adapter_type) && | |
2278 | (lldi->fw_vers >= 0x10d2b00) && | |
2279 | !(cdev->flags & CXGBI_FLAG_DEV_ISO_OFF)) | |
2280 | cdev->skb_iso_txhdr = sizeof(struct cpl_tx_data_iso); | |
2281 | ||
7b36b6e0 | 2282 | rc = cxgb4i_ofld_init(cdev); |
2283 | if (rc) { | |
2284 | pr_info("t4 0x%p ofld init failed.\n", cdev); | |
2285 | goto err_out; | |
2286 | } | |
2287 | ||
e33c2482 VP |
2288 | cxgb4i_host_template.can_queue = max_cmds; |
2289 | rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, max_conn, | |
2290 | &cxgb4i_host_template, cxgb4i_stt); | |
7b36b6e0 | 2291 | if (rc) |
2292 | goto err_out; | |
2293 | ||
2294 | for (i = 0; i < cdev->nports; i++) { | |
2295 | pi = netdev_priv(lldi->ports[i]); | |
2296 | cdev->hbas[i]->port_id = pi->port_id; | |
2297 | } | |
2298 | return cdev; | |
2299 | ||
2300 | err_out: | |
2301 | cxgbi_device_unregister(cdev); | |
2302 | return ERR_PTR(-ENOMEM); | |
2303 | } | |
2304 | ||
2305 | #define RX_PULL_LEN 128 | |
2306 | static int t4_uld_rx_handler(void *handle, const __be64 *rsp, | |
2307 | const struct pkt_gl *pgl) | |
2308 | { | |
2309 | const struct cpl_act_establish *rpl; | |
2310 | struct sk_buff *skb; | |
2311 | unsigned int opc; | |
2312 | struct cxgbi_device *cdev = handle; | |
2313 | ||
2314 | if (pgl == NULL) { | |
2315 | unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; | |
2316 | ||
24d3f95a | 2317 | skb = alloc_wr(len, 0, GFP_ATOMIC); |
7b36b6e0 | 2318 | if (!skb) |
2319 | goto nomem; | |
2320 | skb_copy_to_linear_data(skb, &rsp[1], len); | |
2321 | } else { | |
2322 | if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) { | |
2323 | pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", | |
2324 | pgl->va, be64_to_cpu(*rsp), | |
2325 | be64_to_cpu(*(u64 *)pgl->va), | |
2326 | pgl->tot_len); | |
2327 | return 0; | |
2328 | } | |
2329 | skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN); | |
2330 | if (unlikely(!skb)) | |
2331 | goto nomem; | |
2332 | } | |
2333 | ||
2334 | rpl = (struct cpl_act_establish *)skb->data; | |
2335 | opc = rpl->ot.opcode; | |
2336 | log_debug(1 << CXGBI_DBG_TOE, | |
2337 | "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n", | |
2338 | cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb); | |
ccc495ef | 2339 | if (opc >= ARRAY_SIZE(cxgb4i_cplhandlers) || !cxgb4i_cplhandlers[opc]) { |
7b36b6e0 | 2340 | pr_err("No handler for opcode 0x%x.\n", opc); |
2341 | __kfree_skb(skb); | |
ccc495ef DC |
2342 | } else |
2343 | cxgb4i_cplhandlers[opc](cdev, skb); | |
2344 | ||
7b36b6e0 | 2345 | return 0; |
2346 | nomem: | |
2347 | log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n"); | |
2348 | return 1; | |
2349 | } | |
2350 | ||
2351 | static int t4_uld_state_change(void *handle, enum cxgb4_state state) | |
2352 | { | |
2353 | struct cxgbi_device *cdev = handle; | |
2354 | ||
2355 | switch (state) { | |
2356 | case CXGB4_STATE_UP: | |
2357 | pr_info("cdev 0x%p, UP.\n", cdev); | |
7b36b6e0 | 2358 | break; |
2359 | case CXGB4_STATE_START_RECOVERY: | |
2360 | pr_info("cdev 0x%p, RECOVERY.\n", cdev); | |
2361 | /* close all connections */ | |
2362 | break; | |
2363 | case CXGB4_STATE_DOWN: | |
2364 | pr_info("cdev 0x%p, DOWN.\n", cdev); | |
2365 | break; | |
2366 | case CXGB4_STATE_DETACH: | |
2367 | pr_info("cdev 0x%p, DETACH.\n", cdev); | |
c3b331a3 | 2368 | cxgbi_device_unregister(cdev); |
7b36b6e0 | 2369 | break; |
2370 | default: | |
2371 | pr_info("cdev 0x%p, unknown state %d.\n", cdev, state); | |
2372 | break; | |
2373 | } | |
2374 | return 0; | |
2375 | } | |
2376 | ||
b5a5fe4e VP |
2377 | #ifdef CONFIG_CHELSIO_T4_DCB |
2378 | static int | |
2379 | cxgb4_dcb_change_notify(struct notifier_block *self, unsigned long val, | |
2380 | void *data) | |
2381 | { | |
2382 | int i, port = 0xFF; | |
2383 | struct net_device *ndev; | |
2384 | struct cxgbi_device *cdev = NULL; | |
2385 | struct dcb_app_type *iscsi_app = data; | |
2386 | struct cxgbi_ports_map *pmap; | |
2387 | u8 priority; | |
2388 | ||
2389 | if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) { | |
152e30fc VP |
2390 | if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) && |
2391 | (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY)) | |
b5a5fe4e VP |
2392 | return NOTIFY_DONE; |
2393 | ||
2394 | priority = iscsi_app->app.priority; | |
2395 | } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) { | |
2396 | if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM) | |
2397 | return NOTIFY_DONE; | |
2398 | ||
2399 | if (!iscsi_app->app.priority) | |
2400 | return NOTIFY_DONE; | |
2401 | ||
2402 | priority = ffs(iscsi_app->app.priority) - 1; | |
2403 | } else { | |
2404 | return NOTIFY_DONE; | |
2405 | } | |
2406 | ||
2407 | if (iscsi_app->app.protocol != 3260) | |
2408 | return NOTIFY_DONE; | |
2409 | ||
2410 | log_debug(1 << CXGBI_DBG_ISCSI, "iSCSI priority for ifid %d is %u\n", | |
2411 | iscsi_app->ifindex, priority); | |
2412 | ||
2413 | ndev = dev_get_by_index(&init_net, iscsi_app->ifindex); | |
2414 | if (!ndev) | |
2415 | return NOTIFY_DONE; | |
2416 | ||
2417 | cdev = cxgbi_device_find_by_netdev_rcu(ndev, &port); | |
2418 | ||
2419 | dev_put(ndev); | |
2420 | if (!cdev) | |
2421 | return NOTIFY_DONE; | |
2422 | ||
2423 | pmap = &cdev->pmap; | |
2424 | ||
2425 | for (i = 0; i < pmap->used; i++) { | |
2426 | if (pmap->port_csk[i]) { | |
2427 | struct cxgbi_sock *csk = pmap->port_csk[i]; | |
2428 | ||
2429 | if (csk->dcb_priority != priority) { | |
2430 | iscsi_conn_failure(csk->user_data, | |
2431 | ISCSI_ERR_CONN_FAILED); | |
2432 | pr_info("Restarting iSCSI connection %p with " | |
2433 | "priority %u->%u.\n", csk, | |
2434 | csk->dcb_priority, priority); | |
2435 | } | |
2436 | } | |
2437 | } | |
2438 | return NOTIFY_OK; | |
2439 | } | |
2440 | #endif | |
2441 | ||
7b36b6e0 | 2442 | static int __init cxgb4i_init_module(void) |
2443 | { | |
2444 | int rc; | |
2445 | ||
2446 | printk(KERN_INFO "%s", version); | |
2447 | ||
2448 | rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt); | |
2449 | if (rc < 0) | |
2450 | return rc; | |
2451 | cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info); | |
759a0cc5 | 2452 | |
b5a5fe4e VP |
2453 | #ifdef CONFIG_CHELSIO_T4_DCB |
2454 | pr_info("%s dcb enabled.\n", DRV_MODULE_NAME); | |
2455 | register_dcbevent_notifier(&cxgb4_dcb_change); | |
2456 | #endif | |
7b36b6e0 | 2457 | return 0; |
2458 | } | |
2459 | ||
2460 | static void __exit cxgb4i_exit_module(void) | |
2461 | { | |
b5a5fe4e VP |
2462 | #ifdef CONFIG_CHELSIO_T4_DCB |
2463 | unregister_dcbevent_notifier(&cxgb4_dcb_change); | |
2464 | #endif | |
7b36b6e0 | 2465 | cxgb4_unregister_uld(CXGB4_ULD_ISCSI); |
2466 | cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4); | |
2467 | cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt); | |
2468 | } | |
2469 | ||
2470 | module_init(cxgb4i_init_module); | |
2471 | module_exit(cxgb4i_exit_module); |