]>
Commit | Line | Data |
---|---|---|
9730ffcb VP |
1 | /* |
2 | * Copyright (c) 2016 Chelsio Communications, Inc. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | ||
9 | #include <linux/workqueue.h> | |
10 | #include <linux/kthread.h> | |
174cd4b1 IM |
11 | #include <linux/sched/signal.h> |
12 | ||
9730ffcb | 13 | #include <asm/unaligned.h> |
3bc71e1f | 14 | #include <net/tcp.h> |
9730ffcb VP |
15 | #include <target/target_core_base.h> |
16 | #include <target/target_core_fabric.h> | |
17 | #include "cxgbit.h" | |
18 | ||
19 | struct sge_opaque_hdr { | |
20 | void *dev; | |
21 | dma_addr_t addr[MAX_SKB_FRAGS + 1]; | |
22 | }; | |
23 | ||
24 | static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; | |
25 | ||
26 | #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ | |
27 | sizeof(struct fw_ofld_tx_data_wr)) | |
28 | ||
29 | static struct sk_buff * | |
30 | __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) | |
31 | { | |
32 | struct sk_buff *skb = NULL; | |
33 | u8 submode = 0; | |
34 | int errcode; | |
35 | static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; | |
36 | ||
37 | if (len) { | |
38 | skb = alloc_skb_with_frags(hdr_len, len, | |
39 | 0, &errcode, | |
40 | GFP_KERNEL); | |
41 | if (!skb) | |
42 | return NULL; | |
43 | ||
44 | skb_reserve(skb, TX_HDR_LEN); | |
45 | skb_reset_transport_header(skb); | |
46 | __skb_put(skb, ISCSI_HDR_LEN); | |
47 | skb->data_len = len; | |
48 | skb->len += len; | |
49 | submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); | |
50 | ||
51 | } else { | |
52 | u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; | |
53 | ||
54 | skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); | |
55 | if (!skb) | |
56 | return NULL; | |
57 | ||
58 | skb_reserve(skb, TX_HDR_LEN + iso_len); | |
59 | skb_reset_transport_header(skb); | |
60 | __skb_put(skb, ISCSI_HDR_LEN); | |
61 | } | |
62 | ||
63 | submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); | |
64 | cxgbit_skcb_submode(skb) = submode; | |
65 | cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; | |
66 | cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; | |
67 | return skb; | |
68 | } | |
69 | ||
70 | static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) | |
71 | { | |
72 | return __cxgbit_alloc_skb(csk, len, false); | |
73 | } | |
74 | ||
75 | /* | |
76 | * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data | |
77 | * @skb: the packet | |
78 | * | |
79 | * Returns true if a packet can be sent as an offload WR with immediate | |
80 | * data. We currently use the same limit as for Ethernet packets. | |
81 | */ | |
82 | static int cxgbit_is_ofld_imm(const struct sk_buff *skb) | |
83 | { | |
84 | int length = skb->len; | |
85 | ||
86 | if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) | |
87 | length += sizeof(struct fw_ofld_tx_data_wr); | |
88 | ||
89 | if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) | |
90 | length += sizeof(struct cpl_tx_data_iso); | |
91 | ||
92 | #define MAX_IMM_TX_PKT_LEN 256 | |
93 | return length <= MAX_IMM_TX_PKT_LEN; | |
94 | } | |
95 | ||
96 | /* | |
97 | * cxgbit_sgl_len - calculates the size of an SGL of the given capacity | |
98 | * @n: the number of SGL entries | |
99 | * Calculates the number of flits needed for a scatter/gather list that | |
100 | * can hold the given number of entries. | |
101 | */ | |
102 | static inline unsigned int cxgbit_sgl_len(unsigned int n) | |
103 | { | |
104 | n--; | |
105 | return (3 * n) / 2 + (n & 1) + 2; | |
106 | } | |
107 | ||
108 | /* | |
109 | * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet | |
110 | * @skb: the packet | |
111 | * | |
112 | * Returns the number of flits needed for the given offload packet. | |
113 | * These packets are already fully constructed and no additional headers | |
114 | * will be added. | |
115 | */ | |
116 | static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) | |
117 | { | |
118 | unsigned int flits, cnt; | |
119 | ||
120 | if (cxgbit_is_ofld_imm(skb)) | |
121 | return DIV_ROUND_UP(skb->len, 8); | |
122 | flits = skb_transport_offset(skb) / 8; | |
123 | cnt = skb_shinfo(skb)->nr_frags; | |
124 | if (skb_tail_pointer(skb) != skb_transport_header(skb)) | |
125 | cnt++; | |
126 | return flits + cxgbit_sgl_len(cnt); | |
127 | } | |
128 | ||
129 | #define CXGBIT_ISO_FSLICE 0x1 | |
130 | #define CXGBIT_ISO_LSLICE 0x2 | |
131 | static void | |
132 | cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) | |
133 | { | |
134 | struct cpl_tx_data_iso *cpl; | |
135 | unsigned int submode = cxgbit_skcb_submode(skb); | |
136 | unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); | |
137 | unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); | |
138 | ||
d58ff351 | 139 | cpl = __skb_push(skb, sizeof(*cpl)); |
9730ffcb VP |
140 | |
141 | cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | | |
142 | CPL_TX_DATA_ISO_FIRST_V(fslice) | | |
143 | CPL_TX_DATA_ISO_LAST_V(lslice) | | |
144 | CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | | |
145 | CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | | |
146 | CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | | |
147 | CPL_TX_DATA_ISO_IMMEDIATE_V(0) | | |
148 | CPL_TX_DATA_ISO_SCSI_V(2)); | |
149 | ||
150 | cpl->ahs_len = 0; | |
151 | cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); | |
152 | cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); | |
153 | cpl->len = htonl(iso_info->len); | |
154 | cpl->reserved2_seglen_offset = htonl(0); | |
155 | cpl->datasn_offset = htonl(0); | |
156 | cpl->buffer_offset = htonl(0); | |
157 | cpl->reserved3 = 0; | |
158 | ||
159 | __skb_pull(skb, sizeof(*cpl)); | |
160 | } | |
161 | ||
162 | static void | |
163 | cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, | |
164 | u32 len, u32 credits, u32 compl) | |
165 | { | |
166 | struct fw_ofld_tx_data_wr *req; | |
bdec5188 | 167 | const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; |
9730ffcb VP |
168 | u32 submode = cxgbit_skcb_submode(skb); |
169 | u32 wr_ulp_mode = 0; | |
170 | u32 hdr_size = sizeof(*req); | |
171 | u32 opcode = FW_OFLD_TX_DATA_WR; | |
172 | u32 immlen = 0; | |
bdec5188 VP |
173 | u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : |
174 | T6_TX_FORCE_F; | |
9730ffcb VP |
175 | |
176 | if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { | |
177 | opcode = FW_ISCSI_TX_DATA_WR; | |
178 | immlen += sizeof(struct cpl_tx_data_iso); | |
179 | hdr_size += sizeof(struct cpl_tx_data_iso); | |
180 | submode |= 8; | |
181 | } | |
182 | ||
183 | if (cxgbit_is_ofld_imm(skb)) | |
184 | immlen += dlen; | |
185 | ||
d58ff351 | 186 | req = __skb_push(skb, hdr_size); |
9730ffcb VP |
187 | req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | |
188 | FW_WR_COMPL_V(compl) | | |
189 | FW_WR_IMMDLEN_V(immlen)); | |
190 | req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | | |
191 | FW_WR_LEN16_V(credits)); | |
192 | req->plen = htonl(len); | |
193 | wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | | |
194 | FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); | |
195 | ||
196 | req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | | |
197 | FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); | |
198 | } | |
199 | ||
200 | static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) | |
201 | { | |
202 | kfree_skb(skb); | |
203 | } | |
204 | ||
205 | void cxgbit_push_tx_frames(struct cxgbit_sock *csk) | |
206 | { | |
207 | struct sk_buff *skb; | |
208 | ||
209 | while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { | |
210 | u32 dlen = skb->len; | |
211 | u32 len = skb->len; | |
212 | u32 credits_needed; | |
213 | u32 compl = 0; | |
214 | u32 flowclen16 = 0; | |
215 | u32 iso_cpl_len = 0; | |
216 | ||
217 | if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) | |
218 | iso_cpl_len = sizeof(struct cpl_tx_data_iso); | |
219 | ||
220 | if (cxgbit_is_ofld_imm(skb)) | |
221 | credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); | |
222 | else | |
223 | credits_needed = DIV_ROUND_UP((8 * | |
224 | cxgbit_calc_tx_flits_ofld(skb)) + | |
225 | iso_cpl_len, 16); | |
226 | ||
227 | if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) | |
228 | credits_needed += DIV_ROUND_UP( | |
229 | sizeof(struct fw_ofld_tx_data_wr), 16); | |
230 | /* | |
231 | * Assumes the initial credits is large enough to support | |
232 | * fw_flowc_wr plus largest possible first payload | |
233 | */ | |
234 | ||
235 | if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { | |
236 | flowclen16 = cxgbit_send_tx_flowc_wr(csk); | |
237 | csk->wr_cred -= flowclen16; | |
238 | csk->wr_una_cred += flowclen16; | |
239 | } | |
240 | ||
241 | if (csk->wr_cred < credits_needed) { | |
242 | pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", | |
243 | csk, skb->len, skb->data_len, | |
244 | credits_needed, csk->wr_cred); | |
245 | break; | |
246 | } | |
247 | __skb_unlink(skb, &csk->txq); | |
248 | set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); | |
5cadafb2 | 249 | skb->csum = (__force __wsum)(credits_needed + flowclen16); |
9730ffcb VP |
250 | csk->wr_cred -= credits_needed; |
251 | csk->wr_una_cred += credits_needed; | |
252 | ||
253 | pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", | |
254 | csk, skb->len, skb->data_len, credits_needed, | |
255 | csk->wr_cred, csk->wr_una_cred); | |
256 | ||
257 | if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { | |
258 | len += cxgbit_skcb_tx_extralen(skb); | |
259 | ||
260 | if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || | |
261 | (!before(csk->write_seq, | |
262 | csk->snd_una + csk->snd_win))) { | |
263 | compl = 1; | |
264 | csk->wr_una_cred = 0; | |
265 | } | |
266 | ||
267 | cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, | |
268 | compl); | |
269 | csk->snd_nxt += len; | |
270 | ||
271 | } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || | |
272 | (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { | |
273 | struct cpl_close_con_req *req = | |
274 | (struct cpl_close_con_req *)skb->data; | |
275 | req->wr.wr_hi |= htonl(FW_WR_COMPL_F); | |
276 | csk->wr_una_cred = 0; | |
277 | } | |
278 | ||
279 | cxgbit_sock_enqueue_wr(csk, skb); | |
280 | t4_set_arp_err_handler(skb, csk, | |
281 | cxgbit_arp_failure_skb_discard); | |
282 | ||
283 | pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", | |
284 | csk, csk->tid, skb, len); | |
285 | ||
286 | cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); | |
287 | } | |
288 | } | |
289 | ||
290 | static bool cxgbit_lock_sock(struct cxgbit_sock *csk) | |
291 | { | |
292 | spin_lock_bh(&csk->lock); | |
293 | ||
294 | if (before(csk->write_seq, csk->snd_una + csk->snd_win)) | |
295 | csk->lock_owner = true; | |
296 | ||
297 | spin_unlock_bh(&csk->lock); | |
298 | ||
299 | return csk->lock_owner; | |
300 | } | |
301 | ||
302 | static void cxgbit_unlock_sock(struct cxgbit_sock *csk) | |
303 | { | |
304 | struct sk_buff_head backlogq; | |
305 | struct sk_buff *skb; | |
306 | void (*fn)(struct cxgbit_sock *, struct sk_buff *); | |
307 | ||
308 | skb_queue_head_init(&backlogq); | |
309 | ||
310 | spin_lock_bh(&csk->lock); | |
311 | while (skb_queue_len(&csk->backlogq)) { | |
312 | skb_queue_splice_init(&csk->backlogq, &backlogq); | |
313 | spin_unlock_bh(&csk->lock); | |
314 | ||
315 | while ((skb = __skb_dequeue(&backlogq))) { | |
316 | fn = cxgbit_skcb_rx_backlog_fn(skb); | |
317 | fn(csk, skb); | |
318 | } | |
319 | ||
320 | spin_lock_bh(&csk->lock); | |
321 | } | |
322 | ||
323 | csk->lock_owner = false; | |
324 | spin_unlock_bh(&csk->lock); | |
325 | } | |
326 | ||
327 | static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) | |
328 | { | |
329 | int ret = 0; | |
330 | ||
331 | wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); | |
332 | ||
333 | if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || | |
334 | signal_pending(current))) { | |
335 | __kfree_skb(skb); | |
336 | __skb_queue_purge(&csk->ppodq); | |
337 | ret = -1; | |
338 | spin_lock_bh(&csk->lock); | |
339 | if (csk->lock_owner) { | |
340 | spin_unlock_bh(&csk->lock); | |
341 | goto unlock; | |
342 | } | |
343 | spin_unlock_bh(&csk->lock); | |
344 | return ret; | |
345 | } | |
346 | ||
347 | csk->write_seq += skb->len + | |
348 | cxgbit_skcb_tx_extralen(skb); | |
349 | ||
350 | skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); | |
351 | __skb_queue_tail(&csk->txq, skb); | |
352 | cxgbit_push_tx_frames(csk); | |
353 | ||
354 | unlock: | |
355 | cxgbit_unlock_sock(csk); | |
356 | return ret; | |
357 | } | |
358 | ||
359 | static int | |
360 | cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, | |
361 | u32 data_length) | |
362 | { | |
363 | u32 i = 0, nr_frags = MAX_SKB_FRAGS; | |
364 | u32 padding = ((-data_length) & 3); | |
365 | struct scatterlist *sg; | |
366 | struct page *page; | |
367 | unsigned int page_off; | |
368 | ||
369 | if (padding) | |
370 | nr_frags--; | |
371 | ||
372 | /* | |
373 | * We know each entry in t_data_sg contains a page. | |
374 | */ | |
375 | sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; | |
376 | page_off = (data_offset % PAGE_SIZE); | |
377 | ||
378 | while (data_length && (i < nr_frags)) { | |
379 | u32 cur_len = min_t(u32, data_length, sg->length - page_off); | |
380 | ||
381 | page = sg_page(sg); | |
382 | ||
383 | get_page(page); | |
384 | skb_fill_page_desc(skb, i, page, sg->offset + page_off, | |
385 | cur_len); | |
386 | skb->data_len += cur_len; | |
387 | skb->len += cur_len; | |
388 | skb->truesize += cur_len; | |
389 | ||
390 | data_length -= cur_len; | |
391 | page_off = 0; | |
392 | sg = sg_next(sg); | |
393 | i++; | |
394 | } | |
395 | ||
396 | if (data_length) | |
397 | return -1; | |
398 | ||
399 | if (padding) { | |
400 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); | |
401 | if (!page) | |
402 | return -1; | |
403 | skb_fill_page_desc(skb, i, page, 0, padding); | |
404 | skb->data_len += padding; | |
405 | skb->len += padding; | |
406 | skb->truesize += padding; | |
407 | } | |
408 | ||
409 | return 0; | |
410 | } | |
411 | ||
412 | static int | |
413 | cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, | |
414 | struct iscsi_datain_req *dr) | |
415 | { | |
416 | struct iscsi_conn *conn = csk->conn; | |
417 | struct sk_buff *skb; | |
418 | struct iscsi_datain datain; | |
419 | struct cxgbit_iso_info iso_info; | |
420 | u32 data_length = cmd->se_cmd.data_length; | |
421 | u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; | |
422 | u32 num_pdu, plen, tx_data = 0; | |
423 | bool task_sense = !!(cmd->se_cmd.se_cmd_flags & | |
424 | SCF_TRANSPORT_TASK_SENSE); | |
425 | bool set_statsn = false; | |
426 | int ret = -1; | |
427 | ||
428 | while (data_length) { | |
429 | num_pdu = (data_length + mrdsl - 1) / mrdsl; | |
430 | if (num_pdu > csk->max_iso_npdu) | |
431 | num_pdu = csk->max_iso_npdu; | |
432 | ||
433 | plen = num_pdu * mrdsl; | |
434 | if (plen > data_length) | |
435 | plen = data_length; | |
436 | ||
437 | skb = __cxgbit_alloc_skb(csk, 0, true); | |
438 | if (unlikely(!skb)) | |
439 | return -ENOMEM; | |
440 | ||
441 | memset(skb->data, 0, ISCSI_HDR_LEN); | |
442 | cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; | |
443 | cxgbit_skcb_submode(skb) |= (csk->submode & | |
444 | CXGBIT_SUBMODE_DCRC); | |
445 | cxgbit_skcb_tx_extralen(skb) = (num_pdu * | |
446 | cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + | |
447 | ((num_pdu - 1) * ISCSI_HDR_LEN); | |
448 | ||
449 | memset(&datain, 0, sizeof(struct iscsi_datain)); | |
450 | memset(&iso_info, 0, sizeof(iso_info)); | |
451 | ||
452 | if (!tx_data) | |
453 | iso_info.flags |= CXGBIT_ISO_FSLICE; | |
454 | ||
455 | if (!(data_length - plen)) { | |
456 | iso_info.flags |= CXGBIT_ISO_LSLICE; | |
457 | if (!task_sense) { | |
458 | datain.flags = ISCSI_FLAG_DATA_STATUS; | |
459 | iscsit_increment_maxcmdsn(cmd, conn->sess); | |
460 | cmd->stat_sn = conn->stat_sn++; | |
461 | set_statsn = true; | |
462 | } | |
463 | } | |
464 | ||
465 | iso_info.burst_len = num_pdu * mrdsl; | |
466 | iso_info.mpdu = mrdsl; | |
467 | iso_info.len = ISCSI_HDR_LEN + plen; | |
468 | ||
469 | cxgbit_cpl_tx_data_iso(skb, &iso_info); | |
470 | ||
471 | datain.offset = tx_data; | |
472 | datain.data_sn = cmd->data_sn - 1; | |
473 | ||
474 | iscsit_build_datain_pdu(cmd, conn, &datain, | |
475 | (struct iscsi_data_rsp *)skb->data, | |
476 | set_statsn); | |
477 | ||
478 | ret = cxgbit_map_skb(cmd, skb, tx_data, plen); | |
479 | if (unlikely(ret)) { | |
480 | __kfree_skb(skb); | |
481 | goto out; | |
482 | } | |
483 | ||
484 | ret = cxgbit_queue_skb(csk, skb); | |
485 | if (unlikely(ret)) | |
486 | goto out; | |
487 | ||
488 | tx_data += plen; | |
489 | data_length -= plen; | |
490 | ||
491 | cmd->read_data_done += plen; | |
492 | cmd->data_sn += num_pdu; | |
493 | } | |
494 | ||
495 | dr->dr_complete = DATAIN_COMPLETE_NORMAL; | |
496 | ||
497 | return 0; | |
498 | ||
499 | out: | |
500 | return ret; | |
501 | } | |
502 | ||
503 | static int | |
504 | cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, | |
505 | const struct iscsi_datain *datain) | |
506 | { | |
507 | struct sk_buff *skb; | |
508 | int ret = 0; | |
509 | ||
510 | skb = cxgbit_alloc_skb(csk, 0); | |
511 | if (unlikely(!skb)) | |
512 | return -ENOMEM; | |
513 | ||
514 | memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); | |
515 | ||
516 | if (datain->length) { | |
517 | cxgbit_skcb_submode(skb) |= (csk->submode & | |
518 | CXGBIT_SUBMODE_DCRC); | |
519 | cxgbit_skcb_tx_extralen(skb) = | |
520 | cxgbit_digest_len[cxgbit_skcb_submode(skb)]; | |
521 | } | |
522 | ||
523 | ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); | |
524 | if (ret < 0) { | |
525 | __kfree_skb(skb); | |
526 | return ret; | |
527 | } | |
528 | ||
529 | return cxgbit_queue_skb(csk, skb); | |
530 | } | |
531 | ||
532 | static int | |
533 | cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, | |
534 | struct iscsi_datain_req *dr, | |
535 | const struct iscsi_datain *datain) | |
536 | { | |
537 | struct cxgbit_sock *csk = conn->context; | |
538 | u32 data_length = cmd->se_cmd.data_length; | |
539 | u32 padding = ((-data_length) & 3); | |
540 | u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; | |
541 | ||
542 | if ((data_length > mrdsl) && (!dr->recovery) && | |
543 | (!padding) && (!datain->offset) && csk->max_iso_npdu) { | |
544 | atomic_long_add(data_length - datain->length, | |
545 | &conn->sess->tx_data_octets); | |
546 | return cxgbit_tx_datain_iso(csk, cmd, dr); | |
547 | } | |
548 | ||
549 | return cxgbit_tx_datain(csk, cmd, datain); | |
550 | } | |
551 | ||
552 | static int | |
553 | cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, | |
554 | const void *data_buf, u32 data_buf_len) | |
555 | { | |
556 | struct cxgbit_sock *csk = conn->context; | |
557 | struct sk_buff *skb; | |
558 | u32 padding = ((-data_buf_len) & 3); | |
559 | ||
560 | skb = cxgbit_alloc_skb(csk, data_buf_len + padding); | |
561 | if (unlikely(!skb)) | |
562 | return -ENOMEM; | |
563 | ||
564 | memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); | |
565 | ||
566 | if (data_buf_len) { | |
567 | u32 pad_bytes = 0; | |
568 | ||
569 | skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); | |
570 | ||
571 | if (padding) | |
572 | skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, | |
573 | &pad_bytes, padding); | |
574 | } | |
575 | ||
576 | cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ | |
577 | cxgbit_skcb_submode(skb)]; | |
578 | ||
579 | return cxgbit_queue_skb(csk, skb); | |
580 | } | |
581 | ||
582 | int | |
583 | cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, | |
584 | struct iscsi_datain_req *dr, const void *buf, u32 buf_len) | |
585 | { | |
586 | if (dr) | |
587 | return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); | |
588 | else | |
589 | return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); | |
590 | } | |
591 | ||
592 | int cxgbit_validate_params(struct iscsi_conn *conn) | |
593 | { | |
594 | struct cxgbit_sock *csk = conn->context; | |
595 | struct cxgbit_device *cdev = csk->com.cdev; | |
596 | struct iscsi_param *param; | |
597 | u32 max_xmitdsl; | |
598 | ||
599 | param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, | |
600 | conn->param_list); | |
601 | if (!param) | |
602 | return -1; | |
603 | ||
604 | if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) | |
605 | return -1; | |
606 | ||
607 | if (max_xmitdsl > cdev->mdsl) { | |
608 | if (iscsi_change_param_sprintf( | |
609 | conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) | |
610 | return -1; | |
611 | } | |
612 | ||
613 | return 0; | |
614 | } | |
615 | ||
616 | static int cxgbit_set_digest(struct cxgbit_sock *csk) | |
617 | { | |
618 | struct iscsi_conn *conn = csk->conn; | |
619 | struct iscsi_param *param; | |
620 | ||
621 | param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); | |
622 | if (!param) { | |
623 | pr_err("param not found key %s\n", HEADERDIGEST); | |
624 | return -1; | |
625 | } | |
626 | ||
627 | if (!strcmp(param->value, CRC32C)) | |
628 | csk->submode |= CXGBIT_SUBMODE_HCRC; | |
629 | ||
630 | param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); | |
631 | if (!param) { | |
632 | csk->submode = 0; | |
633 | pr_err("param not found key %s\n", DATADIGEST); | |
634 | return -1; | |
635 | } | |
636 | ||
637 | if (!strcmp(param->value, CRC32C)) | |
638 | csk->submode |= CXGBIT_SUBMODE_DCRC; | |
639 | ||
640 | if (cxgbit_setup_conn_digest(csk)) { | |
641 | csk->submode = 0; | |
642 | return -1; | |
643 | } | |
644 | ||
645 | return 0; | |
646 | } | |
647 | ||
648 | static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) | |
649 | { | |
650 | struct iscsi_conn *conn = csk->conn; | |
651 | struct iscsi_conn_ops *conn_ops = conn->conn_ops; | |
652 | struct iscsi_param *param; | |
653 | u32 mrdsl, mbl; | |
654 | u32 max_npdu, max_iso_npdu; | |
655 | ||
656 | if (conn->login->leading_connection) { | |
9730ffcb VP |
657 | param = iscsi_find_param_from_key(MAXBURSTLENGTH, |
658 | conn->param_list); | |
659 | if (!param) { | |
660 | pr_err("param not found key %s\n", MAXBURSTLENGTH); | |
661 | return -1; | |
662 | } | |
663 | ||
664 | if (kstrtou32(param->value, 0, &mbl) < 0) | |
665 | return -1; | |
666 | } else { | |
9730ffcb VP |
667 | mbl = conn->sess->sess_ops->MaxBurstLength; |
668 | } | |
669 | ||
670 | mrdsl = conn_ops->MaxRecvDataSegmentLength; | |
671 | max_npdu = mbl / mrdsl; | |
672 | ||
673 | max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / | |
674 | (ISCSI_HDR_LEN + mrdsl + | |
675 | cxgbit_digest_len[csk->submode]); | |
676 | ||
677 | csk->max_iso_npdu = min(max_npdu, max_iso_npdu); | |
678 | ||
679 | if (csk->max_iso_npdu <= 1) | |
680 | csk->max_iso_npdu = 0; | |
681 | ||
682 | return 0; | |
683 | } | |
684 | ||
5248788e VP |
685 | /* |
686 | * cxgbit_seq_pdu_inorder() | |
687 | * @csk: pointer to cxgbit socket structure | |
688 | * | |
689 | * This function checks whether data sequence and data | |
690 | * pdu are in order. | |
691 | * | |
692 | * Return: returns -1 on error, 0 if data sequence and | |
693 | * data pdu are in order, 1 if data sequence or data pdu | |
694 | * is not in order. | |
695 | */ | |
696 | static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk) | |
697 | { | |
698 | struct iscsi_conn *conn = csk->conn; | |
699 | struct iscsi_param *param; | |
700 | ||
701 | if (conn->login->leading_connection) { | |
702 | param = iscsi_find_param_from_key(DATASEQUENCEINORDER, | |
703 | conn->param_list); | |
704 | if (!param) { | |
705 | pr_err("param not found key %s\n", DATASEQUENCEINORDER); | |
706 | return -1; | |
707 | } | |
708 | ||
709 | if (strcmp(param->value, YES)) | |
710 | return 1; | |
711 | ||
712 | param = iscsi_find_param_from_key(DATAPDUINORDER, | |
713 | conn->param_list); | |
714 | if (!param) { | |
715 | pr_err("param not found key %s\n", DATAPDUINORDER); | |
716 | return -1; | |
717 | } | |
718 | ||
719 | if (strcmp(param->value, YES)) | |
720 | return 1; | |
721 | ||
722 | } else { | |
723 | if (!conn->sess->sess_ops->DataSequenceInOrder) | |
724 | return 1; | |
725 | if (!conn->sess->sess_ops->DataPDUInOrder) | |
726 | return 1; | |
727 | } | |
728 | ||
729 | return 0; | |
730 | } | |
731 | ||
9730ffcb VP |
732 | static int cxgbit_set_params(struct iscsi_conn *conn) |
733 | { | |
734 | struct cxgbit_sock *csk = conn->context; | |
735 | struct cxgbit_device *cdev = csk->com.cdev; | |
736 | struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; | |
737 | struct iscsi_conn_ops *conn_ops = conn->conn_ops; | |
738 | struct iscsi_param *param; | |
739 | u8 erl; | |
740 | ||
741 | if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) | |
742 | conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; | |
743 | ||
744 | if (conn->login->leading_connection) { | |
745 | param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, | |
746 | conn->param_list); | |
747 | if (!param) { | |
748 | pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); | |
749 | return -1; | |
750 | } | |
751 | if (kstrtou8(param->value, 0, &erl) < 0) | |
752 | return -1; | |
753 | } else { | |
754 | erl = conn->sess->sess_ops->ErrorRecoveryLevel; | |
755 | } | |
756 | ||
757 | if (!erl) { | |
5248788e VP |
758 | int ret; |
759 | ||
760 | ret = cxgbit_seq_pdu_inorder(csk); | |
761 | if (ret < 0) { | |
762 | return -1; | |
763 | } else if (ret > 0) { | |
764 | if (is_t5(cdev->lldi.adapter_type)) | |
765 | goto enable_ddp; | |
766 | else | |
767 | goto enable_digest; | |
768 | } | |
769 | ||
9730ffcb VP |
770 | if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { |
771 | if (cxgbit_set_iso_npdu(csk)) | |
772 | return -1; | |
773 | } | |
774 | ||
5248788e | 775 | enable_ddp: |
9730ffcb VP |
776 | if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { |
777 | if (cxgbit_setup_conn_pgidx(csk, | |
778 | ppm->tformat.pgsz_idx_dflt)) | |
779 | return -1; | |
780 | set_bit(CSK_DDP_ENABLE, &csk->com.flags); | |
781 | } | |
782 | } | |
783 | ||
5248788e | 784 | enable_digest: |
9730ffcb VP |
785 | if (cxgbit_set_digest(csk)) |
786 | return -1; | |
787 | ||
788 | return 0; | |
789 | } | |
790 | ||
791 | int | |
792 | cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, | |
793 | u32 length) | |
794 | { | |
795 | struct cxgbit_sock *csk = conn->context; | |
796 | struct sk_buff *skb; | |
797 | u32 padding_buf = 0; | |
798 | u8 padding = ((-length) & 3); | |
799 | ||
800 | skb = cxgbit_alloc_skb(csk, length + padding); | |
801 | if (!skb) | |
802 | return -ENOMEM; | |
803 | skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); | |
804 | skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); | |
805 | ||
806 | if (padding) | |
807 | skb_store_bits(skb, ISCSI_HDR_LEN + length, | |
808 | &padding_buf, padding); | |
809 | ||
810 | if (login->login_complete) { | |
811 | if (cxgbit_set_params(conn)) { | |
812 | kfree_skb(skb); | |
813 | return -1; | |
814 | } | |
815 | ||
816 | set_bit(CSK_LOGIN_DONE, &csk->com.flags); | |
817 | } | |
818 | ||
819 | if (cxgbit_queue_skb(csk, skb)) | |
820 | return -1; | |
821 | ||
822 | if ((!login->login_complete) && (!login->login_failed)) | |
823 | schedule_delayed_work(&conn->login_work, 0); | |
824 | ||
825 | return 0; | |
826 | } | |
827 | ||
828 | static void | |
829 | cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, | |
830 | unsigned int nents) | |
831 | { | |
832 | struct skb_seq_state st; | |
833 | const u8 *buf; | |
834 | unsigned int consumed = 0, buf_len; | |
835 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); | |
836 | ||
837 | skb_prepare_seq_read(skb, pdu_cb->doffset, | |
838 | pdu_cb->doffset + pdu_cb->dlen, | |
839 | &st); | |
840 | ||
841 | while (true) { | |
842 | buf_len = skb_seq_read(consumed, &buf, &st); | |
843 | if (!buf_len) { | |
844 | skb_abort_seq_read(&st); | |
845 | break; | |
846 | } | |
847 | ||
848 | consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, | |
849 | buf_len, consumed); | |
850 | } | |
851 | } | |
852 | ||
853 | static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) | |
854 | { | |
855 | struct iscsi_conn *conn = csk->conn; | |
856 | struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); | |
857 | struct cxgbit_cmd *ccmd; | |
858 | struct iscsi_cmd *cmd; | |
859 | ||
860 | cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); | |
861 | if (!cmd) { | |
862 | pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); | |
863 | return NULL; | |
864 | } | |
865 | ||
866 | ccmd = iscsit_priv_cmd(cmd); | |
867 | ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; | |
868 | ccmd->setup_ddp = true; | |
869 | ||
870 | return cmd; | |
871 | } | |
872 | ||
873 | static int | |
874 | cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, | |
875 | u32 length) | |
876 | { | |
877 | struct iscsi_conn *conn = cmd->conn; | |
878 | struct cxgbit_sock *csk = conn->context; | |
879 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
880 | ||
881 | if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { | |
882 | pr_err("ImmediateData CRC32C DataDigest error\n"); | |
883 | if (!conn->sess->sess_ops->ErrorRecoveryLevel) { | |
884 | pr_err("Unable to recover from" | |
885 | " Immediate Data digest failure while" | |
886 | " in ERL=0.\n"); | |
887 | iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, | |
888 | (unsigned char *)hdr); | |
889 | return IMMEDIATE_DATA_CANNOT_RECOVER; | |
890 | } | |
891 | ||
892 | iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, | |
893 | (unsigned char *)hdr); | |
894 | return IMMEDIATE_DATA_ERL1_CRC_FAILURE; | |
895 | } | |
896 | ||
897 | if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { | |
898 | struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); | |
899 | struct skb_shared_info *ssi = skb_shinfo(csk->skb); | |
900 | skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; | |
901 | ||
902 | sg_init_table(&ccmd->sg, 1); | |
903 | sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), | |
904 | dfrag->page_offset); | |
905 | get_page(dfrag->page.p); | |
906 | ||
907 | cmd->se_cmd.t_data_sg = &ccmd->sg; | |
908 | cmd->se_cmd.t_data_nents = 1; | |
909 | ||
910 | ccmd->release = true; | |
911 | } else { | |
912 | struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; | |
913 | u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); | |
914 | ||
915 | cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); | |
916 | } | |
917 | ||
918 | cmd->write_data_done += pdu_cb->dlen; | |
919 | ||
920 | if (cmd->write_data_done == cmd->se_cmd.data_length) { | |
921 | spin_lock_bh(&cmd->istate_lock); | |
922 | cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; | |
923 | cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; | |
924 | spin_unlock_bh(&cmd->istate_lock); | |
925 | } | |
926 | ||
927 | return IMMEDIATE_DATA_NORMAL_OPERATION; | |
928 | } | |
929 | ||
930 | static int | |
931 | cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, | |
932 | bool dump_payload) | |
933 | { | |
934 | struct iscsi_conn *conn = cmd->conn; | |
935 | int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; | |
936 | /* | |
937 | * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. | |
938 | */ | |
939 | if (dump_payload) | |
940 | goto after_immediate_data; | |
941 | ||
942 | immed_ret = cxgbit_handle_immediate_data(cmd, hdr, | |
943 | cmd->first_burst_len); | |
944 | after_immediate_data: | |
945 | if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { | |
946 | /* | |
947 | * A PDU/CmdSN carrying Immediate Data passed | |
948 | * DataCRC, check against ExpCmdSN/MaxCmdSN if | |
949 | * Immediate Bit is not set. | |
950 | */ | |
951 | cmdsn_ret = iscsit_sequence_cmd(conn, cmd, | |
952 | (unsigned char *)hdr, | |
953 | hdr->cmdsn); | |
954 | if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) | |
955 | return -1; | |
956 | ||
957 | if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { | |
958 | target_put_sess_cmd(&cmd->se_cmd); | |
959 | return 0; | |
960 | } else if (cmd->unsolicited_data) { | |
961 | iscsit_set_unsoliticed_dataout(cmd); | |
962 | } | |
963 | ||
964 | } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { | |
965 | /* | |
966 | * Immediate Data failed DataCRC and ERL>=1, | |
967 | * silently drop this PDU and let the initiator | |
968 | * plug the CmdSN gap. | |
969 | * | |
970 | * FIXME: Send Unsolicited NOPIN with reserved | |
971 | * TTT here to help the initiator figure out | |
972 | * the missing CmdSN, although they should be | |
973 | * intelligent enough to determine the missing | |
974 | * CmdSN and issue a retry to plug the sequence. | |
975 | */ | |
976 | cmd->i_state = ISTATE_REMOVE; | |
977 | iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); | |
978 | } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ | |
979 | return -1; | |
980 | ||
981 | return 0; | |
982 | } | |
983 | ||
984 | static int | |
985 | cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) | |
986 | { | |
987 | struct iscsi_conn *conn = csk->conn; | |
988 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
989 | struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; | |
990 | int rc; | |
991 | bool dump_payload = false; | |
992 | ||
993 | rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); | |
994 | if (rc < 0) | |
995 | return rc; | |
996 | ||
997 | if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && | |
998 | (pdu_cb->nr_dfrags == 1)) | |
999 | cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; | |
1000 | ||
1001 | rc = iscsit_process_scsi_cmd(conn, cmd, hdr); | |
1002 | if (rc < 0) | |
1003 | return 0; | |
1004 | else if (rc > 0) | |
1005 | dump_payload = true; | |
1006 | ||
1007 | if (!pdu_cb->dlen) | |
1008 | return 0; | |
1009 | ||
1010 | return cxgbit_get_immediate_data(cmd, hdr, dump_payload); | |
1011 | } | |
1012 | ||
1013 | static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) | |
1014 | { | |
1015 | struct scatterlist *sg_start; | |
1016 | struct iscsi_conn *conn = csk->conn; | |
1017 | struct iscsi_cmd *cmd = NULL; | |
1018 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
1019 | struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; | |
1020 | u32 data_offset = be32_to_cpu(hdr->offset); | |
1021 | u32 data_len = pdu_cb->dlen; | |
1022 | int rc, sg_nents, sg_off; | |
1023 | bool dcrc_err = false; | |
1024 | ||
79e57cfe VP |
1025 | if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) { |
1026 | u32 offset = be32_to_cpu(hdr->offset); | |
1027 | u32 ddp_data_len; | |
1028 | u32 payload_length = ntoh24(hdr->dlength); | |
1029 | bool success = false; | |
1030 | ||
1031 | cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0); | |
1032 | if (!cmd) | |
1033 | return 0; | |
1034 | ||
1035 | ddp_data_len = offset - cmd->write_data_done; | |
1036 | atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets); | |
1037 | ||
1038 | cmd->write_data_done = offset; | |
1039 | cmd->next_burst_len = ddp_data_len; | |
1040 | cmd->data_sn = be32_to_cpu(hdr->datasn); | |
1041 | ||
1042 | rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, | |
1043 | cmd, payload_length, &success); | |
1044 | if (rc < 0) | |
1045 | return rc; | |
1046 | else if (!success) | |
1047 | return 0; | |
1048 | } else { | |
1049 | rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); | |
1050 | if (rc < 0) | |
1051 | return rc; | |
1052 | else if (!cmd) | |
1053 | return 0; | |
1054 | } | |
9730ffcb VP |
1055 | |
1056 | if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { | |
1057 | pr_err("ITT: 0x%08x, Offset: %u, Length: %u," | |
1058 | " DataSN: 0x%08x\n", | |
1059 | hdr->itt, hdr->offset, data_len, | |
1060 | hdr->datasn); | |
1061 | ||
1062 | dcrc_err = true; | |
1063 | goto check_payload; | |
1064 | } | |
1065 | ||
1066 | pr_debug("DataOut data_len: %u, " | |
1067 | "write_data_done: %u, data_length: %u\n", | |
1068 | data_len, cmd->write_data_done, | |
1069 | cmd->se_cmd.data_length); | |
1070 | ||
1071 | if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { | |
1072 | sg_off = data_offset / PAGE_SIZE; | |
1073 | sg_start = &cmd->se_cmd.t_data_sg[sg_off]; | |
1074 | sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); | |
1075 | ||
1076 | cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); | |
1077 | } | |
1078 | ||
1079 | check_payload: | |
1080 | ||
1081 | rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); | |
1082 | if (rc < 0) | |
1083 | return rc; | |
1084 | ||
1085 | return 0; | |
1086 | } | |
1087 | ||
1088 | static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) | |
1089 | { | |
1090 | struct iscsi_conn *conn = csk->conn; | |
1091 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
1092 | struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; | |
1093 | unsigned char *ping_data = NULL; | |
1094 | u32 payload_length = pdu_cb->dlen; | |
1095 | int ret; | |
1096 | ||
1097 | ret = iscsit_setup_nop_out(conn, cmd, hdr); | |
1098 | if (ret < 0) | |
1099 | return 0; | |
1100 | ||
1101 | if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { | |
1102 | if (!conn->sess->sess_ops->ErrorRecoveryLevel) { | |
1103 | pr_err("Unable to recover from" | |
1104 | " NOPOUT Ping DataCRC failure while in" | |
1105 | " ERL=0.\n"); | |
1106 | ret = -1; | |
1107 | goto out; | |
1108 | } else { | |
1109 | /* | |
1110 | * drop this PDU and let the | |
1111 | * initiator plug the CmdSN gap. | |
1112 | */ | |
1113 | pr_info("Dropping NOPOUT" | |
1114 | " Command CmdSN: 0x%08x due to" | |
1115 | " DataCRC error.\n", hdr->cmdsn); | |
1116 | ret = 0; | |
1117 | goto out; | |
1118 | } | |
1119 | } | |
1120 | ||
1121 | /* | |
1122 | * Handle NOP-OUT payload for traditional iSCSI sockets | |
1123 | */ | |
1124 | if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { | |
1125 | ping_data = kzalloc(payload_length + 1, GFP_KERNEL); | |
1126 | if (!ping_data) { | |
1127 | pr_err("Unable to allocate memory for" | |
1128 | " NOPOUT ping data.\n"); | |
1129 | ret = -1; | |
1130 | goto out; | |
1131 | } | |
1132 | ||
1133 | skb_copy_bits(csk->skb, pdu_cb->doffset, | |
1134 | ping_data, payload_length); | |
1135 | ||
1136 | ping_data[payload_length] = '\0'; | |
1137 | /* | |
1138 | * Attach ping data to struct iscsi_cmd->buf_ptr. | |
1139 | */ | |
1140 | cmd->buf_ptr = ping_data; | |
1141 | cmd->buf_ptr_size = payload_length; | |
1142 | ||
1143 | pr_debug("Got %u bytes of NOPOUT ping" | |
1144 | " data.\n", payload_length); | |
1145 | pr_debug("Ping Data: \"%s\"\n", ping_data); | |
1146 | } | |
1147 | ||
1148 | return iscsit_process_nop_out(conn, cmd, hdr); | |
1149 | out: | |
1150 | if (cmd) | |
1151 | iscsit_free_cmd(cmd, false); | |
1152 | return ret; | |
1153 | } | |
1154 | ||
1155 | static int | |
1156 | cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) | |
1157 | { | |
1158 | struct iscsi_conn *conn = csk->conn; | |
1159 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
1160 | struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; | |
1161 | u32 payload_length = pdu_cb->dlen; | |
1162 | int rc; | |
1163 | unsigned char *text_in = NULL; | |
1164 | ||
1165 | rc = iscsit_setup_text_cmd(conn, cmd, hdr); | |
1166 | if (rc < 0) | |
1167 | return rc; | |
1168 | ||
1169 | if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { | |
1170 | if (!conn->sess->sess_ops->ErrorRecoveryLevel) { | |
1171 | pr_err("Unable to recover from" | |
1172 | " Text Data digest failure while in" | |
1173 | " ERL=0.\n"); | |
1174 | goto reject; | |
1175 | } else { | |
1176 | /* | |
1177 | * drop this PDU and let the | |
1178 | * initiator plug the CmdSN gap. | |
1179 | */ | |
1180 | pr_info("Dropping Text" | |
1181 | " Command CmdSN: 0x%08x due to" | |
1182 | " DataCRC error.\n", hdr->cmdsn); | |
1183 | return 0; | |
1184 | } | |
1185 | } | |
1186 | ||
1187 | if (payload_length) { | |
1188 | text_in = kzalloc(payload_length, GFP_KERNEL); | |
1189 | if (!text_in) { | |
1190 | pr_err("Unable to allocate text_in of payload_length: %u\n", | |
1191 | payload_length); | |
1192 | return -ENOMEM; | |
1193 | } | |
1194 | skb_copy_bits(csk->skb, pdu_cb->doffset, | |
1195 | text_in, payload_length); | |
1196 | ||
1197 | text_in[payload_length - 1] = '\0'; | |
1198 | ||
1199 | cmd->text_in_ptr = text_in; | |
1200 | } | |
1201 | ||
1202 | return iscsit_process_text_cmd(conn, cmd, hdr); | |
1203 | ||
1204 | reject: | |
1205 | return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, | |
1206 | pdu_cb->hdr); | |
1207 | } | |
1208 | ||
1209 | static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) | |
1210 | { | |
1211 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
1212 | struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; | |
1213 | struct iscsi_conn *conn = csk->conn; | |
1214 | struct iscsi_cmd *cmd = NULL; | |
1215 | u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); | |
1216 | int ret = -EINVAL; | |
1217 | ||
1218 | switch (opcode) { | |
1219 | case ISCSI_OP_SCSI_CMD: | |
1220 | cmd = cxgbit_allocate_cmd(csk); | |
1221 | if (!cmd) | |
1222 | goto reject; | |
1223 | ||
1224 | ret = cxgbit_handle_scsi_cmd(csk, cmd); | |
1225 | break; | |
1226 | case ISCSI_OP_SCSI_DATA_OUT: | |
1227 | ret = cxgbit_handle_iscsi_dataout(csk); | |
1228 | break; | |
1229 | case ISCSI_OP_NOOP_OUT: | |
1230 | if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { | |
1231 | cmd = cxgbit_allocate_cmd(csk); | |
1232 | if (!cmd) | |
1233 | goto reject; | |
1234 | } | |
1235 | ||
1236 | ret = cxgbit_handle_nop_out(csk, cmd); | |
1237 | break; | |
1238 | case ISCSI_OP_SCSI_TMFUNC: | |
1239 | cmd = cxgbit_allocate_cmd(csk); | |
1240 | if (!cmd) | |
1241 | goto reject; | |
1242 | ||
1243 | ret = iscsit_handle_task_mgt_cmd(conn, cmd, | |
1244 | (unsigned char *)hdr); | |
1245 | break; | |
1246 | case ISCSI_OP_TEXT: | |
1247 | if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { | |
1248 | cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); | |
1249 | if (!cmd) | |
1250 | goto reject; | |
1251 | } else { | |
1252 | cmd = cxgbit_allocate_cmd(csk); | |
1253 | if (!cmd) | |
1254 | goto reject; | |
1255 | } | |
1256 | ||
1257 | ret = cxgbit_handle_text_cmd(csk, cmd); | |
1258 | break; | |
1259 | case ISCSI_OP_LOGOUT: | |
1260 | cmd = cxgbit_allocate_cmd(csk); | |
1261 | if (!cmd) | |
1262 | goto reject; | |
1263 | ||
1264 | ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); | |
1265 | if (ret > 0) | |
1266 | wait_for_completion_timeout(&conn->conn_logout_comp, | |
1267 | SECONDS_FOR_LOGOUT_COMP | |
1268 | * HZ); | |
1269 | break; | |
1270 | case ISCSI_OP_SNACK: | |
1271 | ret = iscsit_handle_snack(conn, (unsigned char *)hdr); | |
1272 | break; | |
1273 | default: | |
1274 | pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); | |
1275 | dump_stack(); | |
1276 | break; | |
1277 | } | |
1278 | ||
1279 | return ret; | |
1280 | ||
1281 | reject: | |
1282 | return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, | |
1283 | (unsigned char *)hdr); | |
1284 | return ret; | |
1285 | } | |
1286 | ||
1287 | static int cxgbit_rx_opcode(struct cxgbit_sock *csk) | |
1288 | { | |
1289 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
1290 | struct iscsi_conn *conn = csk->conn; | |
1291 | struct iscsi_hdr *hdr = pdu_cb->hdr; | |
1292 | u8 opcode; | |
1293 | ||
1294 | if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { | |
1295 | atomic_long_inc(&conn->sess->conn_digest_errors); | |
1296 | goto transport_err; | |
1297 | } | |
1298 | ||
1299 | if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) | |
1300 | goto transport_err; | |
1301 | ||
1302 | opcode = hdr->opcode & ISCSI_OPCODE_MASK; | |
1303 | ||
1304 | if (conn->sess->sess_ops->SessionType && | |
1305 | ((!(opcode & ISCSI_OP_TEXT)) || | |
1306 | (!(opcode & ISCSI_OP_LOGOUT)))) { | |
1307 | pr_err("Received illegal iSCSI Opcode: 0x%02x" | |
1308 | " while in Discovery Session, rejecting.\n", opcode); | |
1309 | iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, | |
1310 | (unsigned char *)hdr); | |
1311 | goto transport_err; | |
1312 | } | |
1313 | ||
1314 | if (cxgbit_target_rx_opcode(csk) < 0) | |
1315 | goto transport_err; | |
1316 | ||
1317 | return 0; | |
1318 | ||
1319 | transport_err: | |
1320 | return -1; | |
1321 | } | |
1322 | ||
1323 | static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) | |
1324 | { | |
1325 | struct iscsi_conn *conn = csk->conn; | |
1326 | struct iscsi_login *login = conn->login; | |
1327 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); | |
1328 | struct iscsi_login_req *login_req; | |
1329 | ||
1330 | login_req = (struct iscsi_login_req *)login->req; | |
1331 | memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); | |
1332 | ||
1333 | pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," | |
1334 | " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", | |
1335 | login_req->flags, login_req->itt, login_req->cmdsn, | |
1336 | login_req->exp_statsn, login_req->cid, pdu_cb->dlen); | |
1337 | /* | |
1338 | * Setup the initial iscsi_login values from the leading | |
1339 | * login request PDU. | |
1340 | */ | |
1341 | if (login->first_request) { | |
1342 | login_req = (struct iscsi_login_req *)login->req; | |
1343 | login->leading_connection = (!login_req->tsih) ? 1 : 0; | |
1344 | login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( | |
1345 | login_req->flags); | |
1346 | login->version_min = login_req->min_version; | |
1347 | login->version_max = login_req->max_version; | |
1348 | memcpy(login->isid, login_req->isid, 6); | |
1349 | login->cmd_sn = be32_to_cpu(login_req->cmdsn); | |
1350 | login->init_task_tag = login_req->itt; | |
1351 | login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); | |
1352 | login->cid = be16_to_cpu(login_req->cid); | |
1353 | login->tsih = be16_to_cpu(login_req->tsih); | |
1354 | } | |
1355 | ||
1356 | if (iscsi_target_check_login_request(conn, login) < 0) | |
1357 | return -1; | |
1358 | ||
1359 | memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); | |
1360 | skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); | |
1361 | ||
1362 | return 0; | |
1363 | } | |
1364 | ||
1365 | static int | |
1366 | cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) | |
1367 | { | |
1368 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); | |
1369 | int ret; | |
1370 | ||
1371 | cxgbit_rx_pdu_cb(skb) = pdu_cb; | |
1372 | ||
1373 | csk->skb = skb; | |
1374 | ||
1375 | if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { | |
1376 | ret = cxgbit_rx_login_pdu(csk); | |
1377 | set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); | |
1378 | } else { | |
1379 | ret = cxgbit_rx_opcode(csk); | |
1380 | } | |
1381 | ||
1382 | return ret; | |
1383 | } | |
1384 | ||
1385 | static void cxgbit_lro_skb_dump(struct sk_buff *skb) | |
1386 | { | |
1387 | struct skb_shared_info *ssi = skb_shinfo(skb); | |
1388 | struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); | |
1389 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); | |
1390 | u8 i; | |
1391 | ||
1392 | pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", | |
1393 | skb, skb->head, skb->data, skb->len, skb->data_len, | |
1394 | ssi->nr_frags); | |
1395 | pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", | |
1396 | skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); | |
1397 | ||
1398 | for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) | |
1399 | pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " | |
1400 | "frags %u.\n", | |
1401 | skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, | |
1402 | pdu_cb->ddigest, pdu_cb->frags); | |
1403 | for (i = 0; i < ssi->nr_frags; i++) | |
1404 | pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", | |
1405 | skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); | |
1406 | } | |
1407 | ||
1408 | static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) | |
1409 | { | |
1410 | struct sk_buff *skb = csk->lro_hskb; | |
1411 | struct skb_shared_info *ssi = skb_shinfo(skb); | |
1412 | u8 i; | |
1413 | ||
1414 | memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); | |
1415 | for (i = 0; i < ssi->nr_frags; i++) | |
1416 | put_page(skb_frag_page(&ssi->frags[i])); | |
1417 | ssi->nr_frags = 0; | |
79e57cfe VP |
1418 | skb->data_len = 0; |
1419 | skb->truesize -= skb->len; | |
1420 | skb->len = 0; | |
9730ffcb VP |
1421 | } |
1422 | ||
1423 | static void | |
1424 | cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) | |
1425 | { | |
1426 | struct sk_buff *hskb = csk->lro_hskb; | |
1427 | struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); | |
1428 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); | |
1429 | struct skb_shared_info *hssi = skb_shinfo(hskb); | |
1430 | struct skb_shared_info *ssi = skb_shinfo(skb); | |
1431 | unsigned int len = 0; | |
1432 | ||
1433 | if (pdu_cb->flags & PDUCBF_RX_HDR) { | |
79e57cfe VP |
1434 | u8 hfrag_idx = hssi->nr_frags; |
1435 | ||
1436 | hpdu_cb->flags |= pdu_cb->flags; | |
9730ffcb VP |
1437 | hpdu_cb->seq = pdu_cb->seq; |
1438 | hpdu_cb->hdr = pdu_cb->hdr; | |
1439 | hpdu_cb->hlen = pdu_cb->hlen; | |
1440 | ||
79e57cfe | 1441 | memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx], |
9730ffcb VP |
1442 | sizeof(skb_frag_t)); |
1443 | ||
79e57cfe VP |
1444 | get_page(skb_frag_page(&hssi->frags[hfrag_idx])); |
1445 | hssi->nr_frags++; | |
1446 | hpdu_cb->frags++; | |
1447 | hpdu_cb->hfrag_idx = hfrag_idx; | |
9730ffcb | 1448 | |
79e57cfe VP |
1449 | len = hssi->frags[hfrag_idx].size; |
1450 | hskb->len += len; | |
1451 | hskb->data_len += len; | |
1452 | hskb->truesize += len; | |
9730ffcb VP |
1453 | } |
1454 | ||
1455 | if (pdu_cb->flags & PDUCBF_RX_DATA) { | |
79e57cfe | 1456 | u8 dfrag_idx = hssi->nr_frags, i; |
9730ffcb VP |
1457 | |
1458 | hpdu_cb->flags |= pdu_cb->flags; | |
79e57cfe | 1459 | hpdu_cb->dfrag_idx = dfrag_idx; |
9730ffcb VP |
1460 | |
1461 | len = 0; | |
79e57cfe VP |
1462 | for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) { |
1463 | memcpy(&hssi->frags[dfrag_idx], | |
9730ffcb VP |
1464 | &ssi->frags[pdu_cb->dfrag_idx + i], |
1465 | sizeof(skb_frag_t)); | |
1466 | ||
79e57cfe | 1467 | get_page(skb_frag_page(&hssi->frags[dfrag_idx])); |
9730ffcb | 1468 | |
79e57cfe | 1469 | len += hssi->frags[dfrag_idx].size; |
9730ffcb VP |
1470 | |
1471 | hssi->nr_frags++; | |
1472 | hpdu_cb->frags++; | |
1473 | } | |
1474 | ||
1475 | hpdu_cb->dlen = pdu_cb->dlen; | |
1476 | hpdu_cb->doffset = hpdu_cb->hlen; | |
1477 | hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; | |
9730ffcb VP |
1478 | hskb->len += len; |
1479 | hskb->data_len += len; | |
1480 | hskb->truesize += len; | |
1481 | } | |
1482 | ||
1483 | if (pdu_cb->flags & PDUCBF_RX_STATUS) { | |
1484 | hpdu_cb->flags |= pdu_cb->flags; | |
1485 | ||
1486 | if (hpdu_cb->flags & PDUCBF_RX_DATA) | |
1487 | hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; | |
1488 | ||
1489 | hpdu_cb->ddigest = pdu_cb->ddigest; | |
1490 | hpdu_cb->pdulen = pdu_cb->pdulen; | |
1491 | } | |
1492 | } | |
1493 | ||
1494 | static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) | |
1495 | { | |
1496 | struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); | |
1497 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); | |
1498 | u8 pdu_idx = 0, last_idx = 0; | |
1499 | int ret = 0; | |
1500 | ||
1501 | if (!pdu_cb->complete) { | |
1502 | cxgbit_lro_skb_merge(csk, skb, 0); | |
1503 | ||
1504 | if (pdu_cb->flags & PDUCBF_RX_STATUS) { | |
1505 | struct sk_buff *hskb = csk->lro_hskb; | |
1506 | ||
1507 | ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); | |
1508 | ||
1509 | cxgbit_lro_hskb_reset(csk); | |
1510 | ||
1511 | if (ret < 0) | |
1512 | goto out; | |
1513 | } | |
1514 | ||
1515 | pdu_idx = 1; | |
1516 | } | |
1517 | ||
1518 | if (lro_cb->pdu_idx) | |
1519 | last_idx = lro_cb->pdu_idx - 1; | |
1520 | ||
1521 | for (; pdu_idx <= last_idx; pdu_idx++) { | |
1522 | ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); | |
1523 | if (ret < 0) | |
1524 | goto out; | |
1525 | } | |
1526 | ||
1527 | if ((!lro_cb->complete) && lro_cb->pdu_idx) | |
1528 | cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); | |
1529 | ||
1530 | out: | |
1531 | return ret; | |
1532 | } | |
1533 | ||
1534 | static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) | |
1535 | { | |
1536 | struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); | |
1537 | struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); | |
1538 | int ret = -1; | |
1539 | ||
1540 | if ((pdu_cb->flags & PDUCBF_RX_HDR) && | |
1541 | (pdu_cb->seq != csk->rcv_nxt)) { | |
1542 | pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", | |
1543 | csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); | |
1544 | cxgbit_lro_skb_dump(skb); | |
1545 | return ret; | |
1546 | } | |
1547 | ||
1548 | csk->rcv_nxt += lro_cb->pdu_totallen; | |
1549 | ||
1550 | ret = cxgbit_process_lro_skb(csk, skb); | |
1551 | ||
1552 | csk->rx_credits += lro_cb->pdu_totallen; | |
1553 | ||
1554 | if (csk->rx_credits >= (csk->rcv_win / 4)) | |
1555 | cxgbit_rx_data_ack(csk); | |
1556 | ||
1557 | return ret; | |
1558 | } | |
1559 | ||
1560 | static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) | |
1561 | { | |
79e57cfe | 1562 | struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; |
9730ffcb VP |
1563 | int ret = -1; |
1564 | ||
79e57cfe VP |
1565 | if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) { |
1566 | if (is_t5(lldi->adapter_type)) | |
1567 | ret = cxgbit_rx_lro_skb(csk, skb); | |
1568 | else | |
1569 | ret = cxgbit_process_lro_skb(csk, skb); | |
1570 | } | |
9730ffcb VP |
1571 | |
1572 | __kfree_skb(skb); | |
1573 | return ret; | |
1574 | } | |
1575 | ||
1576 | static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) | |
1577 | { | |
1578 | spin_lock_bh(&csk->rxq.lock); | |
1579 | if (skb_queue_len(&csk->rxq)) { | |
1580 | skb_queue_splice_init(&csk->rxq, rxq); | |
1581 | spin_unlock_bh(&csk->rxq.lock); | |
1582 | return true; | |
1583 | } | |
1584 | spin_unlock_bh(&csk->rxq.lock); | |
1585 | return false; | |
1586 | } | |
1587 | ||
1588 | static int cxgbit_wait_rxq(struct cxgbit_sock *csk) | |
1589 | { | |
1590 | struct sk_buff *skb; | |
1591 | struct sk_buff_head rxq; | |
1592 | ||
1593 | skb_queue_head_init(&rxq); | |
1594 | ||
1595 | wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); | |
1596 | ||
1597 | if (signal_pending(current)) | |
1598 | goto out; | |
1599 | ||
1600 | while ((skb = __skb_dequeue(&rxq))) { | |
1601 | if (cxgbit_rx_skb(csk, skb)) | |
1602 | goto out; | |
1603 | } | |
1604 | ||
1605 | return 0; | |
1606 | out: | |
1607 | __skb_queue_purge(&rxq); | |
1608 | return -1; | |
1609 | } | |
1610 | ||
1611 | int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) | |
1612 | { | |
1613 | struct cxgbit_sock *csk = conn->context; | |
1614 | int ret = -1; | |
1615 | ||
1616 | while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { | |
1617 | ret = cxgbit_wait_rxq(csk); | |
1618 | if (ret) { | |
1619 | clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); | |
1620 | break; | |
1621 | } | |
1622 | } | |
1623 | ||
1624 | return ret; | |
1625 | } | |
1626 | ||
1627 | void cxgbit_get_rx_pdu(struct iscsi_conn *conn) | |
1628 | { | |
1629 | struct cxgbit_sock *csk = conn->context; | |
1630 | ||
1631 | while (!kthread_should_stop()) { | |
1632 | iscsit_thread_check_cpumask(conn, current, 0); | |
1633 | if (cxgbit_wait_rxq(csk)) | |
1634 | return; | |
1635 | } | |
1636 | } |