]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/target/iscsi/cxgbit/cxgbit_target.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
sched/headers: Prepare to move signal wakeup & sigpending methods from <linux/sched...
[mirror_ubuntu-artful-kernel.git] / drivers / target / iscsi / cxgbit / cxgbit_target.c
1 /*
2 * Copyright (c) 2016 Chelsio Communications, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9 #include <linux/workqueue.h>
10 #include <linux/kthread.h>
11 #include <linux/sched/signal.h>
12
13 #include <asm/unaligned.h>
14 #include <net/tcp.h>
15 #include <target/target_core_base.h>
16 #include <target/target_core_fabric.h>
17 #include "cxgbit.h"
18
19 struct sge_opaque_hdr {
20 void *dev;
21 dma_addr_t addr[MAX_SKB_FRAGS + 1];
22 };
23
24 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
25
26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
27 sizeof(struct fw_ofld_tx_data_wr))
28
29 static struct sk_buff *
30 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
31 {
32 struct sk_buff *skb = NULL;
33 u8 submode = 0;
34 int errcode;
35 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
36
37 if (len) {
38 skb = alloc_skb_with_frags(hdr_len, len,
39 0, &errcode,
40 GFP_KERNEL);
41 if (!skb)
42 return NULL;
43
44 skb_reserve(skb, TX_HDR_LEN);
45 skb_reset_transport_header(skb);
46 __skb_put(skb, ISCSI_HDR_LEN);
47 skb->data_len = len;
48 skb->len += len;
49 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
50
51 } else {
52 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
53
54 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
55 if (!skb)
56 return NULL;
57
58 skb_reserve(skb, TX_HDR_LEN + iso_len);
59 skb_reset_transport_header(skb);
60 __skb_put(skb, ISCSI_HDR_LEN);
61 }
62
63 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
64 cxgbit_skcb_submode(skb) = submode;
65 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
66 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
67 return skb;
68 }
69
70 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
71 {
72 return __cxgbit_alloc_skb(csk, len, false);
73 }
74
75 /*
76 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
77 * @skb: the packet
78 *
79 * Returns true if a packet can be sent as an offload WR with immediate
80 * data. We currently use the same limit as for Ethernet packets.
81 */
82 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
83 {
84 int length = skb->len;
85
86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
87 length += sizeof(struct fw_ofld_tx_data_wr);
88
89 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
90 length += sizeof(struct cpl_tx_data_iso);
91
92 #define MAX_IMM_TX_PKT_LEN 256
93 return length <= MAX_IMM_TX_PKT_LEN;
94 }
95
96 /*
97 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
98 * @n: the number of SGL entries
99 * Calculates the number of flits needed for a scatter/gather list that
100 * can hold the given number of entries.
101 */
102 static inline unsigned int cxgbit_sgl_len(unsigned int n)
103 {
104 n--;
105 return (3 * n) / 2 + (n & 1) + 2;
106 }
107
108 /*
109 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
110 * @skb: the packet
111 *
112 * Returns the number of flits needed for the given offload packet.
113 * These packets are already fully constructed and no additional headers
114 * will be added.
115 */
116 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
117 {
118 unsigned int flits, cnt;
119
120 if (cxgbit_is_ofld_imm(skb))
121 return DIV_ROUND_UP(skb->len, 8);
122 flits = skb_transport_offset(skb) / 8;
123 cnt = skb_shinfo(skb)->nr_frags;
124 if (skb_tail_pointer(skb) != skb_transport_header(skb))
125 cnt++;
126 return flits + cxgbit_sgl_len(cnt);
127 }
128
129 #define CXGBIT_ISO_FSLICE 0x1
130 #define CXGBIT_ISO_LSLICE 0x2
131 static void
132 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
133 {
134 struct cpl_tx_data_iso *cpl;
135 unsigned int submode = cxgbit_skcb_submode(skb);
136 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
137 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
138
139 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl));
140
141 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
142 CPL_TX_DATA_ISO_FIRST_V(fslice) |
143 CPL_TX_DATA_ISO_LAST_V(lslice) |
144 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
145 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
146 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
147 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
148 CPL_TX_DATA_ISO_SCSI_V(2));
149
150 cpl->ahs_len = 0;
151 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
152 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
153 cpl->len = htonl(iso_info->len);
154 cpl->reserved2_seglen_offset = htonl(0);
155 cpl->datasn_offset = htonl(0);
156 cpl->buffer_offset = htonl(0);
157 cpl->reserved3 = 0;
158
159 __skb_pull(skb, sizeof(*cpl));
160 }
161
162 static void
163 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
164 u32 len, u32 credits, u32 compl)
165 {
166 struct fw_ofld_tx_data_wr *req;
167 u32 submode = cxgbit_skcb_submode(skb);
168 u32 wr_ulp_mode = 0;
169 u32 hdr_size = sizeof(*req);
170 u32 opcode = FW_OFLD_TX_DATA_WR;
171 u32 immlen = 0;
172 u32 force = TX_FORCE_V(!submode);
173
174 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
175 opcode = FW_ISCSI_TX_DATA_WR;
176 immlen += sizeof(struct cpl_tx_data_iso);
177 hdr_size += sizeof(struct cpl_tx_data_iso);
178 submode |= 8;
179 }
180
181 if (cxgbit_is_ofld_imm(skb))
182 immlen += dlen;
183
184 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb,
185 hdr_size);
186 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
187 FW_WR_COMPL_V(compl) |
188 FW_WR_IMMDLEN_V(immlen));
189 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
190 FW_WR_LEN16_V(credits));
191 req->plen = htonl(len);
192 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
193 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
194
195 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
196 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
197 }
198
199 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
200 {
201 kfree_skb(skb);
202 }
203
204 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
205 {
206 struct sk_buff *skb;
207
208 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
209 u32 dlen = skb->len;
210 u32 len = skb->len;
211 u32 credits_needed;
212 u32 compl = 0;
213 u32 flowclen16 = 0;
214 u32 iso_cpl_len = 0;
215
216 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
217 iso_cpl_len = sizeof(struct cpl_tx_data_iso);
218
219 if (cxgbit_is_ofld_imm(skb))
220 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
221 else
222 credits_needed = DIV_ROUND_UP((8 *
223 cxgbit_calc_tx_flits_ofld(skb)) +
224 iso_cpl_len, 16);
225
226 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
227 credits_needed += DIV_ROUND_UP(
228 sizeof(struct fw_ofld_tx_data_wr), 16);
229 /*
230 * Assumes the initial credits is large enough to support
231 * fw_flowc_wr plus largest possible first payload
232 */
233
234 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
235 flowclen16 = cxgbit_send_tx_flowc_wr(csk);
236 csk->wr_cred -= flowclen16;
237 csk->wr_una_cred += flowclen16;
238 }
239
240 if (csk->wr_cred < credits_needed) {
241 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
242 csk, skb->len, skb->data_len,
243 credits_needed, csk->wr_cred);
244 break;
245 }
246 __skb_unlink(skb, &csk->txq);
247 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
248 skb->csum = credits_needed + flowclen16;
249 csk->wr_cred -= credits_needed;
250 csk->wr_una_cred += credits_needed;
251
252 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
253 csk, skb->len, skb->data_len, credits_needed,
254 csk->wr_cred, csk->wr_una_cred);
255
256 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
257 len += cxgbit_skcb_tx_extralen(skb);
258
259 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
260 (!before(csk->write_seq,
261 csk->snd_una + csk->snd_win))) {
262 compl = 1;
263 csk->wr_una_cred = 0;
264 }
265
266 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
267 compl);
268 csk->snd_nxt += len;
269
270 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
271 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
272 struct cpl_close_con_req *req =
273 (struct cpl_close_con_req *)skb->data;
274 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
275 csk->wr_una_cred = 0;
276 }
277
278 cxgbit_sock_enqueue_wr(csk, skb);
279 t4_set_arp_err_handler(skb, csk,
280 cxgbit_arp_failure_skb_discard);
281
282 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
283 csk, csk->tid, skb, len);
284
285 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
286 }
287 }
288
289 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
290 {
291 spin_lock_bh(&csk->lock);
292
293 if (before(csk->write_seq, csk->snd_una + csk->snd_win))
294 csk->lock_owner = true;
295
296 spin_unlock_bh(&csk->lock);
297
298 return csk->lock_owner;
299 }
300
301 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
302 {
303 struct sk_buff_head backlogq;
304 struct sk_buff *skb;
305 void (*fn)(struct cxgbit_sock *, struct sk_buff *);
306
307 skb_queue_head_init(&backlogq);
308
309 spin_lock_bh(&csk->lock);
310 while (skb_queue_len(&csk->backlogq)) {
311 skb_queue_splice_init(&csk->backlogq, &backlogq);
312 spin_unlock_bh(&csk->lock);
313
314 while ((skb = __skb_dequeue(&backlogq))) {
315 fn = cxgbit_skcb_rx_backlog_fn(skb);
316 fn(csk, skb);
317 }
318
319 spin_lock_bh(&csk->lock);
320 }
321
322 csk->lock_owner = false;
323 spin_unlock_bh(&csk->lock);
324 }
325
326 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
327 {
328 int ret = 0;
329
330 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
331
332 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
333 signal_pending(current))) {
334 __kfree_skb(skb);
335 __skb_queue_purge(&csk->ppodq);
336 ret = -1;
337 spin_lock_bh(&csk->lock);
338 if (csk->lock_owner) {
339 spin_unlock_bh(&csk->lock);
340 goto unlock;
341 }
342 spin_unlock_bh(&csk->lock);
343 return ret;
344 }
345
346 csk->write_seq += skb->len +
347 cxgbit_skcb_tx_extralen(skb);
348
349 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
350 __skb_queue_tail(&csk->txq, skb);
351 cxgbit_push_tx_frames(csk);
352
353 unlock:
354 cxgbit_unlock_sock(csk);
355 return ret;
356 }
357
358 static int
359 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
360 u32 data_length)
361 {
362 u32 i = 0, nr_frags = MAX_SKB_FRAGS;
363 u32 padding = ((-data_length) & 3);
364 struct scatterlist *sg;
365 struct page *page;
366 unsigned int page_off;
367
368 if (padding)
369 nr_frags--;
370
371 /*
372 * We know each entry in t_data_sg contains a page.
373 */
374 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
375 page_off = (data_offset % PAGE_SIZE);
376
377 while (data_length && (i < nr_frags)) {
378 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
379
380 page = sg_page(sg);
381
382 get_page(page);
383 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
384 cur_len);
385 skb->data_len += cur_len;
386 skb->len += cur_len;
387 skb->truesize += cur_len;
388
389 data_length -= cur_len;
390 page_off = 0;
391 sg = sg_next(sg);
392 i++;
393 }
394
395 if (data_length)
396 return -1;
397
398 if (padding) {
399 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
400 if (!page)
401 return -1;
402 skb_fill_page_desc(skb, i, page, 0, padding);
403 skb->data_len += padding;
404 skb->len += padding;
405 skb->truesize += padding;
406 }
407
408 return 0;
409 }
410
411 static int
412 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
413 struct iscsi_datain_req *dr)
414 {
415 struct iscsi_conn *conn = csk->conn;
416 struct sk_buff *skb;
417 struct iscsi_datain datain;
418 struct cxgbit_iso_info iso_info;
419 u32 data_length = cmd->se_cmd.data_length;
420 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
421 u32 num_pdu, plen, tx_data = 0;
422 bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
423 SCF_TRANSPORT_TASK_SENSE);
424 bool set_statsn = false;
425 int ret = -1;
426
427 while (data_length) {
428 num_pdu = (data_length + mrdsl - 1) / mrdsl;
429 if (num_pdu > csk->max_iso_npdu)
430 num_pdu = csk->max_iso_npdu;
431
432 plen = num_pdu * mrdsl;
433 if (plen > data_length)
434 plen = data_length;
435
436 skb = __cxgbit_alloc_skb(csk, 0, true);
437 if (unlikely(!skb))
438 return -ENOMEM;
439
440 memset(skb->data, 0, ISCSI_HDR_LEN);
441 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
442 cxgbit_skcb_submode(skb) |= (csk->submode &
443 CXGBIT_SUBMODE_DCRC);
444 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
445 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
446 ((num_pdu - 1) * ISCSI_HDR_LEN);
447
448 memset(&datain, 0, sizeof(struct iscsi_datain));
449 memset(&iso_info, 0, sizeof(iso_info));
450
451 if (!tx_data)
452 iso_info.flags |= CXGBIT_ISO_FSLICE;
453
454 if (!(data_length - plen)) {
455 iso_info.flags |= CXGBIT_ISO_LSLICE;
456 if (!task_sense) {
457 datain.flags = ISCSI_FLAG_DATA_STATUS;
458 iscsit_increment_maxcmdsn(cmd, conn->sess);
459 cmd->stat_sn = conn->stat_sn++;
460 set_statsn = true;
461 }
462 }
463
464 iso_info.burst_len = num_pdu * mrdsl;
465 iso_info.mpdu = mrdsl;
466 iso_info.len = ISCSI_HDR_LEN + plen;
467
468 cxgbit_cpl_tx_data_iso(skb, &iso_info);
469
470 datain.offset = tx_data;
471 datain.data_sn = cmd->data_sn - 1;
472
473 iscsit_build_datain_pdu(cmd, conn, &datain,
474 (struct iscsi_data_rsp *)skb->data,
475 set_statsn);
476
477 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
478 if (unlikely(ret)) {
479 __kfree_skb(skb);
480 goto out;
481 }
482
483 ret = cxgbit_queue_skb(csk, skb);
484 if (unlikely(ret))
485 goto out;
486
487 tx_data += plen;
488 data_length -= plen;
489
490 cmd->read_data_done += plen;
491 cmd->data_sn += num_pdu;
492 }
493
494 dr->dr_complete = DATAIN_COMPLETE_NORMAL;
495
496 return 0;
497
498 out:
499 return ret;
500 }
501
502 static int
503 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
504 const struct iscsi_datain *datain)
505 {
506 struct sk_buff *skb;
507 int ret = 0;
508
509 skb = cxgbit_alloc_skb(csk, 0);
510 if (unlikely(!skb))
511 return -ENOMEM;
512
513 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
514
515 if (datain->length) {
516 cxgbit_skcb_submode(skb) |= (csk->submode &
517 CXGBIT_SUBMODE_DCRC);
518 cxgbit_skcb_tx_extralen(skb) =
519 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
520 }
521
522 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
523 if (ret < 0) {
524 __kfree_skb(skb);
525 return ret;
526 }
527
528 return cxgbit_queue_skb(csk, skb);
529 }
530
531 static int
532 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
533 struct iscsi_datain_req *dr,
534 const struct iscsi_datain *datain)
535 {
536 struct cxgbit_sock *csk = conn->context;
537 u32 data_length = cmd->se_cmd.data_length;
538 u32 padding = ((-data_length) & 3);
539 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
540
541 if ((data_length > mrdsl) && (!dr->recovery) &&
542 (!padding) && (!datain->offset) && csk->max_iso_npdu) {
543 atomic_long_add(data_length - datain->length,
544 &conn->sess->tx_data_octets);
545 return cxgbit_tx_datain_iso(csk, cmd, dr);
546 }
547
548 return cxgbit_tx_datain(csk, cmd, datain);
549 }
550
551 static int
552 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
553 const void *data_buf, u32 data_buf_len)
554 {
555 struct cxgbit_sock *csk = conn->context;
556 struct sk_buff *skb;
557 u32 padding = ((-data_buf_len) & 3);
558
559 skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
560 if (unlikely(!skb))
561 return -ENOMEM;
562
563 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
564
565 if (data_buf_len) {
566 u32 pad_bytes = 0;
567
568 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
569
570 if (padding)
571 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
572 &pad_bytes, padding);
573 }
574
575 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
576 cxgbit_skcb_submode(skb)];
577
578 return cxgbit_queue_skb(csk, skb);
579 }
580
581 int
582 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
583 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
584 {
585 if (dr)
586 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
587 else
588 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
589 }
590
591 int cxgbit_validate_params(struct iscsi_conn *conn)
592 {
593 struct cxgbit_sock *csk = conn->context;
594 struct cxgbit_device *cdev = csk->com.cdev;
595 struct iscsi_param *param;
596 u32 max_xmitdsl;
597
598 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
599 conn->param_list);
600 if (!param)
601 return -1;
602
603 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
604 return -1;
605
606 if (max_xmitdsl > cdev->mdsl) {
607 if (iscsi_change_param_sprintf(
608 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
609 return -1;
610 }
611
612 return 0;
613 }
614
615 static int cxgbit_set_digest(struct cxgbit_sock *csk)
616 {
617 struct iscsi_conn *conn = csk->conn;
618 struct iscsi_param *param;
619
620 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
621 if (!param) {
622 pr_err("param not found key %s\n", HEADERDIGEST);
623 return -1;
624 }
625
626 if (!strcmp(param->value, CRC32C))
627 csk->submode |= CXGBIT_SUBMODE_HCRC;
628
629 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
630 if (!param) {
631 csk->submode = 0;
632 pr_err("param not found key %s\n", DATADIGEST);
633 return -1;
634 }
635
636 if (!strcmp(param->value, CRC32C))
637 csk->submode |= CXGBIT_SUBMODE_DCRC;
638
639 if (cxgbit_setup_conn_digest(csk)) {
640 csk->submode = 0;
641 return -1;
642 }
643
644 return 0;
645 }
646
647 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
648 {
649 struct iscsi_conn *conn = csk->conn;
650 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
651 struct iscsi_param *param;
652 u32 mrdsl, mbl;
653 u32 max_npdu, max_iso_npdu;
654
655 if (conn->login->leading_connection) {
656 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
657 conn->param_list);
658 if (!param) {
659 pr_err("param not found key %s\n", DATASEQUENCEINORDER);
660 return -1;
661 }
662
663 if (strcmp(param->value, YES))
664 return 0;
665
666 param = iscsi_find_param_from_key(DATAPDUINORDER,
667 conn->param_list);
668 if (!param) {
669 pr_err("param not found key %s\n", DATAPDUINORDER);
670 return -1;
671 }
672
673 if (strcmp(param->value, YES))
674 return 0;
675
676 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
677 conn->param_list);
678 if (!param) {
679 pr_err("param not found key %s\n", MAXBURSTLENGTH);
680 return -1;
681 }
682
683 if (kstrtou32(param->value, 0, &mbl) < 0)
684 return -1;
685 } else {
686 if (!conn->sess->sess_ops->DataSequenceInOrder)
687 return 0;
688 if (!conn->sess->sess_ops->DataPDUInOrder)
689 return 0;
690
691 mbl = conn->sess->sess_ops->MaxBurstLength;
692 }
693
694 mrdsl = conn_ops->MaxRecvDataSegmentLength;
695 max_npdu = mbl / mrdsl;
696
697 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD /
698 (ISCSI_HDR_LEN + mrdsl +
699 cxgbit_digest_len[csk->submode]);
700
701 csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
702
703 if (csk->max_iso_npdu <= 1)
704 csk->max_iso_npdu = 0;
705
706 return 0;
707 }
708
709 static int cxgbit_set_params(struct iscsi_conn *conn)
710 {
711 struct cxgbit_sock *csk = conn->context;
712 struct cxgbit_device *cdev = csk->com.cdev;
713 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
714 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
715 struct iscsi_param *param;
716 u8 erl;
717
718 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
719 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
720
721 if (conn->login->leading_connection) {
722 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
723 conn->param_list);
724 if (!param) {
725 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
726 return -1;
727 }
728 if (kstrtou8(param->value, 0, &erl) < 0)
729 return -1;
730 } else {
731 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
732 }
733
734 if (!erl) {
735 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
736 if (cxgbit_set_iso_npdu(csk))
737 return -1;
738 }
739
740 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
741 if (cxgbit_setup_conn_pgidx(csk,
742 ppm->tformat.pgsz_idx_dflt))
743 return -1;
744 set_bit(CSK_DDP_ENABLE, &csk->com.flags);
745 }
746 }
747
748 if (cxgbit_set_digest(csk))
749 return -1;
750
751 return 0;
752 }
753
754 int
755 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
756 u32 length)
757 {
758 struct cxgbit_sock *csk = conn->context;
759 struct sk_buff *skb;
760 u32 padding_buf = 0;
761 u8 padding = ((-length) & 3);
762
763 skb = cxgbit_alloc_skb(csk, length + padding);
764 if (!skb)
765 return -ENOMEM;
766 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
767 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
768
769 if (padding)
770 skb_store_bits(skb, ISCSI_HDR_LEN + length,
771 &padding_buf, padding);
772
773 if (login->login_complete) {
774 if (cxgbit_set_params(conn)) {
775 kfree_skb(skb);
776 return -1;
777 }
778
779 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
780 }
781
782 if (cxgbit_queue_skb(csk, skb))
783 return -1;
784
785 if ((!login->login_complete) && (!login->login_failed))
786 schedule_delayed_work(&conn->login_work, 0);
787
788 return 0;
789 }
790
791 static void
792 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
793 unsigned int nents)
794 {
795 struct skb_seq_state st;
796 const u8 *buf;
797 unsigned int consumed = 0, buf_len;
798 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
799
800 skb_prepare_seq_read(skb, pdu_cb->doffset,
801 pdu_cb->doffset + pdu_cb->dlen,
802 &st);
803
804 while (true) {
805 buf_len = skb_seq_read(consumed, &buf, &st);
806 if (!buf_len) {
807 skb_abort_seq_read(&st);
808 break;
809 }
810
811 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
812 buf_len, consumed);
813 }
814 }
815
816 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
817 {
818 struct iscsi_conn *conn = csk->conn;
819 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
820 struct cxgbit_cmd *ccmd;
821 struct iscsi_cmd *cmd;
822
823 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
824 if (!cmd) {
825 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
826 return NULL;
827 }
828
829 ccmd = iscsit_priv_cmd(cmd);
830 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
831 ccmd->setup_ddp = true;
832
833 return cmd;
834 }
835
836 static int
837 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
838 u32 length)
839 {
840 struct iscsi_conn *conn = cmd->conn;
841 struct cxgbit_sock *csk = conn->context;
842 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
843
844 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
845 pr_err("ImmediateData CRC32C DataDigest error\n");
846 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
847 pr_err("Unable to recover from"
848 " Immediate Data digest failure while"
849 " in ERL=0.\n");
850 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
851 (unsigned char *)hdr);
852 return IMMEDIATE_DATA_CANNOT_RECOVER;
853 }
854
855 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
856 (unsigned char *)hdr);
857 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
858 }
859
860 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
861 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
862 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
863 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
864
865 sg_init_table(&ccmd->sg, 1);
866 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
867 dfrag->page_offset);
868 get_page(dfrag->page.p);
869
870 cmd->se_cmd.t_data_sg = &ccmd->sg;
871 cmd->se_cmd.t_data_nents = 1;
872
873 ccmd->release = true;
874 } else {
875 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
876 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
877
878 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents);
879 }
880
881 cmd->write_data_done += pdu_cb->dlen;
882
883 if (cmd->write_data_done == cmd->se_cmd.data_length) {
884 spin_lock_bh(&cmd->istate_lock);
885 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
886 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
887 spin_unlock_bh(&cmd->istate_lock);
888 }
889
890 return IMMEDIATE_DATA_NORMAL_OPERATION;
891 }
892
893 static int
894 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
895 bool dump_payload)
896 {
897 struct iscsi_conn *conn = cmd->conn;
898 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
899 /*
900 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
901 */
902 if (dump_payload)
903 goto after_immediate_data;
904
905 immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
906 cmd->first_burst_len);
907 after_immediate_data:
908 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
909 /*
910 * A PDU/CmdSN carrying Immediate Data passed
911 * DataCRC, check against ExpCmdSN/MaxCmdSN if
912 * Immediate Bit is not set.
913 */
914 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
915 (unsigned char *)hdr,
916 hdr->cmdsn);
917 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
918 return -1;
919
920 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
921 target_put_sess_cmd(&cmd->se_cmd);
922 return 0;
923 } else if (cmd->unsolicited_data) {
924 iscsit_set_unsoliticed_dataout(cmd);
925 }
926
927 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
928 /*
929 * Immediate Data failed DataCRC and ERL>=1,
930 * silently drop this PDU and let the initiator
931 * plug the CmdSN gap.
932 *
933 * FIXME: Send Unsolicited NOPIN with reserved
934 * TTT here to help the initiator figure out
935 * the missing CmdSN, although they should be
936 * intelligent enough to determine the missing
937 * CmdSN and issue a retry to plug the sequence.
938 */
939 cmd->i_state = ISTATE_REMOVE;
940 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
941 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
942 return -1;
943
944 return 0;
945 }
946
947 static int
948 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
949 {
950 struct iscsi_conn *conn = csk->conn;
951 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
952 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
953 int rc;
954 bool dump_payload = false;
955
956 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
957 if (rc < 0)
958 return rc;
959
960 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
961 (pdu_cb->nr_dfrags == 1))
962 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
963
964 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
965 if (rc < 0)
966 return 0;
967 else if (rc > 0)
968 dump_payload = true;
969
970 if (!pdu_cb->dlen)
971 return 0;
972
973 return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
974 }
975
976 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
977 {
978 struct scatterlist *sg_start;
979 struct iscsi_conn *conn = csk->conn;
980 struct iscsi_cmd *cmd = NULL;
981 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
982 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
983 u32 data_offset = be32_to_cpu(hdr->offset);
984 u32 data_len = pdu_cb->dlen;
985 int rc, sg_nents, sg_off;
986 bool dcrc_err = false;
987
988 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
989 if (rc < 0)
990 return rc;
991 else if (!cmd)
992 return 0;
993
994 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
995 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
996 " DataSN: 0x%08x\n",
997 hdr->itt, hdr->offset, data_len,
998 hdr->datasn);
999
1000 dcrc_err = true;
1001 goto check_payload;
1002 }
1003
1004 pr_debug("DataOut data_len: %u, "
1005 "write_data_done: %u, data_length: %u\n",
1006 data_len, cmd->write_data_done,
1007 cmd->se_cmd.data_length);
1008
1009 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1010 sg_off = data_offset / PAGE_SIZE;
1011 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1012 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE));
1013
1014 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents);
1015 }
1016
1017 check_payload:
1018
1019 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1020 if (rc < 0)
1021 return rc;
1022
1023 return 0;
1024 }
1025
1026 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1027 {
1028 struct iscsi_conn *conn = csk->conn;
1029 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1030 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1031 unsigned char *ping_data = NULL;
1032 u32 payload_length = pdu_cb->dlen;
1033 int ret;
1034
1035 ret = iscsit_setup_nop_out(conn, cmd, hdr);
1036 if (ret < 0)
1037 return 0;
1038
1039 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1040 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1041 pr_err("Unable to recover from"
1042 " NOPOUT Ping DataCRC failure while in"
1043 " ERL=0.\n");
1044 ret = -1;
1045 goto out;
1046 } else {
1047 /*
1048 * drop this PDU and let the
1049 * initiator plug the CmdSN gap.
1050 */
1051 pr_info("Dropping NOPOUT"
1052 " Command CmdSN: 0x%08x due to"
1053 " DataCRC error.\n", hdr->cmdsn);
1054 ret = 0;
1055 goto out;
1056 }
1057 }
1058
1059 /*
1060 * Handle NOP-OUT payload for traditional iSCSI sockets
1061 */
1062 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1063 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1064 if (!ping_data) {
1065 pr_err("Unable to allocate memory for"
1066 " NOPOUT ping data.\n");
1067 ret = -1;
1068 goto out;
1069 }
1070
1071 skb_copy_bits(csk->skb, pdu_cb->doffset,
1072 ping_data, payload_length);
1073
1074 ping_data[payload_length] = '\0';
1075 /*
1076 * Attach ping data to struct iscsi_cmd->buf_ptr.
1077 */
1078 cmd->buf_ptr = ping_data;
1079 cmd->buf_ptr_size = payload_length;
1080
1081 pr_debug("Got %u bytes of NOPOUT ping"
1082 " data.\n", payload_length);
1083 pr_debug("Ping Data: \"%s\"\n", ping_data);
1084 }
1085
1086 return iscsit_process_nop_out(conn, cmd, hdr);
1087 out:
1088 if (cmd)
1089 iscsit_free_cmd(cmd, false);
1090 return ret;
1091 }
1092
1093 static int
1094 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1095 {
1096 struct iscsi_conn *conn = csk->conn;
1097 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1098 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1099 u32 payload_length = pdu_cb->dlen;
1100 int rc;
1101 unsigned char *text_in = NULL;
1102
1103 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1104 if (rc < 0)
1105 return rc;
1106
1107 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1108 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1109 pr_err("Unable to recover from"
1110 " Text Data digest failure while in"
1111 " ERL=0.\n");
1112 goto reject;
1113 } else {
1114 /*
1115 * drop this PDU and let the
1116 * initiator plug the CmdSN gap.
1117 */
1118 pr_info("Dropping Text"
1119 " Command CmdSN: 0x%08x due to"
1120 " DataCRC error.\n", hdr->cmdsn);
1121 return 0;
1122 }
1123 }
1124
1125 if (payload_length) {
1126 text_in = kzalloc(payload_length, GFP_KERNEL);
1127 if (!text_in) {
1128 pr_err("Unable to allocate text_in of payload_length: %u\n",
1129 payload_length);
1130 return -ENOMEM;
1131 }
1132 skb_copy_bits(csk->skb, pdu_cb->doffset,
1133 text_in, payload_length);
1134
1135 text_in[payload_length - 1] = '\0';
1136
1137 cmd->text_in_ptr = text_in;
1138 }
1139
1140 return iscsit_process_text_cmd(conn, cmd, hdr);
1141
1142 reject:
1143 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1144 pdu_cb->hdr);
1145 }
1146
1147 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1148 {
1149 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1150 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1151 struct iscsi_conn *conn = csk->conn;
1152 struct iscsi_cmd *cmd = NULL;
1153 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1154 int ret = -EINVAL;
1155
1156 switch (opcode) {
1157 case ISCSI_OP_SCSI_CMD:
1158 cmd = cxgbit_allocate_cmd(csk);
1159 if (!cmd)
1160 goto reject;
1161
1162 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1163 break;
1164 case ISCSI_OP_SCSI_DATA_OUT:
1165 ret = cxgbit_handle_iscsi_dataout(csk);
1166 break;
1167 case ISCSI_OP_NOOP_OUT:
1168 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1169 cmd = cxgbit_allocate_cmd(csk);
1170 if (!cmd)
1171 goto reject;
1172 }
1173
1174 ret = cxgbit_handle_nop_out(csk, cmd);
1175 break;
1176 case ISCSI_OP_SCSI_TMFUNC:
1177 cmd = cxgbit_allocate_cmd(csk);
1178 if (!cmd)
1179 goto reject;
1180
1181 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1182 (unsigned char *)hdr);
1183 break;
1184 case ISCSI_OP_TEXT:
1185 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1186 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1187 if (!cmd)
1188 goto reject;
1189 } else {
1190 cmd = cxgbit_allocate_cmd(csk);
1191 if (!cmd)
1192 goto reject;
1193 }
1194
1195 ret = cxgbit_handle_text_cmd(csk, cmd);
1196 break;
1197 case ISCSI_OP_LOGOUT:
1198 cmd = cxgbit_allocate_cmd(csk);
1199 if (!cmd)
1200 goto reject;
1201
1202 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1203 if (ret > 0)
1204 wait_for_completion_timeout(&conn->conn_logout_comp,
1205 SECONDS_FOR_LOGOUT_COMP
1206 * HZ);
1207 break;
1208 case ISCSI_OP_SNACK:
1209 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1210 break;
1211 default:
1212 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1213 dump_stack();
1214 break;
1215 }
1216
1217 return ret;
1218
1219 reject:
1220 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1221 (unsigned char *)hdr);
1222 return ret;
1223 }
1224
1225 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1226 {
1227 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1228 struct iscsi_conn *conn = csk->conn;
1229 struct iscsi_hdr *hdr = pdu_cb->hdr;
1230 u8 opcode;
1231
1232 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1233 atomic_long_inc(&conn->sess->conn_digest_errors);
1234 goto transport_err;
1235 }
1236
1237 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1238 goto transport_err;
1239
1240 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1241
1242 if (conn->sess->sess_ops->SessionType &&
1243 ((!(opcode & ISCSI_OP_TEXT)) ||
1244 (!(opcode & ISCSI_OP_LOGOUT)))) {
1245 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1246 " while in Discovery Session, rejecting.\n", opcode);
1247 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1248 (unsigned char *)hdr);
1249 goto transport_err;
1250 }
1251
1252 if (cxgbit_target_rx_opcode(csk) < 0)
1253 goto transport_err;
1254
1255 return 0;
1256
1257 transport_err:
1258 return -1;
1259 }
1260
1261 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1262 {
1263 struct iscsi_conn *conn = csk->conn;
1264 struct iscsi_login *login = conn->login;
1265 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1266 struct iscsi_login_req *login_req;
1267
1268 login_req = (struct iscsi_login_req *)login->req;
1269 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1270
1271 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1272 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1273 login_req->flags, login_req->itt, login_req->cmdsn,
1274 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1275 /*
1276 * Setup the initial iscsi_login values from the leading
1277 * login request PDU.
1278 */
1279 if (login->first_request) {
1280 login_req = (struct iscsi_login_req *)login->req;
1281 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1282 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1283 login_req->flags);
1284 login->version_min = login_req->min_version;
1285 login->version_max = login_req->max_version;
1286 memcpy(login->isid, login_req->isid, 6);
1287 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1288 login->init_task_tag = login_req->itt;
1289 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1290 login->cid = be16_to_cpu(login_req->cid);
1291 login->tsih = be16_to_cpu(login_req->tsih);
1292 }
1293
1294 if (iscsi_target_check_login_request(conn, login) < 0)
1295 return -1;
1296
1297 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1298 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1299
1300 return 0;
1301 }
1302
1303 static int
1304 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1305 {
1306 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1307 int ret;
1308
1309 cxgbit_rx_pdu_cb(skb) = pdu_cb;
1310
1311 csk->skb = skb;
1312
1313 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1314 ret = cxgbit_rx_login_pdu(csk);
1315 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1316 } else {
1317 ret = cxgbit_rx_opcode(csk);
1318 }
1319
1320 return ret;
1321 }
1322
1323 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1324 {
1325 struct skb_shared_info *ssi = skb_shinfo(skb);
1326 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1327 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1328 u8 i;
1329
1330 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1331 skb, skb->head, skb->data, skb->len, skb->data_len,
1332 ssi->nr_frags);
1333 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1334 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1335
1336 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1337 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1338 "frags %u.\n",
1339 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1340 pdu_cb->ddigest, pdu_cb->frags);
1341 for (i = 0; i < ssi->nr_frags; i++)
1342 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1343 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1344 }
1345
1346 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1347 {
1348 struct sk_buff *skb = csk->lro_hskb;
1349 struct skb_shared_info *ssi = skb_shinfo(skb);
1350 u8 i;
1351
1352 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1353 for (i = 0; i < ssi->nr_frags; i++)
1354 put_page(skb_frag_page(&ssi->frags[i]));
1355 ssi->nr_frags = 0;
1356 }
1357
1358 static void
1359 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1360 {
1361 struct sk_buff *hskb = csk->lro_hskb;
1362 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1363 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1364 struct skb_shared_info *hssi = skb_shinfo(hskb);
1365 struct skb_shared_info *ssi = skb_shinfo(skb);
1366 unsigned int len = 0;
1367
1368 if (pdu_cb->flags & PDUCBF_RX_HDR) {
1369 hpdu_cb->flags = pdu_cb->flags;
1370 hpdu_cb->seq = pdu_cb->seq;
1371 hpdu_cb->hdr = pdu_cb->hdr;
1372 hpdu_cb->hlen = pdu_cb->hlen;
1373
1374 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx],
1375 sizeof(skb_frag_t));
1376
1377 get_page(skb_frag_page(&hssi->frags[0]));
1378 hssi->nr_frags = 1;
1379 hpdu_cb->frags = 1;
1380 hpdu_cb->hfrag_idx = 0;
1381
1382 len = hssi->frags[0].size;
1383 hskb->len = len;
1384 hskb->data_len = len;
1385 hskb->truesize = len;
1386 }
1387
1388 if (pdu_cb->flags & PDUCBF_RX_DATA) {
1389 u8 hfrag_idx = 1, i;
1390
1391 hpdu_cb->flags |= pdu_cb->flags;
1392
1393 len = 0;
1394 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) {
1395 memcpy(&hssi->frags[hfrag_idx],
1396 &ssi->frags[pdu_cb->dfrag_idx + i],
1397 sizeof(skb_frag_t));
1398
1399 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1400
1401 len += hssi->frags[hfrag_idx].size;
1402
1403 hssi->nr_frags++;
1404 hpdu_cb->frags++;
1405 }
1406
1407 hpdu_cb->dlen = pdu_cb->dlen;
1408 hpdu_cb->doffset = hpdu_cb->hlen;
1409 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1410 hpdu_cb->dfrag_idx = 1;
1411 hskb->len += len;
1412 hskb->data_len += len;
1413 hskb->truesize += len;
1414 }
1415
1416 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1417 hpdu_cb->flags |= pdu_cb->flags;
1418
1419 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1420 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1421
1422 hpdu_cb->ddigest = pdu_cb->ddigest;
1423 hpdu_cb->pdulen = pdu_cb->pdulen;
1424 }
1425 }
1426
1427 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1428 {
1429 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1430 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1431 u8 pdu_idx = 0, last_idx = 0;
1432 int ret = 0;
1433
1434 if (!pdu_cb->complete) {
1435 cxgbit_lro_skb_merge(csk, skb, 0);
1436
1437 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1438 struct sk_buff *hskb = csk->lro_hskb;
1439
1440 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1441
1442 cxgbit_lro_hskb_reset(csk);
1443
1444 if (ret < 0)
1445 goto out;
1446 }
1447
1448 pdu_idx = 1;
1449 }
1450
1451 if (lro_cb->pdu_idx)
1452 last_idx = lro_cb->pdu_idx - 1;
1453
1454 for (; pdu_idx <= last_idx; pdu_idx++) {
1455 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1456 if (ret < 0)
1457 goto out;
1458 }
1459
1460 if ((!lro_cb->complete) && lro_cb->pdu_idx)
1461 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1462
1463 out:
1464 return ret;
1465 }
1466
1467 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1468 {
1469 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1470 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1471 int ret = -1;
1472
1473 if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1474 (pdu_cb->seq != csk->rcv_nxt)) {
1475 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1476 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1477 cxgbit_lro_skb_dump(skb);
1478 return ret;
1479 }
1480
1481 csk->rcv_nxt += lro_cb->pdu_totallen;
1482
1483 ret = cxgbit_process_lro_skb(csk, skb);
1484
1485 csk->rx_credits += lro_cb->pdu_totallen;
1486
1487 if (csk->rx_credits >= (csk->rcv_win / 4))
1488 cxgbit_rx_data_ack(csk);
1489
1490 return ret;
1491 }
1492
1493 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1494 {
1495 int ret = -1;
1496
1497 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO))
1498 ret = cxgbit_rx_lro_skb(csk, skb);
1499
1500 __kfree_skb(skb);
1501 return ret;
1502 }
1503
1504 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1505 {
1506 spin_lock_bh(&csk->rxq.lock);
1507 if (skb_queue_len(&csk->rxq)) {
1508 skb_queue_splice_init(&csk->rxq, rxq);
1509 spin_unlock_bh(&csk->rxq.lock);
1510 return true;
1511 }
1512 spin_unlock_bh(&csk->rxq.lock);
1513 return false;
1514 }
1515
1516 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1517 {
1518 struct sk_buff *skb;
1519 struct sk_buff_head rxq;
1520
1521 skb_queue_head_init(&rxq);
1522
1523 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1524
1525 if (signal_pending(current))
1526 goto out;
1527
1528 while ((skb = __skb_dequeue(&rxq))) {
1529 if (cxgbit_rx_skb(csk, skb))
1530 goto out;
1531 }
1532
1533 return 0;
1534 out:
1535 __skb_queue_purge(&rxq);
1536 return -1;
1537 }
1538
1539 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1540 {
1541 struct cxgbit_sock *csk = conn->context;
1542 int ret = -1;
1543
1544 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1545 ret = cxgbit_wait_rxq(csk);
1546 if (ret) {
1547 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1548 break;
1549 }
1550 }
1551
1552 return ret;
1553 }
1554
1555 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1556 {
1557 struct cxgbit_sock *csk = conn->context;
1558
1559 while (!kthread_should_stop()) {
1560 iscsit_thread_check_cpumask(conn, current, 0);
1561 if (cxgbit_wait_rxq(csk))
1562 return;
1563 }
1564 }
Ubuntu Artful kernel mirror