]> git.proxmox.com Git - ceph.git/blame - ceph/src/spdk/lib/nvme/nvme_tcp.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / lib / nvme / nvme_tcp.c
CommitLineData
9f95a23c
TL		 1/*-
2 * BSD LICENSE
3 *
f67539c2
TL		 4 * Copyright (c) Intel Corporation. All rights reserved.
5 * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
9f95a23c
TL		 6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34/*
35 * NVMe/TCP transport
36 */
37
38#include "nvme_internal.h"
39
40#include "spdk/endian.h"
41#include "spdk/likely.h"
42#include "spdk/string.h"
43#include "spdk/stdinc.h"
44#include "spdk/crc32.h"
45#include "spdk/endian.h"
46#include "spdk/assert.h"
47#include "spdk/string.h"
48#include "spdk/thread.h"
49#include "spdk/trace.h"
50#include "spdk/util.h"
51
52#include "spdk_internal/nvme_tcp.h"
53
54#define NVME_TCP_RW_BUFFER_SIZE 131072
f67539c2 55#define NVME_TCP_TIME_OUT_IN_SECONDS 2
9f95a23c
TL		 56
57#define NVME_TCP_HPDA_DEFAULT 0
58#define NVME_TCP_MAX_R2T_DEFAULT 1
59#define NVME_TCP_PDU_H2C_MIN_DATA_SIZE 4096
60#define NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE 8192
61
62/* NVMe TCP transport extensions for spdk_nvme_ctrlr */
63struct nvme_tcp_ctrlr {
64 struct spdk_nvme_ctrlr ctrlr;
65};
66
f67539c2
TL		 67struct nvme_tcp_poll_group {
68 struct spdk_nvme_transport_poll_group group;
69 struct spdk_sock_group *sock_group;
70 uint32_t completions_per_qpair;
71 int64_t num_completions;
72};
73
9f95a23c
TL		 74/* NVMe TCP qpair extensions for spdk_nvme_qpair */
75struct nvme_tcp_qpair {
76 struct spdk_nvme_qpair qpair;
77 struct spdk_sock *sock;
78
79 TAILQ_HEAD(, nvme_tcp_req) free_reqs;
80 TAILQ_HEAD(, nvme_tcp_req) outstanding_reqs;
81
82 TAILQ_HEAD(, nvme_tcp_pdu) send_queue;
83 struct nvme_tcp_pdu recv_pdu;
84 struct nvme_tcp_pdu send_pdu; /* only for error pdu and init pdu */
f67539c2 85 struct nvme_tcp_pdu *send_pdus; /* Used by tcp_reqs */
9f95a23c
TL		 86 enum nvme_tcp_pdu_recv_state recv_state;
87
88 struct nvme_tcp_req *tcp_reqs;
89
90 uint16_t num_entries;
91
92 bool host_hdgst_enable;
93 bool host_ddgst_enable;
94
95 /** Specifies the maximum number of PDU-Data bytes per H2C Data Transfer PDU */
96 uint32_t maxh2cdata;
97
98 uint32_t maxr2t;
99
100 /* 0 based value, which is used to guide the padding */
101 uint8_t cpda;
102
103 enum nvme_tcp_qpair_state state;
104};
105
106enum nvme_tcp_req_state {
107 NVME_TCP_REQ_FREE,
108 NVME_TCP_REQ_ACTIVE,
109 NVME_TCP_REQ_ACTIVE_R2T,
110};
111
112struct nvme_tcp_req {
113 struct nvme_request *req;
114 enum nvme_tcp_req_state state;
115 uint16_t cid;
116 uint16_t ttag;
117 uint32_t datao;
118 uint32_t r2tl_remain;
119 uint32_t active_r2ts;
120 bool in_capsule_data;
f67539c2
TL		 121 /* It is used to track whether the req can be safely freed */
122 struct {
123 uint8_t send_ack : 1;
124 uint8_t data_recv : 1;
125 uint8_t r2t_recv : 1;
126 uint8_t reserved : 5;
127 } ordering;
128 struct nvme_tcp_pdu *send_pdu;
9f95a23c
TL		 129 struct iovec iov[NVME_TCP_MAX_SGL_DESCRIPTORS];
130 uint32_t iovcnt;
f67539c2 131 struct nvme_tcp_qpair *tqpair;
9f95a23c
TL		 132 TAILQ_ENTRY(nvme_tcp_req) link;
133};
134
f67539c2 135static void nvme_tcp_send_h2c_data(struct nvme_tcp_req *tcp_req);
9f95a23c
TL		 136
137static inline struct nvme_tcp_qpair *
138nvme_tcp_qpair(struct spdk_nvme_qpair *qpair)
139{
140 assert(qpair->trtype == SPDK_NVME_TRANSPORT_TCP);
141 return SPDK_CONTAINEROF(qpair, struct nvme_tcp_qpair, qpair);
142}
143
f67539c2
TL		 144static inline struct nvme_tcp_poll_group *
145nvme_tcp_poll_group(struct spdk_nvme_transport_poll_group *group)
146{
147 return SPDK_CONTAINEROF(group, struct nvme_tcp_poll_group, group);
148}
149
9f95a23c
TL		 150static inline struct nvme_tcp_ctrlr *
151nvme_tcp_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
152{
153 assert(ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_TCP);
154 return SPDK_CONTAINEROF(ctrlr, struct nvme_tcp_ctrlr, ctrlr);
155}
156
157static struct nvme_tcp_req *
158nvme_tcp_req_get(struct nvme_tcp_qpair *tqpair)
159{
160 struct nvme_tcp_req *tcp_req;
161
162 tcp_req = TAILQ_FIRST(&tqpair->free_reqs);
163 if (!tcp_req) {
164 return NULL;
165 }
166
167 assert(tcp_req->state == NVME_TCP_REQ_FREE);
168 tcp_req->state = NVME_TCP_REQ_ACTIVE;
169 TAILQ_REMOVE(&tqpair->free_reqs, tcp_req, link);
170 tcp_req->datao = 0;
171 tcp_req->req = NULL;
172 tcp_req->in_capsule_data = false;
173 tcp_req->r2tl_remain = 0;
174 tcp_req->active_r2ts = 0;
175 tcp_req->iovcnt = 0;
f67539c2
TL		 176 tcp_req->ordering.send_ack = 0;
177 tcp_req->ordering.data_recv = 0;
178 tcp_req->ordering.r2t_recv = 0;
179 memset(tcp_req->send_pdu, 0, sizeof(struct nvme_tcp_pdu));
9f95a23c
TL		 180 TAILQ_INSERT_TAIL(&tqpair->outstanding_reqs, tcp_req, link);
181
182 return tcp_req;
183}
184
185static void
186nvme_tcp_req_put(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_req *tcp_req)
187{
188 assert(tcp_req->state != NVME_TCP_REQ_FREE);
189 tcp_req->state = NVME_TCP_REQ_FREE;
f67539c2 190 TAILQ_INSERT_HEAD(&tqpair->free_reqs, tcp_req, link);
9f95a23c
TL		 191}
192
193static int
194nvme_tcp_parse_addr(struct sockaddr_storage *sa, int family, const char *addr, const char *service)
195{
196 struct addrinfo *res;
197 struct addrinfo hints;
198 int ret;
199
200 memset(&hints, 0, sizeof(hints));
201 hints.ai_family = family;
202 hints.ai_socktype = SOCK_STREAM;
203 hints.ai_protocol = 0;
204
205 ret = getaddrinfo(addr, service, &hints, &res);
206 if (ret) {
207 SPDK_ERRLOG("getaddrinfo failed: %s (%d)\n", gai_strerror(ret), ret);
208 return ret;
209 }
210
211 if (res->ai_addrlen > sizeof(*sa)) {
212 SPDK_ERRLOG("getaddrinfo() ai_addrlen %zu too large\n", (size_t)res->ai_addrlen);
213 ret = EINVAL;
214 } else {
215 memcpy(sa, res->ai_addr, res->ai_addrlen);
216 }
217
218 freeaddrinfo(res);
219 return ret;
220}
221
222static void
223nvme_tcp_free_reqs(struct nvme_tcp_qpair *tqpair)
224{
225 free(tqpair->tcp_reqs);
226 tqpair->tcp_reqs = NULL;
f67539c2
TL		 227
228 spdk_free(tqpair->send_pdus);
229 tqpair->send_pdus = NULL;
9f95a23c
TL		 230}
231
232static int
233nvme_tcp_alloc_reqs(struct nvme_tcp_qpair *tqpair)
234{
f67539c2 235 uint16_t i;
9f95a23c
TL		 236 struct nvme_tcp_req *tcp_req;
237
238 tqpair->tcp_reqs = calloc(tqpair->num_entries, sizeof(struct nvme_tcp_req));
239 if (tqpair->tcp_reqs == NULL) {
f67539c2
TL		 240 SPDK_ERRLOG("Failed to allocate tcp_reqs on tqpair=%p\n", tqpair);
241 goto fail;
242 }
243
244 tqpair->send_pdus = spdk_zmalloc(tqpair->num_entries * sizeof(struct nvme_tcp_pdu),
245 0x1000, NULL,
246 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
247
248 if (tqpair->send_pdus == NULL) {
249 SPDK_ERRLOG("Failed to allocate send_pdus on tqpair=%p\n", tqpair);
9f95a23c
TL		 250 goto fail;
251 }
252
253 TAILQ_INIT(&tqpair->send_queue);
254 TAILQ_INIT(&tqpair->free_reqs);
255 TAILQ_INIT(&tqpair->outstanding_reqs);
256 for (i = 0; i < tqpair->num_entries; i++) {
257 tcp_req = &tqpair->tcp_reqs[i];
258 tcp_req->cid = i;
f67539c2
TL		 259 tcp_req->tqpair = tqpair;
260 tcp_req->send_pdu = &tqpair->send_pdus[i];
9f95a23c
TL		 261 TAILQ_INSERT_TAIL(&tqpair->free_reqs, tcp_req, link);
262 }
263
264 return 0;
265fail:
266 nvme_tcp_free_reqs(tqpair);
267 return -ENOMEM;
268}
269
270static void
f67539c2 271nvme_tcp_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
9f95a23c
TL		 272{
273 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
274 struct nvme_tcp_pdu *pdu;
275
276 spdk_sock_close(&tqpair->sock);
277
278 /* clear the send_queue */
279 while (!TAILQ_EMPTY(&tqpair->send_queue)) {
280 pdu = TAILQ_FIRST(&tqpair->send_queue);
281 /* Remove the pdu from the send_queue to prevent the wrong sending out
282 * in the next round connection
283 */
284 TAILQ_REMOVE(&tqpair->send_queue, pdu, tailq);
285 }
286}
287
f67539c2
TL		 288static void nvme_tcp_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
289
9f95a23c 290static int
f67539c2 291nvme_tcp_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
9f95a23c
TL		 292{
293 struct nvme_tcp_qpair *tqpair;
294
295 if (!qpair) {
296 return -1;
297 }
298
f67539c2 299 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair);
9f95a23c
TL		 300 nvme_tcp_qpair_abort_reqs(qpair, 1);
301 nvme_qpair_deinit(qpair);
302 tqpair = nvme_tcp_qpair(qpair);
303 nvme_tcp_free_reqs(tqpair);
304 free(tqpair);
305
306 return 0;
307}
308
f67539c2 309static int
9f95a23c
TL		 310nvme_tcp_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr)
311{
312 return 0;
313}
314
f67539c2 315static int
9f95a23c
TL		 316nvme_tcp_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
317{
318 struct nvme_tcp_ctrlr *tctrlr = nvme_tcp_ctrlr(ctrlr);
319
320 if (ctrlr->adminq) {
f67539c2 321 nvme_tcp_ctrlr_delete_io_qpair(ctrlr, ctrlr->adminq);
9f95a23c
TL		 322 }
323
324 nvme_ctrlr_destruct_finish(ctrlr);
325
326 free(tctrlr);
327
328 return 0;
329}
330
f67539c2
TL		 331static void
332_pdu_write_done(void *cb_arg, int err)
9f95a23c 333{
f67539c2
TL		 334 struct nvme_tcp_pdu *pdu = cb_arg;
335 struct nvme_tcp_qpair *tqpair = pdu->qpair;
9f95a23c 336
f67539c2 337 TAILQ_REMOVE(&tqpair->send_queue, pdu, tailq);
9f95a23c 338
f67539c2
TL		 339 if (err != 0) {
340 nvme_transport_ctrlr_disconnect_qpair(tqpair->qpair.ctrlr, &tqpair->qpair);
341 return;
9f95a23c
TL		 342 }
343
f67539c2
TL		 344 assert(pdu->cb_fn != NULL);
345 pdu->cb_fn(pdu->cb_arg);
9f95a23c
TL		 346}
347
348static int
349nvme_tcp_qpair_write_pdu(struct nvme_tcp_qpair *tqpair,
350 struct nvme_tcp_pdu *pdu,
351 nvme_tcp_qpair_xfer_complete_cb cb_fn,
352 void *cb_arg)
353{
9f95a23c
TL		 354 int hlen;
355 uint32_t crc32c;
f67539c2 356 uint32_t mapped_length = 0;
9f95a23c
TL		 357
358 hlen = pdu->hdr.common.hlen;
9f95a23c
TL		 359
360 /* Header Digest */
f67539c2 361 if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) {
9f95a23c
TL		 362 crc32c = nvme_tcp_pdu_calc_header_digest(pdu);
363 MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c);
364 }
365
366 /* Data Digest */
f67539c2 367 if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) {
9f95a23c
TL		 368 crc32c = nvme_tcp_pdu_calc_data_digest(pdu);
369 MAKE_DIGEST_WORD(pdu->data_digest, crc32c);
370 }
371
372 pdu->cb_fn = cb_fn;
373 pdu->cb_arg = cb_arg;
f67539c2
TL		 374
375 pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, NVME_TCP_MAX_SGL_DESCRIPTORS, pdu,
376 tqpair->host_hdgst_enable, tqpair->host_ddgst_enable,
377 &mapped_length);
378 pdu->qpair = tqpair;
379 pdu->sock_req.cb_fn = _pdu_write_done;
380 pdu->sock_req.cb_arg = pdu;
9f95a23c 381 TAILQ_INSERT_TAIL(&tqpair->send_queue, pdu, tailq);
f67539c2
TL		 382 spdk_sock_writev_async(tqpair->sock, &pdu->sock_req);
383
9f95a23c
TL		 384 return 0;
385}
386
387/*
388 * Build SGL describing contiguous payload buffer.
389 */
390static int
391nvme_tcp_build_contig_request(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_req *tcp_req)
392{
393 struct nvme_request *req = tcp_req->req;
394
395 tcp_req->iov[0].iov_base = req->payload.contig_or_cb_arg + req->payload_offset;
396 tcp_req->iov[0].iov_len = req->payload_size;
397 tcp_req->iovcnt = 1;
398
399 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
400
401 assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG);
402
403 return 0;
404}
405
406/*
407 * Build SGL describing scattered payload buffer.
408 */
409static int
410nvme_tcp_build_sgl_request(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_req *tcp_req)
411{
f67539c2
TL		 412 int rc;
413 uint32_t length, remaining_size, iovcnt = 0, max_num_sgl;
9f95a23c
TL		 414 struct nvme_request *req = tcp_req->req;
415
416 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
417
418 assert(req->payload_size != 0);
419 assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
420 assert(req->payload.reset_sgl_fn != NULL);
421 assert(req->payload.next_sge_fn != NULL);
422 req->payload.reset_sgl_fn(req->payload.contig_or_cb_arg, req->payload_offset);
423
f67539c2 424 max_num_sgl = spdk_min(req->qpair->ctrlr->max_sges, NVME_TCP_MAX_SGL_DESCRIPTORS);
9f95a23c 425 remaining_size = req->payload_size;
9f95a23c
TL		 426
427 do {
428 rc = req->payload.next_sge_fn(req->payload.contig_or_cb_arg, &tcp_req->iov[iovcnt].iov_base,
429 &length);
430 if (rc) {
431 return -1;
432 }
433
434 length = spdk_min(length, remaining_size);
435 tcp_req->iov[iovcnt].iov_len = length;
436 remaining_size -= length;
437 iovcnt++;
f67539c2 438 } while (remaining_size > 0 && iovcnt < max_num_sgl);
9f95a23c
TL		 439
440
441 /* Should be impossible if we did our sgl checks properly up the stack, but do a sanity check here. */
442 if (remaining_size > 0) {
f67539c2
TL		 443 SPDK_ERRLOG("Failed to construct tcp_req=%p, and the iovcnt=%u, remaining_size=%u\n",
444 tcp_req, iovcnt, remaining_size);
9f95a23c
TL		 445 return -1;
446 }
447
448 tcp_req->iovcnt = iovcnt;
449
450 return 0;
451}
452
9f95a23c
TL		 453static int
454nvme_tcp_req_init(struct nvme_tcp_qpair *tqpair, struct nvme_request *req,
455 struct nvme_tcp_req *tcp_req)
456{
457 struct spdk_nvme_ctrlr *ctrlr = tqpair->qpair.ctrlr;
458 int rc = 0;
459 enum spdk_nvme_data_transfer xfer;
460 uint32_t max_incapsule_data_size;
461
462 tcp_req->req = req;
463 req->cmd.cid = tcp_req->cid;
464 req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;
465 req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK;
466 req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_TRANSPORT;
467 req->cmd.dptr.sgl1.unkeyed.length = req->payload_size;
468
469 if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG) {
470 rc = nvme_tcp_build_contig_request(tqpair, tcp_req);
471 } else if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL) {
472 rc = nvme_tcp_build_sgl_request(tqpair, tcp_req);
473 } else {
474 rc = -1;
475 }
476
477 if (rc) {
478 return rc;
479 }
480
481 if (req->cmd.opc == SPDK_NVME_OPC_FABRIC) {
482 struct spdk_nvmf_capsule_cmd *nvmf_cmd = (struct spdk_nvmf_capsule_cmd *)&req->cmd;
483
484 xfer = spdk_nvme_opc_get_data_transfer(nvmf_cmd->fctype);
485 } else {
486 xfer = spdk_nvme_opc_get_data_transfer(req->cmd.opc);
487 }
488 if (xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
f67539c2 489 max_incapsule_data_size = ctrlr->ioccsz_bytes;
9f95a23c
TL		 490 if ((req->cmd.opc == SPDK_NVME_OPC_FABRIC) || nvme_qpair_is_admin_queue(&tqpair->qpair)) {
491 max_incapsule_data_size = spdk_min(max_incapsule_data_size, NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE);
492 }
493
494 if (req->payload_size <= max_incapsule_data_size) {
495 req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
496 req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;
497 req->cmd.dptr.sgl1.address = 0;
498 tcp_req->in_capsule_data = true;
499 }
500 }
501
502 return 0;
503}
504
f67539c2
TL		 505static inline void
506nvme_tcp_req_put_safe(struct nvme_tcp_req *tcp_req)
9f95a23c 507{
f67539c2
TL		 508 if (tcp_req->ordering.send_ack && tcp_req->ordering.data_recv) {
509 assert(tcp_req->state == NVME_TCP_REQ_ACTIVE);
510 assert(tcp_req->tqpair != NULL);
511 nvme_tcp_req_put(tcp_req->tqpair, tcp_req);
512 }
9f95a23c
TL		 513}
514
515static void
f67539c2 516nvme_tcp_qpair_cmd_send_complete(void *cb_arg)
9f95a23c 517{
f67539c2 518 struct nvme_tcp_req *tcp_req = cb_arg;
9f95a23c 519
f67539c2
TL		 520 tcp_req->ordering.send_ack = 1;
521 /* Handle the r2t case */
522 if (spdk_unlikely(tcp_req->ordering.r2t_recv)) {
523 nvme_tcp_send_h2c_data(tcp_req);
9f95a23c 524 } else {
f67539c2 525 nvme_tcp_req_put_safe(tcp_req);
9f95a23c
TL		 526 }
527}
528
529static int
530nvme_tcp_qpair_capsule_cmd_send(struct nvme_tcp_qpair *tqpair,
531 struct nvme_tcp_req *tcp_req)
532{
533 struct nvme_tcp_pdu *pdu;
534 struct spdk_nvme_tcp_cmd *capsule_cmd;
535 uint32_t plen = 0, alignment;
536 uint8_t pdo;
537
538 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
f67539c2 539 pdu = tcp_req->send_pdu;
9f95a23c
TL		 540
541 capsule_cmd = &pdu->hdr.capsule_cmd;
542 capsule_cmd->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD;
543 plen = capsule_cmd->common.hlen = sizeof(*capsule_cmd);
544 capsule_cmd->ccsqe = tcp_req->req->cmd;
545
9f95a23c
TL		 546 SPDK_DEBUGLOG(SPDK_LOG_NVME, "capsule_cmd cid=%u on tqpair(%p)\n", tcp_req->req->cmd.cid, tqpair);
547
548 if (tqpair->host_hdgst_enable) {
549 SPDK_DEBUGLOG(SPDK_LOG_NVME, "Header digest is enabled for capsule command on tcp_req=%p\n",
550 tcp_req);
551 capsule_cmd->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF;
552 plen += SPDK_NVME_TCP_DIGEST_LEN;
553 }
554
555 if ((tcp_req->req->payload_size == 0) || !tcp_req->in_capsule_data) {
556 goto end;
557 }
558
559 pdo = plen;
560 pdu->padding_len = 0;
561 if (tqpair->cpda) {
562 alignment = (tqpair->cpda + 1) << 2;
563 if (alignment > plen) {
564 pdu->padding_len = alignment - plen;
565 pdo = alignment;
566 plen = alignment;
567 }
568 }
569
570 capsule_cmd->common.pdo = pdo;
571 plen += tcp_req->req->payload_size;
572 if (tqpair->host_ddgst_enable) {
573 capsule_cmd->common.flags |= SPDK_NVME_TCP_CH_FLAGS_DDGSTF;
574 plen += SPDK_NVME_TCP_DIGEST_LEN;
575 }
576
577 tcp_req->datao = 0;
f67539c2
TL		 578 nvme_tcp_pdu_set_data_buf(pdu, tcp_req->iov, tcp_req->iovcnt,
579 0, tcp_req->req->payload_size);
9f95a23c
TL		 580end:
581 capsule_cmd->common.plen = plen;
f67539c2 582 return nvme_tcp_qpair_write_pdu(tqpair, pdu, nvme_tcp_qpair_cmd_send_complete, tcp_req);
9f95a23c
TL		 583
584}
585
f67539c2 586static int
9f95a23c
TL		 587nvme_tcp_qpair_submit_request(struct spdk_nvme_qpair *qpair,
588 struct nvme_request *req)
589{
590 struct nvme_tcp_qpair *tqpair;
591 struct nvme_tcp_req *tcp_req;
592
593 tqpair = nvme_tcp_qpair(qpair);
594 assert(tqpair != NULL);
595 assert(req != NULL);
596
597 tcp_req = nvme_tcp_req_get(tqpair);
598 if (!tcp_req) {
f67539c2
TL		 599 /* Inform the upper layer to try again later. */
600 return -EAGAIN;
9f95a23c
TL		 601 }
602
603 if (nvme_tcp_req_init(tqpair, req, tcp_req)) {
604 SPDK_ERRLOG("nvme_tcp_req_init() failed\n");
f67539c2 605 TAILQ_REMOVE(&tcp_req->tqpair->outstanding_reqs, tcp_req, link);
9f95a23c
TL		 606 nvme_tcp_req_put(tqpair, tcp_req);
607 return -1;
608 }
609
610 return nvme_tcp_qpair_capsule_cmd_send(tqpair, tcp_req);
611}
612
f67539c2 613static int
9f95a23c
TL		 614nvme_tcp_qpair_reset(struct spdk_nvme_qpair *qpair)
615{
616 return 0;
617}
618
619static void
f67539c2 620nvme_tcp_req_complete(struct nvme_tcp_req *tcp_req,
9f95a23c
TL		 621 struct spdk_nvme_cpl *rsp)
622{
f67539c2
TL		 623 struct nvme_request *req;
624
625 assert(tcp_req->req != NULL);
626 req = tcp_req->req;
627
628 TAILQ_REMOVE(&tcp_req->tqpair->outstanding_reqs, tcp_req, link);
9f95a23c
TL		 629 nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, rsp);
630 nvme_free_request(req);
631}
632
f67539c2 633static void
9f95a23c
TL		 634nvme_tcp_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
635{
636 struct nvme_tcp_req *tcp_req, *tmp;
9f95a23c
TL		 637 struct spdk_nvme_cpl cpl;
638 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
639
640 cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
641 cpl.status.sct = SPDK_NVME_SCT_GENERIC;
642 cpl.status.dnr = dnr;
643
644 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
f67539c2 645 nvme_tcp_req_complete(tcp_req, &cpl);
9f95a23c
TL		 646 nvme_tcp_req_put(tqpair, tcp_req);
647 }
648}
649
650static void
651nvme_tcp_qpair_set_recv_state(struct nvme_tcp_qpair *tqpair,
652 enum nvme_tcp_pdu_recv_state state)
653{
654 if (tqpair->recv_state == state) {
655 SPDK_ERRLOG("The recv state of tqpair=%p is same with the state(%d) to be set\n",
656 tqpair, state);
657 return;
658 }
659
660 tqpair->recv_state = state;
661 switch (state) {
662 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY:
663 case NVME_TCP_PDU_RECV_STATE_ERROR:
664 memset(&tqpair->recv_pdu, 0, sizeof(struct nvme_tcp_pdu));
665 break;
666 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH:
667 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH:
668 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD:
669 default:
670 break;
671 }
672}
673
674static void
675nvme_tcp_qpair_send_h2c_term_req_complete(void *cb_arg)
676{
677 struct nvme_tcp_qpair *tqpair = cb_arg;
678
679 tqpair->state = NVME_TCP_QPAIR_STATE_EXITING;
680}
681
682static void
683nvme_tcp_qpair_send_h2c_term_req(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu,
684 enum spdk_nvme_tcp_term_req_fes fes, uint32_t error_offset)
685{
686 struct nvme_tcp_pdu *rsp_pdu;
687 struct spdk_nvme_tcp_term_req_hdr *h2c_term_req;
688 uint32_t h2c_term_req_hdr_len = sizeof(*h2c_term_req);
689 uint8_t copy_len;
690
691 rsp_pdu = &tqpair->send_pdu;
692 memset(rsp_pdu, 0, sizeof(*rsp_pdu));
693 h2c_term_req = &rsp_pdu->hdr.term_req;
694 h2c_term_req->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ;
695 h2c_term_req->common.hlen = h2c_term_req_hdr_len;
696
697 if ((fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) ||
698 (fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) {
699 DSET32(&h2c_term_req->fei, error_offset);
700 }
701
702 copy_len = pdu->hdr.common.hlen;
703 if (copy_len > SPDK_NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE) {
704 copy_len = SPDK_NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE;
705 }
706
707 /* Copy the error info into the buffer */
708 memcpy((uint8_t *)rsp_pdu->hdr.raw + h2c_term_req_hdr_len, pdu->hdr.raw, copy_len);
709 nvme_tcp_pdu_set_data(rsp_pdu, (uint8_t *)rsp_pdu->hdr.raw + h2c_term_req_hdr_len, copy_len);
710
711 /* Contain the header len of the wrong received pdu */
712 h2c_term_req->common.plen = h2c_term_req->common.hlen + copy_len;
713 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
714 nvme_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvme_tcp_qpair_send_h2c_term_req_complete, NULL);
715
716}
717
718static void
719nvme_tcp_pdu_ch_handle(struct nvme_tcp_qpair *tqpair)
720{
721 struct nvme_tcp_pdu *pdu;
722 uint32_t error_offset = 0;
723 enum spdk_nvme_tcp_term_req_fes fes;
724 uint32_t expected_hlen, hd_len = 0;
725 bool plen_error = false;
726
727 pdu = &tqpair->recv_pdu;
728
729 SPDK_DEBUGLOG(SPDK_LOG_NVME, "pdu type = %d\n", pdu->hdr.common.pdu_type);
730 if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_RESP) {
731 if (tqpair->state != NVME_TCP_QPAIR_STATE_INVALID) {
732 SPDK_ERRLOG("Already received IC_RESP PDU, and we should reject this pdu=%p\n", pdu);
733 fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR;
734 goto err;
735 }
736 expected_hlen = sizeof(struct spdk_nvme_tcp_ic_resp);
737 if (pdu->hdr.common.plen != expected_hlen) {
738 plen_error = true;
739 }
740 } else {
741 if (tqpair->state != NVME_TCP_QPAIR_STATE_RUNNING) {
742 SPDK_ERRLOG("The TCP/IP tqpair connection is not negotitated\n");
743 fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR;
744 goto err;
745 }
746
747 switch (pdu->hdr.common.pdu_type) {
748 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP:
749 expected_hlen = sizeof(struct spdk_nvme_tcp_rsp);
750 if (pdu->hdr.common.flags & SPDK_NVME_TCP_CH_FLAGS_HDGSTF) {
751 hd_len = SPDK_NVME_TCP_DIGEST_LEN;
752 }
753
754 if (pdu->hdr.common.plen != (expected_hlen + hd_len)) {
755 plen_error = true;
756 }
757 break;
758 case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
759 expected_hlen = sizeof(struct spdk_nvme_tcp_c2h_data_hdr);
760 if (pdu->hdr.common.plen < pdu->hdr.common.pdo) {
761 plen_error = true;
762 }
763 break;
764 case SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
765 expected_hlen = sizeof(struct spdk_nvme_tcp_term_req_hdr);
766 if ((pdu->hdr.common.plen <= expected_hlen) ||
767 (pdu->hdr.common.plen > SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE)) {
768 plen_error = true;
769 }
770 break;
771 case SPDK_NVME_TCP_PDU_TYPE_R2T:
772 expected_hlen = sizeof(struct spdk_nvme_tcp_r2t_hdr);
773 if (pdu->hdr.common.flags & SPDK_NVME_TCP_CH_FLAGS_HDGSTF) {
774 hd_len = SPDK_NVME_TCP_DIGEST_LEN;
775 }
776
777 if (pdu->hdr.common.plen != (expected_hlen + hd_len)) {
778 plen_error = true;
779 }
780 break;
781
782 default:
783 SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", tqpair->recv_pdu.hdr.common.pdu_type);
784 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
785 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdu_type);
786 goto err;
787 }
788 }
789
790 if (pdu->hdr.common.hlen != expected_hlen) {
791 SPDK_ERRLOG("Expected PDU header length %u, got %u\n",
792 expected_hlen, pdu->hdr.common.hlen);
793 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
794 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, hlen);
795 goto err;
796
797 } else if (plen_error) {
798 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
799 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, plen);
800 goto err;
801 } else {
802 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH);
f67539c2 803 nvme_tcp_pdu_calc_psh_len(&tqpair->recv_pdu, tqpair->host_hdgst_enable);
9f95a23c
TL		 804 return;
805 }
806err:
807 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
808}
809
810static struct nvme_tcp_req *
811get_nvme_active_req_by_cid(struct nvme_tcp_qpair *tqpair, uint32_t cid)
812{
813 assert(tqpair != NULL);
814 if ((cid >= tqpair->num_entries) || (tqpair->tcp_reqs[cid].state == NVME_TCP_REQ_FREE)) {
815 return NULL;
816 }
817
818 return &tqpair->tcp_reqs[cid];
819}
820
9f95a23c
TL		 821static void
822nvme_tcp_c2h_data_payload_handle(struct nvme_tcp_qpair *tqpair,
823 struct nvme_tcp_pdu *pdu, uint32_t *reaped)
824{
825 struct nvme_tcp_req *tcp_req;
826 struct spdk_nvme_tcp_c2h_data_hdr *c2h_data;
827 struct spdk_nvme_cpl cpl = {};
828 uint8_t flags;
829
f67539c2 830 tcp_req = pdu->req;
9f95a23c
TL		 831 assert(tcp_req != NULL);
832
833 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
834 c2h_data = &pdu->hdr.c2h_data;
835 tcp_req->datao += pdu->data_len;
836 flags = c2h_data->common.flags;
837
838 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
839 if (flags & SPDK_NVME_TCP_C2H_DATA_FLAGS_SUCCESS) {
840 if (tcp_req->datao == tcp_req->req->payload_size) {
841 cpl.status.p = 0;
842 } else {
843 cpl.status.p = 1;
844 }
845
846 cpl.cid = tcp_req->cid;
847 cpl.sqid = tqpair->qpair.id;
f67539c2
TL		 848 nvme_tcp_req_complete(tcp_req, &cpl);
849 if (tcp_req->ordering.send_ack) {
850 (*reaped)++;
851 }
852
853 tcp_req->ordering.data_recv = 1;
854 nvme_tcp_req_put_safe(tcp_req);
9f95a23c
TL		 855 }
856}
857
858static const char *spdk_nvme_tcp_term_req_fes_str[] = {
859 "Invalid PDU Header Field",
860 "PDU Sequence Error",
861 "Header Digest Error",
862 "Data Transfer Out of Range",
863 "Data Transfer Limit Exceeded",
864 "Unsupported parameter",
865};
866
867static void
868nvme_tcp_c2h_term_req_dump(struct spdk_nvme_tcp_term_req_hdr *c2h_term_req)
869{
870 SPDK_ERRLOG("Error info of pdu(%p): %s\n", c2h_term_req,
871 spdk_nvme_tcp_term_req_fes_str[c2h_term_req->fes]);
872 if ((c2h_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) ||
873 (c2h_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) {
874 SPDK_DEBUGLOG(SPDK_LOG_NVME, "The offset from the start of the PDU header is %u\n",
875 DGET32(c2h_term_req->fei));
876 }
877 /* we may also need to dump some other info here */
878}
879
880static void
881nvme_tcp_c2h_term_req_payload_handle(struct nvme_tcp_qpair *tqpair,
882 struct nvme_tcp_pdu *pdu)
883{
884 nvme_tcp_c2h_term_req_dump(&pdu->hdr.term_req);
885 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
886}
887
888static void
889nvme_tcp_pdu_payload_handle(struct nvme_tcp_qpair *tqpair,
890 uint32_t *reaped)
891{
892 int rc = 0;
893 struct nvme_tcp_pdu *pdu;
894 uint32_t crc32c, error_offset = 0;
895 enum spdk_nvme_tcp_term_req_fes fes;
896
897 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD);
898 pdu = &tqpair->recv_pdu;
899
900 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
901
902 /* check data digest if need */
903 if (pdu->ddgst_enable) {
904 crc32c = nvme_tcp_pdu_calc_data_digest(pdu);
905 rc = MATCH_DIGEST_WORD(pdu->data_digest, crc32c);
906 if (rc == 0) {
907 SPDK_ERRLOG("data digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu);
908 fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR;
909 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
910 return;
911 }
912 }
913
914 switch (pdu->hdr.common.pdu_type) {
915 case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
916 nvme_tcp_c2h_data_payload_handle(tqpair, pdu, reaped);
917 break;
918
919 case SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
920 nvme_tcp_c2h_term_req_payload_handle(tqpair, pdu);
921 break;
922
923 default:
924 /* The code should not go to here */
925 SPDK_ERRLOG("The code should not go to here\n");
926 break;
927 }
928}
929
930static void
931nvme_tcp_send_icreq_complete(void *cb_arg)
932{
933 SPDK_DEBUGLOG(SPDK_LOG_NVME, "Complete the icreq send for tqpair=%p\n",
934 (struct nvme_tcp_qpair *)cb_arg);
935}
936
937static void
938nvme_tcp_icresp_handle(struct nvme_tcp_qpair *tqpair,
939 struct nvme_tcp_pdu *pdu)
940{
941 struct spdk_nvme_tcp_ic_resp *ic_resp = &pdu->hdr.ic_resp;
942 uint32_t error_offset = 0;
943 enum spdk_nvme_tcp_term_req_fes fes;
f67539c2 944 int recv_buf_size;
9f95a23c
TL		 945
946 /* Only PFV 0 is defined currently */
947 if (ic_resp->pfv != 0) {
948 SPDK_ERRLOG("Expected ICResp PFV %u, got %u\n", 0u, ic_resp->pfv);
949 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
950 error_offset = offsetof(struct spdk_nvme_tcp_ic_resp, pfv);
951 goto end;
952 }
953
954 if (ic_resp->maxh2cdata < NVME_TCP_PDU_H2C_MIN_DATA_SIZE) {
955 SPDK_ERRLOG("Expected ICResp maxh2cdata >=%u, got %u\n", NVME_TCP_PDU_H2C_MIN_DATA_SIZE,
956 ic_resp->maxh2cdata);
957 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
958 error_offset = offsetof(struct spdk_nvme_tcp_ic_resp, maxh2cdata);
959 goto end;
960 }
961 tqpair->maxh2cdata = ic_resp->maxh2cdata;
962
963 if (ic_resp->cpda > SPDK_NVME_TCP_CPDA_MAX) {
964 SPDK_ERRLOG("Expected ICResp cpda <=%u, got %u\n", SPDK_NVME_TCP_CPDA_MAX, ic_resp->cpda);
965 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
966 error_offset = offsetof(struct spdk_nvme_tcp_ic_resp, cpda);
967 goto end;
968 }
969 tqpair->cpda = ic_resp->cpda;
970
971 tqpair->host_hdgst_enable = ic_resp->dgst.bits.hdgst_enable ? true : false;
972 tqpair->host_ddgst_enable = ic_resp->dgst.bits.ddgst_enable ? true : false;
973 SPDK_DEBUGLOG(SPDK_LOG_NVME, "host_hdgst_enable: %u\n", tqpair->host_hdgst_enable);
974 SPDK_DEBUGLOG(SPDK_LOG_NVME, "host_ddgst_enable: %u\n", tqpair->host_ddgst_enable);
975
f67539c2
TL		 976 /* Now that we know whether digests are enabled, properly size the receive buffer to
977 * handle several incoming 4K read commands according to SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR
978 * parameter. */
979 recv_buf_size = 0x1000 + sizeof(struct spdk_nvme_tcp_c2h_data_hdr);
980
981 if (tqpair->host_hdgst_enable) {
982 recv_buf_size += SPDK_NVME_TCP_DIGEST_LEN;
983 }
984
985 if (tqpair->host_ddgst_enable) {
986 recv_buf_size += SPDK_NVME_TCP_DIGEST_LEN;
987 }
988
989 if (spdk_sock_set_recvbuf(tqpair->sock, recv_buf_size * SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR) < 0) {
990 SPDK_WARNLOG("Unable to allocate enough memory for receive buffer on tqpair=%p with size=%d\n",
991 tqpair,
992 recv_buf_size);
993 /* Not fatal. */
994 }
995
9f95a23c
TL		 996 tqpair->state = NVME_TCP_QPAIR_STATE_RUNNING;
997 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
998 return;
999end:
1000 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1001 return;
1002}
1003
1004static void
1005nvme_tcp_capsule_resp_hdr_handle(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu,
1006 uint32_t *reaped)
1007{
1008 struct nvme_tcp_req *tcp_req;
1009 struct spdk_nvme_tcp_rsp *capsule_resp = &pdu->hdr.capsule_resp;
1010 uint32_t cid, error_offset = 0;
1011 enum spdk_nvme_tcp_term_req_fes fes;
1012 struct spdk_nvme_cpl cpl;
1013
1014 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
1015 cpl = capsule_resp->rccqe;
1016 cid = cpl.cid;
1017
1018 /* Recv the pdu again */
1019 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
1020
1021 tcp_req = get_nvme_active_req_by_cid(tqpair, cid);
1022 if (!tcp_req) {
1023 SPDK_ERRLOG("no tcp_req is found with cid=%u for tqpair=%p\n", cid, tqpair);
1024 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1025 error_offset = offsetof(struct spdk_nvme_tcp_rsp, rccqe);
1026 goto end;
1027
1028 }
1029
f67539c2
TL		 1030 nvme_tcp_req_complete(tcp_req, &cpl);
1031 if (tcp_req->ordering.send_ack) {
1032 (*reaped)++;
1033 }
1034
1035 tcp_req->ordering.data_recv = 1;
1036 nvme_tcp_req_put_safe(tcp_req);
9f95a23c
TL		 1037
1038 SPDK_DEBUGLOG(SPDK_LOG_NVME, "complete tcp_req(%p) on tqpair=%p\n", tcp_req, tqpair);
1039
1040 return;
1041
1042end:
1043 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1044 return;
1045}
1046
1047static void
1048nvme_tcp_c2h_term_req_hdr_handle(struct nvme_tcp_qpair *tqpair,
1049 struct nvme_tcp_pdu *pdu)
1050{
1051 struct spdk_nvme_tcp_term_req_hdr *c2h_term_req = &pdu->hdr.term_req;
1052 uint32_t error_offset = 0;
1053 enum spdk_nvme_tcp_term_req_fes fes;
1054
1055 if (c2h_term_req->fes > SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER) {
1056 SPDK_ERRLOG("Fatal Error Stauts(FES) is unknown for c2h_term_req pdu=%p\n", pdu);
1057 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1058 error_offset = offsetof(struct spdk_nvme_tcp_term_req_hdr, fes);
1059 goto end;
1060 }
1061
1062 /* set the data buffer */
1063 nvme_tcp_pdu_set_data(pdu, (uint8_t *)pdu->hdr.raw + c2h_term_req->common.hlen,
1064 c2h_term_req->common.plen - c2h_term_req->common.hlen);
1065 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD);
1066 return;
1067end:
1068 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1069 return;
1070}
1071
1072static void
1073nvme_tcp_c2h_data_hdr_handle(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu)
1074{
1075 struct nvme_tcp_req *tcp_req;
1076 struct spdk_nvme_tcp_c2h_data_hdr *c2h_data = &pdu->hdr.c2h_data;
1077 uint32_t error_offset = 0;
1078 enum spdk_nvme_tcp_term_req_fes fes;
1079
1080 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
1081 SPDK_DEBUGLOG(SPDK_LOG_NVME, "c2h_data info on tqpair(%p): datao=%u, datal=%u, cccid=%d\n",
1082 tqpair, c2h_data->datao, c2h_data->datal, c2h_data->cccid);
1083 tcp_req = get_nvme_active_req_by_cid(tqpair, c2h_data->cccid);
1084 if (!tcp_req) {
1085 SPDK_ERRLOG("no tcp_req found for c2hdata cid=%d\n", c2h_data->cccid);
1086 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1087 error_offset = offsetof(struct spdk_nvme_tcp_c2h_data_hdr, cccid);
1088 goto end;
1089
1090 }
1091
1092 SPDK_DEBUGLOG(SPDK_LOG_NVME, "tcp_req(%p) on tqpair(%p): datao=%u, payload_size=%u\n",
1093 tcp_req, tqpair, tcp_req->datao, tcp_req->req->payload_size);
1094
1095 if (c2h_data->datal > tcp_req->req->payload_size) {
1096 SPDK_ERRLOG("Invalid datal for tcp_req(%p), datal(%u) exceeds payload_size(%u)\n",
1097 tcp_req, c2h_data->datal, tcp_req->req->payload_size);
1098 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE;
1099 goto end;
1100 }
1101
1102 if (tcp_req->datao != c2h_data->datao) {
1103 SPDK_ERRLOG("Invalid datao for tcp_req(%p), received datal(%u) != datao(%u) in tcp_req\n",
1104 tcp_req, c2h_data->datao, tcp_req->datao);
1105 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1106 error_offset = offsetof(struct spdk_nvme_tcp_c2h_data_hdr, datao);
1107 goto end;
1108 }
1109
1110 if ((c2h_data->datao + c2h_data->datal) > tcp_req->req->payload_size) {
1111 SPDK_ERRLOG("Invalid data range for tcp_req(%p), received (datao(%u) + datal(%u)) > datao(%u) in tcp_req\n",
1112 tcp_req, c2h_data->datao, c2h_data->datal, tcp_req->req->payload_size);
1113 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE;
1114 error_offset = offsetof(struct spdk_nvme_tcp_c2h_data_hdr, datal);
1115 goto end;
1116
1117 }
1118
f67539c2
TL		 1119 nvme_tcp_pdu_set_data_buf(pdu, tcp_req->iov, tcp_req->iovcnt,
1120 c2h_data->datao, c2h_data->datal);
1121 pdu->req = tcp_req;
9f95a23c
TL		 1122
1123 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD);
1124 return;
1125
1126end:
1127 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1128 return;
1129}
1130
1131static void
1132nvme_tcp_qpair_h2c_data_send_complete(void *cb_arg)
1133{
1134 struct nvme_tcp_req *tcp_req = cb_arg;
1135
1136 assert(tcp_req != NULL);
1137
f67539c2 1138 tcp_req->ordering.send_ack = 1;
9f95a23c 1139 if (tcp_req->r2tl_remain) {
f67539c2 1140 nvme_tcp_send_h2c_data(tcp_req);
9f95a23c
TL		 1141 } else {
1142 assert(tcp_req->active_r2ts > 0);
1143 tcp_req->active_r2ts--;
1144 tcp_req->state = NVME_TCP_REQ_ACTIVE;
f67539c2
TL		 1145 /* Need also call this function to free the resource */
1146 nvme_tcp_req_put_safe(tcp_req);
9f95a23c
TL		 1147 }
1148}
1149
1150static void
f67539c2 1151nvme_tcp_send_h2c_data(struct nvme_tcp_req *tcp_req)
9f95a23c
TL		 1152{
1153 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(tcp_req->req->qpair);
1154 struct nvme_tcp_pdu *rsp_pdu;
1155 struct spdk_nvme_tcp_h2c_data_hdr *h2c_data;
1156 uint32_t plen, pdo, alignment;
1157
f67539c2
TL		 1158 /* Reinit the send_ack and r2t_recv bits */
1159 tcp_req->ordering.send_ack = 0;
1160 tcp_req->ordering.r2t_recv = 0;
1161 rsp_pdu = tcp_req->send_pdu;
9f95a23c
TL		 1162 memset(rsp_pdu, 0, sizeof(*rsp_pdu));
1163 h2c_data = &rsp_pdu->hdr.h2c_data;
1164
1165 h2c_data->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_H2C_DATA;
1166 plen = h2c_data->common.hlen = sizeof(*h2c_data);
1167 h2c_data->cccid = tcp_req->cid;
1168 h2c_data->ttag = tcp_req->ttag;
1169 h2c_data->datao = tcp_req->datao;
1170
1171 h2c_data->datal = spdk_min(tcp_req->r2tl_remain, tqpair->maxh2cdata);
f67539c2
TL		 1172 nvme_tcp_pdu_set_data_buf(rsp_pdu, tcp_req->iov, tcp_req->iovcnt,
1173 h2c_data->datao, h2c_data->datal);
9f95a23c
TL		 1174 tcp_req->r2tl_remain -= h2c_data->datal;
1175
1176 if (tqpair->host_hdgst_enable) {
1177 h2c_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF;
1178 plen += SPDK_NVME_TCP_DIGEST_LEN;
1179 }
1180
1181 rsp_pdu->padding_len = 0;
1182 pdo = plen;
1183 if (tqpair->cpda) {
1184 alignment = (tqpair->cpda + 1) << 2;
1185 if (alignment > plen) {
1186 rsp_pdu->padding_len = alignment - plen;
1187 pdo = plen = alignment;
1188 }
1189 }
1190
1191 h2c_data->common.pdo = pdo;
1192 plen += h2c_data->datal;
1193 if (tqpair->host_ddgst_enable) {
1194 h2c_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_DDGSTF;
1195 plen += SPDK_NVME_TCP_DIGEST_LEN;
1196 }
1197
1198 h2c_data->common.plen = plen;
1199 tcp_req->datao += h2c_data->datal;
1200 if (!tcp_req->r2tl_remain) {
1201 h2c_data->common.flags |= SPDK_NVME_TCP_H2C_DATA_FLAGS_LAST_PDU;
1202 }
1203
1204 SPDK_DEBUGLOG(SPDK_LOG_NVME, "h2c_data info: datao=%u, datal=%u, pdu_len=%u for tqpair=%p\n",
1205 h2c_data->datao, h2c_data->datal, h2c_data->common.plen, tqpair);
1206
1207 nvme_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvme_tcp_qpair_h2c_data_send_complete, tcp_req);
1208}
1209
1210static void
1211nvme_tcp_r2t_hdr_handle(struct nvme_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu)
1212{
1213 struct nvme_tcp_req *tcp_req;
1214 struct spdk_nvme_tcp_r2t_hdr *r2t = &pdu->hdr.r2t;
1215 uint32_t cid, error_offset = 0;
1216 enum spdk_nvme_tcp_term_req_fes fes;
1217
1218 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter\n");
1219 cid = r2t->cccid;
1220 tcp_req = get_nvme_active_req_by_cid(tqpair, cid);
1221 if (!tcp_req) {
1222 SPDK_ERRLOG("Cannot find tcp_req for tqpair=%p\n", tqpair);
1223 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1224 error_offset = offsetof(struct spdk_nvme_tcp_r2t_hdr, cccid);
1225 goto end;
1226 }
1227
f67539c2 1228 tcp_req->ordering.r2t_recv = 1;
9f95a23c
TL		 1229 SPDK_DEBUGLOG(SPDK_LOG_NVME, "r2t info: r2to=%u, r2tl=%u for tqpair=%p\n", r2t->r2to, r2t->r2tl,
1230 tqpair);
1231
1232 if (tcp_req->state == NVME_TCP_REQ_ACTIVE) {
1233 assert(tcp_req->active_r2ts == 0);
1234 tcp_req->state = NVME_TCP_REQ_ACTIVE_R2T;
1235 }
1236
1237 tcp_req->active_r2ts++;
1238 if (tcp_req->active_r2ts > tqpair->maxr2t) {
1239 fes = SPDK_NVME_TCP_TERM_REQ_FES_R2T_LIMIT_EXCEEDED;
1240 SPDK_ERRLOG("Invalid R2T: it exceeds the R2T maixmal=%u for tqpair=%p\n", tqpair->maxr2t, tqpair);
1241 goto end;
1242 }
1243
1244 if (tcp_req->datao != r2t->r2to) {
1245 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1246 error_offset = offsetof(struct spdk_nvme_tcp_r2t_hdr, r2to);
1247 goto end;
1248
1249 }
1250
1251 if ((r2t->r2tl + r2t->r2to) > tcp_req->req->payload_size) {
1252 SPDK_ERRLOG("Invalid R2T info for tcp_req=%p: (r2to(%u) + r2tl(%u)) exceeds payload_size(%u)\n",
1253 tcp_req, r2t->r2to, r2t->r2tl, tqpair->maxh2cdata);
1254 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE;
1255 error_offset = offsetof(struct spdk_nvme_tcp_r2t_hdr, r2tl);
1256 goto end;
1257
1258 }
1259
1260 tcp_req->ttag = r2t->ttag;
1261 tcp_req->r2tl_remain = r2t->r2tl;
1262 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
1263
f67539c2
TL		 1264 if (spdk_likely(tcp_req->ordering.send_ack)) {
1265 nvme_tcp_send_h2c_data(tcp_req);
1266 }
9f95a23c
TL		 1267 return;
1268
1269end:
1270 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1271 return;
1272
1273}
1274
1275static void
1276nvme_tcp_pdu_psh_handle(struct nvme_tcp_qpair *tqpair, uint32_t *reaped)
1277{
1278 struct nvme_tcp_pdu *pdu;
1279 int rc;
1280 uint32_t crc32c, error_offset = 0;
1281 enum spdk_nvme_tcp_term_req_fes fes;
1282
1283 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH);
1284 pdu = &tqpair->recv_pdu;
1285
1286 SPDK_DEBUGLOG(SPDK_LOG_NVME, "enter: pdu type =%u\n", pdu->hdr.common.pdu_type);
1287 /* check header digest if needed */
1288 if (pdu->has_hdgst) {
1289 crc32c = nvme_tcp_pdu_calc_header_digest(pdu);
1290 rc = MATCH_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, crc32c);
1291 if (rc == 0) {
1292 SPDK_ERRLOG("header digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu);
1293 fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR;
1294 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1295 return;
1296
1297 }
1298 }
1299
1300 switch (pdu->hdr.common.pdu_type) {
1301 case SPDK_NVME_TCP_PDU_TYPE_IC_RESP:
1302 nvme_tcp_icresp_handle(tqpair, pdu);
1303 break;
1304 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP:
1305 nvme_tcp_capsule_resp_hdr_handle(tqpair, pdu, reaped);
1306 break;
1307 case SPDK_NVME_TCP_PDU_TYPE_C2H_DATA:
1308 nvme_tcp_c2h_data_hdr_handle(tqpair, pdu);
1309 break;
1310
1311 case SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ:
1312 nvme_tcp_c2h_term_req_hdr_handle(tqpair, pdu);
1313 break;
1314 case SPDK_NVME_TCP_PDU_TYPE_R2T:
1315 nvme_tcp_r2t_hdr_handle(tqpair, pdu);
1316 break;
1317
1318 default:
1319 SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", tqpair->recv_pdu.hdr.common.pdu_type);
1320 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD;
1321 error_offset = 1;
1322 nvme_tcp_qpair_send_h2c_term_req(tqpair, pdu, fes, error_offset);
1323 break;
1324 }
1325
1326}
1327
1328static int
1329nvme_tcp_read_pdu(struct nvme_tcp_qpair *tqpair, uint32_t *reaped)
1330{
1331 int rc = 0;
1332 struct nvme_tcp_pdu *pdu;
1333 uint32_t data_len;
9f95a23c
TL		 1334 enum nvme_tcp_pdu_recv_state prev_state;
1335
1336 /* The loop here is to allow for several back-to-back state changes. */
1337 do {
1338 prev_state = tqpair->recv_state;
1339 switch (tqpair->recv_state) {
1340 /* If in a new state */
1341 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY:
1342 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH);
1343 break;
1344 /* common header */
1345 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH:
1346 pdu = &tqpair->recv_pdu;
1347 if (pdu->ch_valid_bytes < sizeof(struct spdk_nvme_tcp_common_pdu_hdr)) {
1348 rc = nvme_tcp_read_data(tqpair->sock,
1349 sizeof(struct spdk_nvme_tcp_common_pdu_hdr) - pdu->ch_valid_bytes,
1350 (uint8_t *)&pdu->hdr.common + pdu->ch_valid_bytes);
1351 if (rc < 0) {
1352 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
1353 break;
1354 }
1355 pdu->ch_valid_bytes += rc;
1356 if (pdu->ch_valid_bytes < sizeof(struct spdk_nvme_tcp_common_pdu_hdr)) {
1357 return NVME_TCP_PDU_IN_PROGRESS;
1358 }
1359 }
1360
1361 /* The command header of this PDU has now been read from the socket. */
1362 nvme_tcp_pdu_ch_handle(tqpair);
1363 break;
1364 /* Wait for the pdu specific header */
1365 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH:
1366 pdu = &tqpair->recv_pdu;
f67539c2
TL		 1367 rc = nvme_tcp_read_data(tqpair->sock,
1368 pdu->psh_len - pdu->psh_valid_bytes,
1369 (uint8_t *)&pdu->hdr.raw + sizeof(struct spdk_nvme_tcp_common_pdu_hdr) + pdu->psh_valid_bytes);
1370 if (rc < 0) {
1371 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
1372 break;
9f95a23c
TL		 1373 }
1374
f67539c2
TL		 1375 pdu->psh_valid_bytes += rc;
1376 if (pdu->psh_valid_bytes < pdu->psh_len) {
1377 return NVME_TCP_PDU_IN_PROGRESS;
9f95a23c
TL		 1378 }
1379
1380 /* All header(ch, psh, head digist) of this PDU has now been read from the socket. */
1381 nvme_tcp_pdu_psh_handle(tqpair, reaped);
1382 break;
1383 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD:
1384 pdu = &tqpair->recv_pdu;
1385 /* check whether the data is valid, if not we just return */
1386 if (!pdu->data_len) {
1387 return NVME_TCP_PDU_IN_PROGRESS;
1388 }
1389
1390 data_len = pdu->data_len;
1391 /* data digest */
1392 if (spdk_unlikely((pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_C2H_DATA) &&
1393 tqpair->host_ddgst_enable)) {
1394 data_len += SPDK_NVME_TCP_DIGEST_LEN;
1395 pdu->ddgst_enable = true;
9f95a23c
TL		 1396 }
1397
1398 rc = nvme_tcp_read_payload_data(tqpair->sock, pdu);
1399 if (rc < 0) {
1400 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR);
1401 break;
1402 }
1403
1404 pdu->readv_offset += rc;
1405 if (pdu->readv_offset < data_len) {
1406 return NVME_TCP_PDU_IN_PROGRESS;
1407 }
1408
1409 assert(pdu->readv_offset == data_len);
1410 /* All of this PDU has now been read from the socket. */
1411 nvme_tcp_pdu_payload_handle(tqpair, reaped);
1412 break;
1413 case NVME_TCP_PDU_RECV_STATE_ERROR:
1414 rc = NVME_TCP_PDU_FATAL;
1415 break;
1416 default:
1417 assert(0);
1418 break;
1419 }
1420 } while (prev_state != tqpair->recv_state);
1421
1422 return rc;
1423}
1424
1425static void
1426nvme_tcp_qpair_check_timeout(struct spdk_nvme_qpair *qpair)
1427{
1428 uint64_t t02;
1429 struct nvme_tcp_req *tcp_req, *tmp;
1430 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1431 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
1432 struct spdk_nvme_ctrlr_process *active_proc;
1433
1434 /* Don't check timeouts during controller initialization. */
1435 if (ctrlr->state != NVME_CTRLR_STATE_READY) {
1436 return;
1437 }
1438
1439 if (nvme_qpair_is_admin_queue(qpair)) {
f67539c2 1440 active_proc = nvme_ctrlr_get_current_process(ctrlr);
9f95a23c
TL		 1441 } else {
1442 active_proc = qpair->active_proc;
1443 }
1444
1445 /* Only check timeouts if the current process has a timeout callback. */
1446 if (active_proc == NULL || active_proc->timeout_cb_fn == NULL) {
1447 return;
1448 }
1449
1450 t02 = spdk_get_ticks();
1451 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
1452 assert(tcp_req->req != NULL);
1453
1454 if (nvme_request_check_timeout(tcp_req->req, tcp_req->cid, active_proc, t02)) {
1455 /*
1456 * The requests are in order, so as soon as one has not timed out,
1457 * stop iterating.
1458 */
1459 break;
1460 }
1461 }
1462}
1463
f67539c2 1464static int
9f95a23c
TL		 1465nvme_tcp_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
1466{
1467 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1468 uint32_t reaped;
1469 int rc;
1470
f67539c2 1471 rc = spdk_sock_flush(tqpair->sock);
9f95a23c
TL		 1472 if (rc < 0) {
1473 return rc;
1474 }
1475
1476 if (max_completions == 0) {
1477 max_completions = tqpair->num_entries;
1478 } else {
1479 max_completions = spdk_min(max_completions, tqpair->num_entries);
1480 }
1481
1482 reaped = 0;
1483 do {
1484 rc = nvme_tcp_read_pdu(tqpair, &reaped);
1485 if (rc < 0) {
f67539c2
TL		 1486 SPDK_DEBUGLOG(SPDK_LOG_NVME, "Error polling CQ! (%d): %s\n",
1487 errno, spdk_strerror(errno));
1488 goto fail;
9f95a23c
TL		 1489 } else if (rc == 0) {
1490 /* Partial PDU is read */
1491 break;
1492 }
1493
1494 } while (reaped < max_completions);
1495
1496 if (spdk_unlikely(tqpair->qpair.ctrlr->timeout_enabled)) {
1497 nvme_tcp_qpair_check_timeout(qpair);
1498 }
1499
1500 return reaped;
f67539c2
TL		 1501fail:
1502
1503 /*
1504 * Since admin queues take the ctrlr_lock before entering this function,
1505 * we can call nvme_transport_ctrlr_disconnect_qpair. For other qpairs we need
1506 * to call the generic function which will take the lock for us.
1507 */
1508 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_UNKNOWN;
1509
1510 if (nvme_qpair_is_admin_queue(qpair)) {
1511 nvme_transport_ctrlr_disconnect_qpair(qpair->ctrlr, qpair);
1512 } else {
1513 nvme_ctrlr_disconnect_qpair(qpair);
1514 }
1515 return -ENXIO;
1516}
1517
1518static void
1519nvme_tcp_qpair_sock_cb(void *ctx, struct spdk_sock_group *group, struct spdk_sock *sock)
1520{
1521 struct spdk_nvme_qpair *qpair = ctx;
1522 struct nvme_tcp_poll_group *pgroup = nvme_tcp_poll_group(qpair->poll_group);
1523 int32_t num_completions;
1524
1525 num_completions = spdk_nvme_qpair_process_completions(qpair, pgroup->completions_per_qpair);
1526
1527 if (pgroup->num_completions >= 0 && num_completions >= 0) {
1528 pgroup->num_completions += num_completions;
1529 } else {
1530 pgroup->num_completions = -ENXIO;
1531 }
9f95a23c
TL		 1532}
1533
1534static int
1535nvme_tcp_qpair_icreq_send(struct nvme_tcp_qpair *tqpair)
1536{
1537 struct spdk_nvme_tcp_ic_req *ic_req;
1538 struct nvme_tcp_pdu *pdu;
f67539c2
TL		 1539 uint64_t icreq_timeout_tsc;
1540 int rc;
9f95a23c
TL		 1541
1542 pdu = &tqpair->send_pdu;
1543 memset(&tqpair->send_pdu, 0, sizeof(tqpair->send_pdu));
1544 ic_req = &pdu->hdr.ic_req;
1545
1546 ic_req->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_IC_REQ;
1547 ic_req->common.hlen = ic_req->common.plen = sizeof(*ic_req);
1548 ic_req->pfv = 0;
1549 ic_req->maxr2t = NVME_TCP_MAX_R2T_DEFAULT - 1;
1550 ic_req->hpda = NVME_TCP_HPDA_DEFAULT;
1551
1552 ic_req->dgst.bits.hdgst_enable = tqpair->qpair.ctrlr->opts.header_digest;
1553 ic_req->dgst.bits.ddgst_enable = tqpair->qpair.ctrlr->opts.data_digest;
1554
1555 nvme_tcp_qpair_write_pdu(tqpair, pdu, nvme_tcp_send_icreq_complete, tqpair);
1556
f67539c2
TL		 1557 icreq_timeout_tsc = spdk_get_ticks() + (NVME_TCP_TIME_OUT_IN_SECONDS * spdk_get_ticks_hz());
1558 do {
1559 rc = nvme_tcp_qpair_process_completions(&tqpair->qpair, 0);
1560 } while ((tqpair->state == NVME_TCP_QPAIR_STATE_INVALID) &&
1561 (rc == 0) && (spdk_get_ticks() <= icreq_timeout_tsc));
9f95a23c
TL		 1562
1563 if (tqpair->state != NVME_TCP_QPAIR_STATE_RUNNING) {
1564 SPDK_ERRLOG("Failed to construct the tqpair=%p via correct icresp\n", tqpair);
1565 return -1;
1566 }
1567
1568 SPDK_DEBUGLOG(SPDK_LOG_NVME, "Succesfully construct the tqpair=%p via correct icresp\n", tqpair);
1569
1570 return 0;
1571}
1572
1573static int
f67539c2 1574nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
9f95a23c
TL		 1575{
1576 struct sockaddr_storage dst_addr;
1577 struct sockaddr_storage src_addr;
1578 int rc;
f67539c2 1579 struct nvme_tcp_qpair *tqpair;
9f95a23c
TL		 1580 int family;
1581 long int port;
f67539c2 1582 struct spdk_sock_opts opts;
9f95a23c 1583
f67539c2 1584 tqpair = nvme_tcp_qpair(qpair);
9f95a23c
TL		 1585
1586 switch (ctrlr->trid.adrfam) {
1587 case SPDK_NVMF_ADRFAM_IPV4:
1588 family = AF_INET;
1589 break;
1590 case SPDK_NVMF_ADRFAM_IPV6:
1591 family = AF_INET6;
1592 break;
1593 default:
1594 SPDK_ERRLOG("Unhandled ADRFAM %d\n", ctrlr->trid.adrfam);
1595 return -1;
1596 }
1597
1598 SPDK_DEBUGLOG(SPDK_LOG_NVME, "adrfam %d ai_family %d\n", ctrlr->trid.adrfam, family);
1599
1600 memset(&dst_addr, 0, sizeof(dst_addr));
1601
1602 SPDK_DEBUGLOG(SPDK_LOG_NVME, "trsvcid is %s\n", ctrlr->trid.trsvcid);
1603 rc = nvme_tcp_parse_addr(&dst_addr, family, ctrlr->trid.traddr, ctrlr->trid.trsvcid);
1604 if (rc != 0) {
1605 SPDK_ERRLOG("dst_addr nvme_tcp_parse_addr() failed\n");
1606 return -1;
1607 }
1608
1609 if (ctrlr->opts.src_addr[0] || ctrlr->opts.src_svcid[0]) {
1610 memset(&src_addr, 0, sizeof(src_addr));
1611 rc = nvme_tcp_parse_addr(&src_addr, family, ctrlr->opts.src_addr, ctrlr->opts.src_svcid);
1612 if (rc != 0) {
1613 SPDK_ERRLOG("src_addr nvme_tcp_parse_addr() failed\n");
1614 return -1;
1615 }
1616 }
1617
1618 port = spdk_strtol(ctrlr->trid.trsvcid, 10);
1619 if (port <= 0 || port >= INT_MAX) {
1620 SPDK_ERRLOG("Invalid port: %s\n", ctrlr->trid.trsvcid);
1621 return -1;
1622 }
1623
f67539c2
TL		 1624 opts.opts_size = sizeof(opts);
1625 spdk_sock_get_default_opts(&opts);
1626 opts.priority = ctrlr->trid.priority;
1627 tqpair->sock = spdk_sock_connect_ext(ctrlr->trid.traddr, port, NULL, &opts);
9f95a23c
TL		 1628 if (!tqpair->sock) {
1629 SPDK_ERRLOG("sock connection error of tqpair=%p with addr=%s, port=%ld\n",
1630 tqpair, ctrlr->trid.traddr, port);
1631 return -1;
1632 }
1633
1634 tqpair->maxr2t = NVME_TCP_MAX_R2T_DEFAULT;
1635 /* Explicitly set the state and recv_state of tqpair */
1636 tqpair->state = NVME_TCP_QPAIR_STATE_INVALID;
1637 if (tqpair->recv_state != NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY) {
1638 nvme_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY);
1639 }
1640 rc = nvme_tcp_qpair_icreq_send(tqpair);
1641 if (rc != 0) {
1642 SPDK_ERRLOG("Unable to connect the tqpair\n");
1643 return -1;
1644 }
1645
1646 rc = nvme_fabric_qpair_connect(&tqpair->qpair, tqpair->num_entries);
1647 if (rc < 0) {
1648 SPDK_ERRLOG("Failed to send an NVMe-oF Fabric CONNECT command\n");
1649 return -1;
1650 }
1651
1652 return 0;
1653}
1654
9f95a23c
TL		 1655static struct spdk_nvme_qpair *
1656nvme_tcp_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr,
1657 uint16_t qid, uint32_t qsize,
1658 enum spdk_nvme_qprio qprio,
1659 uint32_t num_requests)
1660{
1661 struct nvme_tcp_qpair *tqpair;
1662 struct spdk_nvme_qpair *qpair;
1663 int rc;
1664
1665 tqpair = calloc(1, sizeof(struct nvme_tcp_qpair));
1666 if (!tqpair) {
1667 SPDK_ERRLOG("failed to get create tqpair\n");
1668 return NULL;
1669 }
1670
1671 tqpair->num_entries = qsize;
1672 qpair = &tqpair->qpair;
9f95a23c
TL		 1673 rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, num_requests);
1674 if (rc != 0) {
1675 free(tqpair);
1676 return NULL;
1677 }
1678
1679 rc = nvme_tcp_alloc_reqs(tqpair);
1680 if (rc) {
f67539c2 1681 nvme_tcp_ctrlr_delete_io_qpair(ctrlr, qpair);
9f95a23c
TL		 1682 return NULL;
1683 }
1684
1685 return qpair;
1686}
1687
f67539c2 1688static struct spdk_nvme_qpair *
9f95a23c
TL		 1689nvme_tcp_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
1690 const struct spdk_nvme_io_qpair_opts *opts)
1691{
1692 return nvme_tcp_ctrlr_create_qpair(ctrlr, qid, opts->io_queue_size, opts->qprio,
1693 opts->io_queue_requests);
1694}
1695
f67539c2 1696static struct spdk_nvme_ctrlr *nvme_tcp_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
9f95a23c
TL		 1697 const struct spdk_nvme_ctrlr_opts *opts,
1698 void *devhandle)
1699{
1700 struct nvme_tcp_ctrlr *tctrlr;
1701 union spdk_nvme_cap_register cap;
1702 union spdk_nvme_vs_register vs;
1703 int rc;
1704
1705 tctrlr = calloc(1, sizeof(*tctrlr));
1706 if (tctrlr == NULL) {
1707 SPDK_ERRLOG("could not allocate ctrlr\n");
1708 return NULL;
1709 }
1710
9f95a23c
TL		 1711 tctrlr->ctrlr.opts = *opts;
1712 tctrlr->ctrlr.trid = *trid;
1713
1714 rc = nvme_ctrlr_construct(&tctrlr->ctrlr);
1715 if (rc != 0) {
1716 free(tctrlr);
1717 return NULL;
1718 }
1719
1720 tctrlr->ctrlr.adminq = nvme_tcp_ctrlr_create_qpair(&tctrlr->ctrlr, 0,
f67539c2
TL		 1721 tctrlr->ctrlr.opts.admin_queue_size, 0,
1722 tctrlr->ctrlr.opts.admin_queue_size);
9f95a23c
TL		 1723 if (!tctrlr->ctrlr.adminq) {
1724 SPDK_ERRLOG("failed to create admin qpair\n");
1725 nvme_tcp_ctrlr_destruct(&tctrlr->ctrlr);
1726 return NULL;
1727 }
1728
f67539c2
TL		 1729 rc = nvme_transport_ctrlr_connect_qpair(&tctrlr->ctrlr, tctrlr->ctrlr.adminq);
1730 if (rc < 0) {
1731 SPDK_ERRLOG("failed to connect admin qpair\n");
1732 nvme_tcp_ctrlr_destruct(&tctrlr->ctrlr);
1733 return NULL;
1734 }
1735
9f95a23c
TL		 1736 if (nvme_ctrlr_get_cap(&tctrlr->ctrlr, &cap)) {
1737 SPDK_ERRLOG("get_cap() failed\n");
1738 nvme_ctrlr_destruct(&tctrlr->ctrlr);
1739 return NULL;
1740 }
1741
1742 if (nvme_ctrlr_get_vs(&tctrlr->ctrlr, &vs)) {
1743 SPDK_ERRLOG("get_vs() failed\n");
1744 nvme_ctrlr_destruct(&tctrlr->ctrlr);
1745 return NULL;
1746 }
1747
1748 if (nvme_ctrlr_add_process(&tctrlr->ctrlr, 0) != 0) {
1749 SPDK_ERRLOG("nvme_ctrlr_add_process() failed\n");
1750 nvme_ctrlr_destruct(&tctrlr->ctrlr);
1751 return NULL;
1752 }
1753
1754 nvme_ctrlr_init_cap(&tctrlr->ctrlr, &cap, &vs);
1755
1756 return &tctrlr->ctrlr;
1757}
1758
f67539c2 1759static uint32_t
9f95a23c
TL		 1760nvme_tcp_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
1761{
f67539c2
TL		 1762 /* TCP transport doens't limit maximum IO transfer size. */
1763 return UINT32_MAX;
9f95a23c
TL		 1764}
1765
f67539c2 1766static uint16_t
9f95a23c
TL		 1767nvme_tcp_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
1768{
1769 /*
1770 * We do not support >1 SGE in the initiator currently,
1771 * so we can only return 1 here. Once that support is
1772 * added, this should return ctrlr->cdata.nvmf_specific.msdbd
1773 * instead.
1774 */
1775 return 1;
1776}
1777
f67539c2
TL		 1778static int
1779nvme_tcp_qpair_iterate_requests(struct spdk_nvme_qpair *qpair,
1780 int (*iter_fn)(struct nvme_request *req, void *arg),
1781 void *arg)
9f95a23c 1782{
f67539c2
TL		 1783 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1784 struct nvme_tcp_req *tcp_req, *tmp;
1785 int rc;
9f95a23c 1786
f67539c2
TL		 1787 assert(iter_fn != NULL);
1788
1789 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
1790 assert(tcp_req->req != NULL);
1791
1792 rc = iter_fn(tcp_req->req, arg);
1793 if (rc != 0) {
1794 return rc;
1795 }
1796 }
9f95a23c 1797
9f95a23c
TL		 1798 return 0;
1799}
1800
f67539c2 1801static void
9f95a23c
TL		 1802nvme_tcp_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
1803{
1804 struct nvme_tcp_req *tcp_req, *tmp;
9f95a23c
TL		 1805 struct spdk_nvme_cpl cpl;
1806 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1807
1808 cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION;
1809 cpl.status.sct = SPDK_NVME_SCT_GENERIC;
1810
1811 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->outstanding_reqs, link, tmp) {
f67539c2 1812 assert(tcp_req->req != NULL);
9f95a23c
TL		 1813 if (tcp_req->req->cmd.opc != SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
1814 continue;
1815 }
9f95a23c 1816
f67539c2 1817 nvme_tcp_req_complete(tcp_req, &cpl);
9f95a23c
TL		 1818 nvme_tcp_req_put(tqpair, tcp_req);
1819 }
1820}
f67539c2
TL		 1821
1822static struct spdk_nvme_transport_poll_group *
1823nvme_tcp_poll_group_create(void)
1824{
1825 struct nvme_tcp_poll_group *group = calloc(1, sizeof(*group));
1826
1827 if (group == NULL) {
1828 SPDK_ERRLOG("Unable to allocate poll group.\n");
1829 return NULL;
1830 }
1831
1832 group->sock_group = spdk_sock_group_create(group);
1833 if (group->sock_group == NULL) {
1834 free(group);
1835 SPDK_ERRLOG("Unable to allocate sock group.\n");
1836 return NULL;
1837 }
1838
1839 return &group->group;
1840}
1841
1842static int
1843nvme_tcp_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
1844{
1845 struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(qpair->poll_group);
1846 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1847
1848 if (spdk_sock_group_add_sock(group->sock_group, tqpair->sock, nvme_tcp_qpair_sock_cb, qpair)) {
1849 return -EPROTO;
1850 }
1851 return 0;
1852}
1853
1854static int
1855nvme_tcp_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
1856{
1857 struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(qpair->poll_group);
1858 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1859
1860 if (tqpair->sock && group->sock_group) {
1861 if (spdk_sock_group_remove_sock(group->sock_group, tqpair->sock)) {
1862 return -EPROTO;
1863 }
1864 }
1865 return 0;
1866}
1867
1868static int
1869nvme_tcp_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup,
1870 struct spdk_nvme_qpair *qpair)
1871{
1872 struct nvme_tcp_qpair *tqpair = nvme_tcp_qpair(qpair);
1873 struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(tgroup);
1874
1875 /* disconnected qpairs won't have a sock to add. */
1876 if (nvme_qpair_get_state(qpair) >= NVME_QPAIR_CONNECTED) {
1877 if (spdk_sock_group_add_sock(group->sock_group, tqpair->sock, nvme_tcp_qpair_sock_cb, qpair)) {
1878 return -EPROTO;
1879 }
1880 }
1881
1882 return 0;
1883}
1884
1885static int
1886nvme_tcp_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,
1887 struct spdk_nvme_qpair *qpair)
1888{
1889 if (qpair->poll_group_tailq_head == &tgroup->connected_qpairs) {
1890 return nvme_poll_group_disconnect_qpair(qpair);
1891 }
1892
1893 return 0;
1894}
1895
1896static int64_t
1897nvme_tcp_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,
1898 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)
1899{
1900 struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(tgroup);
1901 struct spdk_nvme_qpair *qpair, *tmp_qpair;
1902
1903 group->completions_per_qpair = completions_per_qpair;
1904 group->num_completions = 0;
1905
1906 spdk_sock_group_poll(group->sock_group);
1907
1908 STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
1909 disconnected_qpair_cb(qpair, tgroup->group->ctx);
1910 }
1911
1912 return group->num_completions;
1913}
1914
1915static int
1916nvme_tcp_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup)
1917{
1918 int rc;
1919 struct nvme_tcp_poll_group *group = nvme_tcp_poll_group(tgroup);
1920
1921 if (!STAILQ_EMPTY(&tgroup->connected_qpairs) || !STAILQ_EMPTY(&tgroup->disconnected_qpairs)) {
1922 return -EBUSY;
1923 }
1924
1925 rc = spdk_sock_group_close(&group->sock_group);
1926 if (rc != 0) {
1927 SPDK_ERRLOG("Failed to close the sock group for a tcp poll group.\n");
1928 assert(false);
1929 }
1930
1931 free(tgroup);
1932
1933 return 0;
1934}
1935
1936const struct spdk_nvme_transport_ops tcp_ops = {
1937 .name = "TCP",
1938 .type = SPDK_NVME_TRANSPORT_TCP,
1939 .ctrlr_construct = nvme_tcp_ctrlr_construct,
1940 .ctrlr_scan = nvme_fabric_ctrlr_scan,
1941 .ctrlr_destruct = nvme_tcp_ctrlr_destruct,
1942 .ctrlr_enable = nvme_tcp_ctrlr_enable,
1943
1944 .ctrlr_set_reg_4 = nvme_fabric_ctrlr_set_reg_4,
1945 .ctrlr_set_reg_8 = nvme_fabric_ctrlr_set_reg_8,
1946 .ctrlr_get_reg_4 = nvme_fabric_ctrlr_get_reg_4,
1947 .ctrlr_get_reg_8 = nvme_fabric_ctrlr_get_reg_8,
1948
1949 .ctrlr_get_max_xfer_size = nvme_tcp_ctrlr_get_max_xfer_size,
1950 .ctrlr_get_max_sges = nvme_tcp_ctrlr_get_max_sges,
1951
1952 .ctrlr_create_io_qpair = nvme_tcp_ctrlr_create_io_qpair,
1953 .ctrlr_delete_io_qpair = nvme_tcp_ctrlr_delete_io_qpair,
1954 .ctrlr_connect_qpair = nvme_tcp_ctrlr_connect_qpair,
1955 .ctrlr_disconnect_qpair = nvme_tcp_ctrlr_disconnect_qpair,
1956
1957 .qpair_abort_reqs = nvme_tcp_qpair_abort_reqs,
1958 .qpair_reset = nvme_tcp_qpair_reset,
1959 .qpair_submit_request = nvme_tcp_qpair_submit_request,
1960 .qpair_process_completions = nvme_tcp_qpair_process_completions,
1961 .qpair_iterate_requests = nvme_tcp_qpair_iterate_requests,
1962 .admin_qpair_abort_aers = nvme_tcp_admin_qpair_abort_aers,
1963
1964 .poll_group_create = nvme_tcp_poll_group_create,
1965 .poll_group_connect_qpair = nvme_tcp_poll_group_connect_qpair,
1966 .poll_group_disconnect_qpair = nvme_tcp_poll_group_disconnect_qpair,
1967 .poll_group_add = nvme_tcp_poll_group_add,
1968 .poll_group_remove = nvme_tcp_poll_group_remove,
1969 .poll_group_process_completions = nvme_tcp_poll_group_process_completions,
1970 .poll_group_destroy = nvme_tcp_poll_group_destroy,
1971};
1972
1973SPDK_NVME_TRANSPORT_REGISTER(tcp, &tcp_ops);
Ceph