1 /* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
9 #include <rdma/ib_verbs.h>
10 #include <linux/socket.h>
11 #include <linux/skbuff.h>
12 #include <crypto/hash.h>
13 #include <linux/crc32.h>
14 #include <linux/crc32c.h>
16 #include <rdma/siw-abi.h>
19 #define SIW_VENDOR_ID 0x626d74 /* ascii 'bmt' for now */
20 #define SIW_VENDORT_PART_ID 0
21 #define SIW_MAX_QP (1024 * 100)
22 #define SIW_MAX_QP_WR (1024 * 32)
23 #define SIW_MAX_ORD_QP 128
24 #define SIW_MAX_IRD_QP 128
25 #define SIW_MAX_SGE_PBL 256 /* max num sge's for PBL */
26 #define SIW_MAX_SGE_RD 1 /* iwarp limitation. we could relax */
27 #define SIW_MAX_CQ (1024 * 100)
28 #define SIW_MAX_CQE (SIW_MAX_QP_WR * 100)
29 #define SIW_MAX_MR (SIW_MAX_QP * 10)
30 #define SIW_MAX_PD SIW_MAX_QP
31 #define SIW_MAX_MW 0 /* to be set if MW's are supported */
32 #define SIW_MAX_FMR SIW_MAX_MR
33 #define SIW_MAX_SRQ SIW_MAX_QP
34 #define SIW_MAX_SRQ_WR (SIW_MAX_QP_WR * 10)
35 #define SIW_MAX_CONTEXT SIW_MAX_PD
37 /* Min number of bytes for using zero copy transmit */
38 #define SENDPAGE_THRESH PAGE_SIZE
40 /* Maximum number of frames which can be send in one SQ processing */
41 #define SQ_USER_MAXBURST 100
43 /* Maximum number of consecutive IRQ elements which get served
44 * if SQ has pending work. Prevents starving local SQ processing
45 * by serving peer Read Requests.
47 #define SIW_IRQ_MAXBURST_SQ_ACTIVE 4
52 int max_ord
; /* max. outbound read queue depth */
53 int max_ird
; /* max. inbound read queue depth */
72 struct ib_device base_dev
;
73 struct net_device
*netdev
;
74 struct siw_dev_cap attrs
;
79 /* physical port state (only one port per device) */
80 enum ib_port_state state
;
87 struct list_head cep_list
;
88 struct list_head qp_list
;
90 /* active objects statistics to enforce limits */
98 struct work_struct netdev_down
;
106 struct siw_ucontext
{
107 struct ib_ucontext base_ucontext
;
108 struct siw_device
*sdev
;
110 /* xarray of user mappable objects */
116 * The RDMA core does not define LOCAL_READ access, which is always
117 * enabled implictely.
119 #define IWARP_ACCESS_MASK \
120 (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | \
121 IB_ACCESS_REMOTE_READ)
124 * siw presentation of user memory registered as source
125 * or target of RDMA operations.
128 struct siw_page_chunk
{
133 struct siw_page_chunk
*page_chunk
;
136 u64 fp_addr
; /* First page base address */
137 struct mm_struct
*owning_mm
;
141 u64 addr
; /* Address of assigned user buffer */
142 u64 size
; /* Size of this entry */
143 u64 pbl_off
; /* Total offset from start of PBL */
147 unsigned int num_buf
;
148 unsigned int max_buf
;
149 struct siw_pble pbe
[1];
155 * Generic memory representation for registered siw memory.
156 * Memory lookup always via higher 24 bit of STag (STag index).
159 struct siw_device
*sdev
;
161 u64 va
; /* VA of memory */
162 u64 len
; /* length of the memory buffer in bytes */
163 u32 stag
; /* iWarp memory access steering tag */
164 u8 stag_valid
; /* VALID or INVALID */
165 u8 is_pbl
; /* PBL or user space mem */
166 u8 is_mw
; /* Memory Region or Memory Window */
167 enum ib_access_flags perms
; /* local/remote READ & WRITE */
169 struct siw_umem
*umem
;
177 struct ib_mr base_mr
;
183 * Error codes for local or remote
184 * access to registered memory
186 enum siw_access_state
{
196 SIW_WR_QUEUED
, /* processing has not started yet */
197 SIW_WR_INPROGRESS
/* initiated processing of the WR */
200 /* The WQE currently being processed (RX or TX) */
202 /* Copy of applications SQE or RQE */
207 struct siw_mem
*mem
[SIW_MAX_SGE
]; /* per sge's resolved mem */
208 enum siw_wr_state wr_status
;
209 enum siw_wc_status wc_status
;
210 u32 bytes
; /* total bytes to process */
211 u32 processed
; /* bytes processed */
215 struct ib_cq base_cq
;
217 struct siw_cq_ctrl
*notify
;
218 struct siw_cqe
*queue
;
223 u32 xa_cq_index
; /* mmap information for CQE array */
224 u32 id
; /* For debugging only */
231 SIW_QP_STATE_CLOSING
,
232 SIW_QP_STATE_TERMINATE
,
238 SIW_RDMA_BIND_ENABLED
= (1 << 0),
239 SIW_RDMA_WRITE_ENABLED
= (1 << 1),
240 SIW_RDMA_READ_ENABLED
= (1 << 2),
241 SIW_SIGNAL_ALL_WR
= (1 << 3),
242 SIW_MPA_CRC
= (1 << 4),
243 SIW_QP_IN_DESTROY
= (1 << 5)
246 enum siw_qp_attr_mask
{
247 SIW_QP_ATTR_STATE
= (1 << 0),
248 SIW_QP_ATTR_ACCESS_FLAGS
= (1 << 1),
249 SIW_QP_ATTR_LLP_HANDLE
= (1 << 2),
250 SIW_QP_ATTR_ORD
= (1 << 3),
251 SIW_QP_ATTR_IRD
= (1 << 4),
252 SIW_QP_ATTR_SQ_SIZE
= (1 << 5),
253 SIW_QP_ATTR_RQ_SIZE
= (1 << 6),
254 SIW_QP_ATTR_MPA
= (1 << 7)
258 struct ib_srq base_srq
;
261 u32 limit
; /* low watermark for async event */
262 struct siw_rqe
*recvq
;
265 u32 num_rqe
; /* max # of wqe's allowed */
266 u32 xa_srq_index
; /* mmap information for SRQ array */
267 char armed
; /* inform user if limit hit */
268 char kernel_verbs
; /* '1' if kernel client */
271 struct siw_qp_attrs
{
272 enum siw_qp_state state
;
279 enum siw_qp_flags flags
;
285 SIW_SEND_HDR
, /* start or continue sending HDR */
286 SIW_SEND_DATA
, /* start or continue sending DDP payload */
287 SIW_SEND_TRAILER
, /* start or continue sending TRAILER */
288 SIW_SEND_SHORT_FPDU
/* send whole FPDU hdr|data|trailer at once */
292 SIW_GET_HDR
, /* await new hdr or within hdr */
293 SIW_GET_DATA_START
, /* start of inbound DDP payload */
294 SIW_GET_DATA_MORE
, /* continuation of (misaligned) DDP payload */
295 SIW_GET_TRAILER
/* await new trailer or within trailer */
298 struct siw_rx_stream
{
300 int skb_new
; /* pending unread bytes in skb */
301 int skb_offset
; /* offset in skb */
302 int skb_copied
; /* processed bytes in skb */
305 struct mpa_trailer trailer
;
307 enum siw_rx_state state
;
310 * For each FPDU, main RX loop runs through 3 stages:
311 * Receiving protocol headers, placing DDP payload and receiving
312 * trailer information (CRC + possibly padding).
313 * Next two variables keep state on receive status of the
314 * current FPDU part (hdr, data, trailer).
316 int fpdu_part_rcvd
; /* bytes in pkt part copied */
317 int fpdu_part_rem
; /* bytes in pkt part not seen */
320 * Next expected DDP MSN for each QN +
321 * expected steering tag +
322 * expected DDP tagget offset (all HBO)
324 u32 ddp_msn
[RDMAP_UNTAGGED_QN_COUNT
];
327 u32 inval_stag
; /* Stag to be invalidated */
329 struct shash_desc
*mpa_crc_hd
;
331 u8 pad
: 2; /* # of pad bytes expected */
332 u8 rdmap_op
: 4; /* opcode of current frame */
337 * Local destination memory of inbound RDMA operation.
338 * Valid, according to wqe->wr_status
340 struct siw_wqe wqe_active
;
342 unsigned int pbl_idx
; /* Index into current PBL */
343 unsigned int sge_idx
; /* current sge in rx */
344 unsigned int sge_off
; /* already rcvd in curr. sge */
346 char first_ddp_seg
; /* this is the first DDP seg */
347 char more_ddp_segs
; /* more DDP segs expected */
348 u8 prev_rdmap_op
: 4; /* opcode of prev frame */
352 * Shorthands for short packets w/o payload
353 * to be transmitted more efficient.
355 struct siw_send_pkt
{
356 struct iwarp_send send
;
360 struct siw_write_pkt
{
361 struct iwarp_rdma_write write
;
365 struct siw_rreq_pkt
{
366 struct iwarp_rdma_rreq rreq
;
370 struct siw_rresp_pkt
{
371 struct iwarp_rdma_rresp rresp
;
375 struct siw_iwarp_tx
{
379 /* Generic part of FPDU header */
380 struct iwarp_ctrl ctrl
;
381 struct iwarp_ctrl_untagged c_untagged
;
382 struct iwarp_ctrl_tagged c_tagged
;
385 struct iwarp_rdma_write rwrite
;
386 struct iwarp_rdma_rreq rreq
;
387 struct iwarp_rdma_rresp rresp
;
388 struct iwarp_terminate terminate
;
389 struct iwarp_send send
;
390 struct iwarp_send_inv send_inv
;
392 /* complete short FPDUs */
393 struct siw_send_pkt send_pkt
;
394 struct siw_write_pkt write_pkt
;
395 struct siw_rreq_pkt rreq_pkt
;
396 struct siw_rresp_pkt rresp_pkt
;
399 struct mpa_trailer trailer
;
400 /* DDP MSN for untagged messages */
401 u32 ddp_msn
[RDMAP_UNTAGGED_QN_COUNT
];
403 enum siw_tx_ctx state
;
404 u16 ctrl_len
; /* ddp+rdmap hdr */
407 int bytes_unsent
; /* ddp payload bytes */
409 struct shash_desc
*mpa_crc_hd
;
411 u8 do_crc
: 1; /* do crc for segment */
412 u8 use_sendpage
: 1; /* send w/o copy */
413 u8 tx_suspend
: 1; /* stop sending DDP segs. */
414 u8 pad
: 2; /* # pad in current fpdu */
415 u8 orq_fence
: 1; /* ORQ full or Send fenced */
416 u8 in_syscall
: 1; /* TX out of user context */
417 u8 zcopy_tx
: 1; /* Use TCP_SENDPAGE if possible */
418 u8 gso_seg_limit
; /* Maximum segments for GSO, 0 = unbound */
420 u16 fpdu_len
; /* len of FPDU to tx */
421 unsigned int tcp_seglen
; /* remaining tcp seg space */
423 struct siw_wqe wqe_active
;
425 int pbl_idx
; /* Index into current PBL */
426 int sge_idx
; /* current sge in tx */
427 u32 sge_off
; /* already sent in curr. sge */
431 struct siw_device
*sdev
;
435 struct list_head devq
;
438 struct siw_qp_attrs attrs
;
441 struct rw_semaphore state_lock
;
448 struct siw_iwarp_tx tx_ctx
; /* Transmit context */
450 struct siw_sqe
*sendq
; /* send queue element array */
451 uint32_t sq_get
; /* consumer index into sq array */
452 uint32_t sq_put
; /* kernel prod. index into sq array */
453 struct llist_node tx_list
;
455 struct siw_sqe
*orq
; /* outbound read queue element array */
457 uint32_t orq_get
; /* consumer index into orq array */
458 uint32_t orq_put
; /* shared producer index for ORQ */
460 struct siw_rx_stream rx_stream
;
461 struct siw_rx_fpdu
*rx_fpdu
;
462 struct siw_rx_fpdu rx_tagged
;
463 struct siw_rx_fpdu rx_untagged
;
465 struct siw_rqe
*recvq
; /* recv queue element array */
466 uint32_t rq_get
; /* consumer index into rq array */
467 uint32_t rq_put
; /* kernel prod. index into rq array */
469 struct siw_sqe
*irq
; /* inbound read queue element array */
470 uint32_t irq_get
; /* consumer index into irq array */
471 uint32_t irq_put
; /* producer index into irq array */
474 struct { /* information to be carried in TERMINATE pkt, if valid */
477 u8 layer
: 4, etype
: 4;
480 u32 xa_sq_index
; /* mmap information for SQE array */
481 u32 xa_rq_index
; /* mmap information for RQE array */
486 struct ib_qp base_qp
;
491 #define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream)
492 #define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx)
493 #define tx_wqe(qp) (&(qp)->tx_ctx.wqe_active)
494 #define rx_wqe(rctx) (&(rctx)->wqe_active)
495 #define rx_mem(rctx) ((rctx)->wqe_active.mem[0])
496 #define tx_type(wqe) ((wqe)->sqe.opcode)
497 #define rx_type(wqe) ((wqe)->rqe.opcode)
498 #define tx_flags(wqe) ((wqe)->sqe.flags)
500 struct iwarp_msg_info
{
502 struct iwarp_ctrl ctrl
;
503 int (*rx_data
)(struct siw_qp
*qp
);
506 /* Global siw parameters. Currently set in siw_main.c */
507 extern const bool zcopy_tx
;
508 extern const bool try_gso
;
509 extern const bool loopback_enabled
;
510 extern const bool mpa_crc_required
;
511 extern const bool mpa_crc_strict
;
512 extern const bool siw_tcp_nagle
;
513 extern u_char mpa_version
;
514 extern const bool peer_to_peer
;
515 extern struct task_struct
*siw_tx_thread
[];
517 extern struct crypto_shash
*siw_crypto_shash
;
518 extern struct iwarp_msg_info iwarp_pktinfo
[RDMAP_TERMINATE
+ 1];
520 /* QP general functions */
521 int siw_qp_modify(struct siw_qp
*qp
, struct siw_qp_attrs
*attr
,
522 enum siw_qp_attr_mask mask
);
523 int siw_qp_mpa_rts(struct siw_qp
*qp
, enum mpa_v2_ctrl ctrl
);
524 void siw_qp_llp_close(struct siw_qp
*qp
);
525 void siw_qp_cm_drop(struct siw_qp
*qp
, int schedule
);
526 void siw_send_terminate(struct siw_qp
*qp
);
528 void siw_qp_get_ref(struct ib_qp
*qp
);
529 void siw_qp_put_ref(struct ib_qp
*qp
);
530 int siw_qp_add(struct siw_device
*sdev
, struct siw_qp
*qp
);
531 void siw_free_qp(struct kref
*ref
);
533 void siw_init_terminate(struct siw_qp
*qp
, enum term_elayer layer
,
534 u8 etype
, u8 ecode
, int in_tx
);
535 enum ddp_ecode
siw_tagged_error(enum siw_access_state state
);
536 enum rdmap_ecode
siw_rdmap_error(enum siw_access_state state
);
538 void siw_read_to_orq(struct siw_sqe
*rreq
, struct siw_sqe
*sqe
);
539 int siw_sqe_complete(struct siw_qp
*qp
, struct siw_sqe
*sqe
, u32 bytes
,
540 enum siw_wc_status status
);
541 int siw_rqe_complete(struct siw_qp
*qp
, struct siw_rqe
*rqe
, u32 bytes
,
542 u32 inval_stag
, enum siw_wc_status status
);
543 void siw_qp_llp_data_ready(struct sock
*sk
);
544 void siw_qp_llp_write_space(struct sock
*sk
);
546 /* QP TX path functions */
547 int siw_run_sq(void *arg
);
548 int siw_qp_sq_process(struct siw_qp
*qp
);
549 int siw_sq_start(struct siw_qp
*qp
);
550 int siw_activate_tx(struct siw_qp
*qp
);
551 void siw_stop_tx_thread(int nr_cpu
);
552 int siw_get_tx_cpu(struct siw_device
*sdev
);
553 void siw_put_tx_cpu(int cpu
);
555 /* QP RX path functions */
556 int siw_proc_send(struct siw_qp
*qp
);
557 int siw_proc_rreq(struct siw_qp
*qp
);
558 int siw_proc_rresp(struct siw_qp
*qp
);
559 int siw_proc_write(struct siw_qp
*qp
);
560 int siw_proc_terminate(struct siw_qp
*qp
);
562 int siw_tcp_rx_data(read_descriptor_t
*rd_desc
, struct sk_buff
*skb
,
563 unsigned int off
, size_t len
);
565 static inline void set_rx_fpdu_context(struct siw_qp
*qp
, u8 opcode
)
567 if (opcode
== RDMAP_RDMA_WRITE
|| opcode
== RDMAP_RDMA_READ_RESP
)
568 qp
->rx_fpdu
= &qp
->rx_tagged
;
570 qp
->rx_fpdu
= &qp
->rx_untagged
;
572 qp
->rx_stream
.rdmap_op
= opcode
;
575 static inline struct siw_ucontext
*to_siw_ctx(struct ib_ucontext
*base_ctx
)
577 return container_of(base_ctx
, struct siw_ucontext
, base_ucontext
);
580 static inline struct siw_base_qp
*to_siw_base_qp(struct ib_qp
*base_qp
)
582 return container_of(base_qp
, struct siw_base_qp
, base_qp
);
585 static inline struct siw_qp
*to_siw_qp(struct ib_qp
*base_qp
)
587 return to_siw_base_qp(base_qp
)->qp
;
590 static inline struct siw_cq
*to_siw_cq(struct ib_cq
*base_cq
)
592 return container_of(base_cq
, struct siw_cq
, base_cq
);
595 static inline struct siw_srq
*to_siw_srq(struct ib_srq
*base_srq
)
597 return container_of(base_srq
, struct siw_srq
, base_srq
);
600 static inline struct siw_device
*to_siw_dev(struct ib_device
*base_dev
)
602 return container_of(base_dev
, struct siw_device
, base_dev
);
605 static inline struct siw_mr
*to_siw_mr(struct ib_mr
*base_mr
)
607 return container_of(base_mr
, struct siw_mr
, base_mr
);
610 static inline struct siw_qp
*siw_qp_id2obj(struct siw_device
*sdev
, int id
)
615 qp
= xa_load(&sdev
->qp_xa
, id
);
616 if (likely(qp
&& kref_get_unless_zero(&qp
->ref
))) {
624 static inline u32
qp_id(struct siw_qp
*qp
)
629 static inline void siw_qp_get(struct siw_qp
*qp
)
634 static inline void siw_qp_put(struct siw_qp
*qp
)
636 kref_put(&qp
->ref
, siw_free_qp
);
639 static inline int siw_sq_empty(struct siw_qp
*qp
)
641 struct siw_sqe
*sqe
= &qp
->sendq
[qp
->sq_get
% qp
->attrs
.sq_size
];
643 return READ_ONCE(sqe
->flags
) == 0;
646 static inline struct siw_sqe
*sq_get_next(struct siw_qp
*qp
)
648 struct siw_sqe
*sqe
= &qp
->sendq
[qp
->sq_get
% qp
->attrs
.sq_size
];
650 if (READ_ONCE(sqe
->flags
) & SIW_WQE_VALID
)
656 static inline struct siw_sqe
*orq_get_current(struct siw_qp
*qp
)
658 return &qp
->orq
[qp
->orq_get
% qp
->attrs
.orq_size
];
661 static inline struct siw_sqe
*orq_get_tail(struct siw_qp
*qp
)
663 return &qp
->orq
[qp
->orq_put
% qp
->attrs
.orq_size
];
666 static inline struct siw_sqe
*orq_get_free(struct siw_qp
*qp
)
668 struct siw_sqe
*orq_e
= orq_get_tail(qp
);
670 if (orq_e
&& READ_ONCE(orq_e
->flags
) == 0)
676 static inline int siw_orq_empty(struct siw_qp
*qp
)
678 return qp
->orq
[qp
->orq_get
% qp
->attrs
.orq_size
].flags
== 0 ? 1 : 0;
681 static inline struct siw_sqe
*irq_alloc_free(struct siw_qp
*qp
)
683 struct siw_sqe
*irq_e
= &qp
->irq
[qp
->irq_put
% qp
->attrs
.irq_size
];
685 if (READ_ONCE(irq_e
->flags
) == 0) {
692 static inline __wsum
siw_csum_update(const void *buff
, int len
, __wsum sum
)
694 return (__force __wsum
)crc32c((__force __u32
)sum
, buff
, len
);
697 static inline __wsum
siw_csum_combine(__wsum csum
, __wsum csum2
, int offset
,
700 return (__force __wsum
)__crc32c_le_combine((__force __u32
)csum
,
701 (__force __u32
)csum2
, len
);
704 static inline void siw_crc_skb(struct siw_rx_stream
*srx
, unsigned int len
)
706 const struct skb_checksum_ops siw_cs_ops
= {
707 .update
= siw_csum_update
,
708 .combine
= siw_csum_combine
,
710 __wsum crc
= *(u32
*)shash_desc_ctx(srx
->mpa_crc_hd
);
712 crc
= __skb_checksum(srx
->skb
, srx
->skb_offset
, len
, crc
,
714 *(u32
*)shash_desc_ctx(srx
->mpa_crc_hd
) = crc
;
717 #define siw_dbg(ibdev, fmt, ...) \
718 ibdev_dbg(ibdev, "%s: " fmt, __func__, ##__VA_ARGS__)
720 #define siw_dbg_qp(qp, fmt, ...) \
721 ibdev_dbg(&qp->sdev->base_dev, "QP[%u] %s: " fmt, qp_id(qp), __func__, \
724 #define siw_dbg_cq(cq, fmt, ...) \
725 ibdev_dbg(cq->base_cq.device, "CQ[%u] %s: " fmt, cq->id, __func__, \
728 #define siw_dbg_pd(pd, fmt, ...) \
729 ibdev_dbg(pd->device, "PD[%u] %s: " fmt, pd->res.id, __func__, \
732 #define siw_dbg_mem(mem, fmt, ...) \
733 ibdev_dbg(&mem->sdev->base_dev, \
734 "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__)
736 #define siw_dbg_cep(cep, fmt, ...) \
737 ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%p] %s: " fmt, \
738 cep, __func__, ##__VA_ARGS__)
740 void siw_cq_flush(struct siw_cq
*cq
);
741 void siw_sq_flush(struct siw_qp
*qp
);
742 void siw_rq_flush(struct siw_qp
*qp
);
743 int siw_reap_cqe(struct siw_cq
*cq
, struct ib_wc
*wc
);