[mirror_ubuntu-jammy-kernel.git] / drivers / crypto / chelsio / chtls / chtls.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (c) 2018 Chelsio Communications, Inc.
 */

#ifndef __CHTLS_H__
#define __CHTLS_H__

#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#include <crypto/authenc.h>
#include <crypto/ctr.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/aead.h>
#include <crypto/null.h>
#include <crypto/internal/skcipher.h>
#include <crypto/aead.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/hash.h>
#include <linux/tls.h>
#include <net/tls.h>

#include "t4fw_api.h"
#include "t4_msg.h"
#include "cxgb4.h"
#include "cxgb4_uld.h"
#include "l2t.h"
#include "chcr_algo.h"
#include "chcr_core.h"
#include "chcr_crypto.h"

#define MAX_IVS_PAGE			256
#define TLS_KEY_CONTEXT_SZ		64
#define CIPHER_BLOCK_SIZE		16
#define GCM_TAG_SIZE			16
#define KEY_ON_MEM_SZ			16
#define AEAD_EXPLICIT_DATA_SIZE		8
#define TLS_HEADER_LENGTH		5
#define SCMD_CIPH_MODE_AES_GCM		2
/* Any MFS size should work and come from openssl */
#define TLS_MFS				16384

#define RSS_HDR sizeof(struct rss_header)
#define TLS_WR_CPL_LEN \
	(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))

enum {
	CHTLS_KEY_CONTEXT_DSGL,
	CHTLS_KEY_CONTEXT_IMM,
	CHTLS_KEY_CONTEXT_DDR,
};

enum {
	CHTLS_LISTEN_START,
	CHTLS_LISTEN_STOP,
};

/* Flags for return value of CPL message handlers */
enum {
	CPL_RET_BUF_DONE =    1,   /* buffer processing done */
	CPL_RET_BAD_MSG =     2,   /* bad CPL message */
	CPL_RET_UNKNOWN_TID = 4    /* unexpected unknown TID */
};

#define LISTEN_INFO_HASH_SIZE 32
#define RSPQ_HASH_BITS 5
struct listen_info {
	struct listen_info *next;  /* Link to next entry */
	struct sock *sk;           /* The listening socket */
	unsigned int stid;         /* The server TID */
};

enum {
	T4_LISTEN_START_PENDING,
	T4_LISTEN_STARTED
};

enum csk_flags {
	CSK_CALLBACKS_CHKD,	/* socket callbacks have been sanitized */
	CSK_ABORT_REQ_RCVD,	/* received one ABORT_REQ_RSS message */
	CSK_TX_MORE_DATA,	/* sending ULP data; don't set SHOVE bit */
	CSK_TX_WAIT_IDLE,	/* suspend Tx until in-flight data is ACKed */
	CSK_ABORT_SHUTDOWN,	/* shouldn't send more abort requests */
	CSK_ABORT_RPL_PENDING,	/* expecting an abort reply */
	CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
	CSK_TX_DATA_SENT,	/* sent a TX_DATA WR on this connection */
	CSK_TX_FAILOVER,	/* Tx traffic failing over */
	CSK_UPDATE_RCV_WND,	/* Need to update rcv window */
	CSK_RST_ABORTED,	/* outgoing RST was aborted */
	CSK_TLS_HANDSHK,	/* TLS Handshake */
	CSK_CONN_INLINE,	/* Connection on HW */
};

enum chtls_cdev_state {
	CHTLS_CDEV_STATE_UP = 1
};

struct listen_ctx {
	struct sock *lsk;
	struct chtls_dev *cdev;
	struct sk_buff_head synq;
	u32 state;
};

struct key_map {
	unsigned long *addr;
	unsigned int start;
	unsigned int available;
	unsigned int size;
	spinlock_t lock; /* lock for key id request from map */
} __packed;

struct tls_scmd {
	u32 seqno_numivs;
	u32 ivgen_hdrlen;
};

struct chtls_dev {
	struct tls_device tlsdev;
	struct list_head list;
	struct cxgb4_lld_info *lldi;
	struct pci_dev *pdev;
	struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
	spinlock_t listen_lock; /* lock for listen list */
	struct net_device **ports;
	struct tid_info *tids;
	unsigned int pfvf;
	const unsigned short *mtus;

	struct idr hwtid_idr;
	struct idr stid_idr;

	spinlock_t idr_lock ____cacheline_aligned_in_smp;

	struct net_device *egr_dev[NCHAN * 2];
	struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
	struct sk_buff *askb;

	struct sk_buff_head deferq;
	struct work_struct deferq_task;

	struct list_head list_node;
	struct list_head rcu_node;
	struct list_head na_node;
	unsigned int send_page_order;
	int max_host_sndbuf;
	struct key_map kmap;
	unsigned int cdev_state;
};

struct chtls_listen {
	struct chtls_dev *cdev;
	struct sock *sk;
};

struct chtls_hws {
	struct sk_buff_head sk_recv_queue;
	u8 txqid;
	u8 ofld;
	u16 type;
	u16 rstate;
	u16 keyrpl;
	u16 pldlen;
	u16 rcvpld;
	u16 compute;
	u16 expansion;
	u16 keylen;
	u16 pdus;
	u16 adjustlen;
	u16 ivsize;
	u16 txleft;
	u32 mfs;
	s32 txkey;
	s32 rxkey;
	u32 fcplenmax;
	u32 copied_seq;
	u64 tx_seq_no;
	struct tls_scmd scmd;
	struct tls12_crypto_info_aes_gcm_128 crypto_info;
};

struct chtls_sock {
	struct sock *sk;
	struct chtls_dev *cdev;
	struct l2t_entry *l2t_entry;    /* pointer to the L2T entry */
	struct net_device *egress_dev;  /* TX_CHAN for act open retry */

	struct sk_buff_head txq;
	struct sk_buff *wr_skb_head;
	struct sk_buff *wr_skb_tail;
	struct sk_buff *ctrl_skb_cache;
	struct sk_buff *txdata_skb_cache; /* abort path messages */
	struct kref kref;
	unsigned long flags;
	u32 opt2;
	u32 wr_credits;
	u32 wr_unacked;
	u32 wr_max_credits;
	u32 wr_nondata;
	u32 hwtid;               /* TCP Control Block ID */
	u32 txq_idx;
	u32 rss_qid;
	u32 tid;
	u32 idr;
	u32 mss;
	u32 ulp_mode;
	u32 tx_chan;
	u32 rx_chan;
	u32 sndbuf;
	u32 txplen_max;
	u32 mtu_idx;           /* MTU table index */
	u32 smac_idx;
	u8 port_id;
	u8 tos;
	u16 resv2;
	u32 delack_mode;
	u32 delack_seq;
	u32 snd_win;
	u32 rcv_win;

	void *passive_reap_next;        /* placeholder for passive */
	struct chtls_hws tlshws;
	struct synq {
		struct sk_buff *next;
		struct sk_buff *prev;
	} synq;
	struct listen_ctx *listen_ctx;
};

struct tls_hdr {
	u8  type;
	u16 version;
	u16 length;
} __packed;

struct tlsrx_cmp_hdr {
	u8  type;
	u16 version;
	u16 length;

	u64 tls_seq;
	u16 reserved1;
	u8  res_to_mac_error;
} __packed;

/* res_to_mac_error fields */
#define TLSRX_HDR_PKT_INT_ERROR_S   4
#define TLSRX_HDR_PKT_INT_ERROR_M   0x1
#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
	((x) << TLSRX_HDR_PKT_INT_ERROR_S)
#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
	(((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
#define TLSRX_HDR_PKT_INT_ERROR_F   TLSRX_HDR_PKT_INT_ERROR_V(1U)

#define TLSRX_HDR_PKT_SPP_ERROR_S        3
#define TLSRX_HDR_PKT_SPP_ERROR_M        0x1
#define TLSRX_HDR_PKT_SPP_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_SPP_ERROR)
#define TLSRX_HDR_PKT_SPP_ERROR_G(x)     \
	(((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
#define TLSRX_HDR_PKT_SPP_ERROR_F        TLSRX_HDR_PKT_SPP_ERROR_V(1U)

#define TLSRX_HDR_PKT_CCDX_ERROR_S       2
#define TLSRX_HDR_PKT_CCDX_ERROR_M       0x1
#define TLSRX_HDR_PKT_CCDX_ERROR_V(x)    ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
#define TLSRX_HDR_PKT_CCDX_ERROR_G(x)    \
	(((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
#define TLSRX_HDR_PKT_CCDX_ERROR_F       TLSRX_HDR_PKT_CCDX_ERROR_V(1U)

#define TLSRX_HDR_PKT_PAD_ERROR_S        1
#define TLSRX_HDR_PKT_PAD_ERROR_M        0x1
#define TLSRX_HDR_PKT_PAD_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
#define TLSRX_HDR_PKT_PAD_ERROR_G(x)     \
	(((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
#define TLSRX_HDR_PKT_PAD_ERROR_F        TLSRX_HDR_PKT_PAD_ERROR_V(1U)

#define TLSRX_HDR_PKT_MAC_ERROR_S        0
#define TLSRX_HDR_PKT_MAC_ERROR_M        0x1
#define TLSRX_HDR_PKT_MAC_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_MAC_ERROR)
#define TLSRX_HDR_PKT_MAC_ERROR_G(x)     \
	(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
#define TLSRX_HDR_PKT_MAC_ERROR_F        TLSRX_HDR_PKT_MAC_ERROR_V(1U)

#define TLSRX_HDR_PKT_ERROR_M           0x1F
#define CONTENT_TYPE_ERROR		0x7F

struct ulp_mem_rw {
	__be32 cmd;
	__be32 len16;             /* command length */
	__be32 dlen;              /* data length in 32-byte units */
	__be32 lock_addr;
};

struct tls_key_wr {
	__be32 op_to_compl;
	__be32 flowid_len16;
	__be32 ftid;
	u8   reneg_to_write_rx;
	u8   protocol;
	__be16 mfs;
};

struct tls_key_req {
	struct tls_key_wr wr;
	struct ulp_mem_rw req;
	struct ulptx_idata sc_imm;
};

/*
 * This lives in skb->cb and is used to chain WRs in a linked list.
 */
struct wr_skb_cb {
	struct l2t_skb_cb l2t;          /* reserve space for l2t CB */
	struct sk_buff *next_wr;        /* next write request */
};

/* Per-skb backlog handler.  Run when a socket's backlog is processed. */
struct blog_skb_cb {
	void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
	struct chtls_dev *cdev;
};

/*
 * Similar to tcp_skb_cb but with ULP elements added to support TLS,
 * etc.
 */
struct ulp_skb_cb {
	struct wr_skb_cb wr;		/* reserve space for write request */
	u16 flags;			/* TCP-like flags */
	u8 psh;
	u8 ulp_mode;			/* ULP mode/submode of sk_buff */
	u32 seq;			/* TCP sequence number */
	union { /* ULP-specific fields */
		struct {
			u8  type;
			u8  ofld;
			u8  iv;
		} tls;
	} ulp;
};

#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)

/*
 * Flags for ulp_skb_cb.flags.
 */
enum {
	ULPCB_FLAG_NEED_HDR  = 1 << 0,	/* packet needs a TX_DATA_WR header */
	ULPCB_FLAG_NO_APPEND = 1 << 1,	/* don't grow this skb */
	ULPCB_FLAG_BARRIER   = 1 << 2,	/* set TX_WAIT_IDLE after sending */
	ULPCB_FLAG_HOLD      = 1 << 3,	/* skb not ready for Tx yet */
	ULPCB_FLAG_COMPL     = 1 << 4,	/* request WR completion */
	ULPCB_FLAG_URG       = 1 << 5,	/* urgent data */
	ULPCB_FLAG_TLS_HDR   = 1 << 6,  /* payload with tls hdr */
	ULPCB_FLAG_NO_HDR    = 1 << 7,  /* not a ofld wr */
};

/* The ULP mode/submode of an skbuff */
#define skb_ulp_mode(skb)  (ULP_SKB_CB(skb)->ulp_mode)
#define TCP_PAGE(sk)   (sk->sk_frag.page)
#define TCP_OFF(sk)    (sk->sk_frag.offset)

static inline struct chtls_dev *to_chtls_dev(struct tls_device *tlsdev)
{
	return container_of(tlsdev, struct chtls_dev, tlsdev);
}

static inline void csk_set_flag(struct chtls_sock *csk,
				enum csk_flags flag)
{
	__set_bit(flag, &csk->flags);
}

static inline void csk_reset_flag(struct chtls_sock *csk,
				  enum csk_flags flag)
{
	__clear_bit(flag, &csk->flags);
}

static inline bool csk_conn_inline(const struct chtls_sock *csk)
{
	return test_bit(CSK_CONN_INLINE, &csk->flags);
}

static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
{
	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);

	if (!csk_conn_inline(csk))
		return 0;
	return test_bit(flag, &csk->flags);
}

static inline int csk_flag_nochk(const struct chtls_sock *csk,
				 enum csk_flags flag)
{
	return test_bit(flag, &csk->flags);
}

static inline void *cplhdr(struct sk_buff *skb)
{
	return skb->data;
}

static inline int is_neg_adv(unsigned int status)
{
	return status == CPL_ERR_RTX_NEG_ADVICE ||
	       status == CPL_ERR_KEEPALV_NEG_ADVICE ||
	       status == CPL_ERR_PERSIST_NEG_ADVICE;
}

static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
				   struct sock *sk,
				   struct sk_buff *skb)
{
	skb_reset_mac_header(skb);
	skb_reset_network_header(skb);
	skb_reset_transport_header(skb);

	bh_lock_sock(sk);
	if (unlikely(sock_owned_by_user(sk))) {
		BLOG_SKB_CB(skb)->backlog_rcv = fn;
		__sk_add_backlog(sk, skb);
	} else {
		fn(sk, skb);
	}
	bh_unlock_sock(sk);
}

static inline void chtls_sock_free(struct kref *ref)
{
	struct chtls_sock *csk = container_of(ref, struct chtls_sock,
					      kref);
	kfree(csk);
}

static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
{
	kref_put(&csk->kref, chtls_sock_free);
}

static inline void __chtls_sock_get(const char *fn,
				    struct chtls_sock *csk)
{
	kref_get(&csk->kref);
}

static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
				 struct sk_buff *skb, int through_l2t)
{
	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);

	if (through_l2t) {
		/* send through L2T */
		cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
	} else {
		/* send directly */
		cxgb4_ofld_send(csk->egress_dev, skb);
	}
}

typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
void chtls_install_cpl_ops(struct sock *sk);
int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
void chtls_close(struct sock *sk, long timeout);
int chtls_disconnect(struct sock *sk, int flags);
void chtls_shutdown(struct sock *sk, int how);
void chtls_destroy_sock(struct sock *sk);
int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
		  size_t len, int nonblock, int flags, int *addr_len);
int chtls_sendpage(struct sock *sk, struct page *page,
		   int offset, size_t size, int flags);
int send_tx_flowc_wr(struct sock *sk, int compl,
		     u32 snd_nxt, u32 rcv_nxt);
void chtls_tcp_push(struct sock *sk, int flags);
int chtls_push_frames(struct chtls_sock *csk, int comp);
int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode);
void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
unsigned int keyid_to_addr(int start_addr, int keyid);
void free_tls_keyid(struct sock *sk);
#endif
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
a6779341 AG	2	/*
a6779341 AG	3	* Copyright (c) 2018 Chelsio Communications, Inc.
a6779341 AG	4	*/
	5
	6	#ifndef __CHTLS_H__
	7	#define __CHTLS_H__
	8
	9	#include <crypto/aes.h>
	10	#include <crypto/algapi.h>
	11	#include <crypto/hash.h>
	12	#include <crypto/sha.h>
	13	#include <crypto/authenc.h>
	14	#include <crypto/ctr.h>
	15	#include <crypto/gf128mul.h>
	16	#include <crypto/internal/aead.h>
	17	#include <crypto/null.h>
	18	#include <crypto/internal/skcipher.h>
	19	#include <crypto/aead.h>
	20	#include <crypto/scatterwalk.h>
	21	#include <crypto/internal/hash.h>
	22	#include <linux/tls.h>
	23	#include <net/tls.h>
	24
	25	#include "t4fw_api.h"
	26	#include "t4_msg.h"
	27	#include "cxgb4.h"
	28	#include "cxgb4_uld.h"
	29	#include "l2t.h"
	30	#include "chcr_algo.h"
	31	#include "chcr_core.h"
	32	#include "chcr_crypto.h"
	33
	34	#define MAX_IVS_PAGE 256
	35	#define TLS_KEY_CONTEXT_SZ 64
	36	#define CIPHER_BLOCK_SIZE 16
	37	#define GCM_TAG_SIZE 16
	38	#define KEY_ON_MEM_SZ 16
	39	#define AEAD_EXPLICIT_DATA_SIZE 8
	40	#define TLS_HEADER_LENGTH 5
	41	#define SCMD_CIPH_MODE_AES_GCM 2
	42	/* Any MFS size should work and come from openssl */
	43	#define TLS_MFS 16384
	44
	45	#define RSS_HDR sizeof(struct rss_header)
	46	#define TLS_WR_CPL_LEN \
	47	(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
	48
	49	enum {
	50	CHTLS_KEY_CONTEXT_DSGL,
	51	CHTLS_KEY_CONTEXT_IMM,
	52	CHTLS_KEY_CONTEXT_DDR,
	53	};
	54
	55	enum {
	56	CHTLS_LISTEN_START,
	57	CHTLS_LISTEN_STOP,
	58	};
	59
	60	/* Flags for return value of CPL message handlers */
	61	enum {
	62	CPL_RET_BUF_DONE = 1, /* buffer processing done */
	63	CPL_RET_BAD_MSG = 2, /* bad CPL message */
	64	CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
	65	};
	66
a6779341 AG	67	#define LISTEN_INFO_HASH_SIZE 32
	68	#define RSPQ_HASH_BITS 5
	69	struct listen_info {
	70	struct listen_info next; / Link to next entry */
	71	struct sock sk; / The listening socket */
	72	unsigned int stid; /* The server TID */
	73	};
	74
	75	enum {
	76	T4_LISTEN_START_PENDING,
	77	T4_LISTEN_STARTED
	78	};
	79
	80	enum csk_flags {
	81	CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
	82	CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
	83	CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
	84	CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
	85	CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
	86	CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
	87	CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
	88	CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
	89	CSK_TX_FAILOVER, /* Tx traffic failing over */
	90	CSK_UPDATE_RCV_WND, /* Need to update rcv window */
	91	CSK_RST_ABORTED, /* outgoing RST was aborted */
	92	CSK_TLS_HANDSHK, /* TLS Handshake */
	93	CSK_CONN_INLINE, /* Connection on HW */
	94	};
	95
65b2c12d GG	96	enum chtls_cdev_state {
	97	CHTLS_CDEV_STATE_UP = 1
	98	};
	99
a6779341 AG	100	struct listen_ctx {
	101	struct sock *lsk;
	102	struct chtls_dev *cdev;
	103	struct sk_buff_head synq;
	104	u32 state;
	105	};
	106
	107	struct key_map {
	108	unsigned long *addr;
	109	unsigned int start;
	110	unsigned int available;
	111	unsigned int size;
	112	spinlock_t lock; /* lock for key id request from map */
	113	} __packed;
	114
	115	struct tls_scmd {
	116	u32 seqno_numivs;
	117	u32 ivgen_hdrlen;
	118	};
	119
	120	struct chtls_dev {
	121	struct tls_device tlsdev;
	122	struct list_head list;
	123	struct cxgb4_lld_info *lldi;
	124	struct pci_dev *pdev;
	125	struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
	126	spinlock_t listen_lock; /* lock for listen list */
	127	struct net_device **ports;
	128	struct tid_info *tids;
	129	unsigned int pfvf;
	130	const unsigned short *mtus;
	131
	132	struct idr hwtid_idr;
	133	struct idr stid_idr;
	134
	135	spinlock_t idr_lock ____cacheline_aligned_in_smp;
	136
	137	struct net_device egr_dev[NCHAN 2];
	138	struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
	139	struct sk_buff *askb;
	140
	141	struct sk_buff_head deferq;
	142	struct work_struct deferq_task;
	143
	144	struct list_head list_node;
	145	struct list_head rcu_node;
	146	struct list_head na_node;
	147	unsigned int send_page_order;
3b8305f5	148	int max_host_sndbuf;
a6779341	149	struct key_map kmap;
65b2c12d	150	unsigned int cdev_state;
a6779341 AG	151	};
a6779341 AG	152
6422ccc5 AG	153	struct chtls_listen {
	154	struct chtls_dev *cdev;
	155	struct sock *sk;
	156	};
	157
a6779341 AG	158	struct chtls_hws {
	159	struct sk_buff_head sk_recv_queue;
	160	u8 txqid;
	161	u8 ofld;
	162	u16 type;
	163	u16 rstate;
	164	u16 keyrpl;
	165	u16 pldlen;
	166	u16 rcvpld;
	167	u16 compute;
	168	u16 expansion;
	169	u16 keylen;
	170	u16 pdus;
	171	u16 adjustlen;
	172	u16 ivsize;
	173	u16 txleft;
	174	u32 mfs;
	175	s32 txkey;
	176	s32 rxkey;
	177	u32 fcplenmax;
	178	u32 copied_seq;
	179	u64 tx_seq_no;
	180	struct tls_scmd scmd;
	181	struct tls12_crypto_info_aes_gcm_128 crypto_info;
	182	};
	183
	184	struct chtls_sock {
	185	struct sock *sk;
	186	struct chtls_dev *cdev;
	187	struct l2t_entry l2t_entry; / pointer to the L2T entry */
	188	struct net_device egress_dev; / TX_CHAN for act open retry */
	189
	190	struct sk_buff_head txq;
	191	struct sk_buff *wr_skb_head;
	192	struct sk_buff *wr_skb_tail;
	193	struct sk_buff *ctrl_skb_cache;
	194	struct sk_buff txdata_skb_cache; / abort path messages */
	195	struct kref kref;
	196	unsigned long flags;
	197	u32 opt2;
	198	u32 wr_credits;
	199	u32 wr_unacked;
	200	u32 wr_max_credits;
	201	u32 wr_nondata;
	202	u32 hwtid; /* TCP Control Block ID */
	203	u32 txq_idx;
	204	u32 rss_qid;
	205	u32 tid;
	206	u32 idr;
	207	u32 mss;
	208	u32 ulp_mode;
	209	u32 tx_chan;
	210	u32 rx_chan;
	211	u32 sndbuf;
	212	u32 txplen_max;
	213	u32 mtu_idx; /* MTU table index */
	214	u32 smac_idx;
	215	u8 port_id;
	216	u8 tos;
	217	u16 resv2;
	218	u32 delack_mode;
	219	u32 delack_seq;
0c3a16be AG	220	u32 snd_win;
0c3a16be AG	221	u32 rcv_win;
a6779341 AG	222
	223	void passive_reap_next; / placeholder for passive */
	224	struct chtls_hws tlshws;
	225	struct synq {
	226	struct sk_buff *next;
	227	struct sk_buff *prev;
	228	} synq;
	229	struct listen_ctx *listen_ctx;
	230	};
	231
	232	struct tls_hdr {
	233	u8 type;
	234	u16 version;
	235	u16 length;
	236	} __packed;
	237
	238	struct tlsrx_cmp_hdr {
	239	u8 type;
	240	u16 version;
	241	u16 length;
	242
	243	u64 tls_seq;
	244	u16 reserved1;
	245	u8 res_to_mac_error;
	246	} __packed;
	247
	248	/* res_to_mac_error fields */
	249	#define TLSRX_HDR_PKT_INT_ERROR_S 4
	250	#define TLSRX_HDR_PKT_INT_ERROR_M 0x1
	251	#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
	252	((x) << TLSRX_HDR_PKT_INT_ERROR_S)
	253	#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
	254	(((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
	255	#define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U)
	256
	257	#define TLSRX_HDR_PKT_SPP_ERROR_S 3
	258	#define TLSRX_HDR_PKT_SPP_ERROR_M 0x1
	259	#define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR)
	260	#define TLSRX_HDR_PKT_SPP_ERROR_G(x) \
	261	(((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
	262	#define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U)
	263
	264	#define TLSRX_HDR_PKT_CCDX_ERROR_S 2
	265	#define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1
	266	#define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
	267	#define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \
	268	(((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
	269	#define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
	270
	271	#define TLSRX_HDR_PKT_PAD_ERROR_S 1
	272	#define TLSRX_HDR_PKT_PAD_ERROR_M 0x1
	273	#define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
	274	#define TLSRX_HDR_PKT_PAD_ERROR_G(x) \
	275	(((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
	276	#define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U)
	277
	278	#define TLSRX_HDR_PKT_MAC_ERROR_S 0
	279	#define TLSRX_HDR_PKT_MAC_ERROR_M 0x1
	280	#define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR)
	281	#define TLSRX_HDR_PKT_MAC_ERROR_G(x) \
	282	(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
	283	#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
	284
	285	#define TLSRX_HDR_PKT_ERROR_M 0x1F
17a7d24a	286	#define CONTENT_TYPE_ERROR 0x7F
a6779341 AG	287
	288	struct ulp_mem_rw {
	289	__be32 cmd;
	290	__be32 len16; /* command length */
	291	__be32 dlen; /* data length in 32-byte units */
	292	__be32 lock_addr;
	293	};
	294
	295	struct tls_key_wr {
	296	__be32 op_to_compl;
	297	__be32 flowid_len16;
	298	__be32 ftid;
	299	u8 reneg_to_write_rx;
	300	u8 protocol;
	301	__be16 mfs;
	302	};
	303
	304	struct tls_key_req {
	305	struct tls_key_wr wr;
	306	struct ulp_mem_rw req;
	307	struct ulptx_idata sc_imm;
	308	};
	309
	310	/*
	311	* This lives in skb->cb and is used to chain WRs in a linked list.
	312	*/
	313	struct wr_skb_cb {
	314	struct l2t_skb_cb l2t; /* reserve space for l2t CB */
	315	struct sk_buff next_wr; / next write request */
	316	};
	317
	318	/* Per-skb backlog handler. Run when a socket's backlog is processed. */
	319	struct blog_skb_cb {
	320	void (backlog_rcv)(struct sock sk, struct sk_buff *skb);
	321	struct chtls_dev *cdev;
	322	};
	323
	324	/*
	325	* Similar to tcp_skb_cb but with ULP elements added to support TLS,
	326	* etc.
	327	*/
	328	struct ulp_skb_cb {
	329	struct wr_skb_cb wr; /* reserve space for write request */
	330	u16 flags; /* TCP-like flags */
	331	u8 psh;
	332	u8 ulp_mode; /* ULP mode/submode of sk_buff */
	333	u32 seq; /* TCP sequence number */
	334	union { /* ULP-specific fields */
	335	struct {
	336	u8 type;
	337	u8 ofld;
	338	u8 iv;
	339	} tls;
	340	} ulp;
	341	};
	342
	343	#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
	344	#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
	345
	346	/*
	347	* Flags for ulp_skb_cb.flags.
	348	*/
	349	enum {
	350	ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
351	ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
352	ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
353	ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */
354	ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */
355	ULPCB_FLAG_URG = 1 << 5, /* urgent data */
17a7d24a AG	356	ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */
17a7d24a AG	357	ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
a6779341 AG	358	};
	359
	360	/* The ULP mode/submode of an skbuff */
	361	#define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode)
	362	#define TCP_PAGE(sk) (sk->sk_frag.page)
	363	#define TCP_OFF(sk) (sk->sk_frag.offset)
	364
	365	static inline struct chtls_dev to_chtls_dev(struct tls_device tlsdev)
	366	{
	367	return container_of(tlsdev, struct chtls_dev, tlsdev);
	368	}
	369
	370	static inline void csk_set_flag(struct chtls_sock *csk,
	371	enum csk_flags flag)
	372	{
	373	__set_bit(flag, &csk->flags);
	374	}
	375
	376	static inline void csk_reset_flag(struct chtls_sock *csk,
	377	enum csk_flags flag)
	378	{
	379	__clear_bit(flag, &csk->flags);
	380	}
	381
	382	static inline bool csk_conn_inline(const struct chtls_sock *csk)
	383	{
	384	return test_bit(CSK_CONN_INLINE, &csk->flags);
	385	}
	386
	387	static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
	388	{
	389	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
	390
	391	if (!csk_conn_inline(csk))
	392	return 0;
	393	return test_bit(flag, &csk->flags);
	394	}
	395
	396	static inline int csk_flag_nochk(const struct chtls_sock *csk,
	397	enum csk_flags flag)
	398	{
	399	return test_bit(flag, &csk->flags);
	400	}
	401
	402	static inline void cplhdr(struct sk_buff skb)
	403	{
	404	return skb->data;
	405	}
	406
	407	static inline int is_neg_adv(unsigned int status)
	408	{
	409	return status == CPL_ERR_RTX_NEG_ADVICE \|\|
	410	status == CPL_ERR_KEEPALV_NEG_ADVICE \|\|
	411	status == CPL_ERR_PERSIST_NEG_ADVICE;
	412	}
	413
	414	static inline void process_cpl_msg(void (fn)(struct sock , struct sk_buff *),
	415	struct sock *sk,
	416	struct sk_buff *skb)
	417	{
	418	skb_reset_mac_header(skb);
	419	skb_reset_network_header(skb);
	420	skb_reset_transport_header(skb);
	421
422	bh_lock_sock(sk);
423	if (unlikely(sock_owned_by_user(sk))) {
424	BLOG_SKB_CB(skb)->backlog_rcv = fn;
425	__sk_add_backlog(sk, skb);
426	} else {
427	fn(sk, skb);
428	}
429	bh_unlock_sock(sk);
430	}
431
432	static inline void chtls_sock_free(struct kref *ref)
433	{
434	struct chtls_sock *csk = container_of(ref, struct chtls_sock,
435	kref);
436	kfree(csk);
437	}
438
439	static inline void __chtls_sock_put(const char fn, struct chtls_sock csk)
440	{
441	kref_put(&csk->kref, chtls_sock_free);
442	}
443
444	static inline void __chtls_sock_get(const char *fn,
445	struct chtls_sock *csk)
446	{
447	kref_get(&csk->kref);
448	}
449
450	static inline void send_or_defer(struct sock sk, struct tcp_sock tp,
451	struct sk_buff *skb, int through_l2t)
452	{
453	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
454
455	if (through_l2t) {
456	/* send through L2T */
457	cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
458	} else {
459	/* send directly */
460	cxgb4_ofld_send(csk->egress_dev, skb);
461	}
462	}
463
464	typedef int (chtls_handler_func)(struct chtls_dev , struct sk_buff *);
465	extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
466	void chtls_install_cpl_ops(struct sock *sk);
467	int chtls_init_kmap(struct chtls_dev cdev, struct cxgb4_lld_info lldi);
468	void chtls_listen_stop(struct chtls_dev cdev, struct sock sk);
469	int chtls_listen_start(struct chtls_dev cdev, struct sock sk);
470	void chtls_close(struct sock *sk, long timeout);
471	int chtls_disconnect(struct sock *sk, int flags);
472	void chtls_shutdown(struct sock *sk, int how);
473	void chtls_destroy_sock(struct sock *sk);
474	int chtls_sendmsg(struct sock sk, struct msghdr msg, size_t size);
475	int chtls_recvmsg(struct sock sk, struct msghdr msg,
476	size_t len, int nonblock, int flags, int *addr_len);
477	int chtls_sendpage(struct sock sk, struct page page,
478	int offset, size_t size, int flags);
479	int send_tx_flowc_wr(struct sock *sk, int compl,
480	u32 snd_nxt, u32 rcv_nxt);
481	void chtls_tcp_push(struct sock *sk, int flags);
482	int chtls_push_frames(struct chtls_sock *csk, int comp);
483	int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
484	int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode);
485	void skb_entail(struct sock sk, struct sk_buff skb, int flags);
486	unsigned int keyid_to_addr(int start_addr, int keyid);
487	void free_tls_keyid(struct sock *sk);
488	#endif