drivers/target/iscsi/cxgbit/cxgbit.h

   1 /*
   2  * Copyright (c) 2016 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef __CXGBIT_H__
  10 #define __CXGBIT_H__
  11
  12 #include <linux/mutex.h>
  13 #include <linux/list.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/idr.h>
  16 #include <linux/completion.h>
  17 #include <linux/netdevice.h>
  18 #include <linux/sched.h>
  19 #include <linux/pci.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/inet.h>
  22 #include <linux/wait.h>
  23 #include <linux/kref.h>
  24 #include <linux/timer.h>
  25 #include <linux/io.h>
  26
  27 #include <asm/byteorder.h>
  28
  29 #include <net/net_namespace.h>
  30
  31 #include <target/iscsi/iscsi_transport.h>
  32 #include <iscsi_target_parameters.h>
  33 #include <iscsi_target_login.h>
  34
  35 #include "t4_regs.h"
  36 #include "t4_msg.h"
  37 #include "cxgb4.h"
  38 #include "cxgb4_uld.h"
  39 #include "l2t.h"
  40 #include "cxgb4_ppm.h"
  41 #include "cxgbit_lro.h"
  42
  43 extern struct mutex cdev_list_lock;
  44 extern struct list_head cdev_list_head;
  45 struct cxgbit_np;
  46
  47 struct cxgbit_sock;
  48
  49 struct cxgbit_cmd {
  50         struct scatterlist sg;
  51         struct cxgbi_task_tag_info ttinfo;
  52         bool setup_ddp;
  53         bool release;
  54 };
  55
  56 #define CXGBIT_MAX_ISO_PAYLOAD  \
  57         min_t(u32, MAX_SKB_FRAGS * PAGE_SIZE, 65535)
  58
  59 struct cxgbit_iso_info {
  60         u8 flags;
  61         u32 mpdu;
  62         u32 len;
  63         u32 burst_len;
  64 };
  65
  66 enum cxgbit_skcb_flags {
  67         SKCBF_TX_NEED_HDR       = (1 << 0), /* packet needs a header */
  68         SKCBF_TX_FLAG_COMPL     = (1 << 1), /* wr completion flag */
  69         SKCBF_TX_ISO            = (1 << 2), /* iso cpl in tx skb */
  70         SKCBF_RX_LRO            = (1 << 3), /* lro skb */
  71 };
  72
  73 struct cxgbit_skb_rx_cb {
  74         u8 opcode;
  75         void *pdu_cb;
  76         void (*backlog_fn)(struct cxgbit_sock *, struct sk_buff *);
  77 };
  78
  79 struct cxgbit_skb_tx_cb {
  80         u8 submode;
  81         u32 extra_len;
  82 };
  83
  84 union cxgbit_skb_cb {
  85         struct {
  86                 u8 flags;
  87                 union {
  88                         struct cxgbit_skb_tx_cb tx;
  89                         struct cxgbit_skb_rx_cb rx;
  90                 };
  91         };
  92
  93         struct {
  94                 /* This member must be first. */
  95                 struct l2t_skb_cb l2t;
  96                 struct sk_buff *wr_next;
  97         };
  98 };
  99
 100 #define CXGBIT_SKB_CB(skb)      ((union cxgbit_skb_cb *)&((skb)->cb[0]))
 101 #define cxgbit_skcb_flags(skb)          (CXGBIT_SKB_CB(skb)->flags)
 102 #define cxgbit_skcb_submode(skb)        (CXGBIT_SKB_CB(skb)->tx.submode)
 103 #define cxgbit_skcb_tx_wr_next(skb)     (CXGBIT_SKB_CB(skb)->wr_next)
 104 #define cxgbit_skcb_tx_extralen(skb)    (CXGBIT_SKB_CB(skb)->tx.extra_len)
 105 #define cxgbit_skcb_rx_opcode(skb)      (CXGBIT_SKB_CB(skb)->rx.opcode)
 106 #define cxgbit_skcb_rx_backlog_fn(skb)  (CXGBIT_SKB_CB(skb)->rx.backlog_fn)
 107 #define cxgbit_rx_pdu_cb(skb)           (CXGBIT_SKB_CB(skb)->rx.pdu_cb)
 108
 109 static inline void *cplhdr(struct sk_buff *skb)
 110 {
 111         return skb->data;
 112 }
 113
 114 enum cxgbit_cdev_flags {
 115         CDEV_STATE_UP = 0,
 116         CDEV_ISO_ENABLE,
 117         CDEV_DDP_ENABLE,
 118 };
 119
 120 #define NP_INFO_HASH_SIZE 32
 121
 122 struct np_info {
 123         struct np_info *next;
 124         struct cxgbit_np *cnp;
 125         unsigned int stid;
 126 };
 127
 128 struct cxgbit_list_head {
 129         struct list_head list;
 130         /* device lock */
 131         spinlock_t lock;
 132 };
 133
 134 struct cxgbit_device {
 135         struct list_head list;
 136         struct cxgb4_lld_info lldi;
 137         struct np_info *np_hash_tab[NP_INFO_HASH_SIZE];
 138         /* np lock */
 139         spinlock_t np_lock;
 140         u8 selectq[MAX_NPORTS][2];
 141         struct cxgbit_list_head cskq;
 142         u32 mdsl;
 143         struct kref kref;
 144         unsigned long flags;
 145 };
 146
 147 struct cxgbit_wr_wait {
 148         struct completion completion;
 149         int ret;
 150 };
 151
 152 enum cxgbit_csk_state {
 153         CSK_STATE_IDLE = 0,
 154         CSK_STATE_LISTEN,
 155         CSK_STATE_CONNECTING,
 156         CSK_STATE_ESTABLISHED,
 157         CSK_STATE_ABORTING,
 158         CSK_STATE_CLOSING,
 159         CSK_STATE_MORIBUND,
 160         CSK_STATE_DEAD,
 161 };
 162
 163 enum cxgbit_csk_flags {
 164         CSK_TX_DATA_SENT = 0,
 165         CSK_LOGIN_PDU_DONE,
 166         CSK_LOGIN_DONE,
 167         CSK_DDP_ENABLE,
 168 };
 169
 170 struct cxgbit_sock_common {
 171         struct cxgbit_device *cdev;
 172         struct sockaddr_storage local_addr;
 173         struct sockaddr_storage remote_addr;
 174         struct cxgbit_wr_wait wr_wait;
 175         enum cxgbit_csk_state state;
 176         unsigned long flags;
 177 };
 178
 179 struct cxgbit_np {
 180         struct cxgbit_sock_common com;
 181         wait_queue_head_t accept_wait;
 182         struct iscsi_np *np;
 183         struct completion accept_comp;
 184         struct list_head np_accept_list;
 185         /* np accept lock */
 186         spinlock_t np_accept_lock;
 187         struct kref kref;
 188         unsigned int stid;
 189 };
 190
 191 struct cxgbit_sock {
 192         struct cxgbit_sock_common com;
 193         struct cxgbit_np *cnp;
 194         struct iscsi_conn *conn;
 195         struct l2t_entry *l2t;
 196         struct dst_entry *dst;
 197         struct list_head list;
 198         struct sk_buff_head rxq;
 199         struct sk_buff_head txq;
 200         struct sk_buff_head ppodq;
 201         struct sk_buff_head backlogq;
 202         struct sk_buff_head skbq;
 203         struct sk_buff *wr_pending_head;
 204         struct sk_buff *wr_pending_tail;
 205         struct sk_buff *skb;
 206         struct sk_buff *lro_skb;
 207         struct sk_buff *lro_hskb;
 208         struct list_head accept_node;
 209         /* socket lock */
 210         spinlock_t lock;
 211         wait_queue_head_t waitq;
 212         wait_queue_head_t ack_waitq;
 213         bool lock_owner;
 214         struct kref kref;
 215         u32 max_iso_npdu;
 216         u32 wr_cred;
 217         u32 wr_una_cred;
 218         u32 wr_max_cred;
 219         u32 snd_una;
 220         u32 tid;
 221         u32 snd_nxt;
 222         u32 rcv_nxt;
 223         u32 smac_idx;
 224         u32 tx_chan;
 225         u32 mtu;
 226         u32 write_seq;
 227         u32 rx_credits;
 228         u32 snd_win;
 229         u32 rcv_win;
 230         u16 mss;
 231         u16 emss;
 232         u16 plen;
 233         u16 rss_qid;
 234         u16 txq_idx;
 235         u16 ctrlq_idx;
 236         u8 tos;
 237         u8 port_id;
 238 #define CXGBIT_SUBMODE_HCRC 0x1
 239 #define CXGBIT_SUBMODE_DCRC 0x2
 240         u8 submode;
 241 #ifdef CONFIG_CHELSIO_T4_DCB
 242         u8 dcb_priority;
 243 #endif
 244         u8 snd_wscale;
 245 };
 246
 247 void _cxgbit_free_cdev(struct kref *kref);
 248 void _cxgbit_free_csk(struct kref *kref);
 249 void _cxgbit_free_cnp(struct kref *kref);
 250
 251 static inline void cxgbit_get_cdev(struct cxgbit_device *cdev)
 252 {
 253         kref_get(&cdev->kref);
 254 }
 255
 256 static inline void cxgbit_put_cdev(struct cxgbit_device *cdev)
 257 {
 258         kref_put(&cdev->kref, _cxgbit_free_cdev);
 259 }
 260
 261 static inline void cxgbit_get_csk(struct cxgbit_sock *csk)
 262 {
 263         kref_get(&csk->kref);
 264 }
 265
 266 static inline void cxgbit_put_csk(struct cxgbit_sock *csk)
 267 {
 268         kref_put(&csk->kref, _cxgbit_free_csk);
 269 }
 270
 271 static inline void cxgbit_get_cnp(struct cxgbit_np *cnp)
 272 {
 273         kref_get(&cnp->kref);
 274 }
 275
 276 static inline void cxgbit_put_cnp(struct cxgbit_np *cnp)
 277 {
 278         kref_put(&cnp->kref, _cxgbit_free_cnp);
 279 }
 280
 281 static inline void cxgbit_sock_reset_wr_list(struct cxgbit_sock *csk)
 282 {
 283         csk->wr_pending_tail = NULL;
 284         csk->wr_pending_head = NULL;
 285 }
 286
 287 static inline struct sk_buff *cxgbit_sock_peek_wr(const struct cxgbit_sock *csk)
 288 {
 289         return csk->wr_pending_head;
 290 }
 291
 292 static inline void
 293 cxgbit_sock_enqueue_wr(struct cxgbit_sock *csk, struct sk_buff *skb)
 294 {
 295         cxgbit_skcb_tx_wr_next(skb) = NULL;
 296
 297         skb_get(skb);
 298
 299         if (!csk->wr_pending_head)
 300                 csk->wr_pending_head = skb;
 301         else
 302                 cxgbit_skcb_tx_wr_next(csk->wr_pending_tail) = skb;
 303         csk->wr_pending_tail = skb;
 304 }
 305
 306 static inline struct sk_buff *cxgbit_sock_dequeue_wr(struct cxgbit_sock *csk)
 307 {
 308         struct sk_buff *skb = csk->wr_pending_head;
 309
 310         if (likely(skb)) {
 311                 csk->wr_pending_head = cxgbit_skcb_tx_wr_next(skb);
 312                 cxgbit_skcb_tx_wr_next(skb) = NULL;
 313         }
 314         return skb;
 315 }
 316
 317 typedef void (*cxgbit_cplhandler_func)(struct cxgbit_device *,
 318                                        struct sk_buff *);
 319
 320 int cxgbit_setup_np(struct iscsi_np *, struct sockaddr_storage *);
 321 int cxgbit_setup_conn_digest(struct cxgbit_sock *);
 322 int cxgbit_accept_np(struct iscsi_np *, struct iscsi_conn *);
 323 void cxgbit_free_np(struct iscsi_np *);
 324 void cxgbit_free_conn(struct iscsi_conn *);
 325 extern cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS];
 326 int cxgbit_get_login_rx(struct iscsi_conn *, struct iscsi_login *);
 327 int cxgbit_rx_data_ack(struct cxgbit_sock *);
 328 int cxgbit_l2t_send(struct cxgbit_device *, struct sk_buff *,
 329                     struct l2t_entry *);
 330 void cxgbit_push_tx_frames(struct cxgbit_sock *);
 331 int cxgbit_put_login_tx(struct iscsi_conn *, struct iscsi_login *, u32);
 332 int cxgbit_xmit_pdu(struct iscsi_conn *, struct iscsi_cmd *,
 333                     struct iscsi_datain_req *, const void *, u32);
 334 void cxgbit_get_r2t_ttt(struct iscsi_conn *, struct iscsi_cmd *,
 335                         struct iscsi_r2t *);
 336 u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *);
 337 int cxgbit_ofld_send(struct cxgbit_device *, struct sk_buff *);
 338 void cxgbit_get_rx_pdu(struct iscsi_conn *);
 339 int cxgbit_validate_params(struct iscsi_conn *);
 340 struct cxgbit_device *cxgbit_find_device(struct net_device *, u8 *);
 341
 342 /* DDP */
 343 int cxgbit_ddp_init(struct cxgbit_device *);
 344 int cxgbit_setup_conn_pgidx(struct cxgbit_sock *, u32);
 345 int cxgbit_reserve_ttt(struct cxgbit_sock *, struct iscsi_cmd *);
 346 void cxgbit_release_cmd(struct iscsi_conn *, struct iscsi_cmd *);
 347
 348 static inline
 349 struct cxgbi_ppm *cdev2ppm(struct cxgbit_device *cdev)
 350 {
 351         return (struct cxgbi_ppm *)(*cdev->lldi.iscsi_ppm);
 352 }
 353 #endif /* __CXGBIT_H__ */