drivers/infiniband/hw/cxgb3/cxio_wr.h

   1 /*
   2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #ifndef __CXIO_WR_H__
  33 #define __CXIO_WR_H__
  34
  35 #include <asm/io.h>
  36 #include <linux/pci.h>
  37 #include <linux/timer.h>
  38 #include "firmware_exports.h"
  39
  40 #define T3_MAX_SGE      4
  41 #define T3_MAX_INLINE   64
  42
  43 #define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
  44 #define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))>>(size_log2)) && \
  45                                        ((rptr)!=(wptr)) )
  46 #define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
  47 #define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
  48 #define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
  49 #define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
  50
  51 static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)
  52 {
  53         writel(((1<<31) | qpid), doorbell);
  54 }
  55
  56 #define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))
  57
  58 enum t3_wr_flags {
  59         T3_COMPLETION_FLAG = 0x01,
  60         T3_NOTIFY_FLAG = 0x02,
  61         T3_SOLICITED_EVENT_FLAG = 0x04,
  62         T3_READ_FENCE_FLAG = 0x08,
  63         T3_LOCAL_FENCE_FLAG = 0x10
  64 } __attribute__ ((packed));
  65
  66 enum t3_wr_opcode {
  67         T3_WR_BP = FW_WROPCODE_RI_BYPASS,
  68         T3_WR_SEND = FW_WROPCODE_RI_SEND,
  69         T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
  70         T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
  71         T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
  72         T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
  73         T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
  74         T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
  75         T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
  76 } __attribute__ ((packed));
  77
  78 enum t3_rdma_opcode {
  79         T3_RDMA_WRITE,          /* IETF RDMAP v1.0 ... */
  80         T3_READ_REQ,
  81         T3_READ_RESP,
  82         T3_SEND,
  83         T3_SEND_WITH_INV,
  84         T3_SEND_WITH_SE,
  85         T3_SEND_WITH_SE_INV,
  86         T3_TERMINATE,
  87         T3_RDMA_INIT,           /* CHELSIO RI specific ... */
  88         T3_BIND_MW,
  89         T3_FAST_REGISTER,
  90         T3_LOCAL_INV,
  91         T3_QP_MOD,
  92         T3_BYPASS
  93 } __attribute__ ((packed));
  94
  95 static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
  96 {
  97         switch (wrop) {
  98                 case T3_WR_BP: return T3_BYPASS;
  99                 case T3_WR_SEND: return T3_SEND;
 100                 case T3_WR_WRITE: return T3_RDMA_WRITE;
 101                 case T3_WR_READ: return T3_READ_REQ;
 102                 case T3_WR_INV_STAG: return T3_LOCAL_INV;
 103                 case T3_WR_BIND: return T3_BIND_MW;
 104                 case T3_WR_INIT: return T3_RDMA_INIT;
 105                 case T3_WR_QP_MOD: return T3_QP_MOD;
 106                 default: break;
 107         }
 108         return -1;
 109 }
 110
 111
 112 /* Work request id */
 113 union t3_wrid {
 114         struct {
 115                 u32 hi;
 116                 u32 low;
 117         } id0;
 118         u64 id1;
 119 };
 120
 121 #define WRID(wrid)              (wrid.id1)
 122 #define WRID_GEN(wrid)          (wrid.id0.wr_gen)
 123 #define WRID_IDX(wrid)          (wrid.id0.wr_idx)
 124 #define WRID_LO(wrid)           (wrid.id0.wr_lo)
 125
 126 struct fw_riwrh {
 127         __be32 op_seop_flags;
 128         __be32 gen_tid_len;
 129 };
 130
 131 #define S_FW_RIWR_OP            24
 132 #define M_FW_RIWR_OP            0xff
 133 #define V_FW_RIWR_OP(x)         ((x) << S_FW_RIWR_OP)
 134 #define G_FW_RIWR_OP(x) ((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)
 135
 136 #define S_FW_RIWR_SOPEOP        22
 137 #define M_FW_RIWR_SOPEOP        0x3
 138 #define V_FW_RIWR_SOPEOP(x)     ((x) << S_FW_RIWR_SOPEOP)
 139
 140 #define S_FW_RIWR_FLAGS         8
 141 #define M_FW_RIWR_FLAGS         0x3fffff
 142 #define V_FW_RIWR_FLAGS(x)      ((x) << S_FW_RIWR_FLAGS)
 143 #define G_FW_RIWR_FLAGS(x)      ((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)
 144
 145 #define S_FW_RIWR_TID           8
 146 #define V_FW_RIWR_TID(x)        ((x) << S_FW_RIWR_TID)
 147
 148 #define S_FW_RIWR_LEN           0
 149 #define V_FW_RIWR_LEN(x)        ((x) << S_FW_RIWR_LEN)
 150
 151 #define S_FW_RIWR_GEN           31
 152 #define V_FW_RIWR_GEN(x)        ((x)  << S_FW_RIWR_GEN)
 153
 154 struct t3_sge {
 155         __be32 stag;
 156         __be32 len;
 157         __be64 to;
 158 };
 159
 160 /* If num_sgle is zero, flit 5+ contains immediate data.*/
 161 struct t3_send_wr {
 162         struct fw_riwrh wrh;    /* 0 */
 163         union t3_wrid wrid;     /* 1 */
 164
 165         u8 rdmaop;              /* 2 */
 166         u8 reserved[3];
 167         __be32 rem_stag;
 168         __be32 plen;            /* 3 */
 169         __be32 num_sgle;
 170         struct t3_sge sgl[T3_MAX_SGE];  /* 4+ */
 171 };
 172
 173 struct t3_local_inv_wr {
 174         struct fw_riwrh wrh;    /* 0 */
 175         union t3_wrid wrid;     /* 1 */
 176         __be32 stag;            /* 2 */
 177         __be32 reserved3;
 178 };
 179
 180 struct t3_rdma_write_wr {
 181         struct fw_riwrh wrh;    /* 0 */
 182         union t3_wrid wrid;     /* 1 */
 183         u8 rdmaop;              /* 2 */
 184         u8 reserved[3];
 185         __be32 stag_sink;
 186         __be64 to_sink;         /* 3 */
 187         __be32 plen;            /* 4 */
 188         __be32 num_sgle;
 189         struct t3_sge sgl[T3_MAX_SGE];  /* 5+ */
 190 };
 191
 192 struct t3_rdma_read_wr {
 193         struct fw_riwrh wrh;    /* 0 */
 194         union t3_wrid wrid;     /* 1 */
 195         u8 rdmaop;              /* 2 */
 196         u8 reserved[3];
 197         __be32 rem_stag;
 198         __be64 rem_to;          /* 3 */
 199         __be32 local_stag;      /* 4 */
 200         __be32 local_len;
 201         __be64 local_to;        /* 5 */
 202 };
 203
 204 enum t3_addr_type {
 205         T3_VA_BASED_TO = 0x0,
 206         T3_ZERO_BASED_TO = 0x1
 207 } __attribute__ ((packed));
 208
 209 enum t3_mem_perms {
 210         T3_MEM_ACCESS_LOCAL_READ = 0x1,
 211         T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
 212         T3_MEM_ACCESS_REM_READ = 0x4,
 213         T3_MEM_ACCESS_REM_WRITE = 0x8
 214 } __attribute__ ((packed));
 215
 216 struct t3_bind_mw_wr {
 217         struct fw_riwrh wrh;    /* 0 */
 218         union t3_wrid wrid;     /* 1 */
 219         u16 reserved;           /* 2 */
 220         u8 type;
 221         u8 perms;
 222         __be32 mr_stag;
 223         __be32 mw_stag;         /* 3 */
 224         __be32 mw_len;
 225         __be64 mw_va;           /* 4 */
 226         __be32 mr_pbl_addr;     /* 5 */
 227         u8 reserved2[3];
 228         u8 mr_pagesz;
 229 };
 230
 231 struct t3_receive_wr {
 232         struct fw_riwrh wrh;    /* 0 */
 233         union t3_wrid wrid;     /* 1 */
 234         u8 pagesz[T3_MAX_SGE];
 235         __be32 num_sgle;                /* 2 */
 236         struct t3_sge sgl[T3_MAX_SGE];  /* 3+ */
 237         __be32 pbl_addr[T3_MAX_SGE];
 238 };
 239
 240 struct t3_bypass_wr {
 241         struct fw_riwrh wrh;
 242         union t3_wrid wrid;     /* 1 */
 243 };
 244
 245 struct t3_modify_qp_wr {
 246         struct fw_riwrh wrh;    /* 0 */
 247         union t3_wrid wrid;     /* 1 */
 248         __be32 flags;           /* 2 */
 249         __be32 quiesce;         /* 2 */
 250         __be32 max_ird;         /* 3 */
 251         __be32 max_ord;         /* 3 */
 252         __be64 sge_cmd;         /* 4 */
 253         __be64 ctx1;            /* 5 */
 254         __be64 ctx0;            /* 6 */
 255 };
 256
 257 enum t3_modify_qp_flags {
 258         MODQP_QUIESCE  = 0x01,
 259         MODQP_MAX_IRD  = 0x02,
 260         MODQP_MAX_ORD  = 0x04,
 261         MODQP_WRITE_EC = 0x08,
 262         MODQP_READ_EC  = 0x10,
 263 };
 264
 265
 266 enum t3_mpa_attrs {
 267         uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
 268         uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
 269         uP_RI_MPA_CRC_ENABLE = 0x4,
 270         uP_RI_MPA_IETF_ENABLE = 0x8
 271 } __attribute__ ((packed));
 272
 273 enum t3_qp_caps {
 274         uP_RI_QP_RDMA_READ_ENABLE = 0x01,
 275         uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
 276         uP_RI_QP_BIND_ENABLE = 0x04,
 277         uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
 278         uP_RI_QP_STAG0_ENABLE = 0x10
 279 } __attribute__ ((packed));
 280
 281 enum rdma_init_rtr_types {
 282         RTR_READ = 1,
 283         RTR_WRITE = 2,
 284         RTR_SEND = 3,
 285 };
 286
 287 #define S_RTR_TYPE      2
 288 #define M_RTR_TYPE      0x3
 289 #define V_RTR_TYPE(x)   ((x) << S_RTR_TYPE)
 290 #define G_RTR_TYPE(x)   ((((x) >> S_RTR_TYPE)) & M_RTR_TYPE)
 291
 292 struct t3_rdma_init_attr {
 293         u32 tid;
 294         u32 qpid;
 295         u32 pdid;
 296         u32 scqid;
 297         u32 rcqid;
 298         u32 rq_addr;
 299         u32 rq_size;
 300         enum t3_mpa_attrs mpaattrs;
 301         enum t3_qp_caps qpcaps;
 302         u16 tcp_emss;
 303         u32 ord;
 304         u32 ird;
 305         u64 qp_dma_addr;
 306         u32 qp_dma_size;
 307         enum rdma_init_rtr_types rtr_type;
 308         u16 flags;
 309         u16 rqe_count;
 310         u32 irs;
 311 };
 312
 313 struct t3_rdma_init_wr {
 314         struct fw_riwrh wrh;    /* 0 */
 315         union t3_wrid wrid;     /* 1 */
 316         __be32 qpid;            /* 2 */
 317         __be32 pdid;
 318         __be32 scqid;           /* 3 */
 319         __be32 rcqid;
 320         __be32 rq_addr;         /* 4 */
 321         __be32 rq_size;
 322         u8 mpaattrs;            /* 5 */
 323         u8 qpcaps;
 324         __be16 ulpdu_size;
 325         __be16 flags_rtr_type;
 326         __be16 rqe_count;
 327         __be32 ord;             /* 6 */
 328         __be32 ird;
 329         __be64 qp_dma_addr;     /* 7 */
 330         __be32 qp_dma_size;     /* 8 */
 331         __be32 irs;
 332 };
 333
 334 struct t3_genbit {
 335         u64 flit[15];
 336         __be64 genbit;
 337 };
 338
 339 enum rdma_init_wr_flags {
 340         MPA_INITIATOR = (1<<0),
 341         PRIV_QP = (1<<1),
 342 };
 343
 344 union t3_wr {
 345         struct t3_send_wr send;
 346         struct t3_rdma_write_wr write;
 347         struct t3_rdma_read_wr read;
 348         struct t3_receive_wr recv;
 349         struct t3_local_inv_wr local_inv;
 350         struct t3_bind_mw_wr bind;
 351         struct t3_bypass_wr bypass;
 352         struct t3_rdma_init_wr init;
 353         struct t3_modify_qp_wr qp_mod;
 354         struct t3_genbit genbit;
 355         u64 flit[16];
 356 };
 357
 358 #define T3_SQ_CQE_FLIT    13
 359 #define T3_SQ_COOKIE_FLIT 14
 360
 361 #define T3_RQ_COOKIE_FLIT 13
 362 #define T3_RQ_CQE_FLIT    14
 363
 364 static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
 365 {
 366         return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
 367 }
 368
 369 static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
 370                                   enum t3_wr_flags flags, u8 genbit, u32 tid,
 371                                   u8 len)
 372 {
 373         wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
 374                                          V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
 375                                          V_FW_RIWR_FLAGS(flags));
 376         wmb();
 377         wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
 378                                        V_FW_RIWR_TID(tid) |
 379                                        V_FW_RIWR_LEN(len));
 380         /* 2nd gen bit... */
 381         ((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit);
 382 }
 383
 384 /*
 385  * T3 ULP2_TX commands
 386  */
 387 enum t3_utx_mem_op {
 388         T3_UTX_MEM_READ = 2,
 389         T3_UTX_MEM_WRITE = 3
 390 };
 391
 392 /* T3 MC7 RDMA TPT entry format */
 393
 394 enum tpt_mem_type {
 395         TPT_NON_SHARED_MR = 0x0,
 396         TPT_SHARED_MR = 0x1,
 397         TPT_MW = 0x2,
 398         TPT_MW_RELAXED_PROTECTION = 0x3
 399 };
 400
 401 enum tpt_addr_type {
 402         TPT_ZBTO = 0,
 403         TPT_VATO = 1
 404 };
 405
 406 enum tpt_mem_perm {
 407         TPT_LOCAL_READ = 0x8,
 408         TPT_LOCAL_WRITE = 0x4,
 409         TPT_REMOTE_READ = 0x2,
 410         TPT_REMOTE_WRITE = 0x1
 411 };
 412
 413 struct tpt_entry {
 414         __be32 valid_stag_pdid;
 415         __be32 flags_pagesize_qpid;
 416
 417         __be32 rsvd_pbl_addr;
 418         __be32 len;
 419         __be32 va_hi;
 420         __be32 va_low_or_fbo;
 421
 422         __be32 rsvd_bind_cnt_or_pstag;
 423         __be32 rsvd_pbl_size;
 424 };
 425
 426 #define S_TPT_VALID             31
 427 #define V_TPT_VALID(x)          ((x) << S_TPT_VALID)
 428 #define F_TPT_VALID             V_TPT_VALID(1U)
 429
 430 #define S_TPT_STAG_KEY          23
 431 #define M_TPT_STAG_KEY          0xFF
 432 #define V_TPT_STAG_KEY(x)       ((x) << S_TPT_STAG_KEY)
 433 #define G_TPT_STAG_KEY(x)       (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
 434
 435 #define S_TPT_STAG_STATE        22
 436 #define V_TPT_STAG_STATE(x)     ((x) << S_TPT_STAG_STATE)
 437 #define F_TPT_STAG_STATE        V_TPT_STAG_STATE(1U)
 438
 439 #define S_TPT_STAG_TYPE         20
 440 #define M_TPT_STAG_TYPE         0x3
 441 #define V_TPT_STAG_TYPE(x)      ((x) << S_TPT_STAG_TYPE)
 442 #define G_TPT_STAG_TYPE(x)      (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
 443
 444 #define S_TPT_PDID              0
 445 #define M_TPT_PDID              0xFFFFF
 446 #define V_TPT_PDID(x)           ((x) << S_TPT_PDID)
 447 #define G_TPT_PDID(x)           (((x) >> S_TPT_PDID) & M_TPT_PDID)
 448
 449 #define S_TPT_PERM              28
 450 #define M_TPT_PERM              0xF
 451 #define V_TPT_PERM(x)           ((x) << S_TPT_PERM)
 452 #define G_TPT_PERM(x)           (((x) >> S_TPT_PERM) & M_TPT_PERM)
 453
 454 #define S_TPT_REM_INV_DIS       27
 455 #define V_TPT_REM_INV_DIS(x)    ((x) << S_TPT_REM_INV_DIS)
 456 #define F_TPT_REM_INV_DIS       V_TPT_REM_INV_DIS(1U)
 457
 458 #define S_TPT_ADDR_TYPE         26
 459 #define V_TPT_ADDR_TYPE(x)      ((x) << S_TPT_ADDR_TYPE)
 460 #define F_TPT_ADDR_TYPE         V_TPT_ADDR_TYPE(1U)
 461
 462 #define S_TPT_MW_BIND_ENABLE    25
 463 #define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE)
 464 #define F_TPT_MW_BIND_ENABLE    V_TPT_MW_BIND_ENABLE(1U)
 465
 466 #define S_TPT_PAGE_SIZE         20
 467 #define M_TPT_PAGE_SIZE         0x1F
 468 #define V_TPT_PAGE_SIZE(x)      ((x) << S_TPT_PAGE_SIZE)
 469 #define G_TPT_PAGE_SIZE(x)      (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
 470
 471 #define S_TPT_PBL_ADDR          0
 472 #define M_TPT_PBL_ADDR          0x1FFFFFFF
 473 #define V_TPT_PBL_ADDR(x)       ((x) << S_TPT_PBL_ADDR)
 474 #define G_TPT_PBL_ADDR(x)       (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
 475
 476 #define S_TPT_QPID              0
 477 #define M_TPT_QPID              0xFFFFF
 478 #define V_TPT_QPID(x)           ((x) << S_TPT_QPID)
 479 #define G_TPT_QPID(x)           (((x) >> S_TPT_QPID) & M_TPT_QPID)
 480
 481 #define S_TPT_PSTAG             0
 482 #define M_TPT_PSTAG             0xFFFFFF
 483 #define V_TPT_PSTAG(x)          ((x) << S_TPT_PSTAG)
 484 #define G_TPT_PSTAG(x)          (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
 485
 486 #define S_TPT_PBL_SIZE          0
 487 #define M_TPT_PBL_SIZE          0xFFFFF
 488 #define V_TPT_PBL_SIZE(x)       ((x) << S_TPT_PBL_SIZE)
 489 #define G_TPT_PBL_SIZE(x)       (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
 490
 491 /*
 492  * CQE defs
 493  */
 494 struct t3_cqe {
 495         __be32 header;
 496         __be32 len;
 497         union {
 498                 struct {
 499                         __be32 stag;
 500                         __be32 msn;
 501                 } rcqe;
 502                 struct {
 503                         u32 wrid_hi;
 504                         u32 wrid_low;
 505                 } scqe;
 506         } u;
 507 };
 508
 509 #define S_CQE_OOO         31
 510 #define M_CQE_OOO         0x1
 511 #define G_CQE_OOO(x)      ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
 512 #define V_CEQ_OOO(x)      ((x)<<S_CQE_OOO)
 513
 514 #define S_CQE_QPID        12
 515 #define M_CQE_QPID        0x7FFFF
 516 #define G_CQE_QPID(x)     ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
 517 #define V_CQE_QPID(x)     ((x)<<S_CQE_QPID)
 518
 519 #define S_CQE_SWCQE       11
 520 #define M_CQE_SWCQE       0x1
 521 #define G_CQE_SWCQE(x)    ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
 522 #define V_CQE_SWCQE(x)    ((x)<<S_CQE_SWCQE)
 523
 524 #define S_CQE_GENBIT      10
 525 #define M_CQE_GENBIT      0x1
 526 #define G_CQE_GENBIT(x)   (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
 527 #define V_CQE_GENBIT(x)   ((x)<<S_CQE_GENBIT)
 528
 529 #define S_CQE_STATUS      5
 530 #define M_CQE_STATUS      0x1F
 531 #define G_CQE_STATUS(x)   ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
 532 #define V_CQE_STATUS(x)   ((x)<<S_CQE_STATUS)
 533
 534 #define S_CQE_TYPE        4
 535 #define M_CQE_TYPE        0x1
 536 #define G_CQE_TYPE(x)     ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
 537 #define V_CQE_TYPE(x)     ((x)<<S_CQE_TYPE)
 538
 539 #define S_CQE_OPCODE      0
 540 #define M_CQE_OPCODE      0xF
 541 #define G_CQE_OPCODE(x)   ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
 542 #define V_CQE_OPCODE(x)   ((x)<<S_CQE_OPCODE)
 543
 544 #define SW_CQE(x)         (G_CQE_SWCQE(be32_to_cpu((x).header)))
 545 #define CQE_OOO(x)        (G_CQE_OOO(be32_to_cpu((x).header)))
 546 #define CQE_QPID(x)       (G_CQE_QPID(be32_to_cpu((x).header)))
 547 #define CQE_GENBIT(x)     (G_CQE_GENBIT(be32_to_cpu((x).header)))
 548 #define CQE_TYPE(x)       (G_CQE_TYPE(be32_to_cpu((x).header)))
 549 #define SQ_TYPE(x)        (CQE_TYPE((x)))
 550 #define RQ_TYPE(x)        (!CQE_TYPE((x)))
 551 #define CQE_STATUS(x)     (G_CQE_STATUS(be32_to_cpu((x).header)))
 552 #define CQE_OPCODE(x)     (G_CQE_OPCODE(be32_to_cpu((x).header)))
 553
 554 #define CQE_LEN(x)        (be32_to_cpu((x).len))
 555
 556 /* used for RQ completion processing */
 557 #define CQE_WRID_STAG(x)  (be32_to_cpu((x).u.rcqe.stag))
 558 #define CQE_WRID_MSN(x)   (be32_to_cpu((x).u.rcqe.msn))
 559
 560 /* used for SQ completion processing */
 561 #define CQE_WRID_SQ_WPTR(x)     ((x).u.scqe.wrid_hi)
 562 #define CQE_WRID_WPTR(x)        ((x).u.scqe.wrid_low)
 563
 564 /* generic accessor macros */
 565 #define CQE_WRID_HI(x)          ((x).u.scqe.wrid_hi)
 566 #define CQE_WRID_LOW(x)         ((x).u.scqe.wrid_low)
 567
 568 #define TPT_ERR_SUCCESS                     0x0
 569 #define TPT_ERR_STAG                        0x1  /* STAG invalid: either the */
 570                                                  /* STAG is offlimt, being 0, */
 571                                                  /* or STAG_key mismatch */
 572 #define TPT_ERR_PDID                        0x2  /* PDID mismatch */
 573 #define TPT_ERR_QPID                        0x3  /* QPID mismatch */
 574 #define TPT_ERR_ACCESS                      0x4  /* Invalid access right */
 575 #define TPT_ERR_WRAP                        0x5  /* Wrap error */
 576 #define TPT_ERR_BOUND                       0x6  /* base and bounds voilation */
 577 #define TPT_ERR_INVALIDATE_SHARED_MR        0x7  /* attempt to invalidate a  */
 578                                                  /* shared memory region */
 579 #define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8  /* attempt to invalidate a  */
 580                                                  /* shared memory region */
 581 #define TPT_ERR_ECC                         0x9  /* ECC error detected */
 582 #define TPT_ERR_ECC_PSTAG                   0xA  /* ECC error detected when  */
 583                                                  /* reading PSTAG for a MW  */
 584                                                  /* Invalidate */
 585 #define TPT_ERR_PBL_ADDR_BOUND              0xB  /* pbl addr out of bounds:  */
 586                                                  /* software error */
 587 #define TPT_ERR_SWFLUSH                     0xC  /* SW FLUSHED */
 588 #define TPT_ERR_CRC                         0x10 /* CRC error */
 589 #define TPT_ERR_MARKER                      0x11 /* Marker error */
 590 #define TPT_ERR_PDU_LEN_ERR                 0x12 /* invalid PDU length */
 591 #define TPT_ERR_OUT_OF_RQE                  0x13 /* out of RQE */
 592 #define TPT_ERR_DDP_VERSION                 0x14 /* wrong DDP version */
 593 #define TPT_ERR_RDMA_VERSION                0x15 /* wrong RDMA version */
 594 #define TPT_ERR_OPCODE                      0x16 /* invalid rdma opcode */
 595 #define TPT_ERR_DDP_QUEUE_NUM               0x17 /* invalid ddp queue number */
 596 #define TPT_ERR_MSN                         0x18 /* MSN error */
 597 #define TPT_ERR_TBIT                        0x19 /* tag bit not set correctly */
 598 #define TPT_ERR_MO                          0x1A /* MO not 0 for TERMINATE  */
 599                                                  /* or READ_REQ */
 600 #define TPT_ERR_MSN_GAP                     0x1B
 601 #define TPT_ERR_MSN_RANGE                   0x1C
 602 #define TPT_ERR_IRD_OVERFLOW                0x1D
 603 #define TPT_ERR_RQE_ADDR_BOUND              0x1E /* RQE addr out of bounds:  */
 604                                                  /* software error */
 605 #define TPT_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */
 606                                                  /* mismatch) */
 607
 608 struct t3_swsq {
 609         __u64                   wr_id;
 610         struct t3_cqe           cqe;
 611         __u32                   sq_wptr;
 612         __be32                  read_len;
 613         int                     opcode;
 614         int                     complete;
 615         int                     signaled;
 616 };
 617
 618 /*
 619  * A T3 WQ implements both the SQ and RQ.
 620  */
 621 struct t3_wq {
 622         union t3_wr *queue;             /* DMA accessable memory */
 623         dma_addr_t dma_addr;            /* DMA address for HW */
 624         DECLARE_PCI_UNMAP_ADDR(mapping) /* unmap kruft */
 625         u32 error;                      /* 1 once we go to ERROR */
 626         u32 qpid;
 627         u32 wptr;                       /* idx to next available WR slot */
 628         u32 size_log2;                  /* total wq size */
 629         struct t3_swsq *sq;             /* SW SQ */
 630         struct t3_swsq *oldest_read;    /* tracks oldest pending read */
 631         u32 sq_wptr;                    /* sq_wptr - sq_rptr == count of */
 632         u32 sq_rptr;                    /* pending wrs */
 633         u32 sq_size_log2;               /* sq size */
 634         u64 *rq;                        /* SW RQ (holds consumer wr_ids */
 635         u32 rq_wptr;                    /* rq_wptr - rq_rptr == count of */
 636         u32 rq_rptr;                    /* pending wrs */
 637         u64 *rq_oldest_wr;              /* oldest wr on the SW RQ */
 638         u32 rq_size_log2;               /* rq size */
 639         u32 rq_addr;                    /* rq adapter address */
 640         void __iomem *doorbell;         /* kernel db */
 641         u64 udb;                        /* user db if any */
 642 };
 643
 644 struct t3_cq {
 645         u32 cqid;
 646         u32 rptr;
 647         u32 wptr;
 648         u32 size_log2;
 649         dma_addr_t dma_addr;
 650         DECLARE_PCI_UNMAP_ADDR(mapping)
 651         struct t3_cqe *queue;
 652         struct t3_cqe *sw_queue;
 653         u32 sw_rptr;
 654         u32 sw_wptr;
 655 };
 656
 657 #define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
 658                                          CQE_GENBIT(*cqe))
 659
 660 static inline void cxio_set_wq_in_error(struct t3_wq *wq)
 661 {
 662         wq->queue->flit[13] = 1;
 663 }
 664
 665 static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
 666 {
 667         struct t3_cqe *cqe;
 668
 669         cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
 670         if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
 671                 return cqe;
 672         return NULL;
 673 }
 674
 675 static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
 676 {
 677         struct t3_cqe *cqe;
 678
 679         if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
 680                 cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
 681                 return cqe;
 682         }
 683         return NULL;
 684 }
 685
 686 static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
 687 {
 688         struct t3_cqe *cqe;
 689
 690         if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
 691                 cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
 692                 return cqe;
 693         }
 694         cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
 695         if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
 696                 return cqe;
 697         return NULL;
 698 }
 699
 700 #endif