]> git.proxmox.com Git - mirror_ubuntu-kernels.git/commitdiff
IB/hfi1: Add 16B UD support
authorDon Hiatt <don.hiatt@intel.com>
Fri, 4 Aug 2017 20:54:23 +0000 (13:54 -0700)
committerDoug Ledford <dledford@redhat.com>
Tue, 22 Aug 2017 18:22:37 +0000 (14:22 -0400)
Add 16B bypass packet support for UD traffic types.

Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Don Hiatt <don.hiatt@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
drivers/infiniband/hw/hfi1/driver.c
drivers/infiniband/hw/hfi1/hfi.h
drivers/infiniband/hw/hfi1/mad.c
drivers/infiniband/hw/hfi1/ruc.c
drivers/infiniband/hw/hfi1/ud.c
drivers/infiniband/hw/hfi1/verbs.h
include/rdma/opa_addr.h

index ae6a90d2a31c4b11dcd0249ddf98337d9c572452..fc7085d6cf3fd576149aca31b05738915a23a084 100644 (file)
@@ -437,23 +437,33 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
                               bool do_cnp)
 {
        struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
-       struct ib_header *hdr = pkt->hdr;
        struct ib_other_headers *ohdr = pkt->ohdr;
        struct ib_grh *grh = pkt->grh;
        u32 rqpn = 0, bth1;
-       u16 rlid, dlid = ib_get_dlid(hdr);
-       u8 sc, svc_type;
+       u16 pkey, rlid, dlid = ib_get_dlid(pkt->hdr);
+       u8 hdr_type, sc, svc_type;
        bool is_mcast = false;
 
+       if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
+               is_mcast = hfi1_is_16B_mcast(dlid);
+               pkey = hfi1_16B_get_pkey(pkt->hdr);
+               sc = hfi1_16B_get_sc(pkt->hdr);
+               hdr_type = HFI1_PKT_TYPE_16B;
+       } else {
+               is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
+                          (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
+               pkey = ib_bth_get_pkey(ohdr);
+               sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf);
+               hdr_type = HFI1_PKT_TYPE_9B;
+       }
+
        switch (qp->ibqp.qp_type) {
        case IB_QPT_SMI:
        case IB_QPT_GSI:
        case IB_QPT_UD:
-               rlid = ib_get_slid(hdr);
-               rqpn = ib_get_sqpn(ohdr);
+               rlid = ib_get_slid(pkt->hdr);
+               rqpn = ib_get_sqpn(pkt->ohdr);
                svc_type = IB_CC_SVCTYPE_UD;
-               is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
-                       (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
                break;
        case IB_QPT_UC:
                rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
@@ -469,14 +479,11 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
                return;
        }
 
-       sc = hfi1_9B_get_sc5(hdr, pkt->rhf);
-
        bth1 = be32_to_cpu(ohdr->bth[1]);
-       if (do_cnp && (bth1 & IB_FECN_SMASK)) {
-               u16 pkey = ib_bth_get_pkey(ohdr);
-
-               return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
-       }
+       /* Call appropriate CNP handler */
+       if (do_cnp && (bth1 & IB_FECN_SMASK))
+               hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey,
+                                             dlid, rlid, sc, grh);
 
        if (!is_mcast && (bth1 & IB_BECN_SMASK)) {
                struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
index cec9590870bae73288a8da317a0044a95489dbd7..7e21192da8e100e68e4553ec59525ecf2cf743f2 100644 (file)
@@ -831,6 +831,10 @@ struct hfi1_pportdata {
 typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);
 
 typedef void (*opcode_handler)(struct hfi1_packet *packet);
+typedef void (*hfi1_make_req)(struct rvt_qp *qp,
+                             struct hfi1_pkt_state *ps,
+                             struct rvt_swqe *wqe);
+
 
 /* return values for the RHF receive functions */
 #define RHF_RCV_CONTINUE  0    /* keep going */
@@ -1373,6 +1377,13 @@ void hfi1_set_vnic_msix_info(struct hfi1_ctxtdata *rcd);
 void hfi1_reset_vnic_msix_info(struct hfi1_ctxtdata *rcd);
 
 extern const struct pci_device_id hfi1_pci_tbl[];
+void hfi1_make_ud_req_9B(struct rvt_qp *qp,
+                        struct hfi1_pkt_state *ps,
+                        struct rvt_swqe *wqe);
+
+void hfi1_make_ud_req_16B(struct rvt_qp *qp,
+                         struct hfi1_pkt_state *ps,
+                         struct rvt_swqe *wqe);
 
 /* receive packet handler dispositions */
 #define RCV_PKT_OK      0x0 /* keep going */
@@ -1507,6 +1518,18 @@ void process_becn(struct hfi1_pportdata *ppd, u8 sl,  u16 rlid, u32 lqpn,
 void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
                u32 pkey, u32 slid, u32 dlid, u8 sc5,
                const struct ib_grh *old_grh);
+void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
+                   u32 remote_qpn, u32 pkey, u32 slid, u32 dlid,
+                   u8 sc5, const struct ib_grh *old_grh);
+typedef void (*hfi1_handle_cnp)(struct hfi1_ibport *ibp, struct rvt_qp *qp,
+                               u32 remote_qpn, u32 pkey, u32 slid, u32 dlid,
+                               u8 sc5, const struct ib_grh *old_grh);
+
+/* We support only two types - 9B and 16B for now */
+static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = {
+       [HFI1_PKT_TYPE_9B] = &return_cnp,
+       [HFI1_PKT_TYPE_16B] = &return_cnp_16B
+};
 #define PKEY_CHECK_INVALID -1
 int egress_pkey_check(struct hfi1_pportdata *ppd, __be16 *lrh, __be32 *bth,
                      u8 sc5, int8_t s_pkey_index);
@@ -1747,12 +1770,22 @@ static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt,
                               bool do_cnp)
 {
        struct ib_other_headers *ohdr = pkt->ohdr;
-       u32 bth1;
 
-       bth1 = be32_to_cpu(ohdr->bth[1]);
-       if (unlikely(bth1 & (IB_BECN_SMASK | IB_FECN_SMASK))) {
+       u32 bth1;
+       bool becn = false;
+       bool fecn = false;
+
+       if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
+               fecn = hfi1_16B_get_fecn(pkt->hdr);
+               becn = hfi1_16B_get_becn(pkt->hdr);
+       } else {
+               bth1 = be32_to_cpu(ohdr->bth[1]);
+               fecn = bth1 & IB_FECN_SMASK;
+               becn = bth1 & IB_BECN_SMASK;
+       }
+       if (unlikely(fecn || becn)) {
                hfi1_process_ecn_slowpath(qp, pkt, do_cnp);
-               return !!(bth1 & IB_FECN_SMASK);
+               return fecn;
        }
        return false;
 }
@@ -2315,4 +2348,80 @@ static inline bool hfi1_get_hdr_type(u32 lid, struct rdma_ah_attr *attr)
 
        return hfi1_get_packet_type(lid);
 }
+
+static inline void hfi1_make_ext_grh(struct hfi1_packet *packet,
+                                    struct ib_grh *grh, u32 slid,
+                                    u32 dlid)
+{
+       struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
+       struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+
+       if (!ibp)
+               return;
+
+       grh->hop_limit = 1;
+       grh->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
+       if (slid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))
+               grh->sgid.global.interface_id =
+                       OPA_MAKE_ID(be32_to_cpu(OPA_LID_PERMISSIVE));
+       else
+               grh->sgid.global.interface_id = OPA_MAKE_ID(slid);
+
+       /*
+        * Upper layers (like mad) may compare the dgid in the
+        * wc that is obtained here with the sgid_index in
+        * the wr. Since sgid_index in wr is always 0 for
+        * extended lids, set the dgid here to the default
+        * IB gid.
+        */
+       grh->dgid.global.subnet_prefix = ibp->rvp.gid_prefix;
+       grh->dgid.global.interface_id =
+               cpu_to_be64(ppd->guids[HFI1_PORT_GUID_INDEX]);
+}
+
+static inline int hfi1_get_16b_padding(u32 hdr_size, u32 payload)
+{
+       return -(hdr_size + payload + (SIZE_OF_CRC << 2) +
+                    SIZE_OF_LT) & 0x7;
+}
+
+static inline void hfi1_make_ib_hdr(struct ib_header *hdr,
+                                   u16 lrh0, u16 len,
+                                   u16 dlid, u16 slid)
+{
+       hdr->lrh[0] = cpu_to_be16(lrh0);
+       hdr->lrh[1] = cpu_to_be16(dlid);
+       hdr->lrh[2] = cpu_to_be16(len);
+       hdr->lrh[3] = cpu_to_be16(slid);
+}
+
+static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr,
+                                    u32 slid, u32 dlid,
+                                    u16 len, u16 pkey,
+                                    u8 becn, u8 fecn, u8 l4,
+                                    u8 sc)
+{
+       u32 lrh0 = 0;
+       u32 lrh1 = 0x40000000;
+       u32 lrh2 = 0;
+       u32 lrh3 = 0;
+
+       lrh0 = (lrh0 & ~OPA_16B_BECN_MASK) | (becn << OPA_16B_BECN_SHIFT);
+       lrh0 = (lrh0 & ~OPA_16B_LEN_MASK) | (len << OPA_16B_LEN_SHIFT);
+       lrh0 = (lrh0 & ~OPA_16B_LID_MASK)  | (slid & OPA_16B_LID_MASK);
+       lrh1 = (lrh1 & ~OPA_16B_FECN_MASK) | (fecn << OPA_16B_FECN_SHIFT);
+       lrh1 = (lrh1 & ~OPA_16B_SC_MASK) | (sc << OPA_16B_SC_SHIFT);
+       lrh1 = (lrh1 & ~OPA_16B_LID_MASK) | (dlid & OPA_16B_LID_MASK);
+       lrh2 = (lrh2 & ~OPA_16B_SLID_MASK) |
+               ((slid >> OPA_16B_SLID_SHIFT) << OPA_16B_SLID_HIGH_SHIFT);
+       lrh2 = (lrh2 & ~OPA_16B_DLID_MASK) |
+               ((dlid >> OPA_16B_DLID_SHIFT) << OPA_16B_DLID_HIGH_SHIFT);
+       lrh2 = (lrh2 & ~OPA_16B_PKEY_MASK) | (pkey << OPA_16B_PKEY_SHIFT);
+       lrh2 = (lrh2 & ~OPA_16B_L4_MASK) | l4;
+
+       hdr->lrh[0] = lrh0;
+       hdr->lrh[1] = lrh1;
+       hdr->lrh[2] = lrh2;
+       hdr->lrh[3] = lrh3;
+}
 #endif                          /* _HFI1_KERNEL_H */
index 21fadb4b510c58d966cc83791fce88d4a800eb52..1509bc6b76d829788d25be4eabdc24a6ef66d3ce 100644 (file)
@@ -373,12 +373,10 @@ static struct trap_node *create_trap_node(u8 type, __be16 trap_num, u32 lid)
  * Send a bad P_Key trap (ch. 14.3.8).
  */
 void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,
-                  u32 qp1, u32 qp2, u16 lid1, u16 lid2)
+                  u32 qp1, u32 qp2, u32 lid1, u32 lid2)
 {
        struct trap_node *trap;
        u32 lid = ppd_from_ibp(ibp)->lid;
-       u32 _lid1 = lid1;
-       u32 _lid2 = lid2;
 
        ibp->rvp.n_pkt_drops++;
        ibp->rvp.pkey_violations++;
@@ -389,8 +387,8 @@ void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,
                return;
 
        /* Send violation trap */
-       trap->data.ntc_257_258.lid1 = cpu_to_be32(_lid1);
-       trap->data.ntc_257_258.lid2 = cpu_to_be32(_lid2);
+       trap->data.ntc_257_258.lid1 = cpu_to_be32(lid1);
+       trap->data.ntc_257_258.lid2 = cpu_to_be32(lid2);
        trap->data.ntc_257_258.key = cpu_to_be32(key);
        trap->data.ntc_257_258.sl = sl << 3;
        trap->data.ntc_257_258.qp1 = cpu_to_be32(qp1);
index d252f8f2207a5bb4c5063dc6c6aa51add0566fad..6839bfae933c99850a0a9db4d950fe6ea8b1636b 100644 (file)
@@ -649,7 +649,7 @@ done:
  * @ibp: a pointer to the IB port
  * @hdr: a pointer to the GRH header being constructed
  * @grh: the global route address to send to
- * @hwords: the number of 32 bit words of header being sent
+ * @hwords: size of header after grh being sent in dwords
  * @nwords: the number of 32 bit words of data being sent
  *
  * Return the size of the header in 32 bit words.
@@ -661,7 +661,7 @@ u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
                cpu_to_be32((IB_GRH_VERSION << IB_GRH_VERSION_SHIFT) |
                            (grh->traffic_class << IB_GRH_TCLASS_SHIFT) |
                            (grh->flow_label << IB_GRH_FLOW_SHIFT));
-       hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2);
+       hdr->paylen = cpu_to_be16((hwords + nwords) << 2);
        /* next_hdr is defined by C8-7 in ch. 8.4.1 */
        hdr->next_hdr = IB_GRH_NEXT_HDR;
        hdr->hop_limit = grh->hop_limit;
index b708376b67dacf11daba51eba8dca9dc56be978f..2ba74fdd6f153dc33c0c153d09e2be1f73f8ed73 100644 (file)
 #include "verbs_txreq.h"
 #include "qp.h"
 
+/* We support only two types - 9B and 16B for now */
+static const hfi1_make_req hfi1_make_ud_req_tbl[2] = {
+       [HFI1_PKT_TYPE_9B] = &hfi1_make_ud_req_9B,
+       [HFI1_PKT_TYPE_16B] = &hfi1_make_ud_req_16B
+};
+
 /**
  * ud_loopback - handle send on loopback QPs
  * @sqp: the sending QP
@@ -67,6 +73,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 {
        struct hfi1_ibport *ibp = to_iport(sqp->ibqp.device, sqp->port_num);
        struct hfi1_pportdata *ppd;
+       struct hfi1_qp_priv *priv = sqp->priv;
        struct rvt_qp *qp;
        struct rdma_ah_attr *ah_attr;
        unsigned long flags;
@@ -102,7 +109,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 
        if (qp->ibqp.qp_num > 1) {
                u16 pkey;
-               u16 slid;
+               u32 slid;
                u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
 
                pkey = hfi1_get_pkey(ibp, sqp->s_pkey_index);
@@ -176,9 +183,33 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 
        if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
                struct ib_grh grh;
-               const struct ib_global_route *grd = rdma_ah_read_grh(ah_attr);
+               struct ib_global_route grd = *(rdma_ah_read_grh(ah_attr));
+
+               /*
+                * For loopback packets with extended LIDs, the
+                * sgid_index in the GRH is 0 and the dgid is
+                * OPA GID of the sender. While creating a response
+                * to the loopback packet, IB core creates the new
+                * sgid_index from the DGID and that will be the
+                * OPA_GID_INDEX. The new dgid is from the sgid
+                * index and that will be in the IB GID format.
+                *
+                * We now have a case where the sent packet had a
+                * different sgid_index and dgid compared to the
+                * one that was received in response.
+                *
+                * Fix this inconsistency.
+                */
+               if (priv->hdr_type == HFI1_PKT_TYPE_16B) {
+                       if (grd.sgid_index == 0)
+                               grd.sgid_index = OPA_GID_INDEX;
 
-               hfi1_make_grh(ibp, &grh, grd, 0, 0);
+                       if (ib_is_opa_gid(&grd.dgid))
+                               grd.dgid.global.interface_id =
+                               cpu_to_be64(ppd->guids[HFI1_PORT_GUID_INDEX]);
+               }
+
+               hfi1_make_grh(ibp, &grh, &grd, 0, 0);
                hfi1_copy_sge(&qp->r_sge, &grh,
                              sizeof(grh), true, false);
                wc.wc_flags |= IB_WC_GRH;
@@ -235,7 +266,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
                wc.pkey_index = 0;
        }
        wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
-                             ((1 << ppd->lmc) - 1));
+                                  ((1 << ppd->lmc) - 1));
        /* Check for loopback when the port lid is not set */
        if (wc.slid == 0 && sqp->ibqp.qp_type == IB_QPT_GSI)
                wc.slid = be16_to_cpu(IB_LID_PERMISSIVE);
@@ -252,6 +283,183 @@ drop:
        rcu_read_unlock();
 }
 
+static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe,
+                              struct ib_other_headers *ohdr,
+                              u16 *pkey, u32 extra_bytes, bool bypass)
+{
+       u32 bth0;
+       struct hfi1_ibport *ibp;
+
+       ibp = to_iport(qp->ibqp.device, qp->port_num);
+       if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
+               ohdr->u.ud.imm_data = wqe->wr.ex.imm_data;
+               bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
+       } else {
+               bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
+       }
+
+       if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+               bth0 |= IB_BTH_SOLICITED;
+       bth0 |= extra_bytes << 20;
+       if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI)
+               *pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index);
+       else
+               *pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
+       if (!bypass)
+               bth0 |= *pkey;
+       ohdr->bth[0] = cpu_to_be32(bth0);
+       ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn);
+       ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn));
+       /*
+        * Qkeys with the high order bit set mean use the
+        * qkey from the QP context instead of the WR (see 10.2.5).
+        */
+       ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ?
+                                        qp->qkey : wqe->ud_wr.remote_qkey);
+       ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
+}
+
+void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
+                        struct rvt_swqe *wqe)
+{
+       u32 nwords, extra_bytes;
+       u16 len, slid, dlid, pkey;
+       u16 lrh0 = 0;
+       u8 sc5;
+       struct hfi1_qp_priv *priv = qp->priv;
+       struct ib_other_headers *ohdr;
+       struct rdma_ah_attr *ah_attr;
+       struct hfi1_pportdata *ppd;
+       struct hfi1_ibport *ibp;
+       struct ib_grh *grh;
+
+       ibp = to_iport(qp->ibqp.device, qp->port_num);
+       ppd = ppd_from_ibp(ibp);
+       ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
+
+       extra_bytes = -wqe->length & 3;
+       nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC;
+       /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
+       qp->s_hdrwords = 7;
+       if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+               qp->s_hdrwords++;
+
+       if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
+               grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh;
+               qp->s_hdrwords += hfi1_make_grh(ibp, grh,
+                                               rdma_ah_read_grh(ah_attr),
+                                               qp->s_hdrwords - 2, nwords);
+               lrh0 = HFI1_LRH_GRH;
+               ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
+       } else {
+               lrh0 = HFI1_LRH_BTH;
+               ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
+       }
+
+       sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
+       lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
+       if (qp->ibqp.qp_type == IB_QPT_SMI) {
+               lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */
+               priv->s_sc = 0xf;
+       } else {
+               lrh0 |= (sc5 & 0xf) << 12;
+               priv->s_sc = sc5;
+       }
+
+       dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
+       if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
+               slid = be16_to_cpu(IB_LID_PERMISSIVE);
+       } else {
+               u16 lid = (u16)ppd->lid;
+
+               if (lid) {
+                       lid |= rdma_ah_get_path_bits(ah_attr) &
+                               ((1 << ppd->lmc) - 1);
+                       slid = lid;
+               } else {
+                       slid = be16_to_cpu(IB_LID_PERMISSIVE);
+               }
+       }
+       hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false);
+       len = qp->s_hdrwords + nwords;
+
+       /* Setup the packet */
+       ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_9B;
+       hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh,
+                        lrh0, len, dlid, slid);
+}
+
+void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
+                         struct rvt_swqe *wqe)
+{
+       struct hfi1_qp_priv *priv = qp->priv;
+       struct ib_other_headers *ohdr;
+       struct rdma_ah_attr *ah_attr;
+       struct hfi1_pportdata *ppd;
+       struct hfi1_ibport *ibp;
+       u32 dlid, slid, nwords, extra_bytes;
+       u16 len, pkey;
+       u8 l4, sc5;
+
+       ibp = to_iport(qp->ibqp.device, qp->port_num);
+       ppd = ppd_from_ibp(ibp);
+       ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
+       /* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */
+       qp->s_hdrwords = 9;
+       if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+               qp->s_hdrwords++;
+
+       /* SW provides space for CRC and LT for bypass packets. */
+       extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2),
+                                          wqe->length);
+       nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC;
+
+       if ((rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) &&
+           hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) {
+               struct ib_grh *grh;
+               struct ib_global_route *grd = rdma_ah_retrieve_grh(ah_attr);
+               /*
+                * Ensure OPA GIDs are transformed to IB gids
+                * before creating the GRH.
+                */
+               if (grd->sgid_index == OPA_GID_INDEX) {
+                       dd_dev_warn(ppd->dd, "Bad sgid_index. sgid_index: %d\n",
+                                   grd->sgid_index);
+                       grd->sgid_index = 0;
+               }
+               grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh;
+               qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd,
+                                       qp->s_hdrwords - 4, nwords);
+               ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
+               l4 = OPA_16B_L4_IB_GLOBAL;
+       } else {
+               ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth;
+               l4 = OPA_16B_L4_IB_LOCAL;
+       }
+
+       sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
+       if (qp->ibqp.qp_type == IB_QPT_SMI)
+               priv->s_sc = 0xf;
+       else
+               priv->s_sc = sc5;
+
+       dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 16B);
+       if (!ppd->lid)
+               slid = be32_to_cpu(OPA_LID_PERMISSIVE);
+       else
+               slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
+                          ((1 << ppd->lmc) - 1));
+
+       hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true);
+       /* Convert dwords to flits */
+       len = (qp->s_hdrwords + nwords) >> 1;
+
+       /* Setup the packet */
+       ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_16B;
+       hfi1_make_16b_hdr(&ps->s_txreq->phdr.hdr.opah,
+                         slid, dlid, len, pkey, 0, 0, l4, priv->s_sc);
+}
+
 /**
  * hfi1_make_ud_req - construct a UD request packet
  * @qp: the QP
@@ -263,18 +471,12 @@ drop:
 int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
        struct hfi1_qp_priv *priv = qp->priv;
-       struct ib_other_headers *ohdr;
        struct rdma_ah_attr *ah_attr;
        struct hfi1_pportdata *ppd;
        struct hfi1_ibport *ibp;
        struct rvt_swqe *wqe;
-       u32 nwords;
-       u32 extra_bytes;
-       u32 bth0;
-       u16 lrh0;
-       u16 lid;
        int next_cur;
-       u8 sc5;
+       u32 lid;
 
        ps->s_txreq = get_txreq(ps->dev, qp);
        if (IS_ERR(ps->s_txreq))
@@ -311,13 +513,14 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        ibp = to_iport(qp->ibqp.device, qp->port_num);
        ppd = ppd_from_ibp(ibp);
        ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
-       if (rdma_ah_get_dlid(ah_attr) < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
-           rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) {
+       priv->hdr_type = hfi1_get_hdr_type(ppd->lid, ah_attr);
+       if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) ||
+           (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) {
                lid = rdma_ah_get_dlid(ah_attr) & ~((1 << ppd->lmc) - 1);
                if (unlikely(!loopback &&
-                            (lid == ppd->lid ||
-                             (lid == be16_to_cpu(IB_LID_PERMISSIVE) &&
-                             qp->ibqp.qp_type == IB_QPT_GSI)))) {
+                            ((lid == ppd->lid) ||
+                             ((lid == be32_to_cpu(OPA_LID_PERMISSIVE)) &&
+                              (qp->ibqp.qp_type == IB_QPT_GSI))))) {
                        unsigned long tflags = ps->flags;
                        /*
                         * If DMAs are in progress, we can't generate
@@ -341,11 +544,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        }
 
        qp->s_cur = next_cur;
-       extra_bytes = -wqe->length & 3;
-       nwords = (wqe->length + extra_bytes) >> 2;
-
-       /* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
-       qp->s_hdrwords = 7;
        ps->s_txreq->s_cur_size = wqe->length;
        ps->s_txreq->ss = &qp->s_sge;
        qp->s_srate = rdma_ah_get_static_rate(ah_attr);
@@ -356,78 +554,12 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        qp->s_sge.num_sge = wqe->wr.num_sge;
        qp->s_sge.total_len = wqe->length;
 
-       if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
-               /* Header size in 32-bit words. */
-               qp->s_hdrwords +=
-                       hfi1_make_grh(ibp,
-                                     &ps->s_txreq->phdr.hdr.ibh.u.l.grh,
-                                     rdma_ah_read_grh(ah_attr),
-                                     qp->s_hdrwords, nwords);
-               lrh0 = HFI1_LRH_GRH;
-               ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
-               /*
-                * Don't worry about sending to locally attached multicast
-                * QPs.  It is unspecified by the spec. what happens.
-                */
-       } else {
-               /* Header size in 32-bit words. */
-               lrh0 = HFI1_LRH_BTH;
-               ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
-       }
-       if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
-               qp->s_hdrwords++;
-               ohdr->u.ud.imm_data = wqe->wr.ex.imm_data;
-               bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
-       } else {
-               bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
-       }
-       sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
-       lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
-       if (qp->ibqp.qp_type == IB_QPT_SMI) {
-               lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */
-               priv->s_sc = 0xf;
-       } else {
-               lrh0 |= (sc5 & 0xf) << 12;
-               priv->s_sc = sc5;
-       }
+       /* Make the appropriate header */
+       hfi1_make_ud_req_tbl[priv->hdr_type](qp, ps, qp->s_wqe);
        priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
        ps->s_txreq->sde = priv->s_sde;
        priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
        ps->s_txreq->psc = priv->s_sendcontext;
-       ps->s_txreq->phdr.hdr.ibh.lrh[0] = cpu_to_be16(lrh0);
-       ps->s_txreq->phdr.hdr.ibh.lrh[1] =
-               cpu_to_be16(rdma_ah_get_dlid(ah_attr));
-       ps->s_txreq->phdr.hdr.ibh.lrh[2] =
-               cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
-       if (rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) {
-               ps->s_txreq->phdr.hdr.ibh.lrh[3] = IB_LID_PERMISSIVE;
-       } else {
-               lid = ppd->lid;
-               if (lid) {
-                       lid |= rdma_ah_get_path_bits(ah_attr) &
-                               ((1 << ppd->lmc) - 1);
-                       ps->s_txreq->phdr.hdr.ibh.lrh[3] = cpu_to_be16(lid);
-               } else {
-                       ps->s_txreq->phdr.hdr.ibh.lrh[3] = IB_LID_PERMISSIVE;
-               }
-       }
-       if (wqe->wr.send_flags & IB_SEND_SOLICITED)
-               bth0 |= IB_BTH_SOLICITED;
-       bth0 |= extra_bytes << 20;
-       if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI)
-               bth0 |= hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index);
-       else
-               bth0 |= hfi1_get_pkey(ibp, qp->s_pkey_index);
-       ohdr->bth[0] = cpu_to_be32(bth0);
-       ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn);
-       ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn));
-       /*
-        * Qkeys with the high order bit set mean use the
-        * qkey from the QP context instead of the WR (see 10.2.5).
-        */
-       ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ?
-                                        qp->qkey : wqe->ud_wr.remote_qkey);
-       ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
        /* disarm any ahg */
        priv->s_ahg->ahgcount = 0;
        priv->s_ahg->ahgidx = 0;
@@ -497,6 +629,64 @@ int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey)
        return -1;
 }
 
+void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
+                   u32 remote_qpn, u32 pkey, u32 slid, u32 dlid,
+                   u8 sc5, const struct ib_grh *old_grh)
+{
+       u64 pbc, pbc_flags = 0;
+       u32 bth0, plen, vl, hwords = 7;
+       u16 len;
+       u8 l4;
+       struct hfi1_16b_header hdr;
+       struct ib_other_headers *ohdr;
+       struct pio_buf *pbuf;
+       struct send_context *ctxt = qp_to_send_context(qp, sc5);
+       struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+       u32 nwords;
+
+       /* Populate length */
+       nwords = ((hfi1_get_16b_padding(hwords << 2, 0) +
+                  SIZE_OF_LT) >> 2) + SIZE_OF_CRC;
+       if (old_grh) {
+               struct ib_grh *grh = &hdr.u.l.grh;
+
+               grh->version_tclass_flow = old_grh->version_tclass_flow;
+               grh->paylen = cpu_to_be16((hwords - 4 + nwords) << 2);
+               grh->hop_limit = 0xff;
+               grh->sgid = old_grh->dgid;
+               grh->dgid = old_grh->sgid;
+               ohdr = &hdr.u.l.oth;
+               l4 = OPA_16B_L4_IB_GLOBAL;
+               hwords += sizeof(struct ib_grh) / sizeof(u32);
+       } else {
+               ohdr = &hdr.u.oth;
+               l4 = OPA_16B_L4_IB_LOCAL;
+       }
+
+       /* BIT 16 to 19 is TVER. Bit 20 to 22 is pad cnt */
+       bth0 = (IB_OPCODE_CNP << 24) | (1 << 16) |
+              (hfi1_get_16b_padding(hwords << 2, 0) << 20);
+       ohdr->bth[0] = cpu_to_be32(bth0);
+
+       ohdr->bth[1] = cpu_to_be32(remote_qpn);
+       ohdr->bth[2] = 0; /* PSN 0 */
+
+       /* Convert dwords to flits */
+       len = (hwords + nwords) >> 1;
+       hfi1_make_16b_hdr(&hdr, slid, dlid, len, pkey, 1, 0, l4, sc5);
+
+       plen = 2 /* PBC */ + hwords + nwords;
+       pbc_flags |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC;
+       vl = sc_to_vlt(ppd->dd, sc5);
+       pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen);
+       if (ctxt) {
+               pbuf = sc_buffer_alloc(ctxt, plen, NULL, NULL);
+               if (pbuf)
+                       ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc,
+                                                &hdr, hwords);
+       }
+}
+
 void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
                u32 pkey, u32 slid, u32 dlid, u8 sc5,
                const struct ib_grh *old_grh)
@@ -535,11 +725,7 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
        ohdr->bth[1] = cpu_to_be32(remote_qpn | (1 << IB_BECN_SHIFT));
        ohdr->bth[2] = 0; /* PSN 0 */
 
-       hdr.lrh[0] = cpu_to_be16(lrh0);
-       hdr.lrh[1] = cpu_to_be16(dlid);
-       hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
-       hdr.lrh[3] = cpu_to_be16(slid);
-
+       hfi1_make_ib_hdr(&hdr, lrh0, hwords + SIZE_OF_CRC, dlid, slid);
        plen = 2 /* PBC */ + hwords;
        pbc_flags |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
        vl = sc_to_vlt(ppd->dd, sc5);
@@ -672,18 +858,33 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
        void *data = packet->payload;
        u32 tlen = packet->tlen;
        struct rvt_qp *qp = packet->qp;
-       u8 sc5 = hfi1_9B_get_sc5(hdr, packet->rhf);
+       u8 sc5 = packet->sc;
        u8 sl_from_sc;
-       u8 extra_bytes = packet->pad;
        u8 opcode = packet->opcode;
        u8 sl = packet->sl;
        u32 dlid = packet->dlid;
        u32 slid = packet->slid;
+       u8 extra_bytes;
+       bool dlid_is_permissive;
+       bool slid_is_permissive;
 
+       extra_bytes = packet->pad + packet->extra_byte + (SIZE_OF_CRC << 2);
        qkey = ib_get_qkey(ohdr);
        src_qp = ib_get_sqpn(ohdr);
-       pkey = ib_bth_get_pkey(ohdr);
-       extra_bytes += (SIZE_OF_CRC << 2);
+
+       if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+               u32 permissive_lid =
+                       opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B);
+
+               pkey = hfi1_16B_get_pkey(packet->hdr);
+               dlid_is_permissive = (dlid == permissive_lid);
+               slid_is_permissive = (slid == permissive_lid);
+       } else {
+               hdr = packet->hdr;
+               pkey = ib_bth_get_pkey(ohdr);
+               dlid_is_permissive = (dlid == be16_to_cpu(IB_LID_PERMISSIVE));
+               slid_is_permissive = (slid == be16_to_cpu(IB_LID_PERMISSIVE));
+       }
        sl_from_sc = ibp->sc_to_sl[sc5];
 
        process_ecn(qp, packet, (opcode != IB_OPCODE_CNP));
@@ -701,8 +902,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
         * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
         */
        if (qp->ibqp.qp_num) {
-               if (unlikely(hdr->lrh[1] == IB_LID_PERMISSIVE ||
-                            hdr->lrh[3] == IB_LID_PERMISSIVE))
+               if (unlikely(dlid_is_permissive || slid_is_permissive))
                        goto drop;
                if (qp->ibqp.qp_num > 1) {
                        if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) {
@@ -740,8 +940,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 
                if (tlen > 2048)
                        goto drop;
-               if ((hdr->lrh[1] == IB_LID_PERMISSIVE ||
-                    hdr->lrh[3] == IB_LID_PERMISSIVE) &&
+               if ((dlid_is_permissive || slid_is_permissive) &&
                    smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
                        goto drop;
 
@@ -794,7 +993,18 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
                goto drop;
        }
        if (packet->grh) {
-               hfi1_copy_sge(&qp->r_sge, &hdr->u.l.grh,
+               hfi1_copy_sge(&qp->r_sge, packet->grh,
+                             sizeof(struct ib_grh), true, false);
+               wc.wc_flags |= IB_WC_GRH;
+       } else if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+               struct ib_grh grh;
+               /*
+                * Assuming we only created 16B on the send side
+                * if we want to use large LIDs, since GRH was stripped
+                * out when creating 16B, add back the GRH here.
+                */
+               hfi1_make_ext_grh(packet, &grh, slid, dlid);
+               hfi1_copy_sge(&qp->r_sge, &grh,
                              sizeof(struct ib_grh), true, false);
                wc.wc_flags |= IB_WC_GRH;
        } else {
@@ -827,14 +1037,15 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
        } else {
                wc.pkey_index = 0;
        }
-
+       if (slid_is_permissive)
+               slid = be32_to_cpu(OPA_LID_PERMISSIVE);
        wc.slid = slid;
        wc.sl = sl_from_sc;
 
        /*
         * Save the LMC lower bits if the destination LID is a unicast LID.
         */
-       wc.dlid_path_bits = dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE) ? 0 :
+       wc.dlid_path_bits = hfi1_check_mcast(dlid) ? 0 :
                dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1);
        wc.port_num = qp->port_num;
        /* Signal completion event if the solicited bit is set. */
index d3dd0c01b8f6dba28cf636a16ae0d6bba2052f62..4928ee4f92c12b0e158a85de8e51bbf68da71847 100644 (file)
@@ -259,7 +259,7 @@ static inline int hfi1_send_ok(struct rvt_qp *qp)
  * This must be called with s_lock held.
  */
 void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,
-                  u32 qp1, u32 qp2, u16 lid1, u16 lid2);
+                  u32 qp1, u32 qp2, u32 lid1, u32 lid2);
 void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num);
 void hfi1_sys_guid_chg(struct hfi1_ibport *ibp);
 void hfi1_node_desc_chg(struct hfi1_ibport *ibp);
index 9ae126fb8648f7c9414fc60cc9eb456909720736..e6e90f18e6d5b6887fccb2b78d8f0c920cdb1edb 100644 (file)
@@ -54,6 +54,7 @@
 #define OPA_MAKE_ID(x)          (cpu_to_be64(OPA_SPECIAL_OUI << 40 | (x)))
 #define OPA_TO_IB_UCAST_LID(x) (((x) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) \
                                ? 0 : x)
+#define OPA_GID_INDEX          0x1
 /**
  * 0xF8 - 4 bits of multicast range and 1 bit for collective range
  * Example: For 24 bit LID space,