[IB] mthca: Better limit checking and reporting

Check the sizes of CQs, QPs and SRQs when creating objects, and fail
instead of creating too-big queues.  Also return real limits instead
of just plausible-sounding values from mthca_query_device().

Signed-off-by: Jack Morgenstein <jackm@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c
index f6a8ac026557db5639d446bf1188cf9d73b22739..1bd7dc8f778c47c0372946cc547e90e391b352e5 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_cmd.c
+++ b/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -933,9 +933,9 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev *dev,
                goto out;
 
        MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET);
-       dev_lim->max_srq_sz = 1 << field;
+       dev_lim->max_srq_sz = (1 << field) - 1;
        MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_SZ_OFFSET);
-       dev_lim->max_qp_sz = 1 << field;
+       dev_lim->max_qp_sz = (1 << field) - 1;
        MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_QP_OFFSET);
        dev_lim->reserved_qps = 1 << (field & 0xf);
        MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_OFFSET);
@@ -1045,6 +1045,8 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev *dev,
                  dev_lim->max_pds, dev_lim->reserved_pds, dev_lim->reserved_uars);
        mthca_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n",
                  dev_lim->max_pds, dev_lim->reserved_mgms);
+       mthca_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n",
+                 dev_lim->max_cq_sz, dev_lim->max_qp_sz, dev_lim->max_srq_sz);
 
        mthca_dbg(dev, "Flags: %08x\n", dev_lim->flags);
 

diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h
index 6e18c128af4231eae676801face25a6ddb90ac51..f106bac0f92555b38bb3043d9fcb773a124a2ae2 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_dev.h
+++ b/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -83,6 +83,8 @@ enum {
        /* Arbel FW gives us these, but we need them for Tavor */
        MTHCA_MPT_ENTRY_SIZE  =  0x40,
        MTHCA_MTT_SEG_SIZE    =  0x40,
+
+       MTHCA_QP_PER_MGM      = 4 * (MTHCA_MGM_ENTRY_SIZE / 16 - 2)
 };
 
 enum {
@@ -128,12 +130,16 @@ struct mthca_limits {
        int      num_uars;
        int      max_sg;
        int      num_qps;
+       int      max_wqes;
+       int      max_qp_init_rdma;
        int      reserved_qps;
        int      num_srqs;
+       int      max_srq_wqes;
        int      reserved_srqs;
        int      num_eecs;
        int      reserved_eecs;
        int      num_cqs;
+       int      max_cqes;
        int      reserved_cqs;
        int      num_eqs;
        int      reserved_eqs;

diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c
index 576e7fcd053cd3043bbfe460d8b54adc82074c48..7c35abec81ea69720f83c5d9f5b26f1d95ede20e 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_main.c
+++ b/drivers/infiniband/hw/mthca/mthca_main.c
@@ -162,9 +162,18 @@ static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim
        mdev->limits.pkey_table_len     = dev_lim->max_pkeys;
        mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
        mdev->limits.max_sg             = dev_lim->max_sg;
+       mdev->limits.max_wqes           = dev_lim->max_qp_sz;
+       mdev->limits.max_qp_init_rdma   = dev_lim->max_requester_per_qp;
        mdev->limits.reserved_qps       = dev_lim->reserved_qps;
+       mdev->limits.max_srq_wqes       = dev_lim->max_srq_sz;
        mdev->limits.reserved_srqs      = dev_lim->reserved_srqs;
        mdev->limits.reserved_eecs      = dev_lim->reserved_eecs;
+       /*
+        * Subtract 1 from the limit because we need to allocate a
+        * spare CQE so the HCA HW can tell the difference between an
+        * empty CQ and a full CQ.
+        */
+       mdev->limits.max_cqes           = dev_lim->max_cq_sz - 1;
        mdev->limits.reserved_cqs       = dev_lim->reserved_cqs;
        mdev->limits.reserved_eqs       = dev_lim->reserved_eqs;
        mdev->limits.reserved_mtts      = dev_lim->reserved_mtts;

diff --git a/drivers/infiniband/hw/mthca/mthca_mcg.c b/drivers/infiniband/hw/mthca/mthca_mcg.c
index a2707605f4c8ba3319644452f2deb48004e001e9..9a0612a9d3048b94edbf545ccdb362450bf0bf9e 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_mcg.c
+++ b/drivers/infiniband/hw/mthca/mthca_mcg.c
@@ -37,10 +37,6 @@
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
 
-enum {
-       MTHCA_QP_PER_MGM = 4 * (MTHCA_MGM_ENTRY_SIZE / 16 - 2)
-};
-
 struct mthca_mgm {
        __be32 next_gid_index;
        u32    reserved[3];

diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 53b29a0841bf64d443ae816302e55fd9b32ff35c..46864d18827d12556974f3630e971b9b753596e2 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -90,17 +90,26 @@ static int mthca_query_device(struct ib_device *ibdev,
 
        props->max_mr_size         = ~0ull;
        props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
-       props->max_qp_wr           = 0xffff;
+       props->max_qp_wr           = mdev->limits.max_wqes;
        props->max_sge             = mdev->limits.max_sg;
        props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
-       props->max_cqe             = 0xffff;
+       props->max_cqe             = mdev->limits.max_cqes;
        props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
        props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
        props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
-       props->max_qp_init_rd_atom = 1 << mdev->qp_table.rdb_shift;
+       props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
+       props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
+       props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
+       props->max_srq_wr          = mdev->limits.max_srq_wqes;
+       props->max_srq_sge         = mdev->limits.max_sg;
        props->local_ca_ack_delay  = mdev->limits.local_ca_ack_delay;
        props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 
                                        IB_ATOMIC_HCA : IB_ATOMIC_NONE;
+       props->max_pkeys           = mdev->limits.pkey_table_len;
+       props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
+       props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
+       props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 
+                                          props->max_mcast_grp;
 
        err = 0;
  out:
@@ -640,6 +649,9 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
        int nent;
        int err;
 
+       if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
+               return ERR_PTR(-EINVAL);
+
        if (context) {
                if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
                        return ERR_PTR(-EFAULT);

diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 5fa00669f9b8c43d59e9dd70fe1469d707ceb600..2ee0a2b0fd4bea99c603ad3fb9871e0d090e787f 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -1112,8 +1112,10 @@ static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
                             struct mthca_qp *qp)
 {
        /* Sanity check QP size before proceeding */
-       if (cap->max_send_wr  > 65536 || cap->max_recv_wr  > 65536 ||
-           cap->max_send_sge > 64    || cap->max_recv_sge > 64)
+       if (cap->max_send_wr  > dev->limits.max_wqes ||
+           cap->max_recv_wr  > dev->limits.max_wqes ||
+           cap->max_send_sge > dev->limits.max_sg   ||
+           cap->max_recv_sge > dev->limits.max_sg)
                return -EINVAL;
 
        if (mthca_is_memfree(dev)) {

diff --git a/drivers/infiniband/hw/mthca/mthca_srq.c b/drivers/infiniband/hw/mthca/mthca_srq.c
index e464321a7aa7a0512676afd7390d7097a6d3e3a7..64f70aa1b3c09a704cc8885ca622445ed099f1c1 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_srq.c
+++ b/drivers/infiniband/hw/mthca/mthca_srq.c
@@ -186,7 +186,8 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
        int err;
 
        /* Sanity check SRQ size before proceeding */
-       if (attr->max_wr > 16 << 20 || attr->max_sge > 64)
+       if (attr->max_wr  > dev->limits.max_srq_wqes ||
+           attr->max_sge > dev->limits.max_sg)
                return -EINVAL;
 
        srq->max      = attr->max_wr;