RDS/IB: add _to_node() macros for numa and use {k,v}malloc_node()

Allocate send/recv rings in memory that is node-local to the HCA.
This significantly helps performance.

Signed-off-by: Andy Grover <andy.grover@oracle.com>

diff --git a/net/rds/ib.c b/net/rds/ib.c
index 7a2131d37dfbd19614ad116ab4907cb1cdc0aaa3..7d289d7985fed171f14fd36265febafaf71035d0 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -77,7 +77,7 @@ void rds_ib_add_one(struct ib_device *device)
                goto free_attr;
        }
 
-       rds_ibdev = kmalloc(sizeof *rds_ibdev, GFP_KERNEL);
+       rds_ibdev = kmalloc_node(sizeof *rds_ibdev, GFP_KERNEL, ibdev_to_node(device));
        if (!rds_ibdev)
                goto free_attr;
 

diff --git a/net/rds/ib.h b/net/rds/ib.h
index c506604325d515aa9b58f0cefbc336ccf1e21ba7..4bc3e2fba25a2c7dee9b478155c41d3ea2baa646 100644 (file)
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -3,6 +3,8 @@
 
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
 #include "rds.h"
 #include "rdma_transport.h"
 
@@ -167,6 +169,10 @@ struct rds_ib_device {
        spinlock_t              spinlock;       /* protect the above */
 };
 
+#define pcidev_to_node(pcidev) pcibus_to_node(pcidev->bus)
+#define ibdev_to_node(ibdev) pcidev_to_node(to_pci_dev(ibdev->dma_device))
+#define rdsibdev_to_node(rdsibdev) ibdev_to_node(rdsibdev->dev)
+
 /* bits for i_ack_flags */
 #define IB_ACK_IN_FLIGHT       0
 #define IB_ACK_REQUESTED       1

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 75eda9c82135fea944b394d27090ae88ca92face..b5d0b60a26bc965fac847aa572865a8a1215b6cc 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -347,7 +347,8 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
                goto out;
        }
 
-       ic->i_sends = vmalloc(ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work));
+       ic->i_sends = vmalloc_node(ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work),
+                                  ibdev_to_node(dev));
        if (!ic->i_sends) {
                ret = -ENOMEM;
                rdsdebug("send allocation failed\n");
@@ -355,7 +356,8 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
        }
        memset(ic->i_sends, 0, ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work));
 
-       ic->i_recvs = vmalloc(ic->i_recv_ring.w_nr * sizeof(struct rds_ib_recv_work));
+       ic->i_recvs = vmalloc_node(ic->i_recv_ring.w_nr * sizeof(struct rds_ib_recv_work),
+                                  ibdev_to_node(dev));
        if (!ic->i_recvs) {
                ret = -ENOMEM;
                rdsdebug("recv allocation failed\n");

diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c
index 7315fffd3bc8e792441093548c56a8683c1d8605..cc341cd70c87fd7de8bb58c96ee0d701a2366169 100644 (file)
--- a/net/rds/ib_rdma.c
+++ b/net/rds/ib_rdma.c
@@ -297,7 +297,7 @@ static struct rds_ib_mr *rds_ib_alloc_fmr(struct rds_ib_device *rds_ibdev)
                rds_ib_flush_mr_pool(pool, 0);
        }
 
-       ibmr = kzalloc(sizeof(*ibmr), GFP_KERNEL);
+       ibmr = kzalloc_node(sizeof(*ibmr), GFP_KERNEL, rdsibdev_to_node(rds_ibdev));
        if (!ibmr) {
                err = -ENOMEM;
                goto out_no_cigar;
@@ -376,7 +376,8 @@ static int rds_ib_map_fmr(struct rds_ib_device *rds_ibdev, struct rds_ib_mr *ibm
        if (page_cnt > fmr_message_size)
                return -EINVAL;
 
-       dma_pages = kmalloc(sizeof(u64) * page_cnt, GFP_ATOMIC);
+       dma_pages = kmalloc_node(sizeof(u64) * page_cnt, GFP_ATOMIC,
+                                rdsibdev_to_node(rds_ibdev));
        if (!dma_pages)
                return -ENOMEM;