2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
35 #include <linux/slab.h>
36 #include <linux/vmalloc.h>
42 * Set the selected protocol version
44 static void rds_ib_set_protocol(struct rds_connection
*conn
, unsigned int version
)
46 conn
->c_version
= version
;
52 static void rds_ib_set_flow_control(struct rds_connection
*conn
, u32 credits
)
54 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
56 if (rds_ib_sysctl_flow_control
&& credits
!= 0) {
57 /* We're doing flow control */
59 rds_ib_send_add_credits(conn
, credits
);
66 * Tune RNR behavior. Without flow control, we use a rather
67 * low timeout, but not the absolute minimum - this should
70 * We already set the RNR retry count to 7 (which is the
71 * smallest infinite number :-) above.
72 * If flow control is off, we want to change this back to 0
73 * so that we learn quickly when our credit accounting is
76 * Caller passes in a qp_attr pointer - don't waste stack spacv
77 * by allocation this twice.
80 rds_ib_tune_rnr(struct rds_ib_connection
*ic
, struct ib_qp_attr
*attr
)
84 attr
->min_rnr_timer
= IB_RNR_TIMER_000_32
;
85 ret
= ib_modify_qp(ic
->i_cm_id
->qp
, attr
, IB_QP_MIN_RNR_TIMER
);
87 printk(KERN_NOTICE
"ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret
);
91 * Connection established.
92 * We get here for both outgoing and incoming connection.
94 void rds_ib_cm_connect_complete(struct rds_connection
*conn
, struct rdma_cm_event
*event
)
96 const struct rds_ib_connect_private
*dp
= NULL
;
97 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
98 struct rds_ib_device
*rds_ibdev
;
99 struct ib_qp_attr qp_attr
;
102 if (event
->param
.conn
.private_data_len
>= sizeof(*dp
)) {
103 dp
= event
->param
.conn
.private_data
;
105 /* make sure it isn't empty data */
106 if (dp
->dp_protocol_major
) {
107 rds_ib_set_protocol(conn
,
108 RDS_PROTOCOL(dp
->dp_protocol_major
,
109 dp
->dp_protocol_minor
));
110 rds_ib_set_flow_control(conn
, be32_to_cpu(dp
->dp_credit
));
114 printk(KERN_NOTICE
"RDS/IB: connected to %pI4 version %u.%u%s\n",
116 RDS_PROTOCOL_MAJOR(conn
->c_version
),
117 RDS_PROTOCOL_MINOR(conn
->c_version
),
118 ic
->i_flowctl
? ", flow control" : "");
121 * Init rings and fill recv. this needs to wait until protocol negotiation
122 * is complete, since ring layout is different from 3.0 to 3.1.
124 rds_ib_send_init_ring(ic
);
125 rds_ib_recv_init_ring(ic
);
126 /* Post receive buffers - as a side effect, this will update
127 * the posted credit count. */
128 rds_ib_recv_refill(conn
, GFP_KERNEL
, GFP_HIGHUSER
, 1);
130 /* Tune RNR behavior */
131 rds_ib_tune_rnr(ic
, &qp_attr
);
133 qp_attr
.qp_state
= IB_QPS_RTS
;
134 err
= ib_modify_qp(ic
->i_cm_id
->qp
, &qp_attr
, IB_QP_STATE
);
136 printk(KERN_NOTICE
"ib_modify_qp(IB_QP_STATE, RTS): err=%d\n", err
);
138 /* update ib_device with this local ipaddr & conn */
139 rds_ibdev
= ib_get_client_data(ic
->i_cm_id
->device
, &rds_ib_client
);
140 err
= rds_ib_update_ipaddr(rds_ibdev
, conn
->c_laddr
);
142 printk(KERN_ERR
"rds_ib_update_ipaddr failed (%d)\n", err
);
143 rds_ib_add_conn(rds_ibdev
, conn
);
145 /* If the peer gave us the last packet it saw, process this as if
146 * we had received a regular ACK. */
147 if (dp
&& dp
->dp_ack_seq
)
148 rds_send_drop_acked(conn
, be64_to_cpu(dp
->dp_ack_seq
), NULL
);
150 rds_connect_complete(conn
);
153 static void rds_ib_cm_fill_conn_param(struct rds_connection
*conn
,
154 struct rdma_conn_param
*conn_param
,
155 struct rds_ib_connect_private
*dp
,
156 u32 protocol_version
)
158 memset(conn_param
, 0, sizeof(struct rdma_conn_param
));
159 /* XXX tune these? */
160 conn_param
->responder_resources
= 1;
161 conn_param
->initiator_depth
= 1;
162 conn_param
->retry_count
= min_t(unsigned int, rds_ib_retry_count
, 7);
163 conn_param
->rnr_retry_count
= 7;
166 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
168 memset(dp
, 0, sizeof(*dp
));
169 dp
->dp_saddr
= conn
->c_laddr
;
170 dp
->dp_daddr
= conn
->c_faddr
;
171 dp
->dp_protocol_major
= RDS_PROTOCOL_MAJOR(protocol_version
);
172 dp
->dp_protocol_minor
= RDS_PROTOCOL_MINOR(protocol_version
);
173 dp
->dp_protocol_minor_mask
= cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS
);
174 dp
->dp_ack_seq
= rds_ib_piggyb_ack(ic
);
176 /* Advertise flow control */
178 unsigned int credits
;
180 credits
= IB_GET_POST_CREDITS(atomic_read(&ic
->i_credits
));
181 dp
->dp_credit
= cpu_to_be32(credits
);
182 atomic_sub(IB_SET_POST_CREDITS(credits
), &ic
->i_credits
);
185 conn_param
->private_data
= dp
;
186 conn_param
->private_data_len
= sizeof(*dp
);
190 static void rds_ib_cq_event_handler(struct ib_event
*event
, void *data
)
192 rdsdebug("event %u data %p\n", event
->event
, data
);
195 static void rds_ib_qp_event_handler(struct ib_event
*event
, void *data
)
197 struct rds_connection
*conn
= data
;
198 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
200 rdsdebug("conn %p ic %p event %u\n", conn
, ic
, event
->event
);
202 switch (event
->event
) {
203 case IB_EVENT_COMM_EST
:
204 rdma_notify(ic
->i_cm_id
, IB_EVENT_COMM_EST
);
207 rds_ib_conn_error(conn
, "RDS/IB: Fatal QP Event %u "
208 "- connection %pI4->%pI4, reconnecting\n",
209 event
->event
, &conn
->c_laddr
, &conn
->c_faddr
);
215 * This needs to be very careful to not leave IS_ERR pointers around for
216 * cleanup to trip over.
218 static int rds_ib_setup_qp(struct rds_connection
*conn
)
220 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
221 struct ib_device
*dev
= ic
->i_cm_id
->device
;
222 struct ib_qp_init_attr attr
;
223 struct rds_ib_device
*rds_ibdev
;
226 /* rds_ib_add_one creates a rds_ib_device object per IB device,
227 * and allocates a protection domain, memory range and FMR pool
228 * for each. If that fails for any reason, it will not register
229 * the rds_ibdev at all.
231 rds_ibdev
= ib_get_client_data(dev
, &rds_ib_client
);
232 if (rds_ibdev
== NULL
) {
233 if (printk_ratelimit())
234 printk(KERN_NOTICE
"RDS/IB: No client_data for device %s\n",
239 if (rds_ibdev
->max_wrs
< ic
->i_send_ring
.w_nr
+ 1)
240 rds_ib_ring_resize(&ic
->i_send_ring
, rds_ibdev
->max_wrs
- 1);
241 if (rds_ibdev
->max_wrs
< ic
->i_recv_ring
.w_nr
+ 1)
242 rds_ib_ring_resize(&ic
->i_recv_ring
, rds_ibdev
->max_wrs
- 1);
244 /* Protection domain and memory range */
245 ic
->i_pd
= rds_ibdev
->pd
;
246 ic
->i_mr
= rds_ibdev
->mr
;
248 ic
->i_send_cq
= ib_create_cq(dev
, rds_ib_send_cq_comp_handler
,
249 rds_ib_cq_event_handler
, conn
,
250 ic
->i_send_ring
.w_nr
+ 1, 0);
251 if (IS_ERR(ic
->i_send_cq
)) {
252 ret
= PTR_ERR(ic
->i_send_cq
);
253 ic
->i_send_cq
= NULL
;
254 rdsdebug("ib_create_cq send failed: %d\n", ret
);
258 ic
->i_recv_cq
= ib_create_cq(dev
, rds_ib_recv_cq_comp_handler
,
259 rds_ib_cq_event_handler
, conn
,
260 ic
->i_recv_ring
.w_nr
, 0);
261 if (IS_ERR(ic
->i_recv_cq
)) {
262 ret
= PTR_ERR(ic
->i_recv_cq
);
263 ic
->i_recv_cq
= NULL
;
264 rdsdebug("ib_create_cq recv failed: %d\n", ret
);
268 ret
= ib_req_notify_cq(ic
->i_send_cq
, IB_CQ_NEXT_COMP
);
270 rdsdebug("ib_req_notify_cq send failed: %d\n", ret
);
274 ret
= ib_req_notify_cq(ic
->i_recv_cq
, IB_CQ_SOLICITED
);
276 rdsdebug("ib_req_notify_cq recv failed: %d\n", ret
);
280 /* XXX negotiate max send/recv with remote? */
281 memset(&attr
, 0, sizeof(attr
));
282 attr
.event_handler
= rds_ib_qp_event_handler
;
283 attr
.qp_context
= conn
;
284 /* + 1 to allow for the single ack message */
285 attr
.cap
.max_send_wr
= ic
->i_send_ring
.w_nr
+ 1;
286 attr
.cap
.max_recv_wr
= ic
->i_recv_ring
.w_nr
+ 1;
287 attr
.cap
.max_send_sge
= rds_ibdev
->max_sge
;
288 attr
.cap
.max_recv_sge
= RDS_IB_RECV_SGE
;
289 attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
290 attr
.qp_type
= IB_QPT_RC
;
291 attr
.send_cq
= ic
->i_send_cq
;
292 attr
.recv_cq
= ic
->i_recv_cq
;
295 * XXX this can fail if max_*_wr is too large? Are we supposed
296 * to back off until we get a value that the hardware can support?
298 ret
= rdma_create_qp(ic
->i_cm_id
, ic
->i_pd
, &attr
);
300 rdsdebug("rdma_create_qp failed: %d\n", ret
);
304 ic
->i_send_hdrs
= ib_dma_alloc_coherent(dev
,
305 ic
->i_send_ring
.w_nr
*
306 sizeof(struct rds_header
),
307 &ic
->i_send_hdrs_dma
, GFP_KERNEL
);
308 if (ic
->i_send_hdrs
== NULL
) {
310 rdsdebug("ib_dma_alloc_coherent send failed\n");
314 ic
->i_recv_hdrs
= ib_dma_alloc_coherent(dev
,
315 ic
->i_recv_ring
.w_nr
*
316 sizeof(struct rds_header
),
317 &ic
->i_recv_hdrs_dma
, GFP_KERNEL
);
318 if (ic
->i_recv_hdrs
== NULL
) {
320 rdsdebug("ib_dma_alloc_coherent recv failed\n");
324 ic
->i_ack
= ib_dma_alloc_coherent(dev
, sizeof(struct rds_header
),
325 &ic
->i_ack_dma
, GFP_KERNEL
);
326 if (ic
->i_ack
== NULL
) {
328 rdsdebug("ib_dma_alloc_coherent ack failed\n");
332 ic
->i_sends
= vmalloc(ic
->i_send_ring
.w_nr
* sizeof(struct rds_ib_send_work
));
333 if (ic
->i_sends
== NULL
) {
335 rdsdebug("send allocation failed\n");
338 memset(ic
->i_sends
, 0, ic
->i_send_ring
.w_nr
* sizeof(struct rds_ib_send_work
));
340 ic
->i_recvs
= vmalloc(ic
->i_recv_ring
.w_nr
* sizeof(struct rds_ib_recv_work
));
341 if (ic
->i_recvs
== NULL
) {
343 rdsdebug("recv allocation failed\n");
346 memset(ic
->i_recvs
, 0, ic
->i_recv_ring
.w_nr
* sizeof(struct rds_ib_recv_work
));
348 rds_ib_recv_init_ack(ic
);
350 rdsdebug("conn %p pd %p mr %p cq %p %p\n", conn
, ic
->i_pd
, ic
->i_mr
,
351 ic
->i_send_cq
, ic
->i_recv_cq
);
357 static u32
rds_ib_protocol_compatible(struct rdma_cm_event
*event
)
359 const struct rds_ib_connect_private
*dp
= event
->param
.conn
.private_data
;
364 * rdma_cm private data is odd - when there is any private data in the
365 * request, we will be given a pretty large buffer without telling us the
366 * original size. The only way to tell the difference is by looking at
367 * the contents, which are initialized to zero.
368 * If the protocol version fields aren't set, this is a connection attempt
369 * from an older version. This could could be 3.0 or 2.0 - we can't tell.
370 * We really should have changed this for OFED 1.3 :-(
373 /* Be paranoid. RDS always has privdata */
374 if (!event
->param
.conn
.private_data_len
) {
375 printk(KERN_NOTICE
"RDS incoming connection has no private data, "
380 /* Even if len is crap *now* I still want to check it. -ASG */
381 if (event
->param
.conn
.private_data_len
< sizeof (*dp
) ||
382 dp
->dp_protocol_major
== 0)
383 return RDS_PROTOCOL_3_0
;
385 common
= be16_to_cpu(dp
->dp_protocol_minor_mask
) & RDS_IB_SUPPORTED_PROTOCOLS
;
386 if (dp
->dp_protocol_major
== 3 && common
) {
387 version
= RDS_PROTOCOL_3_0
;
388 while ((common
>>= 1) != 0)
390 } else if (printk_ratelimit()) {
391 printk(KERN_NOTICE
"RDS: Connection from %pI4 using "
392 "incompatible protocol version %u.%u\n",
394 dp
->dp_protocol_major
,
395 dp
->dp_protocol_minor
);
400 int rds_ib_cm_handle_connect(struct rdma_cm_id
*cm_id
,
401 struct rdma_cm_event
*event
)
403 __be64 lguid
= cm_id
->route
.path_rec
->sgid
.global
.interface_id
;
404 __be64 fguid
= cm_id
->route
.path_rec
->dgid
.global
.interface_id
;
405 const struct rds_ib_connect_private
*dp
= event
->param
.conn
.private_data
;
406 struct rds_ib_connect_private dp_rep
;
407 struct rds_connection
*conn
= NULL
;
408 struct rds_ib_connection
*ic
= NULL
;
409 struct rdma_conn_param conn_param
;
411 int err
, destroy
= 1;
413 /* Check whether the remote protocol version matches ours. */
414 version
= rds_ib_protocol_compatible(event
);
418 rdsdebug("saddr %pI4 daddr %pI4 RDSv%u.%u lguid 0x%llx fguid "
419 "0x%llx\n", &dp
->dp_saddr
, &dp
->dp_daddr
,
420 RDS_PROTOCOL_MAJOR(version
), RDS_PROTOCOL_MINOR(version
),
421 (unsigned long long)be64_to_cpu(lguid
),
422 (unsigned long long)be64_to_cpu(fguid
));
424 conn
= rds_conn_create(dp
->dp_daddr
, dp
->dp_saddr
, &rds_ib_transport
,
427 rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn
));
433 * The connection request may occur while the
434 * previous connection exist, e.g. in case of failover.
435 * But as connections may be initiated simultaneously
436 * by both hosts, we have a random backoff mechanism -
437 * see the comment above rds_queue_reconnect()
439 mutex_lock(&conn
->c_cm_lock
);
440 if (!rds_conn_transition(conn
, RDS_CONN_DOWN
, RDS_CONN_CONNECTING
)) {
441 if (rds_conn_state(conn
) == RDS_CONN_UP
) {
442 rdsdebug("incoming connect while connecting\n");
444 rds_ib_stats_inc(s_ib_listen_closed_stale
);
446 if (rds_conn_state(conn
) == RDS_CONN_CONNECTING
) {
447 /* Wait and see - our connect may still be succeeding */
448 rds_ib_stats_inc(s_ib_connect_raced
);
450 mutex_unlock(&conn
->c_cm_lock
);
454 ic
= conn
->c_transport_data
;
456 rds_ib_set_protocol(conn
, version
);
457 rds_ib_set_flow_control(conn
, be32_to_cpu(dp
->dp_credit
));
459 /* If the peer gave us the last packet it saw, process this as if
460 * we had received a regular ACK. */
462 rds_send_drop_acked(conn
, be64_to_cpu(dp
->dp_ack_seq
), NULL
);
464 BUG_ON(cm_id
->context
);
468 cm_id
->context
= conn
;
470 /* We got halfway through setting up the ib_connection, if we
471 * fail now, we have to take the long route out of this mess. */
474 err
= rds_ib_setup_qp(conn
);
476 rds_ib_conn_error(conn
, "rds_ib_setup_qp failed (%d)\n", err
);
480 rds_ib_cm_fill_conn_param(conn
, &conn_param
, &dp_rep
, version
);
482 /* rdma_accept() calls rdma_reject() internally if it fails */
483 err
= rdma_accept(cm_id
, &conn_param
);
484 mutex_unlock(&conn
->c_cm_lock
);
486 rds_ib_conn_error(conn
, "rdma_accept failed (%d)\n", err
);
493 rdma_reject(cm_id
, NULL
, 0);
498 int rds_ib_cm_initiate_connect(struct rdma_cm_id
*cm_id
)
500 struct rds_connection
*conn
= cm_id
->context
;
501 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
502 struct rdma_conn_param conn_param
;
503 struct rds_ib_connect_private dp
;
506 /* If the peer doesn't do protocol negotiation, we must
507 * default to RDSv3.0 */
508 rds_ib_set_protocol(conn
, RDS_PROTOCOL_3_0
);
509 ic
->i_flowctl
= rds_ib_sysctl_flow_control
; /* advertise flow control */
511 ret
= rds_ib_setup_qp(conn
);
513 rds_ib_conn_error(conn
, "rds_ib_setup_qp failed (%d)\n", ret
);
517 rds_ib_cm_fill_conn_param(conn
, &conn_param
, &dp
, RDS_PROTOCOL_VERSION
);
519 ret
= rdma_connect(cm_id
, &conn_param
);
521 rds_ib_conn_error(conn
, "rdma_connect failed (%d)\n", ret
);
524 /* Beware - returning non-zero tells the rdma_cm to destroy
525 * the cm_id. We should certainly not do it as long as we still
526 * "own" the cm_id. */
528 if (ic
->i_cm_id
== cm_id
)
534 int rds_ib_conn_connect(struct rds_connection
*conn
)
536 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
537 struct sockaddr_in src
, dest
;
540 /* XXX I wonder what affect the port space has */
541 /* delegate cm event handler to rdma_transport */
542 ic
->i_cm_id
= rdma_create_id(rds_rdma_cm_event_handler
, conn
,
544 if (IS_ERR(ic
->i_cm_id
)) {
545 ret
= PTR_ERR(ic
->i_cm_id
);
547 rdsdebug("rdma_create_id() failed: %d\n", ret
);
551 rdsdebug("created cm id %p for conn %p\n", ic
->i_cm_id
, conn
);
553 src
.sin_family
= AF_INET
;
554 src
.sin_addr
.s_addr
= (__force u32
)conn
->c_laddr
;
555 src
.sin_port
= (__force u16
)htons(0);
557 dest
.sin_family
= AF_INET
;
558 dest
.sin_addr
.s_addr
= (__force u32
)conn
->c_faddr
;
559 dest
.sin_port
= (__force u16
)htons(RDS_PORT
);
561 ret
= rdma_resolve_addr(ic
->i_cm_id
, (struct sockaddr
*)&src
,
562 (struct sockaddr
*)&dest
,
563 RDS_RDMA_RESOLVE_TIMEOUT_MS
);
565 rdsdebug("addr resolve failed for cm id %p: %d\n", ic
->i_cm_id
,
567 rdma_destroy_id(ic
->i_cm_id
);
576 * This is so careful about only cleaning up resources that were built up
577 * so that it can be called at any point during startup. In fact it
578 * can be called multiple times for a given connection.
580 void rds_ib_conn_shutdown(struct rds_connection
*conn
)
582 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
585 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic
->i_cm_id
,
586 ic
->i_pd
, ic
->i_send_cq
, ic
->i_recv_cq
,
587 ic
->i_cm_id
? ic
->i_cm_id
->qp
: NULL
);
590 struct ib_device
*dev
= ic
->i_cm_id
->device
;
592 rdsdebug("disconnecting cm %p\n", ic
->i_cm_id
);
593 err
= rdma_disconnect(ic
->i_cm_id
);
595 /* Actually this may happen quite frequently, when
596 * an outgoing connect raced with an incoming connect.
598 rdsdebug("failed to disconnect, cm: %p err %d\n",
602 wait_event(rds_ib_ring_empty_wait
,
603 rds_ib_ring_empty(&ic
->i_send_ring
) &&
604 rds_ib_ring_empty(&ic
->i_recv_ring
));
607 ib_dma_free_coherent(dev
,
608 ic
->i_send_ring
.w_nr
*
609 sizeof(struct rds_header
),
611 ic
->i_send_hdrs_dma
);
614 ib_dma_free_coherent(dev
,
615 ic
->i_recv_ring
.w_nr
*
616 sizeof(struct rds_header
),
618 ic
->i_recv_hdrs_dma
);
621 ib_dma_free_coherent(dev
, sizeof(struct rds_header
),
622 ic
->i_ack
, ic
->i_ack_dma
);
625 rds_ib_send_clear_ring(ic
);
627 rds_ib_recv_clear_ring(ic
);
630 rdma_destroy_qp(ic
->i_cm_id
);
632 ib_destroy_cq(ic
->i_send_cq
);
634 ib_destroy_cq(ic
->i_recv_cq
);
635 rdma_destroy_id(ic
->i_cm_id
);
638 * Move connection back to the nodev list.
641 rds_ib_remove_conn(ic
->rds_ibdev
, conn
);
646 ic
->i_send_cq
= NULL
;
647 ic
->i_recv_cq
= NULL
;
648 ic
->i_send_hdrs
= NULL
;
649 ic
->i_recv_hdrs
= NULL
;
652 BUG_ON(ic
->rds_ibdev
);
654 /* Clear pending transmit */
656 rds_message_put(ic
->i_rm
);
660 /* Clear the ACK state */
661 clear_bit(IB_ACK_IN_FLIGHT
, &ic
->i_ack_flags
);
662 #ifdef KERNEL_HAS_ATOMIC64
663 atomic64_set(&ic
->i_ack_next
, 0);
669 /* Clear flow control state */
671 atomic_set(&ic
->i_credits
, 0);
673 rds_ib_ring_init(&ic
->i_send_ring
, rds_ib_sysctl_max_send_wr
);
674 rds_ib_ring_init(&ic
->i_recv_ring
, rds_ib_sysctl_max_recv_wr
);
677 rds_inc_put(&ic
->i_ibinc
->ii_inc
);
687 int rds_ib_conn_alloc(struct rds_connection
*conn
, gfp_t gfp
)
689 struct rds_ib_connection
*ic
;
693 ic
= kzalloc(sizeof(struct rds_ib_connection
), GFP_KERNEL
);
697 INIT_LIST_HEAD(&ic
->ib_node
);
698 tasklet_init(&ic
->i_recv_tasklet
, rds_ib_recv_tasklet_fn
,
700 mutex_init(&ic
->i_recv_mutex
);
701 #ifndef KERNEL_HAS_ATOMIC64
702 spin_lock_init(&ic
->i_ack_lock
);
706 * rds_ib_conn_shutdown() waits for these to be emptied so they
707 * must be initialized before it can be called.
709 rds_ib_ring_init(&ic
->i_send_ring
, rds_ib_sysctl_max_send_wr
);
710 rds_ib_ring_init(&ic
->i_recv_ring
, rds_ib_sysctl_max_recv_wr
);
713 conn
->c_transport_data
= ic
;
715 spin_lock_irqsave(&ib_nodev_conns_lock
, flags
);
716 list_add_tail(&ic
->ib_node
, &ib_nodev_conns
);
717 spin_unlock_irqrestore(&ib_nodev_conns_lock
, flags
);
720 rdsdebug("conn %p conn ic %p\n", conn
, conn
->c_transport_data
);
725 * Free a connection. Connection must be shut down and not set for reconnect.
727 void rds_ib_conn_free(void *arg
)
729 struct rds_ib_connection
*ic
= arg
;
730 spinlock_t
*lock_ptr
;
732 rdsdebug("ic %p\n", ic
);
735 * Conn is either on a dev's list or on the nodev list.
736 * A race with shutdown() or connect() would cause problems
737 * (since rds_ibdev would change) but that should never happen.
739 lock_ptr
= ic
->rds_ibdev
? &ic
->rds_ibdev
->spinlock
: &ib_nodev_conns_lock
;
741 spin_lock_irq(lock_ptr
);
742 list_del(&ic
->ib_node
);
743 spin_unlock_irq(lock_ptr
);
750 * An error occurred on the connection
753 __rds_ib_conn_error(struct rds_connection
*conn
, const char *fmt
, ...)