2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
35 #include <linux/slab.h>
36 #include <linux/vmalloc.h>
42 * Set the selected protocol version
44 static void rds_ib_set_protocol(struct rds_connection
*conn
, unsigned int version
)
46 conn
->c_version
= version
;
52 static void rds_ib_set_flow_control(struct rds_connection
*conn
, u32 credits
)
54 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
56 if (rds_ib_sysctl_flow_control
&& credits
!= 0) {
57 /* We're doing flow control */
59 rds_ib_send_add_credits(conn
, credits
);
66 * Tune RNR behavior. Without flow control, we use a rather
67 * low timeout, but not the absolute minimum - this should
70 * We already set the RNR retry count to 7 (which is the
71 * smallest infinite number :-) above.
72 * If flow control is off, we want to change this back to 0
73 * so that we learn quickly when our credit accounting is
76 * Caller passes in a qp_attr pointer - don't waste stack spacv
77 * by allocation this twice.
80 rds_ib_tune_rnr(struct rds_ib_connection
*ic
, struct ib_qp_attr
*attr
)
84 attr
->min_rnr_timer
= IB_RNR_TIMER_000_32
;
85 ret
= ib_modify_qp(ic
->i_cm_id
->qp
, attr
, IB_QP_MIN_RNR_TIMER
);
87 printk(KERN_NOTICE
"ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret
);
91 * Connection established.
92 * We get here for both outgoing and incoming connection.
94 void rds_ib_cm_connect_complete(struct rds_connection
*conn
, struct rdma_cm_event
*event
)
96 const struct rds_ib_connect_private
*dp
= NULL
;
97 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
98 struct ib_qp_attr qp_attr
;
101 if (event
->param
.conn
.private_data_len
>= sizeof(*dp
)) {
102 dp
= event
->param
.conn
.private_data
;
104 /* make sure it isn't empty data */
105 if (dp
->dp_protocol_major
) {
106 rds_ib_set_protocol(conn
,
107 RDS_PROTOCOL(dp
->dp_protocol_major
,
108 dp
->dp_protocol_minor
));
109 rds_ib_set_flow_control(conn
, be32_to_cpu(dp
->dp_credit
));
113 if (conn
->c_version
< RDS_PROTOCOL(3,1)) {
114 printk(KERN_NOTICE
"RDS/IB: Connection to %pI4 version %u.%u failed,"
115 " no longer supported\n",
117 RDS_PROTOCOL_MAJOR(conn
->c_version
),
118 RDS_PROTOCOL_MINOR(conn
->c_version
));
119 rds_conn_destroy(conn
);
122 printk(KERN_NOTICE
"RDS/IB: connected to %pI4 version %u.%u%s\n",
124 RDS_PROTOCOL_MAJOR(conn
->c_version
),
125 RDS_PROTOCOL_MINOR(conn
->c_version
),
126 ic
->i_flowctl
? ", flow control" : "");
130 * Init rings and fill recv. this needs to wait until protocol negotiation
131 * is complete, since ring layout is different from 3.0 to 3.1.
133 rds_ib_send_init_ring(ic
);
134 rds_ib_recv_init_ring(ic
);
135 /* Post receive buffers - as a side effect, this will update
136 * the posted credit count. */
137 rds_ib_recv_refill(conn
, 1);
139 /* Tune RNR behavior */
140 rds_ib_tune_rnr(ic
, &qp_attr
);
142 qp_attr
.qp_state
= IB_QPS_RTS
;
143 err
= ib_modify_qp(ic
->i_cm_id
->qp
, &qp_attr
, IB_QP_STATE
);
145 printk(KERN_NOTICE
"ib_modify_qp(IB_QP_STATE, RTS): err=%d\n", err
);
147 /* update ib_device with this local ipaddr */
148 err
= rds_ib_update_ipaddr(ic
->rds_ibdev
, conn
->c_laddr
);
150 printk(KERN_ERR
"rds_ib_update_ipaddr failed (%d)\n",
153 /* If the peer gave us the last packet it saw, process this as if
154 * we had received a regular ACK. */
155 if (dp
&& dp
->dp_ack_seq
)
156 rds_send_drop_acked(conn
, be64_to_cpu(dp
->dp_ack_seq
), NULL
);
158 rds_connect_complete(conn
);
161 static void rds_ib_cm_fill_conn_param(struct rds_connection
*conn
,
162 struct rdma_conn_param
*conn_param
,
163 struct rds_ib_connect_private
*dp
,
164 u32 protocol_version
,
165 u32 max_responder_resources
,
166 u32 max_initiator_depth
)
168 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
169 struct rds_ib_device
*rds_ibdev
= ic
->rds_ibdev
;
171 memset(conn_param
, 0, sizeof(struct rdma_conn_param
));
173 conn_param
->responder_resources
=
174 min_t(u32
, rds_ibdev
->max_responder_resources
, max_responder_resources
);
175 conn_param
->initiator_depth
=
176 min_t(u32
, rds_ibdev
->max_initiator_depth
, max_initiator_depth
);
177 conn_param
->retry_count
= min_t(unsigned int, rds_ib_retry_count
, 7);
178 conn_param
->rnr_retry_count
= 7;
181 memset(dp
, 0, sizeof(*dp
));
182 dp
->dp_saddr
= conn
->c_laddr
;
183 dp
->dp_daddr
= conn
->c_faddr
;
184 dp
->dp_protocol_major
= RDS_PROTOCOL_MAJOR(protocol_version
);
185 dp
->dp_protocol_minor
= RDS_PROTOCOL_MINOR(protocol_version
);
186 dp
->dp_protocol_minor_mask
= cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS
);
187 dp
->dp_ack_seq
= rds_ib_piggyb_ack(ic
);
189 /* Advertise flow control */
191 unsigned int credits
;
193 credits
= IB_GET_POST_CREDITS(atomic_read(&ic
->i_credits
));
194 dp
->dp_credit
= cpu_to_be32(credits
);
195 atomic_sub(IB_SET_POST_CREDITS(credits
), &ic
->i_credits
);
198 conn_param
->private_data
= dp
;
199 conn_param
->private_data_len
= sizeof(*dp
);
203 static void rds_ib_cq_event_handler(struct ib_event
*event
, void *data
)
205 rdsdebug("event %u data %p\n", event
->event
, data
);
208 static void rds_ib_qp_event_handler(struct ib_event
*event
, void *data
)
210 struct rds_connection
*conn
= data
;
211 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
213 rdsdebug("conn %p ic %p event %u\n", conn
, ic
, event
->event
);
215 switch (event
->event
) {
216 case IB_EVENT_COMM_EST
:
217 rdma_notify(ic
->i_cm_id
, IB_EVENT_COMM_EST
);
220 rdsdebug("Fatal QP Event %u "
221 "- connection %pI4->%pI4, reconnecting\n",
222 event
->event
, &conn
->c_laddr
, &conn
->c_faddr
);
229 * This needs to be very careful to not leave IS_ERR pointers around for
230 * cleanup to trip over.
232 static int rds_ib_setup_qp(struct rds_connection
*conn
)
234 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
235 struct ib_device
*dev
= ic
->i_cm_id
->device
;
236 struct ib_qp_init_attr attr
;
237 struct rds_ib_device
*rds_ibdev
;
241 * It's normal to see a null device if an incoming connection races
242 * with device removal, so we don't print a warning.
244 rds_ibdev
= rds_ib_get_client_data(dev
);
248 /* add the conn now so that connection establishment has the dev */
249 rds_ib_add_conn(rds_ibdev
, conn
);
251 if (rds_ibdev
->max_wrs
< ic
->i_send_ring
.w_nr
+ 1)
252 rds_ib_ring_resize(&ic
->i_send_ring
, rds_ibdev
->max_wrs
- 1);
253 if (rds_ibdev
->max_wrs
< ic
->i_recv_ring
.w_nr
+ 1)
254 rds_ib_ring_resize(&ic
->i_recv_ring
, rds_ibdev
->max_wrs
- 1);
256 /* Protection domain and memory range */
257 ic
->i_pd
= rds_ibdev
->pd
;
258 ic
->i_mr
= rds_ibdev
->mr
;
260 ic
->i_send_cq
= ib_create_cq(dev
, rds_ib_send_cq_comp_handler
,
261 rds_ib_cq_event_handler
, conn
,
262 ic
->i_send_ring
.w_nr
+ 1, 0);
263 if (IS_ERR(ic
->i_send_cq
)) {
264 ret
= PTR_ERR(ic
->i_send_cq
);
265 ic
->i_send_cq
= NULL
;
266 rdsdebug("ib_create_cq send failed: %d\n", ret
);
270 ic
->i_recv_cq
= ib_create_cq(dev
, rds_ib_recv_cq_comp_handler
,
271 rds_ib_cq_event_handler
, conn
,
272 ic
->i_recv_ring
.w_nr
, 0);
273 if (IS_ERR(ic
->i_recv_cq
)) {
274 ret
= PTR_ERR(ic
->i_recv_cq
);
275 ic
->i_recv_cq
= NULL
;
276 rdsdebug("ib_create_cq recv failed: %d\n", ret
);
280 ret
= ib_req_notify_cq(ic
->i_send_cq
, IB_CQ_NEXT_COMP
);
282 rdsdebug("ib_req_notify_cq send failed: %d\n", ret
);
286 ret
= ib_req_notify_cq(ic
->i_recv_cq
, IB_CQ_SOLICITED
);
288 rdsdebug("ib_req_notify_cq recv failed: %d\n", ret
);
292 /* XXX negotiate max send/recv with remote? */
293 memset(&attr
, 0, sizeof(attr
));
294 attr
.event_handler
= rds_ib_qp_event_handler
;
295 attr
.qp_context
= conn
;
296 /* + 1 to allow for the single ack message */
297 attr
.cap
.max_send_wr
= ic
->i_send_ring
.w_nr
+ 1;
298 attr
.cap
.max_recv_wr
= ic
->i_recv_ring
.w_nr
+ 1;
299 attr
.cap
.max_send_sge
= rds_ibdev
->max_sge
;
300 attr
.cap
.max_recv_sge
= RDS_IB_RECV_SGE
;
301 attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
302 attr
.qp_type
= IB_QPT_RC
;
303 attr
.send_cq
= ic
->i_send_cq
;
304 attr
.recv_cq
= ic
->i_recv_cq
;
307 * XXX this can fail if max_*_wr is too large? Are we supposed
308 * to back off until we get a value that the hardware can support?
310 ret
= rdma_create_qp(ic
->i_cm_id
, ic
->i_pd
, &attr
);
312 rdsdebug("rdma_create_qp failed: %d\n", ret
);
316 ic
->i_send_hdrs
= ib_dma_alloc_coherent(dev
,
317 ic
->i_send_ring
.w_nr
*
318 sizeof(struct rds_header
),
319 &ic
->i_send_hdrs_dma
, GFP_KERNEL
);
320 if (!ic
->i_send_hdrs
) {
322 rdsdebug("ib_dma_alloc_coherent send failed\n");
326 ic
->i_recv_hdrs
= ib_dma_alloc_coherent(dev
,
327 ic
->i_recv_ring
.w_nr
*
328 sizeof(struct rds_header
),
329 &ic
->i_recv_hdrs_dma
, GFP_KERNEL
);
330 if (!ic
->i_recv_hdrs
) {
332 rdsdebug("ib_dma_alloc_coherent recv failed\n");
336 ic
->i_ack
= ib_dma_alloc_coherent(dev
, sizeof(struct rds_header
),
337 &ic
->i_ack_dma
, GFP_KERNEL
);
340 rdsdebug("ib_dma_alloc_coherent ack failed\n");
344 ic
->i_sends
= vmalloc_node(ic
->i_send_ring
.w_nr
* sizeof(struct rds_ib_send_work
),
348 rdsdebug("send allocation failed\n");
351 memset(ic
->i_sends
, 0, ic
->i_send_ring
.w_nr
* sizeof(struct rds_ib_send_work
));
353 ic
->i_recvs
= vmalloc_node(ic
->i_recv_ring
.w_nr
* sizeof(struct rds_ib_recv_work
),
357 rdsdebug("recv allocation failed\n");
360 memset(ic
->i_recvs
, 0, ic
->i_recv_ring
.w_nr
* sizeof(struct rds_ib_recv_work
));
362 rds_ib_recv_init_ack(ic
);
364 rdsdebug("conn %p pd %p mr %p cq %p %p\n", conn
, ic
->i_pd
, ic
->i_mr
,
365 ic
->i_send_cq
, ic
->i_recv_cq
);
368 rds_ib_dev_put(rds_ibdev
);
372 static u32
rds_ib_protocol_compatible(struct rdma_cm_event
*event
)
374 const struct rds_ib_connect_private
*dp
= event
->param
.conn
.private_data
;
379 * rdma_cm private data is odd - when there is any private data in the
380 * request, we will be given a pretty large buffer without telling us the
381 * original size. The only way to tell the difference is by looking at
382 * the contents, which are initialized to zero.
383 * If the protocol version fields aren't set, this is a connection attempt
384 * from an older version. This could could be 3.0 or 2.0 - we can't tell.
385 * We really should have changed this for OFED 1.3 :-(
388 /* Be paranoid. RDS always has privdata */
389 if (!event
->param
.conn
.private_data_len
) {
390 printk(KERN_NOTICE
"RDS incoming connection has no private data, "
395 /* Even if len is crap *now* I still want to check it. -ASG */
396 if (event
->param
.conn
.private_data_len
< sizeof (*dp
) ||
397 dp
->dp_protocol_major
== 0)
398 return RDS_PROTOCOL_3_0
;
400 common
= be16_to_cpu(dp
->dp_protocol_minor_mask
) & RDS_IB_SUPPORTED_PROTOCOLS
;
401 if (dp
->dp_protocol_major
== 3 && common
) {
402 version
= RDS_PROTOCOL_3_0
;
403 while ((common
>>= 1) != 0)
405 } else if (printk_ratelimit()) {
406 printk(KERN_NOTICE
"RDS: Connection from %pI4 using "
407 "incompatible protocol version %u.%u\n",
409 dp
->dp_protocol_major
,
410 dp
->dp_protocol_minor
);
415 int rds_ib_cm_handle_connect(struct rdma_cm_id
*cm_id
,
416 struct rdma_cm_event
*event
)
418 __be64 lguid
= cm_id
->route
.path_rec
->sgid
.global
.interface_id
;
419 __be64 fguid
= cm_id
->route
.path_rec
->dgid
.global
.interface_id
;
420 const struct rds_ib_connect_private
*dp
= event
->param
.conn
.private_data
;
421 struct rds_ib_connect_private dp_rep
;
422 struct rds_connection
*conn
= NULL
;
423 struct rds_ib_connection
*ic
= NULL
;
424 struct rdma_conn_param conn_param
;
426 int err
= 1, destroy
= 1;
428 /* Check whether the remote protocol version matches ours. */
429 version
= rds_ib_protocol_compatible(event
);
433 rdsdebug("saddr %pI4 daddr %pI4 RDSv%u.%u lguid 0x%llx fguid "
434 "0x%llx\n", &dp
->dp_saddr
, &dp
->dp_daddr
,
435 RDS_PROTOCOL_MAJOR(version
), RDS_PROTOCOL_MINOR(version
),
436 (unsigned long long)be64_to_cpu(lguid
),
437 (unsigned long long)be64_to_cpu(fguid
));
439 conn
= rds_conn_create(dp
->dp_daddr
, dp
->dp_saddr
, &rds_ib_transport
,
442 rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn
));
448 * The connection request may occur while the
449 * previous connection exist, e.g. in case of failover.
450 * But as connections may be initiated simultaneously
451 * by both hosts, we have a random backoff mechanism -
452 * see the comment above rds_queue_reconnect()
454 mutex_lock(&conn
->c_cm_lock
);
455 if (!rds_conn_transition(conn
, RDS_CONN_DOWN
, RDS_CONN_CONNECTING
)) {
456 if (rds_conn_state(conn
) == RDS_CONN_UP
) {
457 rdsdebug("incoming connect while connecting\n");
459 rds_ib_stats_inc(s_ib_listen_closed_stale
);
461 if (rds_conn_state(conn
) == RDS_CONN_CONNECTING
) {
462 /* Wait and see - our connect may still be succeeding */
463 rds_ib_stats_inc(s_ib_connect_raced
);
468 ic
= conn
->c_transport_data
;
470 rds_ib_set_protocol(conn
, version
);
471 rds_ib_set_flow_control(conn
, be32_to_cpu(dp
->dp_credit
));
473 /* If the peer gave us the last packet it saw, process this as if
474 * we had received a regular ACK. */
476 rds_send_drop_acked(conn
, be64_to_cpu(dp
->dp_ack_seq
), NULL
);
478 BUG_ON(cm_id
->context
);
482 cm_id
->context
= conn
;
484 /* We got halfway through setting up the ib_connection, if we
485 * fail now, we have to take the long route out of this mess. */
488 err
= rds_ib_setup_qp(conn
);
490 rds_ib_conn_error(conn
, "rds_ib_setup_qp failed (%d)\n", err
);
491 mutex_unlock(&conn
->c_cm_lock
);
495 rds_ib_cm_fill_conn_param(conn
, &conn_param
, &dp_rep
, version
,
496 event
->param
.conn
.responder_resources
,
497 event
->param
.conn
.initiator_depth
);
499 /* rdma_accept() calls rdma_reject() internally if it fails */
500 err
= rdma_accept(cm_id
, &conn_param
);
502 rds_ib_conn_error(conn
, "rdma_accept failed (%d)\n", err
);
506 mutex_unlock(&conn
->c_cm_lock
);
508 rdma_reject(cm_id
, NULL
, 0);
513 int rds_ib_cm_initiate_connect(struct rdma_cm_id
*cm_id
)
515 struct rds_connection
*conn
= cm_id
->context
;
516 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
517 struct rdma_conn_param conn_param
;
518 struct rds_ib_connect_private dp
;
521 /* If the peer doesn't do protocol negotiation, we must
522 * default to RDSv3.0 */
523 rds_ib_set_protocol(conn
, RDS_PROTOCOL_3_0
);
524 ic
->i_flowctl
= rds_ib_sysctl_flow_control
; /* advertise flow control */
526 ret
= rds_ib_setup_qp(conn
);
528 rds_ib_conn_error(conn
, "rds_ib_setup_qp failed (%d)\n", ret
);
532 rds_ib_cm_fill_conn_param(conn
, &conn_param
, &dp
, RDS_PROTOCOL_VERSION
,
534 ret
= rdma_connect(cm_id
, &conn_param
);
536 rds_ib_conn_error(conn
, "rdma_connect failed (%d)\n", ret
);
539 /* Beware - returning non-zero tells the rdma_cm to destroy
540 * the cm_id. We should certainly not do it as long as we still
541 * "own" the cm_id. */
543 if (ic
->i_cm_id
== cm_id
)
549 int rds_ib_conn_connect(struct rds_connection
*conn
)
551 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
552 struct sockaddr_in src
, dest
;
555 /* XXX I wonder what affect the port space has */
556 /* delegate cm event handler to rdma_transport */
557 ic
->i_cm_id
= rdma_create_id(rds_rdma_cm_event_handler
, conn
,
559 if (IS_ERR(ic
->i_cm_id
)) {
560 ret
= PTR_ERR(ic
->i_cm_id
);
562 rdsdebug("rdma_create_id() failed: %d\n", ret
);
566 rdsdebug("created cm id %p for conn %p\n", ic
->i_cm_id
, conn
);
568 src
.sin_family
= AF_INET
;
569 src
.sin_addr
.s_addr
= (__force u32
)conn
->c_laddr
;
570 src
.sin_port
= (__force u16
)htons(0);
572 dest
.sin_family
= AF_INET
;
573 dest
.sin_addr
.s_addr
= (__force u32
)conn
->c_faddr
;
574 dest
.sin_port
= (__force u16
)htons(RDS_PORT
);
576 ret
= rdma_resolve_addr(ic
->i_cm_id
, (struct sockaddr
*)&src
,
577 (struct sockaddr
*)&dest
,
578 RDS_RDMA_RESOLVE_TIMEOUT_MS
);
580 rdsdebug("addr resolve failed for cm id %p: %d\n", ic
->i_cm_id
,
582 rdma_destroy_id(ic
->i_cm_id
);
591 * This is so careful about only cleaning up resources that were built up
592 * so that it can be called at any point during startup. In fact it
593 * can be called multiple times for a given connection.
595 void rds_ib_conn_shutdown(struct rds_connection
*conn
)
597 struct rds_ib_connection
*ic
= conn
->c_transport_data
;
600 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic
->i_cm_id
,
601 ic
->i_pd
, ic
->i_send_cq
, ic
->i_recv_cq
,
602 ic
->i_cm_id
? ic
->i_cm_id
->qp
: NULL
);
605 struct ib_device
*dev
= ic
->i_cm_id
->device
;
607 rdsdebug("disconnecting cm %p\n", ic
->i_cm_id
);
608 err
= rdma_disconnect(ic
->i_cm_id
);
610 /* Actually this may happen quite frequently, when
611 * an outgoing connect raced with an incoming connect.
613 rdsdebug("failed to disconnect, cm: %p err %d\n",
618 * Don't wait for the send ring to be empty -- there may be completed
619 * non-signaled entries sitting on there. We unmap these below.
621 wait_event(rds_ib_ring_empty_wait
,
622 rds_ib_ring_empty(&ic
->i_recv_ring
));
625 ib_dma_free_coherent(dev
,
626 ic
->i_send_ring
.w_nr
*
627 sizeof(struct rds_header
),
629 ic
->i_send_hdrs_dma
);
632 ib_dma_free_coherent(dev
,
633 ic
->i_recv_ring
.w_nr
*
634 sizeof(struct rds_header
),
636 ic
->i_recv_hdrs_dma
);
639 ib_dma_free_coherent(dev
, sizeof(struct rds_header
),
640 ic
->i_ack
, ic
->i_ack_dma
);
643 rds_ib_send_clear_ring(ic
);
645 rds_ib_recv_clear_ring(ic
);
648 rdma_destroy_qp(ic
->i_cm_id
);
650 ib_destroy_cq(ic
->i_send_cq
);
652 ib_destroy_cq(ic
->i_recv_cq
);
653 rdma_destroy_id(ic
->i_cm_id
);
656 * Move connection back to the nodev list.
659 rds_ib_remove_conn(ic
->rds_ibdev
, conn
);
664 ic
->i_send_cq
= NULL
;
665 ic
->i_recv_cq
= NULL
;
666 ic
->i_send_hdrs
= NULL
;
667 ic
->i_recv_hdrs
= NULL
;
670 BUG_ON(ic
->rds_ibdev
);
672 /* Clear pending transmit */
674 struct rds_message
*rm
;
676 rm
= container_of(ic
->i_data_op
, struct rds_message
, data
);
678 ic
->i_data_op
= NULL
;
681 /* Clear the ACK state */
682 clear_bit(IB_ACK_IN_FLIGHT
, &ic
->i_ack_flags
);
683 #ifdef KERNEL_HAS_ATOMIC64
684 atomic64_set(&ic
->i_ack_next
, 0);
690 /* Clear flow control state */
692 atomic_set(&ic
->i_credits
, 0);
694 rds_ib_ring_init(&ic
->i_send_ring
, rds_ib_sysctl_max_send_wr
);
695 rds_ib_ring_init(&ic
->i_recv_ring
, rds_ib_sysctl_max_recv_wr
);
698 rds_inc_put(&ic
->i_ibinc
->ii_inc
);
708 int rds_ib_conn_alloc(struct rds_connection
*conn
, gfp_t gfp
)
710 struct rds_ib_connection
*ic
;
715 ic
= kzalloc(sizeof(struct rds_ib_connection
), GFP_KERNEL
);
719 ret
= rds_ib_recv_alloc_caches(ic
);
725 INIT_LIST_HEAD(&ic
->ib_node
);
726 tasklet_init(&ic
->i_recv_tasklet
, rds_ib_recv_tasklet_fn
,
728 mutex_init(&ic
->i_recv_mutex
);
729 #ifndef KERNEL_HAS_ATOMIC64
730 spin_lock_init(&ic
->i_ack_lock
);
734 * rds_ib_conn_shutdown() waits for these to be emptied so they
735 * must be initialized before it can be called.
737 rds_ib_ring_init(&ic
->i_send_ring
, rds_ib_sysctl_max_send_wr
);
738 rds_ib_ring_init(&ic
->i_recv_ring
, rds_ib_sysctl_max_recv_wr
);
741 conn
->c_transport_data
= ic
;
743 spin_lock_irqsave(&ib_nodev_conns_lock
, flags
);
744 list_add_tail(&ic
->ib_node
, &ib_nodev_conns
);
745 spin_unlock_irqrestore(&ib_nodev_conns_lock
, flags
);
748 rdsdebug("conn %p conn ic %p\n", conn
, conn
->c_transport_data
);
753 * Free a connection. Connection must be shut down and not set for reconnect.
755 void rds_ib_conn_free(void *arg
)
757 struct rds_ib_connection
*ic
= arg
;
758 spinlock_t
*lock_ptr
;
760 rdsdebug("ic %p\n", ic
);
763 * Conn is either on a dev's list or on the nodev list.
764 * A race with shutdown() or connect() would cause problems
765 * (since rds_ibdev would change) but that should never happen.
767 lock_ptr
= ic
->rds_ibdev
? &ic
->rds_ibdev
->spinlock
: &ib_nodev_conns_lock
;
769 spin_lock_irq(lock_ptr
);
770 list_del(&ic
->ib_node
);
771 spin_unlock_irq(lock_ptr
);
773 rds_ib_recv_free_caches(ic
);
780 * An error occurred on the connection
783 __rds_ib_conn_error(struct rds_connection
*conn
, const char *fmt
, ...)