]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | #include <poll.h> |
2 | ||
3 | #include "msg/async/net_handler.h" | |
4 | #include "RDMAStack.h" | |
5 | ||
6 | #define dout_subsys ceph_subsys_ms | |
7 | #undef dout_prefix | |
8 | #define dout_prefix *_dout << " RDMAIWARPServerSocketImpl " | |
9 | ||
10 | RDMAIWARPServerSocketImpl::RDMAIWARPServerSocketImpl( | |
f67539c2 TL |
11 | CephContext *cct, std::shared_ptr<Infiniband>& ib, |
12 | std::shared_ptr<RDMADispatcher>& rdma_dispatcher, RDMAWorker *w, | |
9f95a23c TL |
13 | entity_addr_t& a, unsigned addr_slot) |
14 | : RDMAServerSocketImpl(cct, ib, rdma_dispatcher, w, a, addr_slot) | |
11fdf7f2 TL |
15 | { |
16 | } | |
17 | ||
18 | int RDMAIWARPServerSocketImpl::listen(entity_addr_t &sa, | |
19 | const SocketOptions &opt) | |
20 | { | |
21 | ldout(cct, 20) << __func__ << " bind to rdma point" << dendl; | |
22 | cm_channel = rdma_create_event_channel(); | |
23 | rdma_create_id(cm_channel, &cm_id, NULL, RDMA_PS_TCP); | |
24 | ldout(cct, 20) << __func__ << " successfully created cm id: " << cm_id << dendl; | |
25 | int rc = rdma_bind_addr(cm_id, const_cast<struct sockaddr*>(sa.get_sockaddr())); | |
26 | if (rc < 0) { | |
27 | rc = -errno; | |
28 | ldout(cct, 10) << __func__ << " unable to bind to " << sa.get_sockaddr() | |
29 | << " on port " << sa.get_port() << ": " << cpp_strerror(errno) << dendl; | |
30 | goto err; | |
31 | } | |
32 | rc = rdma_listen(cm_id, 128); | |
33 | if (rc < 0) { | |
34 | rc = -errno; | |
35 | ldout(cct, 10) << __func__ << " unable to listen to " << sa.get_sockaddr() | |
36 | << " on port " << sa.get_port() << ": " << cpp_strerror(errno) << dendl; | |
37 | goto err; | |
38 | } | |
39 | server_setup_socket = cm_channel->fd; | |
9f95a23c TL |
40 | rc = net.set_nonblock(server_setup_socket); |
41 | if (rc < 0) { | |
42 | goto err; | |
43 | } | |
11fdf7f2 TL |
44 | ldout(cct, 20) << __func__ << " fd of cm_channel is " << server_setup_socket << dendl; |
45 | return 0; | |
46 | ||
47 | err: | |
48 | server_setup_socket = -1; | |
49 | rdma_destroy_id(cm_id); | |
50 | rdma_destroy_event_channel(cm_channel); | |
51 | return rc; | |
52 | } | |
53 | ||
54 | int RDMAIWARPServerSocketImpl::accept(ConnectedSocket *sock, const SocketOptions &opt, | |
55 | entity_addr_t *out, Worker *w) | |
56 | { | |
57 | ldout(cct, 15) << __func__ << dendl; | |
58 | ||
59 | ceph_assert(sock); | |
60 | struct pollfd pfd = { | |
61 | .fd = cm_channel->fd, | |
62 | .events = POLLIN, | |
9f95a23c | 63 | .revents = 0, |
11fdf7f2 TL |
64 | }; |
65 | int ret = poll(&pfd, 1, 0); | |
66 | ceph_assert(ret >= 0); | |
67 | if (!ret) | |
68 | return -EAGAIN; | |
69 | ||
70 | struct rdma_cm_event *cm_event; | |
71 | rdma_get_cm_event(cm_channel, &cm_event); | |
72 | ldout(cct, 20) << __func__ << " event name: " << rdma_event_str(cm_event->event) << dendl; | |
73 | ||
74 | struct rdma_cm_id *event_cm_id = cm_event->id; | |
75 | struct rdma_event_channel *event_channel = rdma_create_event_channel(); | |
76 | ||
9f95a23c TL |
77 | if (net.set_nonblock(event_channel->fd) < 0) { |
78 | lderr(cct) << __func__ << " failed to switch event channel to non-block, close event channel " << dendl; | |
79 | rdma_destroy_event_channel(event_channel); | |
80 | rdma_ack_cm_event(cm_event); | |
81 | return -errno; | |
82 | } | |
83 | ||
11fdf7f2 TL |
84 | rdma_migrate_id(event_cm_id, event_channel); |
85 | ||
11fdf7f2 TL |
86 | struct rdma_conn_param *remote_conn_param = &cm_event->param.conn; |
87 | struct rdma_conn_param local_conn_param; | |
88 | ||
9f95a23c | 89 | RDMACMInfo info(event_cm_id, event_channel, remote_conn_param->qp_num); |
11fdf7f2 | 90 | RDMAIWARPConnectedSocketImpl* server = |
9f95a23c | 91 | new RDMAIWARPConnectedSocketImpl(cct, ib, dispatcher, dynamic_cast<RDMAWorker*>(w), &info); |
11fdf7f2 | 92 | |
92f5a8d4 | 93 | // FIPS zeroization audit 20191115: this memset is not security related. |
11fdf7f2 TL |
94 | memset(&local_conn_param, 0, sizeof(local_conn_param)); |
95 | local_conn_param.qp_num = server->get_local_qpn(); | |
96 | ||
9f95a23c | 97 | if (rdma_accept(event_cm_id, &local_conn_param)) { |
11fdf7f2 TL |
98 | return -EAGAIN; |
99 | } | |
100 | server->activate(); | |
101 | ldout(cct, 20) << __func__ << " accepted a new QP" << dendl; | |
102 | ||
103 | rdma_ack_cm_event(cm_event); | |
104 | ||
105 | std::unique_ptr<RDMAConnectedSocketImpl> csi(server); | |
106 | *sock = ConnectedSocket(std::move(csi)); | |
9f95a23c | 107 | struct sockaddr *addr = &event_cm_id->route.addr.dst_addr; |
11fdf7f2 TL |
108 | out->set_sockaddr(addr); |
109 | ||
110 | return 0; | |
111 | } | |
112 | ||
113 | void RDMAIWARPServerSocketImpl::abort_accept() | |
114 | { | |
115 | if (server_setup_socket >= 0) { | |
116 | rdma_destroy_id(cm_id); | |
117 | rdma_destroy_event_channel(cm_channel); | |
118 | } | |
119 | } |