]>
Commit | Line | Data |
---|---|---|
1 | #ifndef DEF_RDMA_VT_H | |
2 | #define DEF_RDMA_VT_H | |
3 | ||
4 | /* | |
5 | * Copyright(c) 2016 Intel Corporation. | |
6 | * | |
7 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
8 | * redistributing this file, you may do so under either license. | |
9 | * | |
10 | * GPL LICENSE SUMMARY | |
11 | * | |
12 | * This program is free software; you can redistribute it and/or modify | |
13 | * it under the terms of version 2 of the GNU General Public License as | |
14 | * published by the Free Software Foundation. | |
15 | * | |
16 | * This program is distributed in the hope that it will be useful, but | |
17 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
19 | * General Public License for more details. | |
20 | * | |
21 | * BSD LICENSE | |
22 | * | |
23 | * Redistribution and use in source and binary forms, with or without | |
24 | * modification, are permitted provided that the following conditions | |
25 | * are met: | |
26 | * | |
27 | * - Redistributions of source code must retain the above copyright | |
28 | * notice, this list of conditions and the following disclaimer. | |
29 | * - Redistributions in binary form must reproduce the above copyright | |
30 | * notice, this list of conditions and the following disclaimer in | |
31 | * the documentation and/or other materials provided with the | |
32 | * distribution. | |
33 | * - Neither the name of Intel Corporation nor the names of its | |
34 | * contributors may be used to endorse or promote products derived | |
35 | * from this software without specific prior written permission. | |
36 | * | |
37 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
38 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
39 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
40 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
41 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
42 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
43 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
44 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
45 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
46 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
47 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
48 | * | |
49 | */ | |
50 | ||
51 | /* | |
52 | * Structure that low level drivers will populate in order to register with the | |
53 | * rdmavt layer. | |
54 | */ | |
55 | ||
56 | #include <linux/spinlock.h> | |
57 | #include <linux/list.h> | |
58 | #include <linux/hash.h> | |
59 | #include <rdma/ib_verbs.h> | |
60 | #include <rdma/rdmavt_mr.h> | |
61 | #include <rdma/rdmavt_qp.h> | |
62 | ||
63 | /* | |
64 | * For some of the IBTA objects there will likely be some | |
65 | * initializations required. We need flags to determine whether it is OK | |
66 | * for rdmavt to do this or not. This does not imply any functions of a | |
67 | * partiuclar IBTA object are overridden. | |
68 | */ | |
69 | #define RVT_FLAG_MR_INIT_DRIVER BIT(1) | |
70 | #define RVT_FLAG_QP_INIT_DRIVER BIT(2) | |
71 | #define RVT_FLAG_CQ_INIT_DRIVER BIT(3) | |
72 | ||
73 | #define RVT_MAX_PKEY_VALUES 16 | |
74 | ||
75 | struct rvt_ibport { | |
76 | struct rvt_qp __rcu *qp[2]; | |
77 | struct ib_mad_agent *send_agent; /* agent for SMI (traps) */ | |
78 | struct rb_root mcast_tree; | |
79 | spinlock_t lock; /* protect changes in this struct */ | |
80 | ||
81 | /* non-zero when timer is set */ | |
82 | unsigned long mkey_lease_timeout; | |
83 | unsigned long trap_timeout; | |
84 | __be64 gid_prefix; /* in network order */ | |
85 | __be64 mkey; | |
86 | u64 tid; | |
87 | u32 port_cap_flags; | |
88 | u32 pma_sample_start; | |
89 | u32 pma_sample_interval; | |
90 | __be16 pma_counter_select[5]; | |
91 | u16 pma_tag; | |
92 | u16 mkey_lease_period; | |
93 | u16 sm_lid; | |
94 | u8 sm_sl; | |
95 | u8 mkeyprot; | |
96 | u8 subnet_timeout; | |
97 | u8 vl_high_limit; | |
98 | ||
99 | /* | |
100 | * Driver is expected to keep these up to date. These | |
101 | * counters are informational only and not required to be | |
102 | * completely accurate. | |
103 | */ | |
104 | u64 n_rc_resends; | |
105 | u64 n_seq_naks; | |
106 | u64 n_rdma_seq; | |
107 | u64 n_rnr_naks; | |
108 | u64 n_other_naks; | |
109 | u64 n_loop_pkts; | |
110 | u64 n_pkt_drops; | |
111 | u64 n_vl15_dropped; | |
112 | u64 n_rc_timeouts; | |
113 | u64 n_dmawait; | |
114 | u64 n_unaligned; | |
115 | u64 n_rc_dupreq; | |
116 | u64 n_rc_seqnak; | |
117 | u16 pkey_violations; | |
118 | u16 qkey_violations; | |
119 | u16 mkey_violations; | |
120 | ||
121 | /* Hot-path per CPU counters to avoid cacheline trading to update */ | |
122 | u64 z_rc_acks; | |
123 | u64 z_rc_qacks; | |
124 | u64 z_rc_delayed_comp; | |
125 | u64 __percpu *rc_acks; | |
126 | u64 __percpu *rc_qacks; | |
127 | u64 __percpu *rc_delayed_comp; | |
128 | ||
129 | void *priv; /* driver private data */ | |
130 | ||
131 | /* | |
132 | * The pkey table is allocated and maintained by the driver. Drivers | |
133 | * need to have access to this before registering with rdmav. However | |
134 | * rdmavt will need access to it so drivers need to proviee this during | |
135 | * the attach port API call. | |
136 | */ | |
137 | u16 *pkey_table; | |
138 | ||
139 | struct rvt_ah *sm_ah; | |
140 | struct rvt_ah *smi_ah; | |
141 | }; | |
142 | ||
143 | #define RVT_CQN_MAX 16 /* maximum length of cq name */ | |
144 | ||
145 | /* | |
146 | * Things that are driver specific, module parameters in hfi1 and qib | |
147 | */ | |
148 | struct rvt_driver_params { | |
149 | /* | |
150 | * driver required fields: | |
151 | * node_guid | |
152 | * phys_port_cnt | |
153 | * dma_device | |
154 | * owner | |
155 | * driver optional fields (rvt will provide generic value if blank): | |
156 | * name | |
157 | * node_desc | |
158 | * rvt fields, driver value ignored: | |
159 | * uverbs_abi_ver | |
160 | * node_type | |
161 | * num_comp_vectors | |
162 | * uverbs_cmd_mask | |
163 | */ | |
164 | struct ib_device_attr props; | |
165 | ||
166 | /* | |
167 | * Drivers will need to support a number of notifications to rvt in | |
168 | * accordance with certain events. This structure should contain a mask | |
169 | * of the supported events. Such events that the rvt may need to know | |
170 | * about include: | |
171 | * port errors | |
172 | * port active | |
173 | * lid change | |
174 | * sm change | |
175 | * client reregister | |
176 | * pkey change | |
177 | * | |
178 | * There may also be other events that the rvt layers needs to know | |
179 | * about this is not an exhaustive list. Some events though rvt does not | |
180 | * need to rely on the driver for such as completion queue error. | |
181 | */ | |
182 | int rvt_signal_supported; | |
183 | ||
184 | /* | |
185 | * Anything driver specific that is not covered by props | |
186 | * For instance special module parameters. Goes here. | |
187 | */ | |
188 | unsigned int lkey_table_size; | |
189 | unsigned int qp_table_size; | |
190 | int qpn_start; | |
191 | int qpn_inc; | |
192 | int qpn_res_start; | |
193 | int qpn_res_end; | |
194 | int nports; | |
195 | int npkeys; | |
196 | u8 qos_shift; | |
197 | char cq_name[RVT_CQN_MAX]; | |
198 | int node; | |
199 | int max_rdma_atomic; | |
200 | int psn_mask; | |
201 | int psn_shift; | |
202 | int psn_modify_mask; | |
203 | }; | |
204 | ||
205 | /* Protection domain */ | |
206 | struct rvt_pd { | |
207 | struct ib_pd ibpd; | |
208 | int user; /* non-zero if created from user space */ | |
209 | }; | |
210 | ||
211 | /* Address handle */ | |
212 | struct rvt_ah { | |
213 | struct ib_ah ibah; | |
214 | struct ib_ah_attr attr; | |
215 | atomic_t refcount; | |
216 | u8 vl; | |
217 | u8 log_pmtu; | |
218 | }; | |
219 | ||
220 | struct rvt_dev_info; | |
221 | struct rvt_driver_provided { | |
222 | /* | |
223 | * The work to create port files in /sys/class Infiniband is different | |
224 | * depending on the driver. This should not be extracted away and | |
225 | * instead drivers are responsible for setting the correct callback for | |
226 | * this. | |
227 | */ | |
228 | ||
229 | /* -------------------*/ | |
230 | /* Required functions */ | |
231 | /* -------------------*/ | |
232 | int (*port_callback)(struct ib_device *, u8, struct kobject *); | |
233 | const char * (*get_card_name)(struct rvt_dev_info *rdi); | |
234 | struct pci_dev * (*get_pci_dev)(struct rvt_dev_info *rdi); | |
235 | unsigned (*free_all_qps)(struct rvt_dev_info *rdi); | |
236 | void * (*qp_priv_alloc)(struct rvt_dev_info *rdi, struct rvt_qp *qp, | |
237 | gfp_t gfp); | |
238 | void (*qp_priv_free)(struct rvt_dev_info *rdi, struct rvt_qp *qp); | |
239 | void (*notify_qp_reset)(struct rvt_qp *qp); | |
240 | void (*schedule_send)(struct rvt_qp *qp); | |
241 | void (*do_send)(struct rvt_qp *qp); | |
242 | int (*get_pmtu_from_attr)(struct rvt_dev_info *rdi, struct rvt_qp *qp, | |
243 | struct ib_qp_attr *attr); | |
244 | void (*flush_qp_waiters)(struct rvt_qp *qp); | |
245 | void (*stop_send_queue)(struct rvt_qp *qp); | |
246 | void (*quiesce_qp)(struct rvt_qp *qp); | |
247 | void (*notify_error_qp)(struct rvt_qp *qp); | |
248 | u32 (*mtu_from_qp)(struct rvt_dev_info *rdi, struct rvt_qp *qp, | |
249 | u32 pmtu); | |
250 | int (*mtu_to_path_mtu)(u32 mtu); | |
251 | int (*get_guid_be)(struct rvt_dev_info *rdi, struct rvt_ibport *rvp, | |
252 | int guid_index, __be64 *guid); | |
253 | ||
254 | /*--------------------*/ | |
255 | /* Optional functions */ | |
256 | /*--------------------*/ | |
257 | int (*check_ah)(struct ib_device *, struct ib_ah_attr *); | |
258 | void (*notify_new_ah)(struct ib_device *, struct ib_ah_attr *, | |
259 | struct rvt_ah *); | |
260 | int (*alloc_qpn)(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, | |
261 | enum ib_qp_type type, u8 port_num, gfp_t gfp); | |
262 | /** | |
263 | * Return 0 if modification is valid, -errno otherwise | |
264 | */ | |
265 | int (*check_modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr, | |
266 | int attr_mask, struct ib_udata *udata); | |
267 | void (*modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr, | |
268 | int attr_mask, struct ib_udata *udata); | |
269 | ||
270 | int (*check_send_wr)(struct rvt_qp *qp, struct ib_send_wr *wr); | |
271 | ||
272 | void (*notify_create_mad_agent)(struct rvt_dev_info *rdi, int port_idx); | |
273 | void (*notify_free_mad_agent)(struct rvt_dev_info *rdi, int port_idx); | |
274 | }; | |
275 | ||
276 | struct rvt_dev_info { | |
277 | struct ib_device ibdev; /* Keep this first. Nothing above here */ | |
278 | ||
279 | /* | |
280 | * Prior to calling for registration the driver will be responsible for | |
281 | * allocating space for this structure. | |
282 | * | |
283 | * The driver will also be responsible for filling in certain members of | |
284 | * dparms.props. The driver needs to fill in dparms exactly as it would | |
285 | * want values reported to a ULP. This will be returned to the caller | |
286 | * in rdmavt's device. The driver should also therefore refrain from | |
287 | * modifying this directly after registration with rdmavt. | |
288 | */ | |
289 | ||
290 | /* Driver specific properties */ | |
291 | struct rvt_driver_params dparms; | |
292 | ||
293 | struct rvt_mregion __rcu *dma_mr; | |
294 | struct rvt_lkey_table lkey_table; | |
295 | ||
296 | /* Driver specific helper functions */ | |
297 | struct rvt_driver_provided driver_f; | |
298 | ||
299 | /* Internal use */ | |
300 | int n_pds_allocated; | |
301 | spinlock_t n_pds_lock; /* Protect pd allocated count */ | |
302 | ||
303 | int n_ahs_allocated; | |
304 | spinlock_t n_ahs_lock; /* Protect ah allocated count */ | |
305 | ||
306 | u32 n_srqs_allocated; | |
307 | spinlock_t n_srqs_lock; /* Protect srqs allocated count */ | |
308 | ||
309 | int flags; | |
310 | struct rvt_ibport **ports; | |
311 | ||
312 | /* QP */ | |
313 | struct rvt_qp_ibdev *qp_dev; | |
314 | u32 n_qps_allocated; /* number of QPs allocated for device */ | |
315 | spinlock_t n_qps_lock; /* keep track of number of qps */ | |
316 | ||
317 | /* memory maps */ | |
318 | struct list_head pending_mmaps; | |
319 | spinlock_t mmap_offset_lock; /* protect mmap_offset */ | |
320 | u32 mmap_offset; | |
321 | spinlock_t pending_lock; /* protect pending mmap list */ | |
322 | ||
323 | /* CQ */ | |
324 | struct kthread_worker *worker; /* per device cq worker */ | |
325 | u32 n_cqs_allocated; /* number of CQs allocated for device */ | |
326 | spinlock_t n_cqs_lock; /* protect count of in use cqs */ | |
327 | ||
328 | /* Multicast */ | |
329 | u32 n_mcast_grps_allocated; /* number of mcast groups allocated */ | |
330 | spinlock_t n_mcast_grps_lock; | |
331 | ||
332 | }; | |
333 | ||
334 | static inline struct rvt_pd *ibpd_to_rvtpd(struct ib_pd *ibpd) | |
335 | { | |
336 | return container_of(ibpd, struct rvt_pd, ibpd); | |
337 | } | |
338 | ||
339 | static inline struct rvt_ah *ibah_to_rvtah(struct ib_ah *ibah) | |
340 | { | |
341 | return container_of(ibah, struct rvt_ah, ibah); | |
342 | } | |
343 | ||
344 | static inline struct rvt_dev_info *ib_to_rvt(struct ib_device *ibdev) | |
345 | { | |
346 | return container_of(ibdev, struct rvt_dev_info, ibdev); | |
347 | } | |
348 | ||
349 | static inline struct rvt_srq *ibsrq_to_rvtsrq(struct ib_srq *ibsrq) | |
350 | { | |
351 | return container_of(ibsrq, struct rvt_srq, ibsrq); | |
352 | } | |
353 | ||
354 | static inline struct rvt_qp *ibqp_to_rvtqp(struct ib_qp *ibqp) | |
355 | { | |
356 | return container_of(ibqp, struct rvt_qp, ibqp); | |
357 | } | |
358 | ||
359 | static inline unsigned rvt_get_npkeys(struct rvt_dev_info *rdi) | |
360 | { | |
361 | /* | |
362 | * All ports have same number of pkeys. | |
363 | */ | |
364 | return rdi->dparms.npkeys; | |
365 | } | |
366 | ||
367 | /* | |
368 | * Return the indexed PKEY from the port PKEY table. | |
369 | */ | |
370 | static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi, | |
371 | int port_index, | |
372 | unsigned index) | |
373 | { | |
374 | if (index >= rvt_get_npkeys(rdi)) | |
375 | return 0; | |
376 | else | |
377 | return rdi->ports[port_index]->pkey_table[index]; | |
378 | } | |
379 | ||
380 | /** | |
381 | * rvt_lookup_qpn - return the QP with the given QPN | |
382 | * @ibp: the ibport | |
383 | * @qpn: the QP number to look up | |
384 | * | |
385 | * The caller must hold the rcu_read_lock(), and keep the lock until | |
386 | * the returned qp is no longer in use. | |
387 | */ | |
388 | /* TODO: Remove this and put in rdmavt/qp.h when no longer needed by drivers */ | |
389 | static inline struct rvt_qp *rvt_lookup_qpn(struct rvt_dev_info *rdi, | |
390 | struct rvt_ibport *rvp, | |
391 | u32 qpn) __must_hold(RCU) | |
392 | { | |
393 | struct rvt_qp *qp = NULL; | |
394 | ||
395 | if (unlikely(qpn <= 1)) { | |
396 | qp = rcu_dereference(rvp->qp[qpn]); | |
397 | } else { | |
398 | u32 n = hash_32(qpn, rdi->qp_dev->qp_table_bits); | |
399 | ||
400 | for (qp = rcu_dereference(rdi->qp_dev->qp_table[n]); qp; | |
401 | qp = rcu_dereference(qp->next)) | |
402 | if (qp->ibqp.qp_num == qpn) | |
403 | break; | |
404 | } | |
405 | return qp; | |
406 | } | |
407 | ||
408 | struct rvt_dev_info *rvt_alloc_device(size_t size, int nports); | |
409 | int rvt_register_device(struct rvt_dev_info *rvd); | |
410 | void rvt_unregister_device(struct rvt_dev_info *rvd); | |
411 | int rvt_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr); | |
412 | int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port, | |
413 | int port_index, u16 *pkey_table); | |
414 | int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, | |
415 | u32 len, u64 vaddr, u32 rkey, int acc); | |
416 | int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd, | |
417 | struct rvt_sge *isge, struct ib_sge *sge, int acc); | |
418 | int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); | |
419 | void rvt_release_mmap_info(struct kref *ref); | |
420 | struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi, | |
421 | u32 size, | |
422 | struct ib_ucontext *context, | |
423 | void *obj); | |
424 | void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip, | |
425 | u32 size, void *obj); | |
426 | int rvt_reg_mr(struct rvt_qp *qp, struct ib_reg_wr *wr); | |
427 | struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid); | |
428 | ||
429 | /* Temporary export */ | |
430 | void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, | |
431 | enum ib_qp_type type); | |
432 | ||
433 | #endif /* DEF_RDMA_VT_H */ |