]>
Commit | Line | Data |
---|---|---|
77241056 | 1 | /* |
b6eac931 | 2 | * Copyright(c) 2015 - 2017 Intel Corporation. |
77241056 MM |
3 | * |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
77241056 MM |
9 | * This program is free software; you can redistribute it and/or modify |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
77241056 MM |
20 | * Redistribution and use in source and binary forms, with or without |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | ||
48 | #ifndef HFI1_VERBS_H | |
49 | #define HFI1_VERBS_H | |
50 | ||
51 | #include <linux/types.h> | |
52 | #include <linux/seqlock.h> | |
53 | #include <linux/kernel.h> | |
54 | #include <linux/interrupt.h> | |
55 | #include <linux/kref.h> | |
56 | #include <linux/workqueue.h> | |
57 | #include <linux/kthread.h> | |
58 | #include <linux/completion.h> | |
bb5df5f9 | 59 | #include <linux/slab.h> |
77241056 MM |
60 | #include <rdma/ib_pack.h> |
61 | #include <rdma/ib_user_verbs.h> | |
62 | #include <rdma/ib_mad.h> | |
261a4351 | 63 | #include <rdma/ib_hdrs.h> |
ec3f2c12 | 64 | #include <rdma/rdma_vt.h> |
54d10c1e | 65 | #include <rdma/rdmavt_qp.h> |
abd712da | 66 | #include <rdma/rdmavt_cq.h> |
77241056 MM |
67 | |
68 | struct hfi1_ctxtdata; | |
69 | struct hfi1_pportdata; | |
70 | struct hfi1_devdata; | |
71 | struct hfi1_packet; | |
72 | ||
73 | #include "iowait.h" | |
74 | ||
75 | #define HFI1_MAX_RDMA_ATOMIC 16 | |
77241056 MM |
76 | |
77 | /* | |
78 | * Increment this value if any changes that break userspace ABI | |
79 | * compatibility are made. | |
80 | */ | |
81 | #define HFI1_UVERBS_ABI_VERSION 2 | |
82 | ||
77241056 MM |
83 | /* IB Performance Manager status values */ |
84 | #define IB_PMA_SAMPLE_STATUS_DONE 0x00 | |
85 | #define IB_PMA_SAMPLE_STATUS_STARTED 0x01 | |
86 | #define IB_PMA_SAMPLE_STATUS_RUNNING 0x02 | |
87 | ||
88 | /* Mandatory IB performance counter select values. */ | |
89 | #define IB_PMA_PORT_XMIT_DATA cpu_to_be16(0x0001) | |
90 | #define IB_PMA_PORT_RCV_DATA cpu_to_be16(0x0002) | |
91 | #define IB_PMA_PORT_XMIT_PKTS cpu_to_be16(0x0003) | |
92 | #define IB_PMA_PORT_RCV_PKTS cpu_to_be16(0x0004) | |
93 | #define IB_PMA_PORT_XMIT_WAIT cpu_to_be16(0x0005) | |
94 | ||
95 | #define HFI1_VENDOR_IPG cpu_to_be16(0xFFA0) | |
96 | ||
77241056 MM |
97 | #define IB_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL) |
98 | ||
b374e060 MM |
99 | #define RC_OP(x) IB_OPCODE_RC_##x |
100 | #define UC_OP(x) IB_OPCODE_UC_##x | |
101 | ||
77241056 MM |
102 | /* flags passed by hfi1_ib_rcv() */ |
103 | enum { | |
104 | HFI1_HAS_GRH = (1 << 0), | |
105 | }; | |
106 | ||
a9b6b3bc | 107 | struct hfi1_ahg_info { |
77241056 MM |
108 | u32 ahgdesc[2]; |
109 | u16 tx_flags; | |
110 | u8 ahgcount; | |
111 | u8 ahgidx; | |
77241056 MM |
112 | }; |
113 | ||
d4d602e9 | 114 | struct hfi1_sdma_header { |
77241056 | 115 | __le64 pbc; |
261a4351 | 116 | struct ib_header hdr; |
77241056 MM |
117 | } __packed; |
118 | ||
4c6829c5 DD |
119 | /* |
120 | * hfi1 specific data structures that will be hidden from rvt after the queue | |
121 | * pair is made common | |
122 | */ | |
4c6829c5 | 123 | struct hfi1_qp_priv { |
a9b6b3bc | 124 | struct hfi1_ahg_info *s_ahg; /* ahg info for next header */ |
721d0427 JJ |
125 | struct sdma_engine *s_sde; /* current sde */ |
126 | struct send_context *s_sendcontext; /* current sendcontext */ | |
127 | u8 s_sc; /* SC[0..4] for next packet */ | |
4c6829c5 | 128 | struct iowait s_iowait; |
895420dd | 129 | struct rvt_qp *owner; |
77241056 MM |
130 | }; |
131 | ||
d46e5144 DD |
132 | /* |
133 | * This structure is used to hold commonly lookedup and computed values during | |
134 | * the send engine progress. | |
135 | */ | |
136 | struct hfi1_pkt_state { | |
137 | struct hfi1_ibdev *dev; | |
138 | struct hfi1_ibport *ibp; | |
139 | struct hfi1_pportdata *ppd; | |
bb5df5f9 | 140 | struct verbs_txreq *s_txreq; |
747f4d7a | 141 | unsigned long flags; |
dd1ed108 MM |
142 | unsigned long timeout; |
143 | unsigned long timeout_int; | |
144 | int cpu; | |
145 | bool in_thread; | |
d46e5144 DD |
146 | }; |
147 | ||
77241056 MM |
148 | #define HFI1_PSN_CREDIT 16 |
149 | ||
77241056 MM |
150 | struct hfi1_opcode_stats { |
151 | u64 n_packets; /* number of packets */ | |
152 | u64 n_bytes; /* total number of bytes */ | |
153 | }; | |
154 | ||
155 | struct hfi1_opcode_stats_perctx { | |
156 | struct hfi1_opcode_stats stats[256]; | |
157 | }; | |
158 | ||
159 | static inline void inc_opstats( | |
160 | u32 tlen, | |
161 | struct hfi1_opcode_stats *stats) | |
162 | { | |
163 | #ifdef CONFIG_DEBUG_FS | |
164 | stats->n_bytes += tlen; | |
165 | stats->n_packets++; | |
166 | #endif | |
167 | } | |
168 | ||
169 | struct hfi1_ibport { | |
895420dd | 170 | struct rvt_qp __rcu *qp[2]; |
4eb06882 | 171 | struct rvt_ibport rvp; |
4eb06882 | 172 | |
77241056 MM |
173 | /* the first 16 entries are sl_to_vl for !OPA */ |
174 | u8 sl_to_sc[32]; | |
175 | u8 sc_to_sl[32]; | |
176 | }; | |
177 | ||
77241056 | 178 | struct hfi1_ibdev { |
ec3f2c12 | 179 | struct rvt_dev_info rdi; /* Must be first */ |
77241056 | 180 | |
77241056 | 181 | /* QP numbers are shared by all IB ports */ |
4e045572 MM |
182 | /* protect txwait list */ |
183 | seqlock_t txwait_lock ____cacheline_aligned_in_smp; | |
77241056 MM |
184 | struct list_head txwait; /* list for wait verbs_txreq */ |
185 | struct list_head memwait; /* list for wait kernel memory */ | |
77241056 | 186 | struct kmem_cache *verbs_txreq_cache; |
4e045572 MM |
187 | u64 n_txwait; |
188 | u64 n_kmem_wait; | |
77241056 | 189 | |
4e045572 MM |
190 | /* protect iowait lists */ |
191 | seqlock_t iowait_lock ____cacheline_aligned_in_smp; | |
77241056 | 192 | u64 n_piowait; |
14553ca1 | 193 | u64 n_piodrain; |
4e045572 | 194 | struct timer_list mem_timer; |
77241056 | 195 | |
77241056 MM |
196 | #ifdef CONFIG_DEBUG_FS |
197 | /* per HFI debugfs */ | |
198 | struct dentry *hfi1_ibdev_dbg; | |
199 | /* per HFI symlinks to above */ | |
200 | struct dentry *hfi1_ibdev_link; | |
0181ce31 DH |
201 | #ifdef CONFIG_FAULT_INJECTION |
202 | struct fault_opcode *fault_opcode; | |
203 | struct fault_packet *fault_packet; | |
243d9f43 | 204 | bool fault_suppress_err; |
0181ce31 | 205 | #endif |
77241056 MM |
206 | #endif |
207 | }; | |
208 | ||
77241056 MM |
209 | static inline struct hfi1_ibdev *to_idev(struct ib_device *ibdev) |
210 | { | |
ec3f2c12 DD |
211 | struct rvt_dev_info *rdi; |
212 | ||
213 | rdi = container_of(ibdev, struct rvt_dev_info, ibdev); | |
214 | return container_of(rdi, struct hfi1_ibdev, rdi); | |
77241056 MM |
215 | } |
216 | ||
895420dd | 217 | static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait) |
4c6829c5 DD |
218 | { |
219 | struct hfi1_qp_priv *priv; | |
220 | ||
221 | priv = container_of(s_iowait, struct hfi1_qp_priv, s_iowait); | |
222 | return priv->owner; | |
223 | } | |
224 | ||
77241056 MM |
225 | /* |
226 | * Send if not busy or waiting for I/O and either | |
227 | * a RC response is pending or we can process send work requests. | |
228 | */ | |
895420dd | 229 | static inline int hfi1_send_ok(struct rvt_qp *qp) |
77241056 | 230 | { |
54d10c1e DD |
231 | return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && |
232 | (qp->s_hdrwords || (qp->s_flags & RVT_S_RESP_PENDING) || | |
233 | !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); | |
77241056 MM |
234 | } |
235 | ||
236 | /* | |
237 | * This must be called with s_lock held. | |
238 | */ | |
77241056 | 239 | void hfi1_bad_pqkey(struct hfi1_ibport *ibp, __be16 trap_num, u32 key, u32 sl, |
5cd24119 | 240 | u32 qp1, u32 qp2, u16 lid1, u16 lid2); |
45b59eef | 241 | void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num); |
77241056 MM |
242 | void hfi1_sys_guid_chg(struct hfi1_ibport *ibp); |
243 | void hfi1_node_desc_chg(struct hfi1_ibport *ibp); | |
244 | int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, | |
245 | const struct ib_wc *in_wc, const struct ib_grh *in_grh, | |
246 | const struct ib_mad_hdr *in_mad, size_t in_mad_size, | |
247 | struct ib_mad_hdr *out_mad, size_t *out_mad_size, | |
248 | u16 *out_mad_pkey_index); | |
77241056 MM |
249 | |
250 | /* | |
251 | * The PSN_MASK and PSN_SHIFT allow for | |
252 | * 1) comparing two PSNs | |
253 | * 2) returning the PSN with any upper bits masked | |
254 | * 3) returning the difference between to PSNs | |
255 | * | |
256 | * The number of significant bits in the PSN must | |
257 | * necessarily be at least one bit less than | |
258 | * the container holding the PSN. | |
259 | */ | |
260 | #ifndef CONFIG_HFI1_VERBS_31BIT_PSN | |
261 | #define PSN_MASK 0xFFFFFF | |
262 | #define PSN_SHIFT 8 | |
263 | #else | |
264 | #define PSN_MASK 0x7FFFFFFF | |
265 | #define PSN_SHIFT 1 | |
266 | #endif | |
267 | #define PSN_MODIFY_MASK 0xFFFFFF | |
268 | ||
77241056 MM |
269 | /* |
270 | * Compare two PSNs | |
271 | * Returns an integer <, ==, or > than zero. | |
272 | */ | |
273 | static inline int cmp_psn(u32 a, u32 b) | |
274 | { | |
50e5dcbe | 275 | return (((int)a) - ((int)b)) << PSN_SHIFT; |
77241056 MM |
276 | } |
277 | ||
278 | /* | |
279 | * Return masked PSN | |
280 | */ | |
281 | static inline u32 mask_psn(u32 a) | |
282 | { | |
283 | return a & PSN_MASK; | |
284 | } | |
285 | ||
286 | /* | |
287 | * Return delta between two PSNs | |
288 | */ | |
289 | static inline u32 delta_psn(u32 a, u32 b) | |
290 | { | |
291 | return (((int)a - (int)b) << PSN_SHIFT) >> PSN_SHIFT; | |
292 | } | |
293 | ||
77241056 MM |
294 | struct verbs_txreq; |
295 | void hfi1_put_txreq(struct verbs_txreq *tx); | |
296 | ||
895420dd | 297 | int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
77241056 | 298 | |
895420dd | 299 | void hfi1_copy_sge(struct rvt_sge_state *ss, void *data, u32 length, |
0128fcea | 300 | bool release, bool copy_last); |
77241056 | 301 | |
77241056 MM |
302 | void hfi1_cnp_rcv(struct hfi1_packet *packet); |
303 | ||
304 | void hfi1_uc_rcv(struct hfi1_packet *packet); | |
305 | ||
306 | void hfi1_rc_rcv(struct hfi1_packet *packet); | |
307 | ||
308 | void hfi1_rc_hdrerr( | |
309 | struct hfi1_ctxtdata *rcd, | |
261a4351 | 310 | struct ib_header *hdr, |
77241056 | 311 | u32 rcv_flags, |
895420dd | 312 | struct rvt_qp *qp); |
77241056 | 313 | |
90898850 | 314 | u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr); |
77241056 | 315 | |
77241056 MM |
316 | struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid); |
317 | ||
261a4351 | 318 | void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr); |
77241056 | 319 | |
77241056 MM |
320 | void hfi1_ud_rcv(struct hfi1_packet *packet); |
321 | ||
322 | int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey); | |
323 | ||
ec4274f1 | 324 | int hfi1_rvt_get_rwqe(struct rvt_qp *qp, int wr_id_only); |
895420dd DD |
325 | |
326 | void hfi1_migrate_qp(struct rvt_qp *qp); | |
77241056 | 327 | |
ec4274f1 DD |
328 | int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
329 | int attr_mask, struct ib_udata *udata); | |
330 | ||
331 | void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, | |
332 | int attr_mask, struct ib_udata *udata); | |
56acbbfb | 333 | void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait); |
46a80d62 | 334 | int hfi1_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); |
31e7af1c | 335 | |
14553ca1 MM |
336 | extern const u32 rc_only_opcode; |
337 | extern const u32 uc_only_opcode; | |
338 | ||
261a4351 | 339 | static inline u8 get_opcode(struct ib_header *h) |
14553ca1 MM |
340 | { |
341 | u16 lnh = be16_to_cpu(h->lrh[0]) & 3; | |
342 | ||
343 | if (lnh == IB_LNH_IBA_LOCAL) | |
344 | return be32_to_cpu(h->u.oth.bth[0]) >> 24; | |
345 | else | |
346 | return be32_to_cpu(h->u.l.oth.bth[0]) >> 24; | |
347 | } | |
348 | ||
261a4351 | 349 | int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct ib_header *hdr, |
895420dd | 350 | int has_grh, struct rvt_qp *qp, u32 bth0); |
77241056 MM |
351 | |
352 | u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr, | |
d8966fcd | 353 | const struct ib_global_route *grh, u32 hwords, u32 nwords); |
77241056 | 354 | |
261a4351 | 355 | void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, |
bb5df5f9 DD |
356 | u32 bth0, u32 bth2, int middle, |
357 | struct hfi1_pkt_state *ps); | |
77241056 | 358 | |
83693bd1 DD |
359 | void _hfi1_do_send(struct work_struct *work); |
360 | ||
b6eac931 MM |
361 | void hfi1_do_send_from_rvt(struct rvt_qp *qp); |
362 | ||
363 | void hfi1_do_send(struct rvt_qp *qp, bool in_thread); | |
77241056 | 364 | |
895420dd | 365 | void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, |
77241056 MM |
366 | enum ib_wc_status status); |
367 | ||
895420dd | 368 | void hfi1_send_rc_ack(struct hfi1_ctxtdata *, struct rvt_qp *qp, int is_fecn); |
77241056 | 369 | |
bb5df5f9 | 370 | int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
77241056 | 371 | |
bb5df5f9 | 372 | int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
77241056 | 373 | |
bb5df5f9 | 374 | int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
77241056 MM |
375 | |
376 | int hfi1_register_ib_device(struct hfi1_devdata *); | |
377 | ||
378 | void hfi1_unregister_ib_device(struct hfi1_devdata *); | |
379 | ||
380 | void hfi1_ib_rcv(struct hfi1_packet *packet); | |
381 | ||
382 | unsigned hfi1_get_npkeys(struct hfi1_devdata *); | |
383 | ||
895420dd | 384 | int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
d46e5144 | 385 | u64 pbc); |
77241056 | 386 | |
895420dd | 387 | int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
d46e5144 | 388 | u64 pbc); |
77241056 | 389 | |
528ee9fb DL |
390 | int hfi1_wss_init(void); |
391 | void hfi1_wss_exit(void); | |
392 | ||
393 | /* platform specific: return the lowest level cache (llc) size, in KiB */ | |
394 | static inline int wss_llc_size(void) | |
395 | { | |
396 | /* assume that the boot CPU value is universal for all CPUs */ | |
397 | return boot_cpu_data.x86_cache_size; | |
398 | } | |
399 | ||
400 | /* platform specific: cacheless copy */ | |
401 | static inline void cacheless_memcpy(void *dst, void *src, size_t n) | |
402 | { | |
403 | /* | |
404 | * Use the only available X64 cacheless copy. Add a __user cast | |
405 | * to quiet sparse. The src agument is already in the kernel so | |
406 | * there are no security issues. The extra fault recovery machinery | |
407 | * is not invoked. | |
408 | */ | |
409 | __copy_user_nocache(dst, (void __user *)src, n, 0); | |
410 | } | |
411 | ||
77241056 MM |
412 | extern const enum ib_wc_opcode ib_hfi1_wc_opcode[]; |
413 | ||
414 | extern const u8 hdr_len_by_opcode[]; | |
415 | ||
83693bd1 | 416 | extern const int ib_rvt_state_ops[]; |
77241056 MM |
417 | |
418 | extern __be64 ib_hfi1_sys_image_guid; /* in network order */ | |
419 | ||
77241056 MM |
420 | extern unsigned int hfi1_max_cqes; |
421 | ||
422 | extern unsigned int hfi1_max_cqs; | |
423 | ||
424 | extern unsigned int hfi1_max_qp_wrs; | |
425 | ||
426 | extern unsigned int hfi1_max_qps; | |
427 | ||
428 | extern unsigned int hfi1_max_sges; | |
429 | ||
430 | extern unsigned int hfi1_max_mcast_grps; | |
431 | ||
432 | extern unsigned int hfi1_max_mcast_qp_attached; | |
433 | ||
434 | extern unsigned int hfi1_max_srqs; | |
435 | ||
436 | extern unsigned int hfi1_max_srq_sges; | |
437 | ||
438 | extern unsigned int hfi1_max_srq_wrs; | |
439 | ||
14553ca1 MM |
440 | extern unsigned short piothreshold; |
441 | ||
77241056 MM |
442 | extern const u32 ib_hfi1_rnr_table[]; |
443 | ||
77241056 | 444 | #endif /* HFI1_VERBS_H */ |