]>
Commit | Line | Data |
---|---|---|
77241056 | 1 | /* |
05d6ac1d | 2 | * Copyright(c) 2015, 2016 Intel Corporation. |
77241056 MM |
3 | * |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
77241056 MM |
9 | * This program is free software; you can redistribute it and/or modify |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
77241056 MM |
20 | * Redistribution and use in source and binary forms, with or without |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | ||
48 | #include <rdma/ib_mad.h> | |
49 | #include <rdma/ib_user_verbs.h> | |
50 | #include <linux/io.h> | |
51 | #include <linux/module.h> | |
52 | #include <linux/utsname.h> | |
53 | #include <linux/rculist.h> | |
54 | #include <linux/mm.h> | |
77241056 MM |
55 | #include <linux/vmalloc.h> |
56 | ||
57 | #include "hfi.h" | |
58 | #include "common.h" | |
59 | #include "device.h" | |
60 | #include "trace.h" | |
61 | #include "qp.h" | |
45842abb | 62 | #include "verbs_txreq.h" |
77241056 | 63 | |
895420dd | 64 | static unsigned int hfi1_lkey_table_size = 16; |
77241056 MM |
65 | module_param_named(lkey_table_size, hfi1_lkey_table_size, uint, |
66 | S_IRUGO); | |
67 | MODULE_PARM_DESC(lkey_table_size, | |
68 | "LKEY table size in bits (2^n, 1 <= n <= 23)"); | |
69 | ||
70 | static unsigned int hfi1_max_pds = 0xFFFF; | |
71 | module_param_named(max_pds, hfi1_max_pds, uint, S_IRUGO); | |
72 | MODULE_PARM_DESC(max_pds, | |
73 | "Maximum number of protection domains to support"); | |
74 | ||
75 | static unsigned int hfi1_max_ahs = 0xFFFF; | |
76 | module_param_named(max_ahs, hfi1_max_ahs, uint, S_IRUGO); | |
77 | MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support"); | |
78 | ||
79 | unsigned int hfi1_max_cqes = 0x2FFFF; | |
80 | module_param_named(max_cqes, hfi1_max_cqes, uint, S_IRUGO); | |
81 | MODULE_PARM_DESC(max_cqes, | |
82 | "Maximum number of completion queue entries to support"); | |
83 | ||
84 | unsigned int hfi1_max_cqs = 0x1FFFF; | |
85 | module_param_named(max_cqs, hfi1_max_cqs, uint, S_IRUGO); | |
86 | MODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support"); | |
87 | ||
88 | unsigned int hfi1_max_qp_wrs = 0x3FFF; | |
89 | module_param_named(max_qp_wrs, hfi1_max_qp_wrs, uint, S_IRUGO); | |
90 | MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support"); | |
91 | ||
92 | unsigned int hfi1_max_qps = 16384; | |
93 | module_param_named(max_qps, hfi1_max_qps, uint, S_IRUGO); | |
94 | MODULE_PARM_DESC(max_qps, "Maximum number of QPs to support"); | |
95 | ||
96 | unsigned int hfi1_max_sges = 0x60; | |
97 | module_param_named(max_sges, hfi1_max_sges, uint, S_IRUGO); | |
98 | MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support"); | |
99 | ||
100 | unsigned int hfi1_max_mcast_grps = 16384; | |
101 | module_param_named(max_mcast_grps, hfi1_max_mcast_grps, uint, S_IRUGO); | |
102 | MODULE_PARM_DESC(max_mcast_grps, | |
103 | "Maximum number of multicast groups to support"); | |
104 | ||
105 | unsigned int hfi1_max_mcast_qp_attached = 16; | |
106 | module_param_named(max_mcast_qp_attached, hfi1_max_mcast_qp_attached, | |
107 | uint, S_IRUGO); | |
108 | MODULE_PARM_DESC(max_mcast_qp_attached, | |
109 | "Maximum number of attached QPs to support"); | |
110 | ||
111 | unsigned int hfi1_max_srqs = 1024; | |
112 | module_param_named(max_srqs, hfi1_max_srqs, uint, S_IRUGO); | |
113 | MODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support"); | |
114 | ||
115 | unsigned int hfi1_max_srq_sges = 128; | |
116 | module_param_named(max_srq_sges, hfi1_max_srq_sges, uint, S_IRUGO); | |
117 | MODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support"); | |
118 | ||
119 | unsigned int hfi1_max_srq_wrs = 0x1FFFF; | |
120 | module_param_named(max_srq_wrs, hfi1_max_srq_wrs, uint, S_IRUGO); | |
121 | MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support"); | |
122 | ||
d0e859c3 | 123 | unsigned short piothreshold = 256; |
14553ca1 MM |
124 | module_param(piothreshold, ushort, S_IRUGO); |
125 | MODULE_PARM_DESC(piothreshold, "size used to determine sdma vs. pio"); | |
126 | ||
528ee9fb DL |
127 | #define COPY_CACHELESS 1 |
128 | #define COPY_ADAPTIVE 2 | |
129 | static unsigned int sge_copy_mode; | |
130 | module_param(sge_copy_mode, uint, S_IRUGO); | |
131 | MODULE_PARM_DESC(sge_copy_mode, | |
132 | "Verbs copy mode: 0 use memcpy, 1 use cacheless copy, 2 adapt based on WSS"); | |
133 | ||
77241056 MM |
134 | static void verbs_sdma_complete( |
135 | struct sdma_txreq *cookie, | |
a545f530 | 136 | int status); |
77241056 | 137 | |
14553ca1 MM |
138 | static int pio_wait(struct rvt_qp *qp, |
139 | struct send_context *sc, | |
140 | struct hfi1_pkt_state *ps, | |
141 | u32 flag); | |
142 | ||
64ffd86c JJ |
143 | /* Length of buffer to create verbs txreq cache name */ |
144 | #define TXREQ_NAME_LEN 24 | |
145 | ||
528ee9fb DL |
146 | static uint wss_threshold; |
147 | module_param(wss_threshold, uint, S_IRUGO); | |
148 | MODULE_PARM_DESC(wss_threshold, "Percentage (1-100) of LLC to use as a threshold for a cacheless copy"); | |
149 | static uint wss_clean_period = 256; | |
150 | module_param(wss_clean_period, uint, S_IRUGO); | |
151 | MODULE_PARM_DESC(wss_clean_period, "Count of verbs copies before an entry in the page copy table is cleaned"); | |
152 | ||
153 | /* memory working set size */ | |
154 | struct hfi1_wss { | |
155 | unsigned long *entries; | |
156 | atomic_t total_count; | |
157 | atomic_t clean_counter; | |
158 | atomic_t clean_entry; | |
159 | ||
160 | int threshold; | |
161 | int num_entries; | |
162 | long pages_mask; | |
163 | }; | |
164 | ||
165 | static struct hfi1_wss wss; | |
166 | ||
167 | int hfi1_wss_init(void) | |
168 | { | |
169 | long llc_size; | |
170 | long llc_bits; | |
171 | long table_size; | |
172 | long table_bits; | |
173 | ||
174 | /* check for a valid percent range - default to 80 if none or invalid */ | |
175 | if (wss_threshold < 1 || wss_threshold > 100) | |
176 | wss_threshold = 80; | |
177 | /* reject a wildly large period */ | |
178 | if (wss_clean_period > 1000000) | |
179 | wss_clean_period = 256; | |
180 | /* reject a zero period */ | |
181 | if (wss_clean_period == 0) | |
182 | wss_clean_period = 1; | |
183 | ||
184 | /* | |
185 | * Calculate the table size - the next power of 2 larger than the | |
186 | * LLC size. LLC size is in KiB. | |
187 | */ | |
188 | llc_size = wss_llc_size() * 1024; | |
189 | table_size = roundup_pow_of_two(llc_size); | |
190 | ||
191 | /* one bit per page in rounded up table */ | |
192 | llc_bits = llc_size / PAGE_SIZE; | |
193 | table_bits = table_size / PAGE_SIZE; | |
194 | wss.pages_mask = table_bits - 1; | |
195 | wss.num_entries = table_bits / BITS_PER_LONG; | |
196 | ||
197 | wss.threshold = (llc_bits * wss_threshold) / 100; | |
198 | if (wss.threshold == 0) | |
199 | wss.threshold = 1; | |
200 | ||
201 | atomic_set(&wss.clean_counter, wss_clean_period); | |
202 | ||
203 | wss.entries = kcalloc(wss.num_entries, sizeof(*wss.entries), | |
204 | GFP_KERNEL); | |
205 | if (!wss.entries) { | |
206 | hfi1_wss_exit(); | |
207 | return -ENOMEM; | |
208 | } | |
209 | ||
210 | return 0; | |
211 | } | |
212 | ||
213 | void hfi1_wss_exit(void) | |
214 | { | |
215 | /* coded to handle partially initialized and repeat callers */ | |
216 | kfree(wss.entries); | |
217 | wss.entries = NULL; | |
218 | } | |
219 | ||
220 | /* | |
221 | * Advance the clean counter. When the clean period has expired, | |
222 | * clean an entry. | |
223 | * | |
224 | * This is implemented in atomics to avoid locking. Because multiple | |
225 | * variables are involved, it can be racy which can lead to slightly | |
226 | * inaccurate information. Since this is only a heuristic, this is | |
227 | * OK. Any innaccuracies will clean themselves out as the counter | |
228 | * advances. That said, it is unlikely the entry clean operation will | |
229 | * race - the next possible racer will not start until the next clean | |
230 | * period. | |
231 | * | |
232 | * The clean counter is implemented as a decrement to zero. When zero | |
233 | * is reached an entry is cleaned. | |
234 | */ | |
235 | static void wss_advance_clean_counter(void) | |
236 | { | |
237 | int entry; | |
238 | int weight; | |
239 | unsigned long bits; | |
240 | ||
241 | /* become the cleaner if we decrement the counter to zero */ | |
242 | if (atomic_dec_and_test(&wss.clean_counter)) { | |
243 | /* | |
244 | * Set, not add, the clean period. This avoids an issue | |
245 | * where the counter could decrement below the clean period. | |
246 | * Doing a set can result in lost decrements, slowing the | |
247 | * clean advance. Since this a heuristic, this possible | |
248 | * slowdown is OK. | |
249 | * | |
250 | * An alternative is to loop, advancing the counter by a | |
251 | * clean period until the result is > 0. However, this could | |
252 | * lead to several threads keeping another in the clean loop. | |
253 | * This could be mitigated by limiting the number of times | |
254 | * we stay in the loop. | |
255 | */ | |
256 | atomic_set(&wss.clean_counter, wss_clean_period); | |
257 | ||
258 | /* | |
259 | * Uniquely grab the entry to clean and move to next. | |
260 | * The current entry is always the lower bits of | |
261 | * wss.clean_entry. The table size, wss.num_entries, | |
262 | * is always a power-of-2. | |
263 | */ | |
264 | entry = (atomic_inc_return(&wss.clean_entry) - 1) | |
265 | & (wss.num_entries - 1); | |
266 | ||
267 | /* clear the entry and count the bits */ | |
268 | bits = xchg(&wss.entries[entry], 0); | |
269 | weight = hweight64((u64)bits); | |
270 | /* only adjust the contended total count if needed */ | |
271 | if (weight) | |
272 | atomic_sub(weight, &wss.total_count); | |
273 | } | |
274 | } | |
275 | ||
276 | /* | |
277 | * Insert the given address into the working set array. | |
278 | */ | |
279 | static void wss_insert(void *address) | |
280 | { | |
281 | u32 page = ((unsigned long)address >> PAGE_SHIFT) & wss.pages_mask; | |
282 | u32 entry = page / BITS_PER_LONG; /* assumes this ends up a shift */ | |
283 | u32 nr = page & (BITS_PER_LONG - 1); | |
284 | ||
285 | if (!test_and_set_bit(nr, &wss.entries[entry])) | |
286 | atomic_inc(&wss.total_count); | |
287 | ||
288 | wss_advance_clean_counter(); | |
289 | } | |
290 | ||
291 | /* | |
292 | * Is the working set larger than the threshold? | |
293 | */ | |
294 | static inline int wss_exceeds_threshold(void) | |
295 | { | |
296 | return atomic_read(&wss.total_count) >= wss.threshold; | |
297 | } | |
298 | ||
77241056 MM |
299 | /* |
300 | * Translate ib_wr_opcode into ib_wc_opcode. | |
301 | */ | |
302 | const enum ib_wc_opcode ib_hfi1_wc_opcode[] = { | |
303 | [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE, | |
304 | [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, | |
305 | [IB_WR_SEND] = IB_WC_SEND, | |
306 | [IB_WR_SEND_WITH_IMM] = IB_WC_SEND, | |
307 | [IB_WR_RDMA_READ] = IB_WC_RDMA_READ, | |
308 | [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP, | |
0db3dfa0 JX |
309 | [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD, |
310 | [IB_WR_SEND_WITH_INV] = IB_WC_SEND, | |
311 | [IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV, | |
312 | [IB_WR_REG_MR] = IB_WC_REG_MR | |
77241056 MM |
313 | }; |
314 | ||
315 | /* | |
316 | * Length of header by opcode, 0 --> not supported | |
317 | */ | |
318 | const u8 hdr_len_by_opcode[256] = { | |
319 | /* RC */ | |
320 | [IB_OPCODE_RC_SEND_FIRST] = 12 + 8, | |
321 | [IB_OPCODE_RC_SEND_MIDDLE] = 12 + 8, | |
322 | [IB_OPCODE_RC_SEND_LAST] = 12 + 8, | |
323 | [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, | |
324 | [IB_OPCODE_RC_SEND_ONLY] = 12 + 8, | |
325 | [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = 12 + 8 + 4, | |
326 | [IB_OPCODE_RC_RDMA_WRITE_FIRST] = 12 + 8 + 16, | |
327 | [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = 12 + 8, | |
328 | [IB_OPCODE_RC_RDMA_WRITE_LAST] = 12 + 8, | |
329 | [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, | |
330 | [IB_OPCODE_RC_RDMA_WRITE_ONLY] = 12 + 8 + 16, | |
331 | [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = 12 + 8 + 20, | |
332 | [IB_OPCODE_RC_RDMA_READ_REQUEST] = 12 + 8 + 16, | |
333 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = 12 + 8 + 4, | |
334 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = 12 + 8, | |
335 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = 12 + 8 + 4, | |
336 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = 12 + 8 + 4, | |
337 | [IB_OPCODE_RC_ACKNOWLEDGE] = 12 + 8 + 4, | |
338 | [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = 12 + 8 + 4, | |
339 | [IB_OPCODE_RC_COMPARE_SWAP] = 12 + 8 + 28, | |
340 | [IB_OPCODE_RC_FETCH_ADD] = 12 + 8 + 28, | |
bdd8a98c JX |
341 | [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = 12 + 8 + 4, |
342 | [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = 12 + 8 + 4, | |
77241056 MM |
343 | /* UC */ |
344 | [IB_OPCODE_UC_SEND_FIRST] = 12 + 8, | |
345 | [IB_OPCODE_UC_SEND_MIDDLE] = 12 + 8, | |
346 | [IB_OPCODE_UC_SEND_LAST] = 12 + 8, | |
347 | [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, | |
348 | [IB_OPCODE_UC_SEND_ONLY] = 12 + 8, | |
349 | [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = 12 + 8 + 4, | |
350 | [IB_OPCODE_UC_RDMA_WRITE_FIRST] = 12 + 8 + 16, | |
351 | [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = 12 + 8, | |
352 | [IB_OPCODE_UC_RDMA_WRITE_LAST] = 12 + 8, | |
353 | [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, | |
354 | [IB_OPCODE_UC_RDMA_WRITE_ONLY] = 12 + 8 + 16, | |
355 | [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = 12 + 8 + 20, | |
356 | /* UD */ | |
357 | [IB_OPCODE_UD_SEND_ONLY] = 12 + 8 + 8, | |
358 | [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = 12 + 8 + 12 | |
359 | }; | |
360 | ||
361 | static const opcode_handler opcode_handler_tbl[256] = { | |
362 | /* RC */ | |
363 | [IB_OPCODE_RC_SEND_FIRST] = &hfi1_rc_rcv, | |
364 | [IB_OPCODE_RC_SEND_MIDDLE] = &hfi1_rc_rcv, | |
365 | [IB_OPCODE_RC_SEND_LAST] = &hfi1_rc_rcv, | |
366 | [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = &hfi1_rc_rcv, | |
367 | [IB_OPCODE_RC_SEND_ONLY] = &hfi1_rc_rcv, | |
368 | [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = &hfi1_rc_rcv, | |
369 | [IB_OPCODE_RC_RDMA_WRITE_FIRST] = &hfi1_rc_rcv, | |
370 | [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = &hfi1_rc_rcv, | |
371 | [IB_OPCODE_RC_RDMA_WRITE_LAST] = &hfi1_rc_rcv, | |
372 | [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = &hfi1_rc_rcv, | |
373 | [IB_OPCODE_RC_RDMA_WRITE_ONLY] = &hfi1_rc_rcv, | |
374 | [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = &hfi1_rc_rcv, | |
375 | [IB_OPCODE_RC_RDMA_READ_REQUEST] = &hfi1_rc_rcv, | |
376 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = &hfi1_rc_rcv, | |
377 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = &hfi1_rc_rcv, | |
378 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = &hfi1_rc_rcv, | |
379 | [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = &hfi1_rc_rcv, | |
380 | [IB_OPCODE_RC_ACKNOWLEDGE] = &hfi1_rc_rcv, | |
381 | [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = &hfi1_rc_rcv, | |
382 | [IB_OPCODE_RC_COMPARE_SWAP] = &hfi1_rc_rcv, | |
383 | [IB_OPCODE_RC_FETCH_ADD] = &hfi1_rc_rcv, | |
a2df0c83 JX |
384 | [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = &hfi1_rc_rcv, |
385 | [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = &hfi1_rc_rcv, | |
77241056 MM |
386 | /* UC */ |
387 | [IB_OPCODE_UC_SEND_FIRST] = &hfi1_uc_rcv, | |
388 | [IB_OPCODE_UC_SEND_MIDDLE] = &hfi1_uc_rcv, | |
389 | [IB_OPCODE_UC_SEND_LAST] = &hfi1_uc_rcv, | |
390 | [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = &hfi1_uc_rcv, | |
391 | [IB_OPCODE_UC_SEND_ONLY] = &hfi1_uc_rcv, | |
392 | [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = &hfi1_uc_rcv, | |
393 | [IB_OPCODE_UC_RDMA_WRITE_FIRST] = &hfi1_uc_rcv, | |
394 | [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = &hfi1_uc_rcv, | |
395 | [IB_OPCODE_UC_RDMA_WRITE_LAST] = &hfi1_uc_rcv, | |
396 | [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = &hfi1_uc_rcv, | |
397 | [IB_OPCODE_UC_RDMA_WRITE_ONLY] = &hfi1_uc_rcv, | |
398 | [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = &hfi1_uc_rcv, | |
399 | /* UD */ | |
400 | [IB_OPCODE_UD_SEND_ONLY] = &hfi1_ud_rcv, | |
401 | [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = &hfi1_ud_rcv, | |
402 | /* CNP */ | |
403 | [IB_OPCODE_CNP] = &hfi1_cnp_rcv | |
404 | }; | |
405 | ||
406 | /* | |
407 | * System image GUID. | |
408 | */ | |
409 | __be64 ib_hfi1_sys_image_guid; | |
410 | ||
411 | /** | |
412 | * hfi1_copy_sge - copy data to SGE memory | |
413 | * @ss: the SGE state | |
414 | * @data: the data to copy | |
415 | * @length: the length of the data | |
7b0b01aa | 416 | * @copy_last: do a separate copy of the last 8 bytes |
77241056 MM |
417 | */ |
418 | void hfi1_copy_sge( | |
895420dd | 419 | struct rvt_sge_state *ss, |
77241056 | 420 | void *data, u32 length, |
7b0b01aa DL |
421 | int release, |
422 | int copy_last) | |
77241056 | 423 | { |
895420dd | 424 | struct rvt_sge *sge = &ss->sge; |
7b0b01aa DL |
425 | int in_last = 0; |
426 | int i; | |
528ee9fb DL |
427 | int cacheless_copy = 0; |
428 | ||
429 | if (sge_copy_mode == COPY_CACHELESS) { | |
430 | cacheless_copy = length >= PAGE_SIZE; | |
431 | } else if (sge_copy_mode == COPY_ADAPTIVE) { | |
432 | if (length >= PAGE_SIZE) { | |
433 | /* | |
434 | * NOTE: this *assumes*: | |
435 | * o The first vaddr is the dest. | |
436 | * o If multiple pages, then vaddr is sequential. | |
437 | */ | |
438 | wss_insert(sge->vaddr); | |
439 | if (length >= (2 * PAGE_SIZE)) | |
440 | wss_insert(sge->vaddr + PAGE_SIZE); | |
77241056 | 441 | |
528ee9fb DL |
442 | cacheless_copy = wss_exceeds_threshold(); |
443 | } else { | |
444 | wss_advance_clean_counter(); | |
445 | } | |
446 | } | |
7b0b01aa DL |
447 | if (copy_last) { |
448 | if (length > 8) { | |
449 | length -= 8; | |
450 | } else { | |
451 | copy_last = 0; | |
452 | in_last = 1; | |
453 | } | |
454 | } | |
455 | ||
456 | again: | |
77241056 MM |
457 | while (length) { |
458 | u32 len = sge->length; | |
459 | ||
460 | if (len > length) | |
461 | len = length; | |
462 | if (len > sge->sge_length) | |
463 | len = sge->sge_length; | |
464 | WARN_ON_ONCE(len == 0); | |
528ee9fb DL |
465 | if (unlikely(in_last)) { |
466 | /* enforce byte transfer ordering */ | |
7b0b01aa DL |
467 | for (i = 0; i < len; i++) |
468 | ((u8 *)sge->vaddr)[i] = ((u8 *)data)[i]; | |
528ee9fb DL |
469 | } else if (cacheless_copy) { |
470 | cacheless_memcpy(sge->vaddr, data, len); | |
7b0b01aa DL |
471 | } else { |
472 | memcpy(sge->vaddr, data, len); | |
473 | } | |
77241056 MM |
474 | sge->vaddr += len; |
475 | sge->length -= len; | |
476 | sge->sge_length -= len; | |
477 | if (sge->sge_length == 0) { | |
478 | if (release) | |
895420dd | 479 | rvt_put_mr(sge->mr); |
77241056 MM |
480 | if (--ss->num_sge) |
481 | *sge = *ss->sg_list++; | |
482 | } else if (sge->length == 0 && sge->mr->lkey) { | |
cd4ceee3 | 483 | if (++sge->n >= RVT_SEGSZ) { |
77241056 MM |
484 | if (++sge->m >= sge->mr->mapsz) |
485 | break; | |
486 | sge->n = 0; | |
487 | } | |
488 | sge->vaddr = | |
489 | sge->mr->map[sge->m]->segs[sge->n].vaddr; | |
490 | sge->length = | |
491 | sge->mr->map[sge->m]->segs[sge->n].length; | |
492 | } | |
493 | data += len; | |
494 | length -= len; | |
495 | } | |
7b0b01aa DL |
496 | |
497 | if (copy_last) { | |
498 | copy_last = 0; | |
499 | in_last = 1; | |
500 | length = 8; | |
501 | goto again; | |
502 | } | |
77241056 MM |
503 | } |
504 | ||
505 | /** | |
506 | * hfi1_skip_sge - skip over SGE memory | |
507 | * @ss: the SGE state | |
508 | * @length: the number of bytes to skip | |
509 | */ | |
895420dd | 510 | void hfi1_skip_sge(struct rvt_sge_state *ss, u32 length, int release) |
77241056 | 511 | { |
895420dd | 512 | struct rvt_sge *sge = &ss->sge; |
77241056 MM |
513 | |
514 | while (length) { | |
515 | u32 len = sge->length; | |
516 | ||
517 | if (len > length) | |
518 | len = length; | |
519 | if (len > sge->sge_length) | |
520 | len = sge->sge_length; | |
521 | WARN_ON_ONCE(len == 0); | |
522 | sge->vaddr += len; | |
523 | sge->length -= len; | |
524 | sge->sge_length -= len; | |
525 | if (sge->sge_length == 0) { | |
526 | if (release) | |
895420dd | 527 | rvt_put_mr(sge->mr); |
77241056 MM |
528 | if (--ss->num_sge) |
529 | *sge = *ss->sg_list++; | |
530 | } else if (sge->length == 0 && sge->mr->lkey) { | |
cd4ceee3 | 531 | if (++sge->n >= RVT_SEGSZ) { |
77241056 MM |
532 | if (++sge->m >= sge->mr->mapsz) |
533 | break; | |
534 | sge->n = 0; | |
535 | } | |
536 | sge->vaddr = | |
537 | sge->mr->map[sge->m]->segs[sge->n].vaddr; | |
538 | sge->length = | |
539 | sge->mr->map[sge->m]->segs[sge->n].length; | |
540 | } | |
541 | length -= len; | |
542 | } | |
543 | } | |
544 | ||
77241056 MM |
545 | /* |
546 | * Make sure the QP is ready and able to accept the given opcode. | |
547 | */ | |
71e68e3d | 548 | static inline opcode_handler qp_ok(int opcode, struct hfi1_packet *packet) |
77241056 | 549 | { |
83693bd1 | 550 | if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK)) |
71e68e3d | 551 | return NULL; |
b218f786 | 552 | if (((opcode & RVT_OPCODE_QP_MASK) == packet->qp->allowed_ops) || |
77241056 | 553 | (opcode == IB_OPCODE_CNP)) |
71e68e3d JP |
554 | return opcode_handler_tbl[opcode]; |
555 | ||
556 | return NULL; | |
77241056 MM |
557 | } |
558 | ||
77241056 MM |
559 | /** |
560 | * hfi1_ib_rcv - process an incoming packet | |
561 | * @packet: data packet information | |
562 | * | |
563 | * This is called to process an incoming packet at interrupt level. | |
564 | * | |
565 | * Tlen is the length of the header + data + CRC in bytes. | |
566 | */ | |
567 | void hfi1_ib_rcv(struct hfi1_packet *packet) | |
568 | { | |
569 | struct hfi1_ctxtdata *rcd = packet->rcd; | |
570 | struct hfi1_ib_header *hdr = packet->hdr; | |
571 | u32 tlen = packet->tlen; | |
572 | struct hfi1_pportdata *ppd = rcd->ppd; | |
573 | struct hfi1_ibport *ibp = &ppd->ibport_data; | |
ec4274f1 | 574 | struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi; |
71e68e3d | 575 | opcode_handler packet_handler; |
b77d713a | 576 | unsigned long flags; |
77241056 MM |
577 | u32 qp_num; |
578 | int lnh; | |
579 | u8 opcode; | |
580 | u16 lid; | |
581 | ||
582 | /* Check for GRH */ | |
583 | lnh = be16_to_cpu(hdr->lrh[0]) & 3; | |
e490974e | 584 | if (lnh == HFI1_LRH_BTH) { |
77241056 | 585 | packet->ohdr = &hdr->u.oth; |
e490974e | 586 | } else if (lnh == HFI1_LRH_GRH) { |
77241056 MM |
587 | u32 vtf; |
588 | ||
589 | packet->ohdr = &hdr->u.l.oth; | |
590 | if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR) | |
591 | goto drop; | |
592 | vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow); | |
593 | if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) | |
594 | goto drop; | |
595 | packet->rcv_flags |= HFI1_HAS_GRH; | |
e490974e | 596 | } else { |
77241056 | 597 | goto drop; |
e490974e | 598 | } |
77241056 MM |
599 | |
600 | trace_input_ibhdr(rcd->dd, hdr); | |
601 | ||
602 | opcode = (be32_to_cpu(packet->ohdr->bth[0]) >> 24); | |
603 | inc_opstats(tlen, &rcd->opstats->stats[opcode]); | |
604 | ||
605 | /* Get the destination QP number. */ | |
ec4274f1 | 606 | qp_num = be32_to_cpu(packet->ohdr->bth[1]) & RVT_QPN_MASK; |
77241056 | 607 | lid = be16_to_cpu(hdr->lrh[1]); |
8859b4a6 DD |
608 | if (unlikely((lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) && |
609 | (lid != be16_to_cpu(IB_LID_PERMISSIVE)))) { | |
0facc5a1 DD |
610 | struct rvt_mcast *mcast; |
611 | struct rvt_mcast_qp *p; | |
77241056 MM |
612 | |
613 | if (lnh != HFI1_LRH_GRH) | |
614 | goto drop; | |
0facc5a1 | 615 | mcast = rvt_mcast_find(&ibp->rvp, &hdr->u.l.grh.dgid); |
d125a6c6 | 616 | if (!mcast) |
77241056 MM |
617 | goto drop; |
618 | list_for_each_entry_rcu(p, &mcast->qp_list, list) { | |
619 | packet->qp = p->qp; | |
b77d713a | 620 | spin_lock_irqsave(&packet->qp->r_lock, flags); |
71e68e3d JP |
621 | packet_handler = qp_ok(opcode, packet); |
622 | if (likely(packet_handler)) | |
623 | packet_handler(packet); | |
624 | else | |
625 | ibp->rvp.n_pkt_drops++; | |
b77d713a | 626 | spin_unlock_irqrestore(&packet->qp->r_lock, flags); |
77241056 MM |
627 | } |
628 | /* | |
0facc5a1 | 629 | * Notify rvt_multicast_detach() if it is waiting for us |
77241056 MM |
630 | * to finish. |
631 | */ | |
632 | if (atomic_dec_return(&mcast->refcount) <= 1) | |
633 | wake_up(&mcast->wait); | |
634 | } else { | |
635 | rcu_read_lock(); | |
ec4274f1 | 636 | packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); |
77241056 MM |
637 | if (!packet->qp) { |
638 | rcu_read_unlock(); | |
639 | goto drop; | |
640 | } | |
b77d713a | 641 | spin_lock_irqsave(&packet->qp->r_lock, flags); |
71e68e3d JP |
642 | packet_handler = qp_ok(opcode, packet); |
643 | if (likely(packet_handler)) | |
644 | packet_handler(packet); | |
645 | else | |
646 | ibp->rvp.n_pkt_drops++; | |
b77d713a | 647 | spin_unlock_irqrestore(&packet->qp->r_lock, flags); |
77241056 MM |
648 | rcu_read_unlock(); |
649 | } | |
650 | return; | |
651 | ||
652 | drop: | |
4eb06882 | 653 | ibp->rvp.n_pkt_drops++; |
77241056 MM |
654 | } |
655 | ||
656 | /* | |
657 | * This is called from a timer to check for QPs | |
658 | * which need kernel memory in order to send a packet. | |
659 | */ | |
660 | static void mem_timer(unsigned long data) | |
661 | { | |
662 | struct hfi1_ibdev *dev = (struct hfi1_ibdev *)data; | |
663 | struct list_head *list = &dev->memwait; | |
895420dd | 664 | struct rvt_qp *qp = NULL; |
77241056 MM |
665 | struct iowait *wait; |
666 | unsigned long flags; | |
4c6829c5 | 667 | struct hfi1_qp_priv *priv; |
77241056 MM |
668 | |
669 | write_seqlock_irqsave(&dev->iowait_lock, flags); | |
670 | if (!list_empty(list)) { | |
671 | wait = list_first_entry(list, struct iowait, list); | |
4c6829c5 DD |
672 | qp = iowait_to_qp(wait); |
673 | priv = qp->priv; | |
674 | list_del_init(&priv->s_iowait.list); | |
77241056 MM |
675 | /* refcount held until actual wake up */ |
676 | if (!list_empty(list)) | |
677 | mod_timer(&dev->mem_timer, jiffies + 1); | |
678 | } | |
679 | write_sequnlock_irqrestore(&dev->iowait_lock, flags); | |
680 | ||
681 | if (qp) | |
54d10c1e | 682 | hfi1_qp_wakeup(qp, RVT_S_WAIT_KMEM); |
77241056 MM |
683 | } |
684 | ||
895420dd | 685 | void update_sge(struct rvt_sge_state *ss, u32 length) |
77241056 | 686 | { |
895420dd | 687 | struct rvt_sge *sge = &ss->sge; |
77241056 MM |
688 | |
689 | sge->vaddr += length; | |
690 | sge->length -= length; | |
691 | sge->sge_length -= length; | |
692 | if (sge->sge_length == 0) { | |
693 | if (--ss->num_sge) | |
694 | *sge = *ss->sg_list++; | |
695 | } else if (sge->length == 0 && sge->mr->lkey) { | |
cd4ceee3 | 696 | if (++sge->n >= RVT_SEGSZ) { |
77241056 MM |
697 | if (++sge->m >= sge->mr->mapsz) |
698 | return; | |
699 | sge->n = 0; | |
700 | } | |
701 | sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr; | |
702 | sge->length = sge->mr->map[sge->m]->segs[sge->n].length; | |
703 | } | |
704 | } | |
705 | ||
77241056 MM |
706 | /* |
707 | * This is called with progress side lock held. | |
708 | */ | |
709 | /* New API */ | |
710 | static void verbs_sdma_complete( | |
711 | struct sdma_txreq *cookie, | |
a545f530 | 712 | int status) |
77241056 MM |
713 | { |
714 | struct verbs_txreq *tx = | |
715 | container_of(cookie, struct verbs_txreq, txreq); | |
895420dd | 716 | struct rvt_qp *qp = tx->qp; |
77241056 MM |
717 | |
718 | spin_lock(&qp->s_lock); | |
e490974e | 719 | if (tx->wqe) { |
77241056 | 720 | hfi1_send_complete(qp, tx->wqe, IB_WC_SUCCESS); |
e490974e | 721 | } else if (qp->ibqp.qp_type == IB_QPT_RC) { |
77241056 MM |
722 | struct hfi1_ib_header *hdr; |
723 | ||
724 | hdr = &tx->phdr.hdr; | |
725 | hfi1_rc_send_complete(qp, hdr); | |
726 | } | |
77241056 MM |
727 | spin_unlock(&qp->s_lock); |
728 | ||
729 | hfi1_put_txreq(tx); | |
730 | } | |
731 | ||
711e104d MM |
732 | static int wait_kmem(struct hfi1_ibdev *dev, |
733 | struct rvt_qp *qp, | |
734 | struct hfi1_pkt_state *ps) | |
77241056 | 735 | { |
4c6829c5 | 736 | struct hfi1_qp_priv *priv = qp->priv; |
77241056 MM |
737 | unsigned long flags; |
738 | int ret = 0; | |
739 | ||
740 | spin_lock_irqsave(&qp->s_lock, flags); | |
83693bd1 | 741 | if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { |
77241056 | 742 | write_seqlock(&dev->iowait_lock); |
711e104d MM |
743 | list_add_tail(&ps->s_txreq->txreq.list, |
744 | &priv->s_iowait.tx_head); | |
4c6829c5 | 745 | if (list_empty(&priv->s_iowait.list)) { |
77241056 MM |
746 | if (list_empty(&dev->memwait)) |
747 | mod_timer(&dev->mem_timer, jiffies + 1); | |
54d10c1e | 748 | qp->s_flags |= RVT_S_WAIT_KMEM; |
4c6829c5 | 749 | list_add_tail(&priv->s_iowait.list, &dev->memwait); |
54d10c1e | 750 | trace_hfi1_qpsleep(qp, RVT_S_WAIT_KMEM); |
77241056 MM |
751 | atomic_inc(&qp->refcount); |
752 | } | |
753 | write_sequnlock(&dev->iowait_lock); | |
54d10c1e | 754 | qp->s_flags &= ~RVT_S_BUSY; |
77241056 MM |
755 | ret = -EBUSY; |
756 | } | |
757 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
758 | ||
759 | return ret; | |
760 | } | |
761 | ||
762 | /* | |
763 | * This routine calls txadds for each sg entry. | |
764 | * | |
765 | * Add failures will revert the sge cursor | |
766 | */ | |
711e104d | 767 | static noinline int build_verbs_ulp_payload( |
77241056 | 768 | struct sdma_engine *sde, |
895420dd | 769 | struct rvt_sge_state *ss, |
77241056 MM |
770 | u32 length, |
771 | struct verbs_txreq *tx) | |
772 | { | |
895420dd DD |
773 | struct rvt_sge *sg_list = ss->sg_list; |
774 | struct rvt_sge sge = ss->sge; | |
77241056 MM |
775 | u8 num_sge = ss->num_sge; |
776 | u32 len; | |
777 | int ret = 0; | |
778 | ||
779 | while (length) { | |
780 | len = ss->sge.length; | |
781 | if (len > length) | |
782 | len = length; | |
783 | if (len > ss->sge.sge_length) | |
784 | len = ss->sge.sge_length; | |
785 | WARN_ON_ONCE(len == 0); | |
786 | ret = sdma_txadd_kvaddr( | |
787 | sde->dd, | |
788 | &tx->txreq, | |
789 | ss->sge.vaddr, | |
790 | len); | |
791 | if (ret) | |
792 | goto bail_txadd; | |
793 | update_sge(ss, len); | |
794 | length -= len; | |
795 | } | |
796 | return ret; | |
797 | bail_txadd: | |
798 | /* unwind cursor */ | |
799 | ss->sge = sge; | |
800 | ss->num_sge = num_sge; | |
801 | ss->sg_list = sg_list; | |
802 | return ret; | |
803 | } | |
804 | ||
805 | /* | |
806 | * Build the number of DMA descriptors needed to send length bytes of data. | |
807 | * | |
808 | * NOTE: DMA mapping is held in the tx until completed in the ring or | |
809 | * the tx desc is freed without having been submitted to the ring | |
810 | * | |
bb5df5f9 | 811 | * This routine ensures all the helper routine calls succeed. |
77241056 MM |
812 | */ |
813 | /* New API */ | |
814 | static int build_verbs_tx_desc( | |
815 | struct sdma_engine *sde, | |
895420dd | 816 | struct rvt_sge_state *ss, |
77241056 MM |
817 | u32 length, |
818 | struct verbs_txreq *tx, | |
a9b6b3bc | 819 | struct hfi1_ahg_info *ahg_info, |
77241056 MM |
820 | u64 pbc) |
821 | { | |
822 | int ret = 0; | |
d4d602e9 | 823 | struct hfi1_sdma_header *phdr = &tx->phdr; |
77241056 MM |
824 | u16 hdrbytes = tx->hdr_dwords << 2; |
825 | ||
a9b6b3bc | 826 | if (!ahg_info->ahgcount) { |
77241056 MM |
827 | ret = sdma_txinit_ahg( |
828 | &tx->txreq, | |
a9b6b3bc | 829 | ahg_info->tx_flags, |
77241056 | 830 | hdrbytes + length, |
a9b6b3bc | 831 | ahg_info->ahgidx, |
77241056 MM |
832 | 0, |
833 | NULL, | |
834 | 0, | |
835 | verbs_sdma_complete); | |
836 | if (ret) | |
837 | goto bail_txadd; | |
838 | phdr->pbc = cpu_to_le64(pbc); | |
77241056 MM |
839 | ret = sdma_txadd_kvaddr( |
840 | sde->dd, | |
841 | &tx->txreq, | |
bb5df5f9 DD |
842 | phdr, |
843 | hdrbytes); | |
77241056 MM |
844 | if (ret) |
845 | goto bail_txadd; | |
846 | } else { | |
77241056 MM |
847 | ret = sdma_txinit_ahg( |
848 | &tx->txreq, | |
a9b6b3bc | 849 | ahg_info->tx_flags, |
77241056 | 850 | length, |
a9b6b3bc DC |
851 | ahg_info->ahgidx, |
852 | ahg_info->ahgcount, | |
853 | ahg_info->ahgdesc, | |
77241056 MM |
854 | hdrbytes, |
855 | verbs_sdma_complete); | |
856 | if (ret) | |
857 | goto bail_txadd; | |
858 | } | |
859 | ||
860 | /* add the ulp payload - if any. ss can be NULL for acks */ | |
861 | if (ss) | |
862 | ret = build_verbs_ulp_payload(sde, ss, length, tx); | |
863 | bail_txadd: | |
864 | return ret; | |
865 | } | |
866 | ||
895420dd | 867 | int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
d46e5144 | 868 | u64 pbc) |
77241056 | 869 | { |
4c6829c5 | 870 | struct hfi1_qp_priv *priv = qp->priv; |
a9b6b3bc | 871 | struct hfi1_ahg_info *ahg_info = priv->s_ahg; |
d46e5144 | 872 | u32 hdrwords = qp->s_hdrwords; |
895420dd | 873 | struct rvt_sge_state *ss = qp->s_cur_sge; |
d46e5144 DD |
874 | u32 len = qp->s_cur_size; |
875 | u32 plen = hdrwords + ((len + 3) >> 2) + 2; /* includes pbc */ | |
876 | struct hfi1_ibdev *dev = ps->dev; | |
877 | struct hfi1_pportdata *ppd = ps->ppd; | |
77241056 | 878 | struct verbs_txreq *tx; |
77241056 | 879 | u64 pbc_flags = 0; |
4c6829c5 DD |
880 | u8 sc5 = priv->s_sc; |
881 | ||
77241056 | 882 | int ret; |
77241056 | 883 | |
bb5df5f9 | 884 | tx = ps->s_txreq; |
711e104d MM |
885 | if (!sdma_txreq_built(&tx->txreq)) { |
886 | if (likely(pbc == 0)) { | |
887 | u32 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5); | |
888 | /* No vl15 here */ | |
889 | /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */ | |
890 | pbc_flags |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT; | |
891 | ||
892 | pbc = create_pbc(ppd, | |
893 | pbc_flags, | |
894 | qp->srate_mbps, | |
895 | vl, | |
896 | plen); | |
897 | } | |
898 | tx->wqe = qp->s_wqe; | |
a9b6b3bc | 899 | ret = build_verbs_tx_desc(tx->sde, ss, len, tx, ahg_info, pbc); |
711e104d MM |
900 | if (unlikely(ret)) |
901 | goto bail_build; | |
77241056 | 902 | } |
5326dfbf MM |
903 | ret = sdma_send_txreq(tx->sde, &priv->s_iowait, &tx->txreq); |
904 | if (unlikely(ret < 0)) { | |
905 | if (ret == -ECOMM) | |
906 | goto bail_ecomm; | |
907 | return ret; | |
908 | } | |
1db78eee MM |
909 | trace_sdma_output_ibhdr(dd_from_ibdev(qp->ibqp.device), |
910 | &ps->s_txreq->phdr.hdr); | |
77241056 MM |
911 | return ret; |
912 | ||
77241056 MM |
913 | bail_ecomm: |
914 | /* The current one got "sent" */ | |
915 | return 0; | |
916 | bail_build: | |
711e104d MM |
917 | ret = wait_kmem(dev, qp, ps); |
918 | if (!ret) { | |
919 | /* free txreq - bad state */ | |
920 | hfi1_put_txreq(ps->s_txreq); | |
921 | ps->s_txreq = NULL; | |
922 | } | |
923 | return ret; | |
77241056 MM |
924 | } |
925 | ||
926 | /* | |
927 | * If we are now in the error state, return zero to flush the | |
928 | * send work request. | |
929 | */ | |
14553ca1 MM |
930 | static int pio_wait(struct rvt_qp *qp, |
931 | struct send_context *sc, | |
932 | struct hfi1_pkt_state *ps, | |
933 | u32 flag) | |
77241056 | 934 | { |
4c6829c5 | 935 | struct hfi1_qp_priv *priv = qp->priv; |
77241056 MM |
936 | struct hfi1_devdata *dd = sc->dd; |
937 | struct hfi1_ibdev *dev = &dd->verbs_dev; | |
938 | unsigned long flags; | |
939 | int ret = 0; | |
940 | ||
941 | /* | |
942 | * Note that as soon as want_buffer() is called and | |
943 | * possibly before it returns, sc_piobufavail() | |
944 | * could be called. Therefore, put QP on the I/O wait list before | |
945 | * enabling the PIO avail interrupt. | |
946 | */ | |
947 | spin_lock_irqsave(&qp->s_lock, flags); | |
83693bd1 | 948 | if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { |
77241056 | 949 | write_seqlock(&dev->iowait_lock); |
711e104d MM |
950 | list_add_tail(&ps->s_txreq->txreq.list, |
951 | &priv->s_iowait.tx_head); | |
4c6829c5 | 952 | if (list_empty(&priv->s_iowait.list)) { |
77241056 MM |
953 | struct hfi1_ibdev *dev = &dd->verbs_dev; |
954 | int was_empty; | |
955 | ||
14553ca1 MM |
956 | dev->n_piowait += !!(flag & RVT_S_WAIT_PIO); |
957 | dev->n_piodrain += !!(flag & RVT_S_WAIT_PIO_DRAIN); | |
14553ca1 | 958 | qp->s_flags |= flag; |
77241056 | 959 | was_empty = list_empty(&sc->piowait); |
4c6829c5 | 960 | list_add_tail(&priv->s_iowait.list, &sc->piowait); |
54d10c1e | 961 | trace_hfi1_qpsleep(qp, RVT_S_WAIT_PIO); |
77241056 MM |
962 | atomic_inc(&qp->refcount); |
963 | /* counting: only call wantpiobuf_intr if first user */ | |
964 | if (was_empty) | |
965 | hfi1_sc_wantpiobuf_intr(sc, 1); | |
966 | } | |
967 | write_sequnlock(&dev->iowait_lock); | |
54d10c1e | 968 | qp->s_flags &= ~RVT_S_BUSY; |
77241056 MM |
969 | ret = -EBUSY; |
970 | } | |
971 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
972 | return ret; | |
973 | } | |
974 | ||
14553ca1 MM |
975 | static void verbs_pio_complete(void *arg, int code) |
976 | { | |
977 | struct rvt_qp *qp = (struct rvt_qp *)arg; | |
978 | struct hfi1_qp_priv *priv = qp->priv; | |
979 | ||
980 | if (iowait_pio_dec(&priv->s_iowait)) | |
981 | iowait_drain_wakeup(&priv->s_iowait); | |
982 | } | |
983 | ||
895420dd | 984 | int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
d46e5144 | 985 | u64 pbc) |
77241056 | 986 | { |
4c6829c5 | 987 | struct hfi1_qp_priv *priv = qp->priv; |
d46e5144 | 988 | u32 hdrwords = qp->s_hdrwords; |
895420dd | 989 | struct rvt_sge_state *ss = qp->s_cur_sge; |
d46e5144 DD |
990 | u32 len = qp->s_cur_size; |
991 | u32 dwords = (len + 3) >> 2; | |
992 | u32 plen = hdrwords + dwords + 2; /* includes pbc */ | |
993 | struct hfi1_pportdata *ppd = ps->ppd; | |
bb5df5f9 | 994 | u32 *hdr = (u32 *)&ps->s_txreq->phdr.hdr; |
77241056 | 995 | u64 pbc_flags = 0; |
4f8cc5c0 | 996 | u8 sc5; |
77241056 MM |
997 | unsigned long flags = 0; |
998 | struct send_context *sc; | |
999 | struct pio_buf *pbuf; | |
1000 | int wc_status = IB_WC_SUCCESS; | |
bb5df5f9 | 1001 | int ret = 0; |
14553ca1 MM |
1002 | pio_release_cb cb = NULL; |
1003 | ||
1004 | /* only RC/UC use complete */ | |
1005 | switch (qp->ibqp.qp_type) { | |
1006 | case IB_QPT_RC: | |
1007 | case IB_QPT_UC: | |
1008 | cb = verbs_pio_complete; | |
1009 | break; | |
1010 | default: | |
1011 | break; | |
1012 | } | |
77241056 MM |
1013 | |
1014 | /* vl15 special case taken care of in ud.c */ | |
4c6829c5 | 1015 | sc5 = priv->s_sc; |
cef504c5 | 1016 | sc = ps->s_txreq->psc; |
77241056 | 1017 | |
77241056 | 1018 | if (likely(pbc == 0)) { |
4f8cc5c0 | 1019 | u8 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5); |
77241056 MM |
1020 | /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */ |
1021 | pbc_flags |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT; | |
1022 | pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen); | |
1023 | } | |
14553ca1 MM |
1024 | if (cb) |
1025 | iowait_pio_inc(&priv->s_iowait); | |
1026 | pbuf = sc_buffer_alloc(sc, plen, cb, qp); | |
d125a6c6 | 1027 | if (unlikely(!pbuf)) { |
14553ca1 MM |
1028 | if (cb) |
1029 | verbs_pio_complete(qp, 0); | |
77241056 MM |
1030 | if (ppd->host_link_state != HLS_UP_ACTIVE) { |
1031 | /* | |
1032 | * If we have filled the PIO buffers to capacity and are | |
1033 | * not in an active state this request is not going to | |
1034 | * go out to so just complete it with an error or else a | |
1035 | * ULP or the core may be stuck waiting. | |
1036 | */ | |
1037 | hfi1_cdbg( | |
1038 | PIO, | |
1039 | "alloc failed. state not active, completing"); | |
1040 | wc_status = IB_WC_GENERAL_ERR; | |
1041 | goto pio_bail; | |
1042 | } else { | |
1043 | /* | |
1044 | * This is a normal occurrence. The PIO buffs are full | |
1045 | * up but we are still happily sending, well we could be | |
1046 | * so lets continue to queue the request. | |
1047 | */ | |
1048 | hfi1_cdbg(PIO, "alloc failed. state active, queuing"); | |
14553ca1 | 1049 | ret = pio_wait(qp, sc, ps, RVT_S_WAIT_PIO); |
711e104d | 1050 | if (!ret) |
14553ca1 | 1051 | /* txreq not queued - free */ |
711e104d MM |
1052 | goto bail; |
1053 | /* tx consumed in wait */ | |
1054 | return ret; | |
77241056 MM |
1055 | } |
1056 | } | |
1057 | ||
1058 | if (len == 0) { | |
1059 | pio_copy(ppd->dd, pbuf, pbc, hdr, hdrwords); | |
1060 | } else { | |
1061 | if (ss) { | |
8638b77f | 1062 | seg_pio_copy_start(pbuf, pbc, hdr, hdrwords * 4); |
77241056 MM |
1063 | while (len) { |
1064 | void *addr = ss->sge.vaddr; | |
1065 | u32 slen = ss->sge.length; | |
1066 | ||
1067 | if (slen > len) | |
1068 | slen = len; | |
1069 | update_sge(ss, slen); | |
1070 | seg_pio_copy_mid(pbuf, addr, slen); | |
1071 | len -= slen; | |
1072 | } | |
1073 | seg_pio_copy_end(pbuf); | |
1074 | } | |
1075 | } | |
1076 | ||
1db78eee MM |
1077 | trace_pio_output_ibhdr(dd_from_ibdev(qp->ibqp.device), |
1078 | &ps->s_txreq->phdr.hdr); | |
77241056 | 1079 | |
77241056 MM |
1080 | pio_bail: |
1081 | if (qp->s_wqe) { | |
1082 | spin_lock_irqsave(&qp->s_lock, flags); | |
1083 | hfi1_send_complete(qp, qp->s_wqe, wc_status); | |
1084 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
1085 | } else if (qp->ibqp.qp_type == IB_QPT_RC) { | |
1086 | spin_lock_irqsave(&qp->s_lock, flags); | |
bb5df5f9 | 1087 | hfi1_rc_send_complete(qp, &ps->s_txreq->phdr.hdr); |
77241056 MM |
1088 | spin_unlock_irqrestore(&qp->s_lock, flags); |
1089 | } | |
bb5df5f9 DD |
1090 | |
1091 | ret = 0; | |
1092 | ||
1093 | bail: | |
1094 | hfi1_put_txreq(ps->s_txreq); | |
1095 | return ret; | |
77241056 | 1096 | } |
b91cc573 | 1097 | |
77241056 MM |
1098 | /* |
1099 | * egress_pkey_matches_entry - return 1 if the pkey matches ent (ent | |
e38d1e4f | 1100 | * being an entry from the partition key table), return 0 |
77241056 MM |
1101 | * otherwise. Use the matching criteria for egress partition keys |
1102 | * specified in the OPAv1 spec., section 9.1l.7. | |
1103 | */ | |
1104 | static inline int egress_pkey_matches_entry(u16 pkey, u16 ent) | |
1105 | { | |
1106 | u16 mkey = pkey & PKEY_LOW_15_MASK; | |
e38d1e4f | 1107 | u16 mentry = ent & PKEY_LOW_15_MASK; |
77241056 | 1108 | |
e38d1e4f | 1109 | if (mkey == mentry) { |
77241056 MM |
1110 | /* |
1111 | * If pkey[15] is set (full partition member), | |
1112 | * is bit 15 in the corresponding table element | |
1113 | * clear (limited member)? | |
1114 | */ | |
1115 | if (pkey & PKEY_MEMBER_MASK) | |
1116 | return !!(ent & PKEY_MEMBER_MASK); | |
1117 | return 1; | |
1118 | } | |
1119 | return 0; | |
1120 | } | |
1121 | ||
e38d1e4f SS |
1122 | /** |
1123 | * egress_pkey_check - check P_KEY of a packet | |
1124 | * @ppd: Physical IB port data | |
1125 | * @lrh: Local route header | |
1126 | * @bth: Base transport header | |
1127 | * @sc5: SC for packet | |
1128 | * @s_pkey_index: It will be used for look up optimization for kernel contexts | |
1129 | * only. If it is negative value, then it means user contexts is calling this | |
1130 | * function. | |
1131 | * | |
1132 | * It checks if hdr's pkey is valid. | |
1133 | * | |
1134 | * Return: 0 on success, otherwise, 1 | |
77241056 | 1135 | */ |
e38d1e4f SS |
1136 | int egress_pkey_check(struct hfi1_pportdata *ppd, __be16 *lrh, __be32 *bth, |
1137 | u8 sc5, int8_t s_pkey_index) | |
77241056 | 1138 | { |
77241056 | 1139 | struct hfi1_devdata *dd; |
e38d1e4f | 1140 | int i; |
77241056 | 1141 | u16 pkey; |
e38d1e4f | 1142 | int is_user_ctxt_mechanism = (s_pkey_index < 0); |
77241056 MM |
1143 | |
1144 | if (!(ppd->part_enforce & HFI1_PART_ENFORCE_OUT)) | |
1145 | return 0; | |
1146 | ||
e38d1e4f | 1147 | pkey = (u16)be32_to_cpu(bth[0]); |
77241056 MM |
1148 | |
1149 | /* If SC15, pkey[0:14] must be 0x7fff */ | |
1150 | if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK)) | |
1151 | goto bad; | |
1152 | ||
77241056 MM |
1153 | /* Is the pkey = 0x0, or 0x8000? */ |
1154 | if ((pkey & PKEY_LOW_15_MASK) == 0) | |
1155 | goto bad; | |
1156 | ||
e38d1e4f SS |
1157 | /* |
1158 | * For the kernel contexts only, if a qp is passed into the function, | |
1159 | * the most likely matching pkey has index qp->s_pkey_index | |
1160 | */ | |
1161 | if (!is_user_ctxt_mechanism && | |
1162 | egress_pkey_matches_entry(pkey, ppd->pkeys[s_pkey_index])) { | |
1163 | return 0; | |
77241056 MM |
1164 | } |
1165 | ||
e38d1e4f SS |
1166 | for (i = 0; i < MAX_PKEY_VALUES; i++) { |
1167 | if (egress_pkey_matches_entry(pkey, ppd->pkeys[i])) | |
1168 | return 0; | |
1169 | } | |
77241056 | 1170 | bad: |
e38d1e4f SS |
1171 | /* |
1172 | * For the user-context mechanism, the P_KEY check would only happen | |
1173 | * once per SDMA request, not once per packet. Therefore, there's no | |
1174 | * need to increment the counter for the user-context mechanism. | |
1175 | */ | |
1176 | if (!is_user_ctxt_mechanism) { | |
1177 | incr_cntr64(&ppd->port_xmit_constraint_errors); | |
1178 | dd = ppd->dd; | |
1179 | if (!(dd->err_info_xmit_constraint.status & | |
1180 | OPA_EI_STATUS_SMASK)) { | |
1181 | u16 slid = be16_to_cpu(lrh[3]); | |
1182 | ||
1183 | dd->err_info_xmit_constraint.status |= | |
1184 | OPA_EI_STATUS_SMASK; | |
1185 | dd->err_info_xmit_constraint.slid = slid; | |
1186 | dd->err_info_xmit_constraint.pkey = pkey; | |
1187 | } | |
77241056 MM |
1188 | } |
1189 | return 1; | |
1190 | } | |
1191 | ||
14553ca1 MM |
1192 | /** |
1193 | * get_send_routine - choose an egress routine | |
1194 | * | |
1195 | * Choose an egress routine based on QP type | |
1196 | * and size | |
1197 | */ | |
1198 | static inline send_routine get_send_routine(struct rvt_qp *qp, | |
47177f1b | 1199 | struct verbs_txreq *tx) |
14553ca1 MM |
1200 | { |
1201 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
1202 | struct hfi1_qp_priv *priv = qp->priv; | |
47177f1b | 1203 | struct hfi1_ib_header *h = &tx->phdr.hdr; |
14553ca1 MM |
1204 | |
1205 | if (unlikely(!(dd->flags & HFI1_HAS_SEND_DMA))) | |
1206 | return dd->process_pio_send; | |
1207 | switch (qp->ibqp.qp_type) { | |
1208 | case IB_QPT_SMI: | |
1209 | return dd->process_pio_send; | |
1210 | case IB_QPT_GSI: | |
1211 | case IB_QPT_UD: | |
14553ca1 MM |
1212 | break; |
1213 | case IB_QPT_RC: | |
1214 | if (piothreshold && | |
1215 | qp->s_cur_size <= min(piothreshold, qp->pmtu) && | |
1216 | (BIT(get_opcode(h) & 0x1f) & rc_only_opcode) && | |
47177f1b MM |
1217 | iowait_sdma_pending(&priv->s_iowait) == 0 && |
1218 | !sdma_txreq_built(&tx->txreq)) | |
14553ca1 MM |
1219 | return dd->process_pio_send; |
1220 | break; | |
1221 | case IB_QPT_UC: | |
1222 | if (piothreshold && | |
1223 | qp->s_cur_size <= min(piothreshold, qp->pmtu) && | |
1224 | (BIT(get_opcode(h) & 0x1f) & uc_only_opcode) && | |
47177f1b MM |
1225 | iowait_sdma_pending(&priv->s_iowait) == 0 && |
1226 | !sdma_txreq_built(&tx->txreq)) | |
14553ca1 MM |
1227 | return dd->process_pio_send; |
1228 | break; | |
1229 | default: | |
1230 | break; | |
1231 | } | |
1232 | return dd->process_dma_send; | |
1233 | } | |
1234 | ||
77241056 MM |
1235 | /** |
1236 | * hfi1_verbs_send - send a packet | |
1237 | * @qp: the QP to send on | |
d46e5144 | 1238 | * @ps: the state of the packet to send |
77241056 MM |
1239 | * |
1240 | * Return zero if packet is sent or queued OK. | |
54d10c1e | 1241 | * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise. |
77241056 | 1242 | */ |
895420dd | 1243 | int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps) |
77241056 MM |
1244 | { |
1245 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
47177f1b | 1246 | struct hfi1_qp_priv *priv = qp->priv; |
e38d1e4f SS |
1247 | struct hfi1_other_headers *ohdr; |
1248 | struct hfi1_ib_header *hdr; | |
14553ca1 | 1249 | send_routine sr; |
77241056 | 1250 | int ret; |
e38d1e4f SS |
1251 | u8 lnh; |
1252 | ||
1253 | hdr = &ps->s_txreq->phdr.hdr; | |
1254 | /* locate the pkey within the headers */ | |
1255 | lnh = be16_to_cpu(hdr->lrh[0]) & 3; | |
1256 | if (lnh == HFI1_LRH_GRH) | |
1257 | ohdr = &hdr->u.l.oth; | |
1258 | else | |
1259 | ohdr = &hdr->u.oth; | |
77241056 | 1260 | |
47177f1b | 1261 | sr = get_send_routine(qp, ps->s_txreq); |
e38d1e4f SS |
1262 | ret = egress_pkey_check(dd->pport, |
1263 | hdr->lrh, | |
1264 | ohdr->bth, | |
1265 | priv->s_sc, | |
1266 | qp->s_pkey_index); | |
77241056 MM |
1267 | if (unlikely(ret)) { |
1268 | /* | |
1269 | * The value we are returning here does not get propagated to | |
1270 | * the verbs caller. Thus we need to complete the request with | |
1271 | * error otherwise the caller could be sitting waiting on the | |
1272 | * completion event. Only do this for PIO. SDMA has its own | |
1273 | * mechanism for handling the errors. So for SDMA we can just | |
1274 | * return. | |
1275 | */ | |
14553ca1 MM |
1276 | if (sr == dd->process_pio_send) { |
1277 | unsigned long flags; | |
1278 | ||
77241056 MM |
1279 | hfi1_cdbg(PIO, "%s() Failed. Completing with err", |
1280 | __func__); | |
1281 | spin_lock_irqsave(&qp->s_lock, flags); | |
1282 | hfi1_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); | |
1283 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
1284 | } | |
1285 | return -EINVAL; | |
1286 | } | |
47177f1b MM |
1287 | if (sr == dd->process_dma_send && iowait_pio_pending(&priv->s_iowait)) |
1288 | return pio_wait(qp, | |
1289 | ps->s_txreq->psc, | |
1290 | ps, | |
1291 | RVT_S_WAIT_PIO_DRAIN); | |
14553ca1 | 1292 | return sr(qp, ps, 0); |
77241056 MM |
1293 | } |
1294 | ||
94d5171c HC |
1295 | /** |
1296 | * hfi1_fill_device_attr - Fill in rvt dev info device attributes. | |
1297 | * @dd: the device data structure | |
1298 | */ | |
1299 | static void hfi1_fill_device_attr(struct hfi1_devdata *dd) | |
77241056 | 1300 | { |
94d5171c HC |
1301 | struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; |
1302 | ||
1303 | memset(&rdi->dparms.props, 0, sizeof(rdi->dparms.props)); | |
1304 | ||
1305 | rdi->dparms.props.device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR | | |
1306 | IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT | | |
1307 | IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN | | |
c72cfe3e JX |
1308 | IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE | |
1309 | IB_DEVICE_MEM_MGT_EXTENSIONS; | |
94d5171c HC |
1310 | rdi->dparms.props.page_size_cap = PAGE_SIZE; |
1311 | rdi->dparms.props.vendor_id = dd->oui1 << 16 | dd->oui2 << 8 | dd->oui3; | |
1312 | rdi->dparms.props.vendor_part_id = dd->pcidev->device; | |
1313 | rdi->dparms.props.hw_ver = dd->minrev; | |
1314 | rdi->dparms.props.sys_image_guid = ib_hfi1_sys_image_guid; | |
c72cfe3e JX |
1315 | rdi->dparms.props.max_mr_size = U64_MAX; |
1316 | rdi->dparms.props.max_fast_reg_page_list_len = UINT_MAX; | |
94d5171c HC |
1317 | rdi->dparms.props.max_qp = hfi1_max_qps; |
1318 | rdi->dparms.props.max_qp_wr = hfi1_max_qp_wrs; | |
1319 | rdi->dparms.props.max_sge = hfi1_max_sges; | |
1320 | rdi->dparms.props.max_sge_rd = hfi1_max_sges; | |
1321 | rdi->dparms.props.max_cq = hfi1_max_cqs; | |
1322 | rdi->dparms.props.max_ah = hfi1_max_ahs; | |
1323 | rdi->dparms.props.max_cqe = hfi1_max_cqes; | |
1324 | rdi->dparms.props.max_mr = rdi->lkey_table.max; | |
1325 | rdi->dparms.props.max_fmr = rdi->lkey_table.max; | |
1326 | rdi->dparms.props.max_map_per_fmr = 32767; | |
1327 | rdi->dparms.props.max_pd = hfi1_max_pds; | |
1328 | rdi->dparms.props.max_qp_rd_atom = HFI1_MAX_RDMA_ATOMIC; | |
1329 | rdi->dparms.props.max_qp_init_rd_atom = 255; | |
1330 | rdi->dparms.props.max_srq = hfi1_max_srqs; | |
1331 | rdi->dparms.props.max_srq_wr = hfi1_max_srq_wrs; | |
1332 | rdi->dparms.props.max_srq_sge = hfi1_max_srq_sges; | |
1333 | rdi->dparms.props.atomic_cap = IB_ATOMIC_GLOB; | |
1334 | rdi->dparms.props.max_pkeys = hfi1_get_npkeys(dd); | |
1335 | rdi->dparms.props.max_mcast_grp = hfi1_max_mcast_grps; | |
1336 | rdi->dparms.props.max_mcast_qp_attach = hfi1_max_mcast_qp_attached; | |
1337 | rdi->dparms.props.max_total_mcast_qp_attach = | |
1338 | rdi->dparms.props.max_mcast_qp_attach * | |
1339 | rdi->dparms.props.max_mcast_grp; | |
77241056 MM |
1340 | } |
1341 | ||
1342 | static inline u16 opa_speed_to_ib(u16 in) | |
1343 | { | |
1344 | u16 out = 0; | |
1345 | ||
1346 | if (in & OPA_LINK_SPEED_25G) | |
1347 | out |= IB_SPEED_EDR; | |
1348 | if (in & OPA_LINK_SPEED_12_5G) | |
1349 | out |= IB_SPEED_FDR; | |
1350 | ||
1351 | return out; | |
1352 | } | |
1353 | ||
1354 | /* | |
1355 | * Convert a single OPA link width (no multiple flags) to an IB value. | |
1356 | * A zero OPA link width means link down, which means the IB width value | |
1357 | * is a don't care. | |
1358 | */ | |
1359 | static inline u16 opa_width_to_ib(u16 in) | |
1360 | { | |
1361 | switch (in) { | |
1362 | case OPA_LINK_WIDTH_1X: | |
1363 | /* map 2x and 3x to 1x as they don't exist in IB */ | |
1364 | case OPA_LINK_WIDTH_2X: | |
1365 | case OPA_LINK_WIDTH_3X: | |
1366 | return IB_WIDTH_1X; | |
1367 | default: /* link down or unknown, return our largest width */ | |
1368 | case OPA_LINK_WIDTH_4X: | |
1369 | return IB_WIDTH_4X; | |
1370 | } | |
1371 | } | |
1372 | ||
45b59eef | 1373 | static int query_port(struct rvt_dev_info *rdi, u8 port_num, |
77241056 MM |
1374 | struct ib_port_attr *props) |
1375 | { | |
45b59eef HC |
1376 | struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); |
1377 | struct hfi1_devdata *dd = dd_from_dev(verbs_dev); | |
1378 | struct hfi1_pportdata *ppd = &dd->pport[port_num - 1]; | |
77241056 MM |
1379 | u16 lid = ppd->lid; |
1380 | ||
77241056 MM |
1381 | props->lid = lid ? lid : 0; |
1382 | props->lmc = ppd->lmc; | |
77241056 MM |
1383 | /* OPA logical states match IB logical states */ |
1384 | props->state = driver_lstate(ppd); | |
1385 | props->phys_state = hfi1_ibphys_portstate(ppd); | |
77241056 | 1386 | props->gid_tbl_len = HFI1_GUIDS_PER_PORT; |
77241056 MM |
1387 | props->active_width = (u8)opa_width_to_ib(ppd->link_width_active); |
1388 | /* see rate_show() in ib core/sysfs.c */ | |
1389 | props->active_speed = (u8)opa_speed_to_ib(ppd->link_speed_active); | |
1390 | props->max_vl_num = ppd->vls_supported; | |
77241056 MM |
1391 | |
1392 | /* Once we are a "first class" citizen and have added the OPA MTUs to | |
1393 | * the core we can advertise the larger MTU enum to the ULPs, for now | |
1394 | * advertise only 4K. | |
1395 | * | |
1396 | * Those applications which are either OPA aware or pass the MTU enum | |
1397 | * from the Path Records to us will get the new 8k MTU. Those that | |
1398 | * attempt to process the MTU enum may fail in various ways. | |
1399 | */ | |
1400 | props->max_mtu = mtu_to_enum((!valid_ib_mtu(hfi1_max_mtu) ? | |
1401 | 4096 : hfi1_max_mtu), IB_MTU_4096); | |
1402 | props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu : | |
1403 | mtu_to_enum(ppd->ibmtu, IB_MTU_2048); | |
77241056 MM |
1404 | |
1405 | return 0; | |
1406 | } | |
1407 | ||
1408 | static int modify_device(struct ib_device *device, | |
1409 | int device_modify_mask, | |
1410 | struct ib_device_modify *device_modify) | |
1411 | { | |
1412 | struct hfi1_devdata *dd = dd_from_ibdev(device); | |
1413 | unsigned i; | |
1414 | int ret; | |
1415 | ||
1416 | if (device_modify_mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID | | |
1417 | IB_DEVICE_MODIFY_NODE_DESC)) { | |
1418 | ret = -EOPNOTSUPP; | |
1419 | goto bail; | |
1420 | } | |
1421 | ||
1422 | if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) { | |
1423 | memcpy(device->node_desc, device_modify->node_desc, 64); | |
1424 | for (i = 0; i < dd->num_pports; i++) { | |
1425 | struct hfi1_ibport *ibp = &dd->pport[i].ibport_data; | |
1426 | ||
1427 | hfi1_node_desc_chg(ibp); | |
1428 | } | |
1429 | } | |
1430 | ||
1431 | if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) { | |
1432 | ib_hfi1_sys_image_guid = | |
1433 | cpu_to_be64(device_modify->sys_image_guid); | |
1434 | for (i = 0; i < dd->num_pports; i++) { | |
1435 | struct hfi1_ibport *ibp = &dd->pport[i].ibport_data; | |
1436 | ||
1437 | hfi1_sys_guid_chg(ibp); | |
1438 | } | |
1439 | } | |
1440 | ||
1441 | ret = 0; | |
1442 | ||
1443 | bail: | |
1444 | return ret; | |
1445 | } | |
1446 | ||
45b59eef | 1447 | static int shut_down_port(struct rvt_dev_info *rdi, u8 port_num) |
77241056 | 1448 | { |
45b59eef HC |
1449 | struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); |
1450 | struct hfi1_devdata *dd = dd_from_dev(verbs_dev); | |
1451 | struct hfi1_pportdata *ppd = &dd->pport[port_num - 1]; | |
1452 | int ret; | |
77241056 | 1453 | |
45b59eef HC |
1454 | set_link_down_reason(ppd, OPA_LINKDOWN_REASON_UNKNOWN, 0, |
1455 | OPA_LINKDOWN_REASON_UNKNOWN); | |
1456 | ret = set_link_state(ppd, HLS_DN_DOWNDEF); | |
77241056 MM |
1457 | return ret; |
1458 | } | |
1459 | ||
25131463 DD |
1460 | static int hfi1_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp, |
1461 | int guid_index, __be64 *guid) | |
77241056 | 1462 | { |
25131463 DD |
1463 | struct hfi1_ibport *ibp = container_of(rvp, struct hfi1_ibport, rvp); |
1464 | struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); | |
77241056 | 1465 | |
25131463 DD |
1466 | if (guid_index == 0) |
1467 | *guid = cpu_to_be64(ppd->guid); | |
1468 | else if (guid_index < HFI1_GUIDS_PER_PORT) | |
1469 | *guid = ibp->guids[guid_index - 1]; | |
1470 | else | |
1471 | return -EINVAL; | |
77241056 | 1472 | |
25131463 | 1473 | return 0; |
77241056 MM |
1474 | } |
1475 | ||
77241056 MM |
1476 | /* |
1477 | * convert ah port,sl to sc | |
1478 | */ | |
1479 | u8 ah_to_sc(struct ib_device *ibdev, struct ib_ah_attr *ah) | |
1480 | { | |
1481 | struct hfi1_ibport *ibp = to_iport(ibdev, ah->port_num); | |
1482 | ||
1483 | return ibp->sl_to_sc[ah->sl]; | |
1484 | } | |
1485 | ||
15723f06 | 1486 | static int hfi1_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr) |
77241056 MM |
1487 | { |
1488 | struct hfi1_ibport *ibp; | |
1489 | struct hfi1_pportdata *ppd; | |
1490 | struct hfi1_devdata *dd; | |
1491 | u8 sc5; | |
1492 | ||
77241056 MM |
1493 | /* test the mapping for validity */ |
1494 | ibp = to_iport(ibdev, ah_attr->port_num); | |
1495 | ppd = ppd_from_ibp(ibp); | |
1496 | sc5 = ibp->sl_to_sc[ah_attr->sl]; | |
1497 | dd = dd_from_ppd(ppd); | |
1498 | if (sc_to_vlt(dd, sc5) > num_vls && sc_to_vlt(dd, sc5) != 0xf) | |
15723f06 | 1499 | return -EINVAL; |
77241056 | 1500 | return 0; |
77241056 MM |
1501 | } |
1502 | ||
8f1764fa DD |
1503 | static void hfi1_notify_new_ah(struct ib_device *ibdev, |
1504 | struct ib_ah_attr *ah_attr, | |
1505 | struct rvt_ah *ah) | |
1506 | { | |
1507 | struct hfi1_ibport *ibp; | |
1508 | struct hfi1_pportdata *ppd; | |
1509 | struct hfi1_devdata *dd; | |
1510 | u8 sc5; | |
1511 | ||
1512 | /* | |
1513 | * Do not trust reading anything from rvt_ah at this point as it is not | |
1514 | * done being setup. We can however modify things which we need to set. | |
1515 | */ | |
1516 | ||
1517 | ibp = to_iport(ibdev, ah_attr->port_num); | |
1518 | ppd = ppd_from_ibp(ibp); | |
1519 | sc5 = ibp->sl_to_sc[ah->attr.sl]; | |
1520 | dd = dd_from_ppd(ppd); | |
1521 | ah->vl = sc_to_vlt(dd, sc5); | |
1522 | if (ah->vl < num_vls || ah->vl == 15) | |
1523 | ah->log_pmtu = ilog2(dd->vld[ah->vl].mtu); | |
1524 | } | |
1525 | ||
77241056 MM |
1526 | struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid) |
1527 | { | |
1528 | struct ib_ah_attr attr; | |
1529 | struct ib_ah *ah = ERR_PTR(-EINVAL); | |
895420dd | 1530 | struct rvt_qp *qp0; |
77241056 MM |
1531 | |
1532 | memset(&attr, 0, sizeof(attr)); | |
1533 | attr.dlid = dlid; | |
1534 | attr.port_num = ppd_from_ibp(ibp)->port; | |
1535 | rcu_read_lock(); | |
4eb06882 | 1536 | qp0 = rcu_dereference(ibp->rvp.qp[0]); |
77241056 MM |
1537 | if (qp0) |
1538 | ah = ib_create_ah(qp0->ibqp.pd, &attr); | |
1539 | rcu_read_unlock(); | |
1540 | return ah; | |
1541 | } | |
1542 | ||
77241056 MM |
1543 | /** |
1544 | * hfi1_get_npkeys - return the size of the PKEY table for context 0 | |
1545 | * @dd: the hfi1_ib device | |
1546 | */ | |
1547 | unsigned hfi1_get_npkeys(struct hfi1_devdata *dd) | |
1548 | { | |
1549 | return ARRAY_SIZE(dd->pport[0].pkeys); | |
1550 | } | |
1551 | ||
77241056 MM |
1552 | static void init_ibport(struct hfi1_pportdata *ppd) |
1553 | { | |
1554 | struct hfi1_ibport *ibp = &ppd->ibport_data; | |
1555 | size_t sz = ARRAY_SIZE(ibp->sl_to_sc); | |
1556 | int i; | |
1557 | ||
1558 | for (i = 0; i < sz; i++) { | |
1559 | ibp->sl_to_sc[i] = i; | |
1560 | ibp->sc_to_sl[i] = i; | |
1561 | } | |
1562 | ||
4eb06882 | 1563 | spin_lock_init(&ibp->rvp.lock); |
77241056 | 1564 | /* Set the prefix to the default value (see ch. 4.1.1) */ |
4eb06882 DD |
1565 | ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX; |
1566 | ibp->rvp.sm_lid = 0; | |
77241056 | 1567 | /* Below should only set bits defined in OPA PortInfo.CapabilityMask */ |
4eb06882 | 1568 | ibp->rvp.port_cap_flags = IB_PORT_AUTO_MIGR_SUP | |
77241056 | 1569 | IB_PORT_CAP_MASK_NOTICE_SUP; |
4eb06882 DD |
1570 | ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA; |
1571 | ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA; | |
1572 | ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS; | |
1573 | ibp->rvp.pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS; | |
1574 | ibp->rvp.pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT; | |
1575 | ||
1576 | RCU_INIT_POINTER(ibp->rvp.qp[0], NULL); | |
1577 | RCU_INIT_POINTER(ibp->rvp.qp[1], NULL); | |
77241056 MM |
1578 | } |
1579 | ||
77241056 MM |
1580 | /** |
1581 | * hfi1_register_ib_device - register our device with the infiniband core | |
1582 | * @dd: the device data structure | |
1583 | * Return 0 if successful, errno if unsuccessful. | |
1584 | */ | |
1585 | int hfi1_register_ib_device(struct hfi1_devdata *dd) | |
1586 | { | |
1587 | struct hfi1_ibdev *dev = &dd->verbs_dev; | |
ec3f2c12 | 1588 | struct ib_device *ibdev = &dev->rdi.ibdev; |
77241056 | 1589 | struct hfi1_pportdata *ppd = dd->pport; |
895420dd | 1590 | unsigned i; |
77241056 MM |
1591 | int ret; |
1592 | size_t lcpysz = IB_DEVICE_NAME_MAX; | |
77241056 | 1593 | |
77241056 MM |
1594 | for (i = 0; i < dd->num_pports; i++) |
1595 | init_ibport(ppd + i); | |
1596 | ||
1597 | /* Only need to initialize non-zero fields. */ | |
4f87ccfc | 1598 | |
045277cf | 1599 | setup_timer(&dev->mem_timer, mem_timer, (unsigned long)dev); |
77241056 | 1600 | |
77241056 | 1601 | seqlock_init(&dev->iowait_lock); |
77241056 MM |
1602 | INIT_LIST_HEAD(&dev->txwait); |
1603 | INIT_LIST_HEAD(&dev->memwait); | |
1604 | ||
45842abb MM |
1605 | ret = verbs_txreq_init(dev); |
1606 | if (ret) | |
77241056 | 1607 | goto err_verbs_txreq; |
77241056 MM |
1608 | |
1609 | /* | |
1610 | * The system image GUID is supposed to be the same for all | |
1611 | * HFIs in a single system but since there can be other | |
1612 | * device types in the system, we can't be sure this is unique. | |
1613 | */ | |
1614 | if (!ib_hfi1_sys_image_guid) | |
1615 | ib_hfi1_sys_image_guid = cpu_to_be64(ppd->guid); | |
1616 | lcpysz = strlcpy(ibdev->name, class_name(), lcpysz); | |
1617 | strlcpy(ibdev->name + lcpysz, "_%d", IB_DEVICE_NAME_MAX - lcpysz); | |
1618 | ibdev->owner = THIS_MODULE; | |
1619 | ibdev->node_guid = cpu_to_be64(ppd->guid); | |
77241056 | 1620 | ibdev->phys_port_cnt = dd->num_pports; |
77241056 | 1621 | ibdev->dma_device = &dd->pcidev->dev; |
77241056 | 1622 | ibdev->modify_device = modify_device; |
4331629f DD |
1623 | |
1624 | /* keep process mad in the driver */ | |
77241056 | 1625 | ibdev->process_mad = hfi1_process_mad; |
77241056 MM |
1626 | |
1627 | strncpy(ibdev->node_desc, init_utsname()->nodename, | |
1628 | sizeof(ibdev->node_desc)); | |
1629 | ||
ec3f2c12 DD |
1630 | /* |
1631 | * Fill in rvt info object. | |
1632 | */ | |
1633 | dd->verbs_dev.rdi.driver_f.port_callback = hfi1_create_port_files; | |
49dbb6cf DD |
1634 | dd->verbs_dev.rdi.driver_f.get_card_name = get_card_name; |
1635 | dd->verbs_dev.rdi.driver_f.get_pci_dev = get_pci_dev; | |
15723f06 | 1636 | dd->verbs_dev.rdi.driver_f.check_ah = hfi1_check_ah; |
8f1764fa | 1637 | dd->verbs_dev.rdi.driver_f.notify_new_ah = hfi1_notify_new_ah; |
25131463 | 1638 | dd->verbs_dev.rdi.driver_f.get_guid_be = hfi1_get_guid_be; |
45b59eef HC |
1639 | dd->verbs_dev.rdi.driver_f.query_port_state = query_port; |
1640 | dd->verbs_dev.rdi.driver_f.shut_down_port = shut_down_port; | |
1641 | dd->verbs_dev.rdi.driver_f.cap_mask_chg = hfi1_cap_mask_chg; | |
94d5171c HC |
1642 | /* |
1643 | * Fill in rvt info device attributes. | |
1644 | */ | |
1645 | hfi1_fill_device_attr(dd); | |
a2c2d608 DD |
1646 | |
1647 | /* queue pair */ | |
a2c2d608 DD |
1648 | dd->verbs_dev.rdi.dparms.qp_table_size = hfi1_qp_table_size; |
1649 | dd->verbs_dev.rdi.dparms.qpn_start = 0; | |
1650 | dd->verbs_dev.rdi.dparms.qpn_inc = 1; | |
1651 | dd->verbs_dev.rdi.dparms.qos_shift = dd->qos_shift; | |
1652 | dd->verbs_dev.rdi.dparms.qpn_res_start = kdeth_qp << 16; | |
1653 | dd->verbs_dev.rdi.dparms.qpn_res_end = | |
abd712da | 1654 | dd->verbs_dev.rdi.dparms.qpn_res_start + 65535; |
ec4274f1 DD |
1655 | dd->verbs_dev.rdi.dparms.max_rdma_atomic = HFI1_MAX_RDMA_ATOMIC; |
1656 | dd->verbs_dev.rdi.dparms.psn_mask = PSN_MASK; | |
1657 | dd->verbs_dev.rdi.dparms.psn_shift = PSN_SHIFT; | |
1658 | dd->verbs_dev.rdi.dparms.psn_modify_mask = PSN_MODIFY_MASK; | |
45b59eef HC |
1659 | dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_INTEL_OPA; |
1660 | dd->verbs_dev.rdi.dparms.max_mad_size = OPA_MGMT_MAD_SIZE; | |
1661 | ||
a2c2d608 DD |
1662 | dd->verbs_dev.rdi.driver_f.qp_priv_alloc = qp_priv_alloc; |
1663 | dd->verbs_dev.rdi.driver_f.qp_priv_free = qp_priv_free; | |
1664 | dd->verbs_dev.rdi.driver_f.free_all_qps = free_all_qps; | |
1665 | dd->verbs_dev.rdi.driver_f.notify_qp_reset = notify_qp_reset; | |
83693bd1 DD |
1666 | dd->verbs_dev.rdi.driver_f.do_send = hfi1_do_send; |
1667 | dd->verbs_dev.rdi.driver_f.schedule_send = hfi1_schedule_send; | |
46a80d62 | 1668 | dd->verbs_dev.rdi.driver_f.schedule_send_no_lock = _hfi1_schedule_send; |
ec4274f1 DD |
1669 | dd->verbs_dev.rdi.driver_f.get_pmtu_from_attr = get_pmtu_from_attr; |
1670 | dd->verbs_dev.rdi.driver_f.notify_error_qp = notify_error_qp; | |
1671 | dd->verbs_dev.rdi.driver_f.flush_qp_waiters = flush_qp_waiters; | |
1672 | dd->verbs_dev.rdi.driver_f.stop_send_queue = stop_send_queue; | |
1673 | dd->verbs_dev.rdi.driver_f.quiesce_qp = quiesce_qp; | |
1674 | dd->verbs_dev.rdi.driver_f.notify_error_qp = notify_error_qp; | |
1675 | dd->verbs_dev.rdi.driver_f.mtu_from_qp = mtu_from_qp; | |
1676 | dd->verbs_dev.rdi.driver_f.mtu_to_path_mtu = mtu_to_path_mtu; | |
1677 | dd->verbs_dev.rdi.driver_f.check_modify_qp = hfi1_check_modify_qp; | |
1678 | dd->verbs_dev.rdi.driver_f.modify_qp = hfi1_modify_qp; | |
46a80d62 | 1679 | dd->verbs_dev.rdi.driver_f.check_send_wqe = hfi1_check_send_wqe; |
a2c2d608 | 1680 | |
abd712da DD |
1681 | /* completeion queue */ |
1682 | snprintf(dd->verbs_dev.rdi.dparms.cq_name, | |
1683 | sizeof(dd->verbs_dev.rdi.dparms.cq_name), | |
1684 | "hfi1_cq%d", dd->unit); | |
27807392 | 1685 | dd->verbs_dev.rdi.dparms.node = dd->node; |
abd712da | 1686 | |
a2c2d608 | 1687 | /* misc settings */ |
abd712da | 1688 | dd->verbs_dev.rdi.flags = 0; /* Let rdmavt handle it all */ |
895420dd | 1689 | dd->verbs_dev.rdi.dparms.lkey_table_size = hfi1_lkey_table_size; |
4eb06882 DD |
1690 | dd->verbs_dev.rdi.dparms.nports = dd->num_pports; |
1691 | dd->verbs_dev.rdi.dparms.npkeys = hfi1_get_npkeys(dd); | |
1692 | ||
1ac57c50 MM |
1693 | /* post send table */ |
1694 | dd->verbs_dev.rdi.post_parms = hfi1_post_parms; | |
1695 | ||
4eb06882 DD |
1696 | ppd = dd->pport; |
1697 | for (i = 0; i < dd->num_pports; i++, ppd++) | |
1698 | rvt_init_port(&dd->verbs_dev.rdi, | |
1699 | &ppd->ibport_data.rvp, | |
1700 | i, | |
1701 | ppd->pkeys); | |
ec3f2c12 DD |
1702 | |
1703 | ret = rvt_register_device(&dd->verbs_dev.rdi); | |
77241056 | 1704 | if (ret) |
9c4a311e | 1705 | goto err_verbs_txreq; |
77241056 MM |
1706 | |
1707 | ret = hfi1_verbs_register_sysfs(dd); | |
1708 | if (ret) | |
1709 | goto err_class; | |
1710 | ||
9c4a311e | 1711 | return ret; |
77241056 MM |
1712 | |
1713 | err_class: | |
ec3f2c12 | 1714 | rvt_unregister_device(&dd->verbs_dev.rdi); |
77241056 | 1715 | err_verbs_txreq: |
45842abb | 1716 | verbs_txreq_exit(dev); |
77241056 | 1717 | dd_dev_err(dd, "cannot register verbs: %d!\n", -ret); |
77241056 MM |
1718 | return ret; |
1719 | } | |
1720 | ||
1721 | void hfi1_unregister_ib_device(struct hfi1_devdata *dd) | |
1722 | { | |
1723 | struct hfi1_ibdev *dev = &dd->verbs_dev; | |
77241056 MM |
1724 | |
1725 | hfi1_verbs_unregister_sysfs(dd); | |
1726 | ||
ec3f2c12 | 1727 | rvt_unregister_device(&dd->verbs_dev.rdi); |
77241056 MM |
1728 | |
1729 | if (!list_empty(&dev->txwait)) | |
1730 | dd_dev_err(dd, "txwait list not empty!\n"); | |
1731 | if (!list_empty(&dev->memwait)) | |
1732 | dd_dev_err(dd, "memwait list not empty!\n"); | |
77241056 | 1733 | |
77241056 | 1734 | del_timer_sync(&dev->mem_timer); |
45842abb | 1735 | verbs_txreq_exit(dev); |
77241056 MM |
1736 | } |
1737 | ||
77241056 MM |
1738 | void hfi1_cnp_rcv(struct hfi1_packet *packet) |
1739 | { | |
1740 | struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data; | |
977940b8 AK |
1741 | struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); |
1742 | struct hfi1_ib_header *hdr = packet->hdr; | |
895420dd | 1743 | struct rvt_qp *qp = packet->qp; |
977940b8 AK |
1744 | u32 lqpn, rqpn = 0; |
1745 | u16 rlid = 0; | |
b736a469 | 1746 | u8 sl, sc5, svc_type; |
977940b8 AK |
1747 | |
1748 | switch (packet->qp->ibqp.qp_type) { | |
1749 | case IB_QPT_UC: | |
1750 | rlid = qp->remote_ah_attr.dlid; | |
1751 | rqpn = qp->remote_qpn; | |
1752 | svc_type = IB_CC_SVCTYPE_UC; | |
1753 | break; | |
1754 | case IB_QPT_RC: | |
1755 | rlid = qp->remote_ah_attr.dlid; | |
1756 | rqpn = qp->remote_qpn; | |
1757 | svc_type = IB_CC_SVCTYPE_RC; | |
1758 | break; | |
1759 | case IB_QPT_SMI: | |
1760 | case IB_QPT_GSI: | |
1761 | case IB_QPT_UD: | |
1762 | svc_type = IB_CC_SVCTYPE_UD; | |
1763 | break; | |
1764 | default: | |
4eb06882 | 1765 | ibp->rvp.n_pkt_drops++; |
977940b8 AK |
1766 | return; |
1767 | } | |
1768 | ||
b736a469 | 1769 | sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf); |
977940b8 AK |
1770 | sl = ibp->sc_to_sl[sc5]; |
1771 | lqpn = qp->ibqp.qp_num; | |
1772 | ||
1773 | process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); | |
77241056 | 1774 | } |