]>
Commit | Line | Data |
---|---|---|
77241056 | 1 | /* |
2280740f | 2 | * Copyright(c) 2015-2017 Intel Corporation. |
77241056 MM |
3 | * |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
77241056 MM |
9 | * This program is free software; you can redistribute it and/or modify |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
77241056 MM |
20 | * Redistribution and use in source and binary forms, with or without |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | ||
48 | #include <linux/delay.h> | |
49 | #include "hfi.h" | |
50 | #include "qp.h" | |
51 | #include "trace.h" | |
52 | ||
53 | #define SC_CTXT_PACKET_EGRESS_TIMEOUT 350 /* in chip cycles */ | |
54 | ||
55 | #define SC(name) SEND_CTXT_##name | |
56 | /* | |
57 | * Send Context functions | |
58 | */ | |
59 | static void sc_wait_for_packet_egress(struct send_context *sc, int pause); | |
60 | ||
61 | /* | |
62 | * Set the CM reset bit and wait for it to clear. Use the provided | |
63 | * sendctrl register. This routine has no locking. | |
64 | */ | |
65 | void __cm_reset(struct hfi1_devdata *dd, u64 sendctrl) | |
66 | { | |
67 | write_csr(dd, SEND_CTRL, sendctrl | SEND_CTRL_CM_RESET_SMASK); | |
68 | while (1) { | |
69 | udelay(1); | |
70 | sendctrl = read_csr(dd, SEND_CTRL); | |
71 | if ((sendctrl & SEND_CTRL_CM_RESET_SMASK) == 0) | |
72 | break; | |
73 | } | |
74 | } | |
75 | ||
76 | /* defined in header release 48 and higher */ | |
77 | #ifndef SEND_CTRL_UNSUPPORTED_VL_SHIFT | |
78 | #define SEND_CTRL_UNSUPPORTED_VL_SHIFT 3 | |
79 | #define SEND_CTRL_UNSUPPORTED_VL_MASK 0xffull | |
80 | #define SEND_CTRL_UNSUPPORTED_VL_SMASK (SEND_CTRL_UNSUPPORTED_VL_MASK \ | |
81 | << SEND_CTRL_UNSUPPORTED_VL_SHIFT) | |
82 | #endif | |
83 | ||
84 | /* global control of PIO send */ | |
85 | void pio_send_control(struct hfi1_devdata *dd, int op) | |
86 | { | |
87 | u64 reg, mask; | |
88 | unsigned long flags; | |
89 | int write = 1; /* write sendctrl back */ | |
90 | int flush = 0; /* re-read sendctrl to make sure it is flushed */ | |
91 | ||
92 | spin_lock_irqsave(&dd->sendctrl_lock, flags); | |
93 | ||
94 | reg = read_csr(dd, SEND_CTRL); | |
95 | switch (op) { | |
96 | case PSC_GLOBAL_ENABLE: | |
97 | reg |= SEND_CTRL_SEND_ENABLE_SMASK; | |
98 | /* Fall through */ | |
99 | case PSC_DATA_VL_ENABLE: | |
100 | /* Disallow sending on VLs not enabled */ | |
8638b77f | 101 | mask = (((~0ull) << num_vls) & SEND_CTRL_UNSUPPORTED_VL_MASK) << |
77241056 MM |
102 | SEND_CTRL_UNSUPPORTED_VL_SHIFT; |
103 | reg = (reg & ~SEND_CTRL_UNSUPPORTED_VL_SMASK) | mask; | |
104 | break; | |
105 | case PSC_GLOBAL_DISABLE: | |
106 | reg &= ~SEND_CTRL_SEND_ENABLE_SMASK; | |
107 | break; | |
108 | case PSC_GLOBAL_VLARB_ENABLE: | |
109 | reg |= SEND_CTRL_VL_ARBITER_ENABLE_SMASK; | |
110 | break; | |
111 | case PSC_GLOBAL_VLARB_DISABLE: | |
112 | reg &= ~SEND_CTRL_VL_ARBITER_ENABLE_SMASK; | |
113 | break; | |
114 | case PSC_CM_RESET: | |
115 | __cm_reset(dd, reg); | |
116 | write = 0; /* CSR already written (and flushed) */ | |
117 | break; | |
118 | case PSC_DATA_VL_DISABLE: | |
119 | reg |= SEND_CTRL_UNSUPPORTED_VL_SMASK; | |
120 | flush = 1; | |
121 | break; | |
122 | default: | |
123 | dd_dev_err(dd, "%s: invalid control %d\n", __func__, op); | |
124 | break; | |
125 | } | |
126 | ||
127 | if (write) { | |
128 | write_csr(dd, SEND_CTRL, reg); | |
129 | if (flush) | |
50e5dcbe | 130 | (void)read_csr(dd, SEND_CTRL); /* flush write */ |
77241056 MM |
131 | } |
132 | ||
133 | spin_unlock_irqrestore(&dd->sendctrl_lock, flags); | |
134 | } | |
135 | ||
136 | /* number of send context memory pools */ | |
137 | #define NUM_SC_POOLS 2 | |
138 | ||
139 | /* Send Context Size (SCS) wildcards */ | |
140 | #define SCS_POOL_0 -1 | |
141 | #define SCS_POOL_1 -2 | |
44306f15 | 142 | |
77241056 MM |
143 | /* Send Context Count (SCC) wildcards */ |
144 | #define SCC_PER_VL -1 | |
145 | #define SCC_PER_CPU -2 | |
77241056 | 146 | #define SCC_PER_KRCVQ -3 |
44306f15 JX |
147 | |
148 | /* Send Context Size (SCS) constants */ | |
149 | #define SCS_ACK_CREDITS 32 | |
150 | #define SCS_VL15_CREDITS 102 /* 3 pkts of 2048B data + 128B header */ | |
151 | ||
152 | #define PIO_THRESHOLD_CEILING 4096 | |
77241056 MM |
153 | |
154 | #define PIO_WAIT_BATCH_SIZE 5 | |
155 | ||
156 | /* default send context sizes */ | |
157 | static struct sc_config_sizes sc_config_sizes[SC_MAX] = { | |
158 | [SC_KERNEL] = { .size = SCS_POOL_0, /* even divide, pool 0 */ | |
44306f15 JX |
159 | .count = SCC_PER_VL }, /* one per NUMA */ |
160 | [SC_ACK] = { .size = SCS_ACK_CREDITS, | |
77241056 MM |
161 | .count = SCC_PER_KRCVQ }, |
162 | [SC_USER] = { .size = SCS_POOL_0, /* even divide, pool 0 */ | |
163 | .count = SCC_PER_CPU }, /* one per CPU */ | |
44306f15 JX |
164 | [SC_VL15] = { .size = SCS_VL15_CREDITS, |
165 | .count = 1 }, | |
77241056 MM |
166 | |
167 | }; | |
168 | ||
169 | /* send context memory pool configuration */ | |
170 | struct mem_pool_config { | |
171 | int centipercent; /* % of memory, in 100ths of 1% */ | |
172 | int absolute_blocks; /* absolute block count */ | |
173 | }; | |
174 | ||
175 | /* default memory pool configuration: 100% in pool 0 */ | |
176 | static struct mem_pool_config sc_mem_pool_config[NUM_SC_POOLS] = { | |
177 | /* centi%, abs blocks */ | |
178 | { 10000, -1 }, /* pool 0 */ | |
179 | { 0, -1 }, /* pool 1 */ | |
180 | }; | |
181 | ||
182 | /* memory pool information, used when calculating final sizes */ | |
183 | struct mem_pool_info { | |
4d114fdd JJ |
184 | int centipercent; /* |
185 | * 100th of 1% of memory to use, -1 if blocks | |
186 | * already set | |
187 | */ | |
77241056 MM |
188 | int count; /* count of contexts in the pool */ |
189 | int blocks; /* block size of the pool */ | |
190 | int size; /* context size, in blocks */ | |
191 | }; | |
192 | ||
193 | /* | |
194 | * Convert a pool wildcard to a valid pool index. The wildcards | |
195 | * start at -1 and increase negatively. Map them as: | |
196 | * -1 => 0 | |
197 | * -2 => 1 | |
198 | * etc. | |
199 | * | |
200 | * Return -1 on non-wildcard input, otherwise convert to a pool number. | |
201 | */ | |
202 | static int wildcard_to_pool(int wc) | |
203 | { | |
204 | if (wc >= 0) | |
205 | return -1; /* non-wildcard */ | |
206 | return -wc - 1; | |
207 | } | |
208 | ||
209 | static const char *sc_type_names[SC_MAX] = { | |
210 | "kernel", | |
211 | "ack", | |
44306f15 JX |
212 | "user", |
213 | "vl15" | |
77241056 MM |
214 | }; |
215 | ||
216 | static const char *sc_type_name(int index) | |
217 | { | |
218 | if (index < 0 || index >= SC_MAX) | |
219 | return "unknown"; | |
220 | return sc_type_names[index]; | |
221 | } | |
222 | ||
223 | /* | |
224 | * Read the send context memory pool configuration and send context | |
225 | * size configuration. Replace any wildcards and come up with final | |
226 | * counts and sizes for the send context types. | |
227 | */ | |
228 | int init_sc_pools_and_sizes(struct hfi1_devdata *dd) | |
229 | { | |
230 | struct mem_pool_info mem_pool_info[NUM_SC_POOLS] = { { 0 } }; | |
231 | int total_blocks = (dd->chip_pio_mem_size / PIO_BLOCK_SIZE) - 1; | |
232 | int total_contexts = 0; | |
233 | int fixed_blocks; | |
234 | int pool_blocks; | |
235 | int used_blocks; | |
236 | int cp_total; /* centipercent total */ | |
237 | int ab_total; /* absolute block total */ | |
238 | int extra; | |
239 | int i; | |
240 | ||
44306f15 JX |
241 | /* |
242 | * When SDMA is enabled, kernel context pio packet size is capped by | |
243 | * "piothreshold". Reduce pio buffer allocation for kernel context by | |
244 | * setting it to a fixed size. The allocation allows 3-deep buffering | |
245 | * of the largest pio packets plus up to 128 bytes header, sufficient | |
246 | * to maintain verbs performance. | |
247 | * | |
248 | * When SDMA is disabled, keep the default pooling allocation. | |
249 | */ | |
250 | if (HFI1_CAP_IS_KSET(SDMA)) { | |
251 | u16 max_pkt_size = (piothreshold < PIO_THRESHOLD_CEILING) ? | |
252 | piothreshold : PIO_THRESHOLD_CEILING; | |
253 | sc_config_sizes[SC_KERNEL].size = | |
254 | 3 * (max_pkt_size + 128) / PIO_BLOCK_SIZE; | |
255 | } | |
256 | ||
77241056 MM |
257 | /* |
258 | * Step 0: | |
259 | * - copy the centipercents/absolute sizes from the pool config | |
260 | * - sanity check these values | |
261 | * - add up centipercents, then later check for full value | |
262 | * - add up absolute blocks, then later check for over-commit | |
263 | */ | |
264 | cp_total = 0; | |
265 | ab_total = 0; | |
266 | for (i = 0; i < NUM_SC_POOLS; i++) { | |
267 | int cp = sc_mem_pool_config[i].centipercent; | |
268 | int ab = sc_mem_pool_config[i].absolute_blocks; | |
269 | ||
270 | /* | |
271 | * A negative value is "unused" or "invalid". Both *can* | |
272 | * be valid, but centipercent wins, so check that first | |
273 | */ | |
274 | if (cp >= 0) { /* centipercent valid */ | |
275 | cp_total += cp; | |
276 | } else if (ab >= 0) { /* absolute blocks valid */ | |
277 | ab_total += ab; | |
278 | } else { /* neither valid */ | |
279 | dd_dev_err( | |
280 | dd, | |
281 | "Send context memory pool %d: both the block count and centipercent are invalid\n", | |
282 | i); | |
283 | return -EINVAL; | |
284 | } | |
285 | ||
286 | mem_pool_info[i].centipercent = cp; | |
287 | mem_pool_info[i].blocks = ab; | |
288 | } | |
289 | ||
290 | /* do not use both % and absolute blocks for different pools */ | |
291 | if (cp_total != 0 && ab_total != 0) { | |
292 | dd_dev_err( | |
293 | dd, | |
294 | "All send context memory pools must be described as either centipercent or blocks, no mixing between pools\n"); | |
295 | return -EINVAL; | |
296 | } | |
297 | ||
298 | /* if any percentages are present, they must add up to 100% x 100 */ | |
299 | if (cp_total != 0 && cp_total != 10000) { | |
300 | dd_dev_err( | |
301 | dd, | |
302 | "Send context memory pool centipercent is %d, expecting 10000\n", | |
303 | cp_total); | |
304 | return -EINVAL; | |
305 | } | |
306 | ||
307 | /* the absolute pool total cannot be more than the mem total */ | |
308 | if (ab_total > total_blocks) { | |
309 | dd_dev_err( | |
310 | dd, | |
311 | "Send context memory pool absolute block count %d is larger than the memory size %d\n", | |
312 | ab_total, total_blocks); | |
313 | return -EINVAL; | |
314 | } | |
315 | ||
316 | /* | |
317 | * Step 2: | |
318 | * - copy from the context size config | |
319 | * - replace context type wildcard counts with real values | |
320 | * - add up non-memory pool block sizes | |
321 | * - add up memory pool user counts | |
322 | */ | |
323 | fixed_blocks = 0; | |
324 | for (i = 0; i < SC_MAX; i++) { | |
325 | int count = sc_config_sizes[i].count; | |
326 | int size = sc_config_sizes[i].size; | |
327 | int pool; | |
328 | ||
329 | /* | |
330 | * Sanity check count: Either a positive value or | |
331 | * one of the expected wildcards is valid. The positive | |
332 | * value is checked later when we compare against total | |
333 | * memory available. | |
334 | */ | |
335 | if (i == SC_ACK) { | |
336 | count = dd->n_krcv_queues; | |
337 | } else if (i == SC_KERNEL) { | |
44306f15 | 338 | count = INIT_SC_PER_VL * num_vls; |
77241056 MM |
339 | } else if (count == SCC_PER_CPU) { |
340 | count = dd->num_rcv_contexts - dd->n_krcv_queues; | |
341 | } else if (count < 0) { | |
342 | dd_dev_err( | |
343 | dd, | |
344 | "%s send context invalid count wildcard %d\n", | |
345 | sc_type_name(i), count); | |
346 | return -EINVAL; | |
347 | } | |
348 | if (total_contexts + count > dd->chip_send_contexts) | |
349 | count = dd->chip_send_contexts - total_contexts; | |
350 | ||
351 | total_contexts += count; | |
352 | ||
353 | /* | |
354 | * Sanity check pool: The conversion will return a pool | |
355 | * number or -1 if a fixed (non-negative) value. The fixed | |
356 | * value is checked later when we compare against | |
357 | * total memory available. | |
358 | */ | |
359 | pool = wildcard_to_pool(size); | |
360 | if (pool == -1) { /* non-wildcard */ | |
361 | fixed_blocks += size * count; | |
362 | } else if (pool < NUM_SC_POOLS) { /* valid wildcard */ | |
363 | mem_pool_info[pool].count += count; | |
364 | } else { /* invalid wildcard */ | |
365 | dd_dev_err( | |
366 | dd, | |
367 | "%s send context invalid pool wildcard %d\n", | |
368 | sc_type_name(i), size); | |
369 | return -EINVAL; | |
370 | } | |
371 | ||
372 | dd->sc_sizes[i].count = count; | |
373 | dd->sc_sizes[i].size = size; | |
374 | } | |
375 | if (fixed_blocks > total_blocks) { | |
376 | dd_dev_err( | |
377 | dd, | |
378 | "Send context fixed block count, %u, larger than total block count %u\n", | |
379 | fixed_blocks, total_blocks); | |
380 | return -EINVAL; | |
381 | } | |
382 | ||
383 | /* step 3: calculate the blocks in the pools, and pool context sizes */ | |
384 | pool_blocks = total_blocks - fixed_blocks; | |
385 | if (ab_total > pool_blocks) { | |
386 | dd_dev_err( | |
387 | dd, | |
388 | "Send context fixed pool sizes, %u, larger than pool block count %u\n", | |
389 | ab_total, pool_blocks); | |
390 | return -EINVAL; | |
391 | } | |
392 | /* subtract off the fixed pool blocks */ | |
393 | pool_blocks -= ab_total; | |
394 | ||
395 | for (i = 0; i < NUM_SC_POOLS; i++) { | |
396 | struct mem_pool_info *pi = &mem_pool_info[i]; | |
397 | ||
398 | /* % beats absolute blocks */ | |
399 | if (pi->centipercent >= 0) | |
400 | pi->blocks = (pool_blocks * pi->centipercent) / 10000; | |
401 | ||
402 | if (pi->blocks == 0 && pi->count != 0) { | |
403 | dd_dev_err( | |
404 | dd, | |
405 | "Send context memory pool %d has %u contexts, but no blocks\n", | |
406 | i, pi->count); | |
407 | return -EINVAL; | |
408 | } | |
409 | if (pi->count == 0) { | |
410 | /* warn about wasted blocks */ | |
411 | if (pi->blocks != 0) | |
412 | dd_dev_err( | |
413 | dd, | |
414 | "Send context memory pool %d has %u blocks, but zero contexts\n", | |
415 | i, pi->blocks); | |
416 | pi->size = 0; | |
417 | } else { | |
418 | pi->size = pi->blocks / pi->count; | |
419 | } | |
420 | } | |
421 | ||
422 | /* step 4: fill in the context type sizes from the pool sizes */ | |
423 | used_blocks = 0; | |
424 | for (i = 0; i < SC_MAX; i++) { | |
425 | if (dd->sc_sizes[i].size < 0) { | |
426 | unsigned pool = wildcard_to_pool(dd->sc_sizes[i].size); | |
427 | ||
428 | WARN_ON_ONCE(pool >= NUM_SC_POOLS); | |
429 | dd->sc_sizes[i].size = mem_pool_info[pool].size; | |
430 | } | |
431 | /* make sure we are not larger than what is allowed by the HW */ | |
432 | #define PIO_MAX_BLOCKS 1024 | |
433 | if (dd->sc_sizes[i].size > PIO_MAX_BLOCKS) | |
434 | dd->sc_sizes[i].size = PIO_MAX_BLOCKS; | |
435 | ||
436 | /* calculate our total usage */ | |
437 | used_blocks += dd->sc_sizes[i].size * dd->sc_sizes[i].count; | |
438 | } | |
439 | extra = total_blocks - used_blocks; | |
440 | if (extra != 0) | |
441 | dd_dev_info(dd, "unused send context blocks: %d\n", extra); | |
442 | ||
443 | return total_contexts; | |
444 | } | |
445 | ||
446 | int init_send_contexts(struct hfi1_devdata *dd) | |
447 | { | |
448 | u16 base; | |
449 | int ret, i, j, context; | |
450 | ||
451 | ret = init_credit_return(dd); | |
452 | if (ret) | |
453 | return ret; | |
454 | ||
455 | dd->hw_to_sw = kmalloc_array(TXE_NUM_CONTEXTS, sizeof(u8), | |
456 | GFP_KERNEL); | |
457 | dd->send_contexts = kcalloc(dd->num_send_contexts, | |
458 | sizeof(struct send_context_info), | |
459 | GFP_KERNEL); | |
460 | if (!dd->send_contexts || !dd->hw_to_sw) { | |
77241056 MM |
461 | kfree(dd->hw_to_sw); |
462 | kfree(dd->send_contexts); | |
463 | free_credit_return(dd); | |
464 | return -ENOMEM; | |
465 | } | |
466 | ||
467 | /* hardware context map starts with invalid send context indices */ | |
468 | for (i = 0; i < TXE_NUM_CONTEXTS; i++) | |
469 | dd->hw_to_sw[i] = INVALID_SCI; | |
470 | ||
471 | /* | |
472 | * All send contexts have their credit sizes. Allocate credits | |
473 | * for each context one after another from the global space. | |
474 | */ | |
475 | context = 0; | |
476 | base = 1; | |
477 | for (i = 0; i < SC_MAX; i++) { | |
478 | struct sc_config_sizes *scs = &dd->sc_sizes[i]; | |
479 | ||
480 | for (j = 0; j < scs->count; j++) { | |
481 | struct send_context_info *sci = | |
482 | &dd->send_contexts[context]; | |
483 | sci->type = i; | |
484 | sci->base = base; | |
485 | sci->credits = scs->size; | |
486 | ||
487 | context++; | |
488 | base += scs->size; | |
489 | } | |
490 | } | |
491 | ||
492 | return 0; | |
493 | } | |
494 | ||
495 | /* | |
496 | * Allocate a software index and hardware context of the given type. | |
497 | * | |
498 | * Must be called with dd->sc_lock held. | |
499 | */ | |
500 | static int sc_hw_alloc(struct hfi1_devdata *dd, int type, u32 *sw_index, | |
501 | u32 *hw_context) | |
502 | { | |
503 | struct send_context_info *sci; | |
504 | u32 index; | |
505 | u32 context; | |
506 | ||
507 | for (index = 0, sci = &dd->send_contexts[0]; | |
508 | index < dd->num_send_contexts; index++, sci++) { | |
509 | if (sci->type == type && sci->allocated == 0) { | |
510 | sci->allocated = 1; | |
511 | /* use a 1:1 mapping, but make them non-equal */ | |
512 | context = dd->chip_send_contexts - index - 1; | |
513 | dd->hw_to_sw[context] = index; | |
514 | *sw_index = index; | |
515 | *hw_context = context; | |
516 | return 0; /* success */ | |
517 | } | |
518 | } | |
519 | dd_dev_err(dd, "Unable to locate a free type %d send context\n", type); | |
520 | return -ENOSPC; | |
521 | } | |
522 | ||
523 | /* | |
524 | * Free the send context given by its software index. | |
525 | * | |
526 | * Must be called with dd->sc_lock held. | |
527 | */ | |
528 | static void sc_hw_free(struct hfi1_devdata *dd, u32 sw_index, u32 hw_context) | |
529 | { | |
530 | struct send_context_info *sci; | |
531 | ||
532 | sci = &dd->send_contexts[sw_index]; | |
533 | if (!sci->allocated) { | |
534 | dd_dev_err(dd, "%s: sw_index %u not allocated? hw_context %u\n", | |
17fb4f29 | 535 | __func__, sw_index, hw_context); |
77241056 MM |
536 | } |
537 | sci->allocated = 0; | |
538 | dd->hw_to_sw[hw_context] = INVALID_SCI; | |
539 | } | |
540 | ||
541 | /* return the base context of a context in a group */ | |
542 | static inline u32 group_context(u32 context, u32 group) | |
543 | { | |
544 | return (context >> group) << group; | |
545 | } | |
546 | ||
547 | /* return the size of a group */ | |
548 | static inline u32 group_size(u32 group) | |
549 | { | |
550 | return 1 << group; | |
551 | } | |
552 | ||
553 | /* | |
60368186 | 554 | * Obtain the credit return addresses, kernel virtual and bus, for the |
77241056 MM |
555 | * given sc. |
556 | * | |
557 | * To understand this routine: | |
60368186 | 558 | * o va and dma are arrays of struct credit_return. One for each physical |
77241056 MM |
559 | * send context, per NUMA. |
560 | * o Each send context always looks in its relative location in a struct | |
561 | * credit_return for its credit return. | |
562 | * o Each send context in a group must have its return address CSR programmed | |
563 | * with the same value. Use the address of the first send context in the | |
564 | * group. | |
565 | */ | |
60368186 | 566 | static void cr_group_addresses(struct send_context *sc, dma_addr_t *dma) |
77241056 MM |
567 | { |
568 | u32 gc = group_context(sc->hw_context, sc->group); | |
569 | u32 index = sc->hw_context & 0x7; | |
570 | ||
571 | sc->hw_free = &sc->dd->cr_base[sc->node].va[gc].cr[index]; | |
60368186 TK |
572 | *dma = (unsigned long) |
573 | &((struct credit_return *)sc->dd->cr_base[sc->node].dma)[gc]; | |
77241056 MM |
574 | } |
575 | ||
576 | /* | |
577 | * Work queue function triggered in error interrupt routine for | |
578 | * kernel contexts. | |
579 | */ | |
580 | static void sc_halted(struct work_struct *work) | |
581 | { | |
582 | struct send_context *sc; | |
583 | ||
584 | sc = container_of(work, struct send_context, halt_work); | |
585 | sc_restart(sc); | |
586 | } | |
587 | ||
588 | /* | |
589 | * Calculate PIO block threshold for this send context using the given MTU. | |
590 | * Trigger a return when one MTU plus optional header of credits remain. | |
591 | * | |
592 | * Parameter mtu is in bytes. | |
593 | * Parameter hdrqentsize is in DWORDs. | |
594 | * | |
595 | * Return value is what to write into the CSR: trigger return when | |
596 | * unreturned credits pass this count. | |
597 | */ | |
598 | u32 sc_mtu_to_threshold(struct send_context *sc, u32 mtu, u32 hdrqentsize) | |
599 | { | |
600 | u32 release_credits; | |
601 | u32 threshold; | |
602 | ||
603 | /* add in the header size, then divide by the PIO block size */ | |
604 | mtu += hdrqentsize << 2; | |
605 | release_credits = DIV_ROUND_UP(mtu, PIO_BLOCK_SIZE); | |
606 | ||
607 | /* check against this context's credits */ | |
608 | if (sc->credits <= release_credits) | |
609 | threshold = 1; | |
610 | else | |
611 | threshold = sc->credits - release_credits; | |
612 | ||
613 | return threshold; | |
614 | } | |
615 | ||
616 | /* | |
617 | * Calculate credit threshold in terms of percent of the allocated credits. | |
618 | * Trigger when unreturned credits equal or exceed the percentage of the whole. | |
619 | * | |
620 | * Return value is what to write into the CSR: trigger return when | |
621 | * unreturned credits pass this count. | |
622 | */ | |
44306f15 | 623 | u32 sc_percent_to_threshold(struct send_context *sc, u32 percent) |
77241056 MM |
624 | { |
625 | return (sc->credits * percent) / 100; | |
626 | } | |
627 | ||
628 | /* | |
629 | * Set the credit return threshold. | |
630 | */ | |
631 | void sc_set_cr_threshold(struct send_context *sc, u32 new_threshold) | |
632 | { | |
633 | unsigned long flags; | |
634 | u32 old_threshold; | |
635 | int force_return = 0; | |
636 | ||
637 | spin_lock_irqsave(&sc->credit_ctrl_lock, flags); | |
638 | ||
639 | old_threshold = (sc->credit_ctrl >> | |
640 | SC(CREDIT_CTRL_THRESHOLD_SHIFT)) | |
641 | & SC(CREDIT_CTRL_THRESHOLD_MASK); | |
642 | ||
643 | if (new_threshold != old_threshold) { | |
644 | sc->credit_ctrl = | |
645 | (sc->credit_ctrl | |
646 | & ~SC(CREDIT_CTRL_THRESHOLD_SMASK)) | |
647 | | ((new_threshold | |
648 | & SC(CREDIT_CTRL_THRESHOLD_MASK)) | |
649 | << SC(CREDIT_CTRL_THRESHOLD_SHIFT)); | |
650 | write_kctxt_csr(sc->dd, sc->hw_context, | |
17fb4f29 | 651 | SC(CREDIT_CTRL), sc->credit_ctrl); |
77241056 MM |
652 | |
653 | /* force a credit return on change to avoid a possible stall */ | |
654 | force_return = 1; | |
655 | } | |
656 | ||
657 | spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags); | |
658 | ||
659 | if (force_return) | |
660 | sc_return_credits(sc); | |
661 | } | |
662 | ||
663 | /* | |
664 | * set_pio_integrity | |
665 | * | |
666 | * Set the CHECK_ENABLE register for the send context 'sc'. | |
667 | */ | |
668 | void set_pio_integrity(struct send_context *sc) | |
669 | { | |
670 | struct hfi1_devdata *dd = sc->dd; | |
77241056 MM |
671 | u32 hw_context = sc->hw_context; |
672 | int type = sc->type; | |
673 | ||
d9ac4555 JP |
674 | write_kctxt_csr(dd, hw_context, |
675 | SC(CHECK_ENABLE), | |
676 | hfi1_pkt_default_send_ctxt_mask(dd, type)); | |
77241056 MM |
677 | } |
678 | ||
a054374f MM |
679 | static u32 get_buffers_allocated(struct send_context *sc) |
680 | { | |
681 | int cpu; | |
682 | u32 ret = 0; | |
683 | ||
684 | for_each_possible_cpu(cpu) | |
685 | ret += *per_cpu_ptr(sc->buffers_allocated, cpu); | |
686 | return ret; | |
687 | } | |
688 | ||
689 | static void reset_buffers_allocated(struct send_context *sc) | |
690 | { | |
691 | int cpu; | |
692 | ||
693 | for_each_possible_cpu(cpu) | |
694 | (*per_cpu_ptr(sc->buffers_allocated, cpu)) = 0; | |
695 | } | |
696 | ||
77241056 MM |
697 | /* |
698 | * Allocate a NUMA relative send context structure of the given type along | |
699 | * with a HW context. | |
700 | */ | |
701 | struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, | |
702 | uint hdrqentsize, int numa) | |
703 | { | |
704 | struct send_context_info *sci; | |
a054374f | 705 | struct send_context *sc = NULL; |
2280740f | 706 | int req_type = type; |
60368186 | 707 | dma_addr_t dma; |
77241056 MM |
708 | unsigned long flags; |
709 | u64 reg; | |
710 | u32 thresh; | |
711 | u32 sw_index; | |
712 | u32 hw_context; | |
713 | int ret; | |
714 | u8 opval, opmask; | |
715 | ||
716 | /* do not allocate while frozen */ | |
717 | if (dd->flags & HFI1_FROZEN) | |
718 | return NULL; | |
719 | ||
fcdd76df | 720 | sc = kzalloc_node(sizeof(*sc), GFP_KERNEL, numa); |
806e6e1b | 721 | if (!sc) |
77241056 | 722 | return NULL; |
77241056 | 723 | |
a054374f MM |
724 | sc->buffers_allocated = alloc_percpu(u32); |
725 | if (!sc->buffers_allocated) { | |
726 | kfree(sc); | |
727 | dd_dev_err(dd, | |
728 | "Cannot allocate buffers_allocated per cpu counters\n" | |
729 | ); | |
730 | return NULL; | |
731 | } | |
732 | ||
2280740f VN |
733 | /* |
734 | * VNIC contexts are dynamically allocated. | |
735 | * Hence, pick a user context for VNIC. | |
736 | */ | |
737 | if (type == SC_VNIC) | |
738 | type = SC_USER; | |
739 | ||
77241056 MM |
740 | spin_lock_irqsave(&dd->sc_lock, flags); |
741 | ret = sc_hw_alloc(dd, type, &sw_index, &hw_context); | |
742 | if (ret) { | |
743 | spin_unlock_irqrestore(&dd->sc_lock, flags); | |
a054374f | 744 | free_percpu(sc->buffers_allocated); |
77241056 MM |
745 | kfree(sc); |
746 | return NULL; | |
747 | } | |
748 | ||
2280740f VN |
749 | /* |
750 | * VNIC contexts are used by kernel driver. | |
751 | * Hence, mark them as kernel contexts. | |
752 | */ | |
753 | if (req_type == SC_VNIC) { | |
754 | dd->send_contexts[sw_index].type = SC_KERNEL; | |
755 | type = SC_KERNEL; | |
756 | } | |
757 | ||
77241056 MM |
758 | sci = &dd->send_contexts[sw_index]; |
759 | sci->sc = sc; | |
760 | ||
761 | sc->dd = dd; | |
762 | sc->node = numa; | |
763 | sc->type = type; | |
764 | spin_lock_init(&sc->alloc_lock); | |
765 | spin_lock_init(&sc->release_lock); | |
766 | spin_lock_init(&sc->credit_ctrl_lock); | |
767 | INIT_LIST_HEAD(&sc->piowait); | |
768 | INIT_WORK(&sc->halt_work, sc_halted); | |
77241056 MM |
769 | init_waitqueue_head(&sc->halt_wait); |
770 | ||
771 | /* grouping is always single context for now */ | |
772 | sc->group = 0; | |
773 | ||
774 | sc->sw_index = sw_index; | |
775 | sc->hw_context = hw_context; | |
60368186 | 776 | cr_group_addresses(sc, &dma); |
77241056 | 777 | sc->credits = sci->credits; |
8af8d297 | 778 | sc->size = sc->credits * PIO_BLOCK_SIZE; |
77241056 MM |
779 | |
780 | /* PIO Send Memory Address details */ | |
781 | #define PIO_ADDR_CONTEXT_MASK 0xfful | |
782 | #define PIO_ADDR_CONTEXT_SHIFT 16 | |
783 | sc->base_addr = dd->piobase + ((hw_context & PIO_ADDR_CONTEXT_MASK) | |
784 | << PIO_ADDR_CONTEXT_SHIFT); | |
785 | ||
786 | /* set base and credits */ | |
787 | reg = ((sci->credits & SC(CTRL_CTXT_DEPTH_MASK)) | |
788 | << SC(CTRL_CTXT_DEPTH_SHIFT)) | |
789 | | ((sci->base & SC(CTRL_CTXT_BASE_MASK)) | |
790 | << SC(CTRL_CTXT_BASE_SHIFT)); | |
791 | write_kctxt_csr(dd, hw_context, SC(CTRL), reg); | |
792 | ||
793 | set_pio_integrity(sc); | |
794 | ||
795 | /* unmask all errors */ | |
796 | write_kctxt_csr(dd, hw_context, SC(ERR_MASK), (u64)-1); | |
797 | ||
798 | /* set the default partition key */ | |
799 | write_kctxt_csr(dd, hw_context, SC(CHECK_PARTITION_KEY), | |
bf640096 HC |
800 | (SC(CHECK_PARTITION_KEY_VALUE_MASK) & |
801 | DEFAULT_PKEY) << | |
17fb4f29 | 802 | SC(CHECK_PARTITION_KEY_VALUE_SHIFT)); |
77241056 MM |
803 | |
804 | /* per context type checks */ | |
805 | if (type == SC_USER) { | |
806 | opval = USER_OPCODE_CHECK_VAL; | |
807 | opmask = USER_OPCODE_CHECK_MASK; | |
808 | } else { | |
809 | opval = OPCODE_CHECK_VAL_DISABLED; | |
810 | opmask = OPCODE_CHECK_MASK_DISABLED; | |
811 | } | |
812 | ||
813 | /* set the send context check opcode mask and value */ | |
814 | write_kctxt_csr(dd, hw_context, SC(CHECK_OPCODE), | |
17fb4f29 JJ |
815 | ((u64)opmask << SC(CHECK_OPCODE_MASK_SHIFT)) | |
816 | ((u64)opval << SC(CHECK_OPCODE_VALUE_SHIFT))); | |
77241056 MM |
817 | |
818 | /* set up credit return */ | |
60368186 | 819 | reg = dma & SC(CREDIT_RETURN_ADDR_ADDRESS_SMASK); |
77241056 MM |
820 | write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), reg); |
821 | ||
822 | /* | |
823 | * Calculate the initial credit return threshold. | |
824 | * | |
825 | * For Ack contexts, set a threshold for half the credits. | |
826 | * For User contexts use the given percentage. This has been | |
827 | * sanitized on driver start-up. | |
44306f15 JX |
828 | * For Kernel contexts, use the default MTU plus a header |
829 | * or half the credits, whichever is smaller. This should | |
830 | * work for both the 3-deep buffering allocation and the | |
831 | * pooling allocation. | |
77241056 MM |
832 | */ |
833 | if (type == SC_ACK) { | |
834 | thresh = sc_percent_to_threshold(sc, 50); | |
835 | } else if (type == SC_USER) { | |
836 | thresh = sc_percent_to_threshold(sc, | |
17fb4f29 | 837 | user_credit_return_threshold); |
77241056 | 838 | } else { /* kernel */ |
44306f15 JX |
839 | thresh = min(sc_percent_to_threshold(sc, 50), |
840 | sc_mtu_to_threshold(sc, hfi1_max_mtu, | |
841 | hdrqentsize)); | |
77241056 MM |
842 | } |
843 | reg = thresh << SC(CREDIT_CTRL_THRESHOLD_SHIFT); | |
844 | /* add in early return */ | |
845 | if (type == SC_USER && HFI1_CAP_IS_USET(EARLY_CREDIT_RETURN)) | |
846 | reg |= SC(CREDIT_CTRL_EARLY_RETURN_SMASK); | |
847 | else if (HFI1_CAP_IS_KSET(EARLY_CREDIT_RETURN)) /* kernel, ack */ | |
848 | reg |= SC(CREDIT_CTRL_EARLY_RETURN_SMASK); | |
849 | ||
850 | /* set up write-through credit_ctrl */ | |
851 | sc->credit_ctrl = reg; | |
852 | write_kctxt_csr(dd, hw_context, SC(CREDIT_CTRL), reg); | |
853 | ||
854 | /* User send contexts should not allow sending on VL15 */ | |
855 | if (type == SC_USER) { | |
856 | reg = 1ULL << 15; | |
857 | write_kctxt_csr(dd, hw_context, SC(CHECK_VL), reg); | |
858 | } | |
859 | ||
860 | spin_unlock_irqrestore(&dd->sc_lock, flags); | |
861 | ||
862 | /* | |
863 | * Allocate shadow ring to track outstanding PIO buffers _after_ | |
864 | * unlocking. We don't know the size until the lock is held and | |
865 | * we can't allocate while the lock is held. No one is using | |
866 | * the context yet, so allocate it now. | |
867 | * | |
868 | * User contexts do not get a shadow ring. | |
869 | */ | |
870 | if (type != SC_USER) { | |
871 | /* | |
872 | * Size the shadow ring 1 larger than the number of credits | |
873 | * so head == tail can mean empty. | |
874 | */ | |
875 | sc->sr_size = sci->credits + 1; | |
876 | sc->sr = kzalloc_node(sizeof(union pio_shadow_ring) * | |
877 | sc->sr_size, GFP_KERNEL, numa); | |
878 | if (!sc->sr) { | |
77241056 MM |
879 | sc_free(sc); |
880 | return NULL; | |
881 | } | |
882 | } | |
883 | ||
6c63e423 SS |
884 | hfi1_cdbg(PIO, |
885 | "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n", | |
886 | sw_index, | |
887 | hw_context, | |
888 | sc_type_name(type), | |
889 | sc->group, | |
890 | sc->credits, | |
891 | sc->credit_ctrl, | |
892 | thresh); | |
893 | ||
77241056 MM |
894 | return sc; |
895 | } | |
896 | ||
897 | /* free a per-NUMA send context structure */ | |
898 | void sc_free(struct send_context *sc) | |
899 | { | |
900 | struct hfi1_devdata *dd; | |
901 | unsigned long flags; | |
902 | u32 sw_index; | |
903 | u32 hw_context; | |
904 | ||
905 | if (!sc) | |
906 | return; | |
907 | ||
908 | sc->flags |= SCF_IN_FREE; /* ensure no restarts */ | |
909 | dd = sc->dd; | |
910 | if (!list_empty(&sc->piowait)) | |
911 | dd_dev_err(dd, "piowait list not empty!\n"); | |
912 | sw_index = sc->sw_index; | |
913 | hw_context = sc->hw_context; | |
914 | sc_disable(sc); /* make sure the HW is disabled */ | |
915 | flush_work(&sc->halt_work); | |
916 | ||
917 | spin_lock_irqsave(&dd->sc_lock, flags); | |
918 | dd->send_contexts[sw_index].sc = NULL; | |
919 | ||
920 | /* clear/disable all registers set in sc_alloc */ | |
921 | write_kctxt_csr(dd, hw_context, SC(CTRL), 0); | |
922 | write_kctxt_csr(dd, hw_context, SC(CHECK_ENABLE), 0); | |
923 | write_kctxt_csr(dd, hw_context, SC(ERR_MASK), 0); | |
924 | write_kctxt_csr(dd, hw_context, SC(CHECK_PARTITION_KEY), 0); | |
925 | write_kctxt_csr(dd, hw_context, SC(CHECK_OPCODE), 0); | |
926 | write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), 0); | |
927 | write_kctxt_csr(dd, hw_context, SC(CREDIT_CTRL), 0); | |
928 | ||
929 | /* release the index and context for re-use */ | |
930 | sc_hw_free(dd, sw_index, hw_context); | |
931 | spin_unlock_irqrestore(&dd->sc_lock, flags); | |
932 | ||
933 | kfree(sc->sr); | |
a054374f | 934 | free_percpu(sc->buffers_allocated); |
77241056 MM |
935 | kfree(sc); |
936 | } | |
937 | ||
938 | /* disable the context */ | |
939 | void sc_disable(struct send_context *sc) | |
940 | { | |
941 | u64 reg; | |
942 | unsigned long flags; | |
943 | struct pio_buf *pbuf; | |
944 | ||
945 | if (!sc) | |
946 | return; | |
947 | ||
948 | /* do all steps, even if already disabled */ | |
949 | spin_lock_irqsave(&sc->alloc_lock, flags); | |
950 | reg = read_kctxt_csr(sc->dd, sc->hw_context, SC(CTRL)); | |
951 | reg &= ~SC(CTRL_CTXT_ENABLE_SMASK); | |
952 | sc->flags &= ~SCF_ENABLED; | |
953 | sc_wait_for_packet_egress(sc, 1); | |
954 | write_kctxt_csr(sc->dd, sc->hw_context, SC(CTRL), reg); | |
955 | spin_unlock_irqrestore(&sc->alloc_lock, flags); | |
956 | ||
957 | /* | |
958 | * Flush any waiters. Once the context is disabled, | |
959 | * credit return interrupts are stopped (although there | |
960 | * could be one in-process when the context is disabled). | |
961 | * Wait one microsecond for any lingering interrupts, then | |
962 | * proceed with the flush. | |
963 | */ | |
964 | udelay(1); | |
965 | spin_lock_irqsave(&sc->release_lock, flags); | |
966 | if (sc->sr) { /* this context has a shadow ring */ | |
967 | while (sc->sr_tail != sc->sr_head) { | |
968 | pbuf = &sc->sr[sc->sr_tail].pbuf; | |
969 | if (pbuf->cb) | |
970 | (*pbuf->cb)(pbuf->arg, PRC_SC_DISABLE); | |
971 | sc->sr_tail++; | |
972 | if (sc->sr_tail >= sc->sr_size) | |
973 | sc->sr_tail = 0; | |
974 | } | |
975 | } | |
976 | spin_unlock_irqrestore(&sc->release_lock, flags); | |
977 | } | |
978 | ||
979 | /* return SendEgressCtxtStatus.PacketOccupancy */ | |
980 | #define packet_occupancy(r) \ | |
981 | (((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SMASK)\ | |
982 | >> SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SHIFT) | |
983 | ||
984 | /* is egress halted on the context? */ | |
985 | #define egress_halted(r) \ | |
986 | ((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_HALT_STATUS_SMASK) | |
987 | ||
988 | /* wait for packet egress, optionally pause for credit return */ | |
989 | static void sc_wait_for_packet_egress(struct send_context *sc, int pause) | |
990 | { | |
991 | struct hfi1_devdata *dd = sc->dd; | |
25d97dd5 VM |
992 | u64 reg = 0; |
993 | u64 reg_prev; | |
77241056 MM |
994 | u32 loop = 0; |
995 | ||
996 | while (1) { | |
25d97dd5 | 997 | reg_prev = reg; |
77241056 MM |
998 | reg = read_csr(dd, sc->hw_context * 8 + |
999 | SEND_EGRESS_CTXT_STATUS); | |
1000 | /* done if egress is stopped */ | |
1001 | if (egress_halted(reg)) | |
1002 | break; | |
1003 | reg = packet_occupancy(reg); | |
1004 | if (reg == 0) | |
1005 | break; | |
25d97dd5 VM |
1006 | /* counter is reset if occupancy count changes */ |
1007 | if (reg != reg_prev) | |
1008 | loop = 0; | |
c3c64a95 | 1009 | if (loop > 50000) { |
25d97dd5 | 1010 | /* timed out - bounce the link */ |
77241056 | 1011 | dd_dev_err(dd, |
17fb4f29 JJ |
1012 | "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u, bouncing link\n", |
1013 | __func__, sc->sw_index, | |
1014 | sc->hw_context, (u32)reg); | |
71d47008 | 1015 | queue_work(dd->pport->link_wq, |
17fb4f29 | 1016 | &dd->pport->link_bounce_work); |
77241056 MM |
1017 | break; |
1018 | } | |
1019 | loop++; | |
1020 | udelay(1); | |
1021 | } | |
1022 | ||
1023 | if (pause) | |
1024 | /* Add additional delay to ensure chip returns all credits */ | |
1025 | pause_for_credit_return(dd); | |
1026 | } | |
1027 | ||
1028 | void sc_wait(struct hfi1_devdata *dd) | |
1029 | { | |
1030 | int i; | |
1031 | ||
1032 | for (i = 0; i < dd->num_send_contexts; i++) { | |
1033 | struct send_context *sc = dd->send_contexts[i].sc; | |
1034 | ||
1035 | if (!sc) | |
1036 | continue; | |
1037 | sc_wait_for_packet_egress(sc, 0); | |
1038 | } | |
1039 | } | |
1040 | ||
1041 | /* | |
1042 | * Restart a context after it has been halted due to error. | |
1043 | * | |
1044 | * If the first step fails - wait for the halt to be asserted, return early. | |
1045 | * Otherwise complain about timeouts but keep going. | |
1046 | * | |
1047 | * It is expected that allocations (enabled flag bit) have been shut off | |
1048 | * already (only applies to kernel contexts). | |
1049 | */ | |
1050 | int sc_restart(struct send_context *sc) | |
1051 | { | |
1052 | struct hfi1_devdata *dd = sc->dd; | |
1053 | u64 reg; | |
1054 | u32 loop; | |
1055 | int count; | |
1056 | ||
1057 | /* bounce off if not halted, or being free'd */ | |
1058 | if (!(sc->flags & SCF_HALTED) || (sc->flags & SCF_IN_FREE)) | |
1059 | return -EINVAL; | |
1060 | ||
1061 | dd_dev_info(dd, "restarting send context %u(%u)\n", sc->sw_index, | |
17fb4f29 | 1062 | sc->hw_context); |
77241056 MM |
1063 | |
1064 | /* | |
1065 | * Step 1: Wait for the context to actually halt. | |
1066 | * | |
1067 | * The error interrupt is asynchronous to actually setting halt | |
1068 | * on the context. | |
1069 | */ | |
1070 | loop = 0; | |
1071 | while (1) { | |
1072 | reg = read_kctxt_csr(dd, sc->hw_context, SC(STATUS)); | |
1073 | if (reg & SC(STATUS_CTXT_HALTED_SMASK)) | |
1074 | break; | |
1075 | if (loop > 100) { | |
1076 | dd_dev_err(dd, "%s: context %u(%u) not halting, skipping\n", | |
17fb4f29 | 1077 | __func__, sc->sw_index, sc->hw_context); |
77241056 MM |
1078 | return -ETIME; |
1079 | } | |
1080 | loop++; | |
1081 | udelay(1); | |
1082 | } | |
1083 | ||
1084 | /* | |
1085 | * Step 2: Ensure no users are still trying to write to PIO. | |
1086 | * | |
1087 | * For kernel contexts, we have already turned off buffer allocation. | |
1088 | * Now wait for the buffer count to go to zero. | |
1089 | * | |
1090 | * For user contexts, the user handling code has cut off write access | |
1091 | * to the context's PIO pages before calling this routine and will | |
1092 | * restore write access after this routine returns. | |
1093 | */ | |
1094 | if (sc->type != SC_USER) { | |
1095 | /* kernel context */ | |
1096 | loop = 0; | |
1097 | while (1) { | |
a054374f | 1098 | count = get_buffers_allocated(sc); |
77241056 MM |
1099 | if (count == 0) |
1100 | break; | |
1101 | if (loop > 100) { | |
1102 | dd_dev_err(dd, | |
17fb4f29 JJ |
1103 | "%s: context %u(%u) timeout waiting for PIO buffers to zero, remaining %d\n", |
1104 | __func__, sc->sw_index, | |
1105 | sc->hw_context, count); | |
77241056 MM |
1106 | } |
1107 | loop++; | |
1108 | udelay(1); | |
1109 | } | |
1110 | } | |
1111 | ||
1112 | /* | |
1113 | * Step 3: Wait for all packets to egress. | |
1114 | * This is done while disabling the send context | |
1115 | * | |
1116 | * Step 4: Disable the context | |
1117 | * | |
1118 | * This is a superset of the halt. After the disable, the | |
1119 | * errors can be cleared. | |
1120 | */ | |
1121 | sc_disable(sc); | |
1122 | ||
1123 | /* | |
1124 | * Step 5: Enable the context | |
1125 | * | |
1126 | * This enable will clear the halted flag and per-send context | |
1127 | * error flags. | |
1128 | */ | |
1129 | return sc_enable(sc); | |
1130 | } | |
1131 | ||
1132 | /* | |
1133 | * PIO freeze processing. To be called after the TXE block is fully frozen. | |
1134 | * Go through all frozen send contexts and disable them. The contexts are | |
1135 | * already stopped by the freeze. | |
1136 | */ | |
1137 | void pio_freeze(struct hfi1_devdata *dd) | |
1138 | { | |
1139 | struct send_context *sc; | |
1140 | int i; | |
1141 | ||
1142 | for (i = 0; i < dd->num_send_contexts; i++) { | |
1143 | sc = dd->send_contexts[i].sc; | |
1144 | /* | |
1145 | * Don't disable unallocated, unfrozen, or user send contexts. | |
1146 | * User send contexts will be disabled when the process | |
1147 | * calls into the driver to reset its context. | |
1148 | */ | |
1149 | if (!sc || !(sc->flags & SCF_FROZEN) || sc->type == SC_USER) | |
1150 | continue; | |
1151 | ||
1152 | /* only need to disable, the context is already stopped */ | |
1153 | sc_disable(sc); | |
1154 | } | |
1155 | } | |
1156 | ||
1157 | /* | |
1158 | * Unfreeze PIO for kernel send contexts. The precondition for calling this | |
1159 | * is that all PIO send contexts have been disabled and the SPC freeze has | |
1160 | * been cleared. Now perform the last step and re-enable each kernel context. | |
1161 | * User (PSM) processing will occur when PSM calls into the kernel to | |
1162 | * acknowledge the freeze. | |
1163 | */ | |
1164 | void pio_kernel_unfreeze(struct hfi1_devdata *dd) | |
1165 | { | |
1166 | struct send_context *sc; | |
1167 | int i; | |
1168 | ||
1169 | for (i = 0; i < dd->num_send_contexts; i++) { | |
1170 | sc = dd->send_contexts[i].sc; | |
1171 | if (!sc || !(sc->flags & SCF_FROZEN) || sc->type == SC_USER) | |
1172 | continue; | |
1173 | ||
1174 | sc_enable(sc); /* will clear the sc frozen flag */ | |
1175 | } | |
1176 | } | |
1177 | ||
1178 | /* | |
1179 | * Wait for the SendPioInitCtxt.PioInitInProgress bit to clear. | |
1180 | * Returns: | |
1181 | * -ETIMEDOUT - if we wait too long | |
1182 | * -EIO - if there was an error | |
1183 | */ | |
1184 | static int pio_init_wait_progress(struct hfi1_devdata *dd) | |
1185 | { | |
1186 | u64 reg; | |
1187 | int max, count = 0; | |
1188 | ||
1189 | /* max is the longest possible HW init time / delay */ | |
1190 | max = (dd->icode == ICODE_FPGA_EMULATION) ? 120 : 5; | |
1191 | while (1) { | |
1192 | reg = read_csr(dd, SEND_PIO_INIT_CTXT); | |
1193 | if (!(reg & SEND_PIO_INIT_CTXT_PIO_INIT_IN_PROGRESS_SMASK)) | |
1194 | break; | |
1195 | if (count >= max) | |
1196 | return -ETIMEDOUT; | |
1197 | udelay(5); | |
1198 | count++; | |
1199 | } | |
1200 | ||
1201 | return reg & SEND_PIO_INIT_CTXT_PIO_INIT_ERR_SMASK ? -EIO : 0; | |
1202 | } | |
1203 | ||
1204 | /* | |
1205 | * Reset all of the send contexts to their power-on state. Used | |
1206 | * only during manual init - no lock against sc_enable needed. | |
1207 | */ | |
1208 | void pio_reset_all(struct hfi1_devdata *dd) | |
1209 | { | |
1210 | int ret; | |
1211 | ||
1212 | /* make sure the init engine is not busy */ | |
1213 | ret = pio_init_wait_progress(dd); | |
1214 | /* ignore any timeout */ | |
1215 | if (ret == -EIO) { | |
1216 | /* clear the error */ | |
1217 | write_csr(dd, SEND_PIO_ERR_CLEAR, | |
17fb4f29 | 1218 | SEND_PIO_ERR_CLEAR_PIO_INIT_SM_IN_ERR_SMASK); |
77241056 MM |
1219 | } |
1220 | ||
1221 | /* reset init all */ | |
1222 | write_csr(dd, SEND_PIO_INIT_CTXT, | |
17fb4f29 | 1223 | SEND_PIO_INIT_CTXT_PIO_ALL_CTXT_INIT_SMASK); |
77241056 MM |
1224 | udelay(2); |
1225 | ret = pio_init_wait_progress(dd); | |
1226 | if (ret < 0) { | |
1227 | dd_dev_err(dd, | |
17fb4f29 JJ |
1228 | "PIO send context init %s while initializing all PIO blocks\n", |
1229 | ret == -ETIMEDOUT ? "is stuck" : "had an error"); | |
77241056 MM |
1230 | } |
1231 | } | |
1232 | ||
1233 | /* enable the context */ | |
1234 | int sc_enable(struct send_context *sc) | |
1235 | { | |
1236 | u64 sc_ctrl, reg, pio; | |
1237 | struct hfi1_devdata *dd; | |
1238 | unsigned long flags; | |
1239 | int ret = 0; | |
1240 | ||
1241 | if (!sc) | |
1242 | return -EINVAL; | |
1243 | dd = sc->dd; | |
1244 | ||
1245 | /* | |
1246 | * Obtain the allocator lock to guard against any allocation | |
1247 | * attempts (which should not happen prior to context being | |
1248 | * enabled). On the release/disable side we don't need to | |
1249 | * worry about locking since the releaser will not do anything | |
1250 | * if the context accounting values have not changed. | |
1251 | */ | |
1252 | spin_lock_irqsave(&sc->alloc_lock, flags); | |
1253 | sc_ctrl = read_kctxt_csr(dd, sc->hw_context, SC(CTRL)); | |
1254 | if ((sc_ctrl & SC(CTRL_CTXT_ENABLE_SMASK))) | |
1255 | goto unlock; /* already enabled */ | |
1256 | ||
1257 | /* IMPORTANT: only clear free and fill if transitioning 0 -> 1 */ | |
1258 | ||
1259 | *sc->hw_free = 0; | |
1260 | sc->free = 0; | |
1261 | sc->alloc_free = 0; | |
1262 | sc->fill = 0; | |
2474d775 | 1263 | sc->fill_wrap = 0; |
77241056 MM |
1264 | sc->sr_head = 0; |
1265 | sc->sr_tail = 0; | |
1266 | sc->flags = 0; | |
a054374f MM |
1267 | /* the alloc lock insures no fast path allocation */ |
1268 | reset_buffers_allocated(sc); | |
77241056 MM |
1269 | |
1270 | /* | |
1271 | * Clear all per-context errors. Some of these will be set when | |
1272 | * we are re-enabling after a context halt. Now that the context | |
1273 | * is disabled, the halt will not clear until after the PIO init | |
1274 | * engine runs below. | |
1275 | */ | |
1276 | reg = read_kctxt_csr(dd, sc->hw_context, SC(ERR_STATUS)); | |
1277 | if (reg) | |
17fb4f29 | 1278 | write_kctxt_csr(dd, sc->hw_context, SC(ERR_CLEAR), reg); |
77241056 MM |
1279 | |
1280 | /* | |
1281 | * The HW PIO initialization engine can handle only one init | |
1282 | * request at a time. Serialize access to each device's engine. | |
1283 | */ | |
1284 | spin_lock(&dd->sc_init_lock); | |
1285 | /* | |
1286 | * Since access to this code block is serialized and | |
1287 | * each access waits for the initialization to complete | |
1288 | * before releasing the lock, the PIO initialization engine | |
1289 | * should not be in use, so we don't have to wait for the | |
1290 | * InProgress bit to go down. | |
1291 | */ | |
1292 | pio = ((sc->hw_context & SEND_PIO_INIT_CTXT_PIO_CTXT_NUM_MASK) << | |
1293 | SEND_PIO_INIT_CTXT_PIO_CTXT_NUM_SHIFT) | | |
1294 | SEND_PIO_INIT_CTXT_PIO_SINGLE_CTXT_INIT_SMASK; | |
1295 | write_csr(dd, SEND_PIO_INIT_CTXT, pio); | |
1296 | /* | |
1297 | * Wait until the engine is done. Give the chip the required time | |
1298 | * so, hopefully, we read the register just once. | |
1299 | */ | |
1300 | udelay(2); | |
1301 | ret = pio_init_wait_progress(dd); | |
1302 | spin_unlock(&dd->sc_init_lock); | |
1303 | if (ret) { | |
1304 | dd_dev_err(dd, | |
1305 | "sctxt%u(%u): Context not enabled due to init failure %d\n", | |
1306 | sc->sw_index, sc->hw_context, ret); | |
1307 | goto unlock; | |
1308 | } | |
1309 | ||
1310 | /* | |
1311 | * All is well. Enable the context. | |
1312 | */ | |
1313 | sc_ctrl |= SC(CTRL_CTXT_ENABLE_SMASK); | |
1314 | write_kctxt_csr(dd, sc->hw_context, SC(CTRL), sc_ctrl); | |
1315 | /* | |
1316 | * Read SendCtxtCtrl to force the write out and prevent a timing | |
1317 | * hazard where a PIO write may reach the context before the enable. | |
1318 | */ | |
1319 | read_kctxt_csr(dd, sc->hw_context, SC(CTRL)); | |
1320 | sc->flags |= SCF_ENABLED; | |
1321 | ||
1322 | unlock: | |
1323 | spin_unlock_irqrestore(&sc->alloc_lock, flags); | |
1324 | ||
1325 | return ret; | |
1326 | } | |
1327 | ||
1328 | /* force a credit return on the context */ | |
1329 | void sc_return_credits(struct send_context *sc) | |
1330 | { | |
1331 | if (!sc) | |
1332 | return; | |
1333 | ||
1334 | /* a 0->1 transition schedules a credit return */ | |
1335 | write_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE), | |
17fb4f29 | 1336 | SC(CREDIT_FORCE_FORCE_RETURN_SMASK)); |
77241056 MM |
1337 | /* |
1338 | * Ensure that the write is flushed and the credit return is | |
1339 | * scheduled. We care more about the 0 -> 1 transition. | |
1340 | */ | |
1341 | read_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE)); | |
1342 | /* set back to 0 for next time */ | |
1343 | write_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE), 0); | |
1344 | } | |
1345 | ||
1346 | /* allow all in-flight packets to drain on the context */ | |
1347 | void sc_flush(struct send_context *sc) | |
1348 | { | |
1349 | if (!sc) | |
1350 | return; | |
1351 | ||
1352 | sc_wait_for_packet_egress(sc, 1); | |
1353 | } | |
1354 | ||
1355 | /* drop all packets on the context, no waiting until they are sent */ | |
1356 | void sc_drop(struct send_context *sc) | |
1357 | { | |
1358 | if (!sc) | |
1359 | return; | |
1360 | ||
1361 | dd_dev_info(sc->dd, "%s: context %u(%u) - not implemented\n", | |
17fb4f29 | 1362 | __func__, sc->sw_index, sc->hw_context); |
77241056 MM |
1363 | } |
1364 | ||
1365 | /* | |
1366 | * Start the software reaction to a context halt or SPC freeze: | |
1367 | * - mark the context as halted or frozen | |
1368 | * - stop buffer allocations | |
1369 | * | |
1370 | * Called from the error interrupt. Other work is deferred until | |
1371 | * out of the interrupt. | |
1372 | */ | |
1373 | void sc_stop(struct send_context *sc, int flag) | |
1374 | { | |
1375 | unsigned long flags; | |
1376 | ||
1377 | /* mark the context */ | |
1378 | sc->flags |= flag; | |
1379 | ||
1380 | /* stop buffer allocations */ | |
1381 | spin_lock_irqsave(&sc->alloc_lock, flags); | |
1382 | sc->flags &= ~SCF_ENABLED; | |
1383 | spin_unlock_irqrestore(&sc->alloc_lock, flags); | |
1384 | wake_up(&sc->halt_wait); | |
1385 | } | |
1386 | ||
8638b77f | 1387 | #define BLOCK_DWORDS (PIO_BLOCK_SIZE / sizeof(u32)) |
77241056 MM |
1388 | #define dwords_to_blocks(x) DIV_ROUND_UP(x, BLOCK_DWORDS) |
1389 | ||
1390 | /* | |
1391 | * The send context buffer "allocator". | |
1392 | * | |
1393 | * @sc: the PIO send context we are allocating from | |
1394 | * @len: length of whole packet - including PBC - in dwords | |
1395 | * @cb: optional callback to call when the buffer is finished sending | |
1396 | * @arg: argument for cb | |
1397 | * | |
1398 | * Return a pointer to a PIO buffer if successful, NULL if not enough room. | |
1399 | */ | |
1400 | struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len, | |
1401 | pio_release_cb cb, void *arg) | |
1402 | { | |
1403 | struct pio_buf *pbuf = NULL; | |
1404 | unsigned long flags; | |
1405 | unsigned long avail; | |
1406 | unsigned long blocks = dwords_to_blocks(dw_len); | |
2474d775 | 1407 | u32 fill_wrap; |
77241056 MM |
1408 | int trycount = 0; |
1409 | u32 head, next; | |
1410 | ||
1411 | spin_lock_irqsave(&sc->alloc_lock, flags); | |
1412 | if (!(sc->flags & SCF_ENABLED)) { | |
1413 | spin_unlock_irqrestore(&sc->alloc_lock, flags); | |
1414 | goto done; | |
1415 | } | |
1416 | ||
1417 | retry: | |
1418 | avail = (unsigned long)sc->credits - (sc->fill - sc->alloc_free); | |
1419 | if (blocks > avail) { | |
1420 | /* not enough room */ | |
1421 | if (unlikely(trycount)) { /* already tried to get more room */ | |
1422 | spin_unlock_irqrestore(&sc->alloc_lock, flags); | |
1423 | goto done; | |
1424 | } | |
1425 | /* copy from receiver cache line and recalculate */ | |
6aa7de05 | 1426 | sc->alloc_free = READ_ONCE(sc->free); |
77241056 MM |
1427 | avail = |
1428 | (unsigned long)sc->credits - | |
1429 | (sc->fill - sc->alloc_free); | |
1430 | if (blocks > avail) { | |
1431 | /* still no room, actively update */ | |
77241056 | 1432 | sc_release_update(sc); |
6aa7de05 | 1433 | sc->alloc_free = READ_ONCE(sc->free); |
77241056 MM |
1434 | trycount++; |
1435 | goto retry; | |
1436 | } | |
1437 | } | |
1438 | ||
1439 | /* there is enough room */ | |
1440 | ||
a054374f MM |
1441 | preempt_disable(); |
1442 | this_cpu_inc(*sc->buffers_allocated); | |
77241056 MM |
1443 | |
1444 | /* read this once */ | |
1445 | head = sc->sr_head; | |
1446 | ||
1447 | /* "allocate" the buffer */ | |
77241056 | 1448 | sc->fill += blocks; |
2474d775 SS |
1449 | fill_wrap = sc->fill_wrap; |
1450 | sc->fill_wrap += blocks; | |
1451 | if (sc->fill_wrap >= sc->credits) | |
1452 | sc->fill_wrap = sc->fill_wrap - sc->credits; | |
77241056 MM |
1453 | |
1454 | /* | |
1455 | * Fill the parts that the releaser looks at before moving the head. | |
1456 | * The only necessary piece is the sent_at field. The credits | |
1457 | * we have just allocated cannot have been returned yet, so the | |
1458 | * cb and arg will not be looked at for a "while". Put them | |
1459 | * on this side of the memory barrier anyway. | |
1460 | */ | |
1461 | pbuf = &sc->sr[head].pbuf; | |
1462 | pbuf->sent_at = sc->fill; | |
1463 | pbuf->cb = cb; | |
1464 | pbuf->arg = arg; | |
1465 | pbuf->sc = sc; /* could be filled in at sc->sr init time */ | |
1466 | /* make sure this is in memory before updating the head */ | |
1467 | ||
1468 | /* calculate next head index, do not store */ | |
1469 | next = head + 1; | |
1470 | if (next >= sc->sr_size) | |
1471 | next = 0; | |
4d114fdd JJ |
1472 | /* |
1473 | * update the head - must be last! - the releaser can look at fields | |
1474 | * in pbuf once we move the head | |
1475 | */ | |
77241056 MM |
1476 | smp_wmb(); |
1477 | sc->sr_head = next; | |
1478 | spin_unlock_irqrestore(&sc->alloc_lock, flags); | |
1479 | ||
1480 | /* finish filling in the buffer outside the lock */ | |
2474d775 | 1481 | pbuf->start = sc->base_addr + fill_wrap * PIO_BLOCK_SIZE; |
8af8d297 | 1482 | pbuf->end = sc->base_addr + sc->size; |
77241056 MM |
1483 | pbuf->qw_written = 0; |
1484 | pbuf->carry_bytes = 0; | |
1485 | pbuf->carry.val64 = 0; | |
1486 | done: | |
1487 | return pbuf; | |
1488 | } | |
1489 | ||
1490 | /* | |
1491 | * There are at least two entities that can turn on credit return | |
1492 | * interrupts and they can overlap. Avoid problems by implementing | |
1493 | * a count scheme that is enforced by a lock. The lock is needed because | |
1494 | * the count and CSR write must be paired. | |
1495 | */ | |
1496 | ||
1497 | /* | |
1498 | * Start credit return interrupts. This is managed by a count. If already | |
1499 | * on, just increment the count. | |
1500 | */ | |
1501 | void sc_add_credit_return_intr(struct send_context *sc) | |
1502 | { | |
1503 | unsigned long flags; | |
1504 | ||
1505 | /* lock must surround both the count change and the CSR update */ | |
1506 | spin_lock_irqsave(&sc->credit_ctrl_lock, flags); | |
1507 | if (sc->credit_intr_count == 0) { | |
1508 | sc->credit_ctrl |= SC(CREDIT_CTRL_CREDIT_INTR_SMASK); | |
1509 | write_kctxt_csr(sc->dd, sc->hw_context, | |
17fb4f29 | 1510 | SC(CREDIT_CTRL), sc->credit_ctrl); |
77241056 MM |
1511 | } |
1512 | sc->credit_intr_count++; | |
1513 | spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags); | |
1514 | } | |
1515 | ||
1516 | /* | |
1517 | * Stop credit return interrupts. This is managed by a count. Decrement the | |
1518 | * count, if the last user, then turn the credit interrupts off. | |
1519 | */ | |
1520 | void sc_del_credit_return_intr(struct send_context *sc) | |
1521 | { | |
1522 | unsigned long flags; | |
1523 | ||
1524 | WARN_ON(sc->credit_intr_count == 0); | |
1525 | ||
1526 | /* lock must surround both the count change and the CSR update */ | |
1527 | spin_lock_irqsave(&sc->credit_ctrl_lock, flags); | |
1528 | sc->credit_intr_count--; | |
1529 | if (sc->credit_intr_count == 0) { | |
1530 | sc->credit_ctrl &= ~SC(CREDIT_CTRL_CREDIT_INTR_SMASK); | |
1531 | write_kctxt_csr(sc->dd, sc->hw_context, | |
17fb4f29 | 1532 | SC(CREDIT_CTRL), sc->credit_ctrl); |
77241056 MM |
1533 | } |
1534 | spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags); | |
1535 | } | |
1536 | ||
1537 | /* | |
1538 | * The caller must be careful when calling this. All needint calls | |
1539 | * must be paired with !needint. | |
1540 | */ | |
1541 | void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint) | |
1542 | { | |
1543 | if (needint) | |
1544 | sc_add_credit_return_intr(sc); | |
1545 | else | |
1546 | sc_del_credit_return_intr(sc); | |
1547 | trace_hfi1_wantpiointr(sc, needint, sc->credit_ctrl); | |
1548 | if (needint) { | |
1549 | mmiowb(); | |
1550 | sc_return_credits(sc); | |
1551 | } | |
1552 | } | |
1553 | ||
1554 | /** | |
1555 | * sc_piobufavail - callback when a PIO buffer is available | |
1556 | * @sc: the send context | |
1557 | * | |
1558 | * This is called from the interrupt handler when a PIO buffer is | |
1559 | * available after hfi1_verbs_send() returned an error that no buffers were | |
1560 | * available. Disable the interrupt if there are no more QPs waiting. | |
1561 | */ | |
1562 | static void sc_piobufavail(struct send_context *sc) | |
1563 | { | |
1564 | struct hfi1_devdata *dd = sc->dd; | |
1565 | struct hfi1_ibdev *dev = &dd->verbs_dev; | |
1566 | struct list_head *list; | |
895420dd DD |
1567 | struct rvt_qp *qps[PIO_WAIT_BATCH_SIZE]; |
1568 | struct rvt_qp *qp; | |
4c6829c5 | 1569 | struct hfi1_qp_priv *priv; |
77241056 | 1570 | unsigned long flags; |
bcad2913 KW |
1571 | uint i, n = 0, max_idx = 0; |
1572 | u8 max_starved_cnt = 0; | |
77241056 | 1573 | |
44306f15 JX |
1574 | if (dd->send_contexts[sc->sw_index].type != SC_KERNEL && |
1575 | dd->send_contexts[sc->sw_index].type != SC_VL15) | |
77241056 MM |
1576 | return; |
1577 | list = &sc->piowait; | |
1578 | /* | |
1579 | * Note: checking that the piowait list is empty and clearing | |
1580 | * the buffer available interrupt needs to be atomic or we | |
1581 | * could end up with QPs on the wait list with the interrupt | |
1582 | * disabled. | |
1583 | */ | |
1584 | write_seqlock_irqsave(&dev->iowait_lock, flags); | |
1585 | while (!list_empty(list)) { | |
1586 | struct iowait *wait; | |
1587 | ||
1588 | if (n == ARRAY_SIZE(qps)) | |
60df2958 | 1589 | break; |
77241056 | 1590 | wait = list_first_entry(list, struct iowait, list); |
4c6829c5 DD |
1591 | qp = iowait_to_qp(wait); |
1592 | priv = qp->priv; | |
1593 | list_del_init(&priv->s_iowait.list); | |
4e045572 | 1594 | priv->s_iowait.lock = NULL; |
bcad2913 | 1595 | iowait_starve_find_max(wait, &max_starved_cnt, n, &max_idx); |
77241056 MM |
1596 | /* refcount held until actual wake up */ |
1597 | qps[n++] = qp; | |
1598 | } | |
1599 | /* | |
60df2958 MM |
1600 | * If there had been waiters and there are more |
1601 | * insure that we redo the force to avoid a potential hang. | |
77241056 | 1602 | */ |
60df2958 | 1603 | if (n) { |
77241056 | 1604 | hfi1_sc_wantpiobuf_intr(sc, 0); |
60df2958 MM |
1605 | if (!list_empty(list)) |
1606 | hfi1_sc_wantpiobuf_intr(sc, 1); | |
1607 | } | |
77241056 MM |
1608 | write_sequnlock_irqrestore(&dev->iowait_lock, flags); |
1609 | ||
bcad2913 KW |
1610 | /* Wake up the most starved one first */ |
1611 | if (n) | |
1612 | hfi1_qp_wakeup(qps[max_idx], | |
14553ca1 | 1613 | RVT_S_WAIT_PIO | RVT_S_WAIT_PIO_DRAIN); |
bcad2913 KW |
1614 | for (i = 0; i < n; i++) |
1615 | if (i != max_idx) | |
1616 | hfi1_qp_wakeup(qps[i], | |
1617 | RVT_S_WAIT_PIO | RVT_S_WAIT_PIO_DRAIN); | |
77241056 MM |
1618 | } |
1619 | ||
1620 | /* translate a send credit update to a bit code of reasons */ | |
1621 | static inline int fill_code(u64 hw_free) | |
1622 | { | |
1623 | int code = 0; | |
1624 | ||
1625 | if (hw_free & CR_STATUS_SMASK) | |
1626 | code |= PRC_STATUS_ERR; | |
1627 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_PBC_SMASK) | |
1628 | code |= PRC_PBC; | |
1629 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_THRESHOLD_SMASK) | |
1630 | code |= PRC_THRESHOLD; | |
1631 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_ERR_SMASK) | |
1632 | code |= PRC_FILL_ERR; | |
1633 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_FORCE_SMASK) | |
1634 | code |= PRC_SC_DISABLE; | |
1635 | return code; | |
1636 | } | |
1637 | ||
1638 | /* use the jiffies compare to get the wrap right */ | |
1639 | #define sent_before(a, b) time_before(a, b) /* a < b */ | |
1640 | ||
1641 | /* | |
1642 | * The send context buffer "releaser". | |
1643 | */ | |
1644 | void sc_release_update(struct send_context *sc) | |
1645 | { | |
1646 | struct pio_buf *pbuf; | |
1647 | u64 hw_free; | |
1648 | u32 head, tail; | |
1649 | unsigned long old_free; | |
e607a221 | 1650 | unsigned long free; |
77241056 MM |
1651 | unsigned long extra; |
1652 | unsigned long flags; | |
1653 | int code; | |
1654 | ||
1655 | if (!sc) | |
1656 | return; | |
1657 | ||
1658 | spin_lock_irqsave(&sc->release_lock, flags); | |
1659 | /* update free */ | |
1660 | hw_free = le64_to_cpu(*sc->hw_free); /* volatile read */ | |
1661 | old_free = sc->free; | |
1662 | extra = (((hw_free & CR_COUNTER_SMASK) >> CR_COUNTER_SHIFT) | |
1663 | - (old_free & CR_COUNTER_MASK)) | |
1664 | & CR_COUNTER_MASK; | |
e607a221 | 1665 | free = old_free + extra; |
77241056 MM |
1666 | trace_hfi1_piofree(sc, extra); |
1667 | ||
1668 | /* call sent buffer callbacks */ | |
1669 | code = -1; /* code not yet set */ | |
6aa7de05 | 1670 | head = READ_ONCE(sc->sr_head); /* snapshot the head */ |
77241056 MM |
1671 | tail = sc->sr_tail; |
1672 | while (head != tail) { | |
1673 | pbuf = &sc->sr[tail].pbuf; | |
1674 | ||
e607a221 | 1675 | if (sent_before(free, pbuf->sent_at)) { |
77241056 MM |
1676 | /* not sent yet */ |
1677 | break; | |
1678 | } | |
1679 | if (pbuf->cb) { | |
1680 | if (code < 0) /* fill in code on first user */ | |
1681 | code = fill_code(hw_free); | |
1682 | (*pbuf->cb)(pbuf->arg, code); | |
1683 | } | |
1684 | ||
1685 | tail++; | |
1686 | if (tail >= sc->sr_size) | |
1687 | tail = 0; | |
1688 | } | |
77241056 | 1689 | sc->sr_tail = tail; |
e607a221 MM |
1690 | /* make sure tail is updated before free */ |
1691 | smp_wmb(); | |
1692 | sc->free = free; | |
77241056 MM |
1693 | spin_unlock_irqrestore(&sc->release_lock, flags); |
1694 | sc_piobufavail(sc); | |
1695 | } | |
1696 | ||
1697 | /* | |
1698 | * Send context group releaser. Argument is the send context that caused | |
1699 | * the interrupt. Called from the send context interrupt handler. | |
1700 | * | |
1701 | * Call release on all contexts in the group. | |
1702 | * | |
1703 | * This routine takes the sc_lock without an irqsave because it is only | |
1704 | * called from an interrupt handler. Adjust if that changes. | |
1705 | */ | |
1706 | void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context) | |
1707 | { | |
1708 | struct send_context *sc; | |
1709 | u32 sw_index; | |
1710 | u32 gc, gc_end; | |
1711 | ||
1712 | spin_lock(&dd->sc_lock); | |
1713 | sw_index = dd->hw_to_sw[hw_context]; | |
1714 | if (unlikely(sw_index >= dd->num_send_contexts)) { | |
1715 | dd_dev_err(dd, "%s: invalid hw (%u) to sw (%u) mapping\n", | |
17fb4f29 | 1716 | __func__, hw_context, sw_index); |
77241056 MM |
1717 | goto done; |
1718 | } | |
1719 | sc = dd->send_contexts[sw_index].sc; | |
1720 | if (unlikely(!sc)) | |
1721 | goto done; | |
1722 | ||
1723 | gc = group_context(hw_context, sc->group); | |
1724 | gc_end = gc + group_size(sc->group); | |
1725 | for (; gc < gc_end; gc++) { | |
1726 | sw_index = dd->hw_to_sw[gc]; | |
1727 | if (unlikely(sw_index >= dd->num_send_contexts)) { | |
1728 | dd_dev_err(dd, | |
17fb4f29 JJ |
1729 | "%s: invalid hw (%u) to sw (%u) mapping\n", |
1730 | __func__, hw_context, sw_index); | |
77241056 MM |
1731 | continue; |
1732 | } | |
1733 | sc_release_update(dd->send_contexts[sw_index].sc); | |
1734 | } | |
1735 | done: | |
1736 | spin_unlock(&dd->sc_lock); | |
1737 | } | |
1738 | ||
35f6befc JJ |
1739 | /* |
1740 | * pio_select_send_context_vl() - select send context | |
1741 | * @dd: devdata | |
1742 | * @selector: a spreading factor | |
1743 | * @vl: this vl | |
1744 | * | |
1745 | * This function returns a send context based on the selector and a vl. | |
1746 | * The mapping fields are protected by RCU | |
1747 | */ | |
1748 | struct send_context *pio_select_send_context_vl(struct hfi1_devdata *dd, | |
1749 | u32 selector, u8 vl) | |
1750 | { | |
1751 | struct pio_vl_map *m; | |
1752 | struct pio_map_elem *e; | |
1753 | struct send_context *rval; | |
1754 | ||
1755 | /* | |
1756 | * NOTE This should only happen if SC->VL changed after the initial | |
1757 | * checks on the QP/AH | |
1758 | * Default will return VL0's send context below | |
1759 | */ | |
1760 | if (unlikely(vl >= num_vls)) { | |
1761 | rval = NULL; | |
1762 | goto done; | |
1763 | } | |
1764 | ||
1765 | rcu_read_lock(); | |
1766 | m = rcu_dereference(dd->pio_map); | |
1767 | if (unlikely(!m)) { | |
1768 | rcu_read_unlock(); | |
1769 | return dd->vld[0].sc; | |
1770 | } | |
1771 | e = m->map[vl & m->mask]; | |
1772 | rval = e->ksc[selector & e->mask]; | |
1773 | rcu_read_unlock(); | |
1774 | ||
1775 | done: | |
1776 | rval = !rval ? dd->vld[0].sc : rval; | |
1777 | return rval; | |
1778 | } | |
1779 | ||
1780 | /* | |
1781 | * pio_select_send_context_sc() - select send context | |
1782 | * @dd: devdata | |
1783 | * @selector: a spreading factor | |
1784 | * @sc5: the 5 bit sc | |
1785 | * | |
1786 | * This function returns an send context based on the selector and an sc | |
1787 | */ | |
1788 | struct send_context *pio_select_send_context_sc(struct hfi1_devdata *dd, | |
1789 | u32 selector, u8 sc5) | |
1790 | { | |
1791 | u8 vl = sc_to_vlt(dd, sc5); | |
1792 | ||
1793 | return pio_select_send_context_vl(dd, selector, vl); | |
1794 | } | |
1795 | ||
1796 | /* | |
1797 | * Free the indicated map struct | |
1798 | */ | |
1799 | static void pio_map_free(struct pio_vl_map *m) | |
1800 | { | |
1801 | int i; | |
1802 | ||
1803 | for (i = 0; m && i < m->actual_vls; i++) | |
1804 | kfree(m->map[i]); | |
1805 | kfree(m); | |
1806 | } | |
1807 | ||
1808 | /* | |
1809 | * Handle RCU callback | |
1810 | */ | |
1811 | static void pio_map_rcu_callback(struct rcu_head *list) | |
1812 | { | |
1813 | struct pio_vl_map *m = container_of(list, struct pio_vl_map, list); | |
1814 | ||
1815 | pio_map_free(m); | |
1816 | } | |
1817 | ||
b4ba6633 JJ |
1818 | /* |
1819 | * Set credit return threshold for the kernel send context | |
1820 | */ | |
1821 | static void set_threshold(struct hfi1_devdata *dd, int scontext, int i) | |
1822 | { | |
1823 | u32 thres; | |
1824 | ||
1825 | thres = min(sc_percent_to_threshold(dd->kernel_send_context[scontext], | |
1826 | 50), | |
1827 | sc_mtu_to_threshold(dd->kernel_send_context[scontext], | |
1828 | dd->vld[i].mtu, | |
1829 | dd->rcd[0]->rcvhdrqentsize)); | |
1830 | sc_set_cr_threshold(dd->kernel_send_context[scontext], thres); | |
1831 | } | |
1832 | ||
35f6befc JJ |
1833 | /* |
1834 | * pio_map_init - called when #vls change | |
1835 | * @dd: hfi1_devdata | |
1836 | * @port: port number | |
1837 | * @num_vls: number of vls | |
1838 | * @vl_scontexts: per vl send context mapping (optional) | |
1839 | * | |
1840 | * This routine changes the mapping based on the number of vls. | |
1841 | * | |
1842 | * vl_scontexts is used to specify a non-uniform vl/send context | |
1843 | * loading. NULL implies auto computing the loading and giving each | |
1844 | * VL an uniform distribution of send contexts per VL. | |
1845 | * | |
1846 | * The auto algorithm computers the sc_per_vl and the number of extra | |
1847 | * send contexts. Any extra send contexts are added from the last VL | |
1848 | * on down | |
1849 | * | |
1850 | * rcu locking is used here to control access to the mapping fields. | |
1851 | * | |
1852 | * If either the num_vls or num_send_contexts are non-power of 2, the | |
1853 | * array sizes in the struct pio_vl_map and the struct pio_map_elem are | |
1854 | * rounded up to the next highest power of 2 and the first entry is | |
1855 | * reused in a round robin fashion. | |
1856 | * | |
1857 | * If an error occurs the map change is not done and the mapping is not | |
1858 | * chaged. | |
1859 | * | |
1860 | */ | |
1861 | int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts) | |
1862 | { | |
1863 | int i, j; | |
1864 | int extra, sc_per_vl; | |
1865 | int scontext = 1; | |
1866 | int num_kernel_send_contexts = 0; | |
1867 | u8 lvl_scontexts[OPA_MAX_VLS]; | |
1868 | struct pio_vl_map *oldmap, *newmap; | |
1869 | ||
1870 | if (!vl_scontexts) { | |
f1584865 | 1871 | for (i = 0; i < dd->num_send_contexts; i++) |
35f6befc JJ |
1872 | if (dd->send_contexts[i].type == SC_KERNEL) |
1873 | num_kernel_send_contexts++; | |
1874 | /* truncate divide */ | |
1875 | sc_per_vl = num_kernel_send_contexts / num_vls; | |
1876 | /* extras */ | |
1877 | extra = num_kernel_send_contexts % num_vls; | |
1878 | vl_scontexts = lvl_scontexts; | |
1879 | /* add extras from last vl down */ | |
1880 | for (i = num_vls - 1; i >= 0; i--, extra--) | |
1881 | vl_scontexts[i] = sc_per_vl + (extra > 0 ? 1 : 0); | |
1882 | } | |
1883 | /* build new map */ | |
1884 | newmap = kzalloc(sizeof(*newmap) + | |
1885 | roundup_pow_of_two(num_vls) * | |
1886 | sizeof(struct pio_map_elem *), | |
1887 | GFP_KERNEL); | |
1888 | if (!newmap) | |
1889 | goto bail; | |
1890 | newmap->actual_vls = num_vls; | |
1891 | newmap->vls = roundup_pow_of_two(num_vls); | |
1892 | newmap->mask = (1 << ilog2(newmap->vls)) - 1; | |
1893 | for (i = 0; i < newmap->vls; i++) { | |
1894 | /* save for wrap around */ | |
1895 | int first_scontext = scontext; | |
1896 | ||
1897 | if (i < newmap->actual_vls) { | |
1898 | int sz = roundup_pow_of_two(vl_scontexts[i]); | |
1899 | ||
1900 | /* only allocate once */ | |
1901 | newmap->map[i] = kzalloc(sizeof(*newmap->map[i]) + | |
1902 | sz * sizeof(struct | |
1903 | send_context *), | |
1904 | GFP_KERNEL); | |
1905 | if (!newmap->map[i]) | |
1906 | goto bail; | |
1907 | newmap->map[i]->mask = (1 << ilog2(sz)) - 1; | |
b4ba6633 JJ |
1908 | /* |
1909 | * assign send contexts and | |
1910 | * adjust credit return threshold | |
1911 | */ | |
35f6befc | 1912 | for (j = 0; j < sz; j++) { |
b4ba6633 | 1913 | if (dd->kernel_send_context[scontext]) { |
35f6befc JJ |
1914 | newmap->map[i]->ksc[j] = |
1915 | dd->kernel_send_context[scontext]; | |
b4ba6633 JJ |
1916 | set_threshold(dd, scontext, i); |
1917 | } | |
35f6befc JJ |
1918 | if (++scontext >= first_scontext + |
1919 | vl_scontexts[i]) | |
1920 | /* wrap back to first send context */ | |
1921 | scontext = first_scontext; | |
1922 | } | |
1923 | } else { | |
1924 | /* just re-use entry without allocating */ | |
1925 | newmap->map[i] = newmap->map[i % num_vls]; | |
1926 | } | |
1927 | scontext = first_scontext + vl_scontexts[i]; | |
1928 | } | |
1929 | /* newmap in hand, save old map */ | |
1930 | spin_lock_irq(&dd->pio_map_lock); | |
1931 | oldmap = rcu_dereference_protected(dd->pio_map, | |
1932 | lockdep_is_held(&dd->pio_map_lock)); | |
1933 | ||
1934 | /* publish newmap */ | |
1935 | rcu_assign_pointer(dd->pio_map, newmap); | |
1936 | ||
1937 | spin_unlock_irq(&dd->pio_map_lock); | |
1938 | /* success, free any old map after grace period */ | |
1939 | if (oldmap) | |
1940 | call_rcu(&oldmap->list, pio_map_rcu_callback); | |
1941 | return 0; | |
1942 | bail: | |
1943 | /* free any partial allocation */ | |
1944 | pio_map_free(newmap); | |
1945 | return -ENOMEM; | |
1946 | } | |
1947 | ||
1948 | void free_pio_map(struct hfi1_devdata *dd) | |
1949 | { | |
1950 | /* Free PIO map if allocated */ | |
1951 | if (rcu_access_pointer(dd->pio_map)) { | |
1952 | spin_lock_irq(&dd->pio_map_lock); | |
79d0c088 | 1953 | pio_map_free(rcu_access_pointer(dd->pio_map)); |
35f6befc JJ |
1954 | RCU_INIT_POINTER(dd->pio_map, NULL); |
1955 | spin_unlock_irq(&dd->pio_map_lock); | |
1956 | synchronize_rcu(); | |
1957 | } | |
1958 | kfree(dd->kernel_send_context); | |
1959 | dd->kernel_send_context = NULL; | |
1960 | } | |
1961 | ||
77241056 MM |
1962 | int init_pervl_scs(struct hfi1_devdata *dd) |
1963 | { | |
1964 | int i; | |
35f6befc JJ |
1965 | u64 mask, all_vl_mask = (u64)0x80ff; /* VLs 0-7, 15 */ |
1966 | u64 data_vls_mask = (u64)0x00ff; /* VLs 0-7 */ | |
77241056 | 1967 | u32 ctxt; |
35f6befc | 1968 | struct hfi1_pportdata *ppd = dd->pport; |
77241056 | 1969 | |
44306f15 | 1970 | dd->vld[15].sc = sc_alloc(dd, SC_VL15, |
77241056 MM |
1971 | dd->rcd[0]->rcvhdrqentsize, dd->node); |
1972 | if (!dd->vld[15].sc) | |
042b0159 IW |
1973 | return -ENOMEM; |
1974 | ||
77241056 MM |
1975 | hfi1_init_ctxt(dd->vld[15].sc); |
1976 | dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048); | |
35f6befc | 1977 | |
1b23f02c | 1978 | dd->kernel_send_context = kzalloc_node(dd->num_send_contexts * |
35f6befc JJ |
1979 | sizeof(struct send_context *), |
1980 | GFP_KERNEL, dd->node); | |
042b0159 IW |
1981 | if (!dd->kernel_send_context) |
1982 | goto freesc15; | |
1983 | ||
35f6befc JJ |
1984 | dd->kernel_send_context[0] = dd->vld[15].sc; |
1985 | ||
77241056 MM |
1986 | for (i = 0; i < num_vls; i++) { |
1987 | /* | |
1988 | * Since this function does not deal with a specific | |
1989 | * receive context but we need the RcvHdrQ entry size, | |
1990 | * use the size from rcd[0]. It is guaranteed to be | |
1991 | * valid at this point and will remain the same for all | |
1992 | * receive contexts. | |
1993 | */ | |
1994 | dd->vld[i].sc = sc_alloc(dd, SC_KERNEL, | |
1995 | dd->rcd[0]->rcvhdrqentsize, dd->node); | |
1996 | if (!dd->vld[i].sc) | |
1997 | goto nomem; | |
35f6befc | 1998 | dd->kernel_send_context[i + 1] = dd->vld[i].sc; |
77241056 | 1999 | hfi1_init_ctxt(dd->vld[i].sc); |
77241056 MM |
2000 | /* non VL15 start with the max MTU */ |
2001 | dd->vld[i].mtu = hfi1_max_mtu; | |
2002 | } | |
35f6befc JJ |
2003 | for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++) { |
2004 | dd->kernel_send_context[i + 1] = | |
2005 | sc_alloc(dd, SC_KERNEL, dd->rcd[0]->rcvhdrqentsize, dd->node); | |
2006 | if (!dd->kernel_send_context[i + 1]) | |
2007 | goto nomem; | |
2008 | hfi1_init_ctxt(dd->kernel_send_context[i + 1]); | |
2009 | } | |
2010 | ||
77241056 MM |
2011 | sc_enable(dd->vld[15].sc); |
2012 | ctxt = dd->vld[15].sc->hw_context; | |
2013 | mask = all_vl_mask & ~(1LL << 15); | |
2014 | write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask); | |
2015 | dd_dev_info(dd, | |
2016 | "Using send context %u(%u) for VL15\n", | |
2017 | dd->vld[15].sc->sw_index, ctxt); | |
35f6befc | 2018 | |
77241056 MM |
2019 | for (i = 0; i < num_vls; i++) { |
2020 | sc_enable(dd->vld[i].sc); | |
2021 | ctxt = dd->vld[i].sc->hw_context; | |
35f6befc | 2022 | mask = all_vl_mask & ~(data_vls_mask); |
77241056 MM |
2023 | write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask); |
2024 | } | |
35f6befc JJ |
2025 | for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++) { |
2026 | sc_enable(dd->kernel_send_context[i + 1]); | |
2027 | ctxt = dd->kernel_send_context[i + 1]->hw_context; | |
2028 | mask = all_vl_mask & ~(data_vls_mask); | |
2029 | write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask); | |
2030 | } | |
2031 | ||
2032 | if (pio_map_init(dd, ppd->port - 1, num_vls, NULL)) | |
2033 | goto nomem; | |
77241056 | 2034 | return 0; |
042b0159 | 2035 | |
77241056 | 2036 | nomem: |
042b0159 | 2037 | for (i = 0; i < num_vls; i++) { |
77241056 | 2038 | sc_free(dd->vld[i].sc); |
042b0159 IW |
2039 | dd->vld[i].sc = NULL; |
2040 | } | |
2041 | ||
35f6befc JJ |
2042 | for (i = num_vls; i < INIT_SC_PER_VL * num_vls; i++) |
2043 | sc_free(dd->kernel_send_context[i + 1]); | |
042b0159 IW |
2044 | |
2045 | kfree(dd->kernel_send_context); | |
2046 | dd->kernel_send_context = NULL; | |
2047 | ||
2048 | freesc15: | |
2049 | sc_free(dd->vld[15].sc); | |
77241056 MM |
2050 | return -ENOMEM; |
2051 | } | |
2052 | ||
2053 | int init_credit_return(struct hfi1_devdata *dd) | |
2054 | { | |
2055 | int ret; | |
77241056 MM |
2056 | int i; |
2057 | ||
77241056 | 2058 | dd->cr_base = kcalloc( |
9d8145a6 | 2059 | node_affinity.num_possible_nodes, |
77241056 MM |
2060 | sizeof(struct credit_return_base), |
2061 | GFP_KERNEL); | |
2062 | if (!dd->cr_base) { | |
77241056 MM |
2063 | ret = -ENOMEM; |
2064 | goto done; | |
2065 | } | |
9d8145a6 | 2066 | for_each_node_with_cpus(i) { |
77241056 MM |
2067 | int bytes = TXE_NUM_CONTEXTS * sizeof(struct credit_return); |
2068 | ||
2069 | set_dev_node(&dd->pcidev->dev, i); | |
2070 | dd->cr_base[i].va = dma_zalloc_coherent( | |
2071 | &dd->pcidev->dev, | |
2072 | bytes, | |
60368186 | 2073 | &dd->cr_base[i].dma, |
77241056 | 2074 | GFP_KERNEL); |
d125a6c6 | 2075 | if (!dd->cr_base[i].va) { |
77241056 MM |
2076 | set_dev_node(&dd->pcidev->dev, dd->node); |
2077 | dd_dev_err(dd, | |
17fb4f29 JJ |
2078 | "Unable to allocate credit return DMA range for NUMA %d\n", |
2079 | i); | |
77241056 MM |
2080 | ret = -ENOMEM; |
2081 | goto done; | |
2082 | } | |
2083 | } | |
2084 | set_dev_node(&dd->pcidev->dev, dd->node); | |
2085 | ||
2086 | ret = 0; | |
2087 | done: | |
2088 | return ret; | |
2089 | } | |
2090 | ||
2091 | void free_credit_return(struct hfi1_devdata *dd) | |
2092 | { | |
77241056 MM |
2093 | int i; |
2094 | ||
2095 | if (!dd->cr_base) | |
2096 | return; | |
9d8145a6 | 2097 | for (i = 0; i < node_affinity.num_possible_nodes; i++) { |
77241056 MM |
2098 | if (dd->cr_base[i].va) { |
2099 | dma_free_coherent(&dd->pcidev->dev, | |
17fb4f29 JJ |
2100 | TXE_NUM_CONTEXTS * |
2101 | sizeof(struct credit_return), | |
2102 | dd->cr_base[i].va, | |
60368186 | 2103 | dd->cr_base[i].dma); |
77241056 MM |
2104 | } |
2105 | } | |
2106 | kfree(dd->cr_base); | |
2107 | dd->cr_base = NULL; | |
2108 | } |