]>
Commit | Line | Data |
---|---|---|
1812924b CW |
1 | /* |
2 | * SGI UltraViolet TLB flush routines. | |
3 | * | |
4 | * (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI. | |
5 | * | |
6 | * This code is released under the GNU General Public License version 2 or | |
7 | * later. | |
8 | */ | |
aef8f5b8 | 9 | #include <linux/seq_file.h> |
1812924b CW |
10 | #include <linux/proc_fs.h> |
11 | #include <linux/kernel.h> | |
5a0e3ad6 | 12 | #include <linux/slab.h> |
1812924b | 13 | |
1812924b | 14 | #include <asm/mmu_context.h> |
bdbcdd48 | 15 | #include <asm/uv/uv.h> |
1812924b | 16 | #include <asm/uv/uv_mmrs.h> |
b4c286e6 | 17 | #include <asm/uv/uv_hub.h> |
1812924b | 18 | #include <asm/uv/uv_bau.h> |
7b6aa335 | 19 | #include <asm/apic.h> |
b4c286e6 | 20 | #include <asm/idle.h> |
b194b120 | 21 | #include <asm/tsc.h> |
99dd8713 | 22 | #include <asm/irq_vectors.h> |
1812924b | 23 | |
b4c286e6 IM |
24 | static struct bau_control **uv_bau_table_bases __read_mostly; |
25 | static int uv_bau_retry_limit __read_mostly; | |
26 | ||
94ca8e48 CW |
27 | /* base pnode in this partition */ |
28 | static int uv_partition_base_pnode __read_mostly; | |
b4c286e6 IM |
29 | |
30 | static unsigned long uv_mmask __read_mostly; | |
1812924b | 31 | |
dc163a41 IM |
32 | static DEFINE_PER_CPU(struct ptc_stats, ptcstats); |
33 | static DEFINE_PER_CPU(struct bau_control, bau_control); | |
1812924b | 34 | |
9674f35b CW |
35 | /* |
36 | * Determine the first node on a blade. | |
37 | */ | |
38 | static int __init blade_to_first_node(int blade) | |
39 | { | |
40 | int node, b; | |
41 | ||
42 | for_each_online_node(node) { | |
43 | b = uv_node_to_blade_id(node); | |
44 | if (blade == b) | |
45 | return node; | |
46 | } | |
94ca8e48 | 47 | return -1; /* shouldn't happen */ |
9674f35b CW |
48 | } |
49 | ||
50 | /* | |
51 | * Determine the apicid of the first cpu on a blade. | |
52 | */ | |
53 | static int __init blade_to_first_apicid(int blade) | |
54 | { | |
55 | int cpu; | |
56 | ||
57 | for_each_present_cpu(cpu) | |
58 | if (blade == uv_cpu_to_blade_id(cpu)) | |
59 | return per_cpu(x86_cpu_to_apicid, cpu); | |
60 | return -1; | |
61 | } | |
62 | ||
1812924b CW |
63 | /* |
64 | * Free a software acknowledge hardware resource by clearing its Pending | |
65 | * bit. This will return a reply to the sender. | |
66 | * If the message has timed out, a reply has already been sent by the | |
67 | * hardware but the resource has not been released. In that case our | |
68 | * clear of the Timeout bit (as well) will free the resource. No reply will | |
69 | * be sent (the hardware will only do one reply per message). | |
70 | */ | |
b194b120 | 71 | static void uv_reply_to_message(int resource, |
b4c286e6 IM |
72 | struct bau_payload_queue_entry *msg, |
73 | struct bau_msg_status *msp) | |
1812924b | 74 | { |
b194b120 | 75 | unsigned long dw; |
1812924b | 76 | |
b194b120 | 77 | dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); |
1812924b CW |
78 | msg->replied_to = 1; |
79 | msg->sw_ack_vector = 0; | |
80 | if (msp) | |
81 | msp->seen_by.bits = 0; | |
b194b120 | 82 | uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw); |
1812924b CW |
83 | } |
84 | ||
85 | /* | |
86 | * Do all the things a cpu should do for a TLB shootdown message. | |
87 | * Other cpu's may come here at the same time for this message. | |
88 | */ | |
b194b120 | 89 | static void uv_bau_process_message(struct bau_payload_queue_entry *msg, |
b4c286e6 | 90 | int msg_slot, int sw_ack_slot) |
1812924b | 91 | { |
1812924b CW |
92 | unsigned long this_cpu_mask; |
93 | struct bau_msg_status *msp; | |
b4c286e6 | 94 | int cpu; |
1812924b CW |
95 | |
96 | msp = __get_cpu_var(bau_control).msg_statuses + msg_slot; | |
97 | cpu = uv_blade_processor_id(); | |
98 | msg->number_of_cpus = | |
9674f35b | 99 | uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id())); |
dc163a41 | 100 | this_cpu_mask = 1UL << cpu; |
1812924b CW |
101 | if (msp->seen_by.bits & this_cpu_mask) |
102 | return; | |
103 | atomic_or_long(&msp->seen_by.bits, this_cpu_mask); | |
104 | ||
105 | if (msg->replied_to == 1) | |
106 | return; | |
107 | ||
108 | if (msg->address == TLB_FLUSH_ALL) { | |
109 | local_flush_tlb(); | |
110 | __get_cpu_var(ptcstats).alltlb++; | |
111 | } else { | |
112 | __flush_tlb_one(msg->address); | |
113 | __get_cpu_var(ptcstats).onetlb++; | |
114 | } | |
115 | ||
116 | __get_cpu_var(ptcstats).requestee++; | |
117 | ||
118 | atomic_inc_short(&msg->acknowledge_count); | |
119 | if (msg->number_of_cpus == msg->acknowledge_count) | |
120 | uv_reply_to_message(sw_ack_slot, msg, msp); | |
1812924b CW |
121 | } |
122 | ||
123 | /* | |
dc163a41 | 124 | * Examine the payload queue on one distribution node to see |
1812924b CW |
125 | * which messages have not been seen, and which cpu(s) have not seen them. |
126 | * | |
127 | * Returns the number of cpu's that have not responded. | |
128 | */ | |
dc163a41 | 129 | static int uv_examine_destination(struct bau_control *bau_tablesp, int sender) |
1812924b | 130 | { |
1812924b CW |
131 | struct bau_payload_queue_entry *msg; |
132 | struct bau_msg_status *msp; | |
b4c286e6 IM |
133 | int count = 0; |
134 | int i; | |
135 | int j; | |
1812924b | 136 | |
dc163a41 IM |
137 | for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE; |
138 | msg++, i++) { | |
139 | if ((msg->sending_cpu == sender) && (!msg->replied_to)) { | |
140 | msp = bau_tablesp->msg_statuses + i; | |
141 | printk(KERN_DEBUG | |
142 | "blade %d: address:%#lx %d of %d, not cpu(s): ", | |
143 | i, msg->address, msg->acknowledge_count, | |
144 | msg->number_of_cpus); | |
145 | for (j = 0; j < msg->number_of_cpus; j++) { | |
b4c286e6 | 146 | if (!((1L << j) & msp->seen_by.bits)) { |
dc163a41 IM |
147 | count++; |
148 | printk("%d ", j); | |
149 | } | |
150 | } | |
151 | printk("\n"); | |
152 | } | |
153 | } | |
154 | return count; | |
155 | } | |
156 | ||
157 | /* | |
158 | * Examine the payload queue on all the distribution nodes to see | |
159 | * which messages have not been seen, and which cpu(s) have not seen them. | |
160 | * | |
161 | * Returns the number of cpu's that have not responded. | |
162 | */ | |
163 | static int uv_examine_destinations(struct bau_target_nodemask *distribution) | |
164 | { | |
165 | int sender; | |
166 | int i; | |
167 | int count = 0; | |
168 | ||
1812924b | 169 | sender = smp_processor_id(); |
b4c286e6 | 170 | for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) { |
b194b120 CW |
171 | if (!bau_node_isset(i, distribution)) |
172 | continue; | |
dc163a41 | 173 | count += uv_examine_destination(uv_bau_table_bases[i], sender); |
1812924b CW |
174 | } |
175 | return count; | |
176 | } | |
177 | ||
b194b120 CW |
178 | /* |
179 | * wait for completion of a broadcast message | |
180 | * | |
181 | * return COMPLETE, RETRY or GIVEUP | |
182 | */ | |
dc163a41 | 183 | static int uv_wait_completion(struct bau_desc *bau_desc, |
b194b120 CW |
184 | unsigned long mmr_offset, int right_shift) |
185 | { | |
186 | int exams = 0; | |
187 | long destination_timeouts = 0; | |
188 | long source_timeouts = 0; | |
189 | unsigned long descriptor_status; | |
190 | ||
191 | while ((descriptor_status = (((unsigned long) | |
192 | uv_read_local_mmr(mmr_offset) >> | |
193 | right_shift) & UV_ACT_STATUS_MASK)) != | |
194 | DESC_STATUS_IDLE) { | |
195 | if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) { | |
196 | source_timeouts++; | |
197 | if (source_timeouts > SOURCE_TIMEOUT_LIMIT) | |
198 | source_timeouts = 0; | |
199 | __get_cpu_var(ptcstats).s_retry++; | |
200 | return FLUSH_RETRY; | |
201 | } | |
202 | /* | |
203 | * spin here looking for progress at the destinations | |
204 | */ | |
205 | if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) { | |
206 | destination_timeouts++; | |
207 | if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) { | |
208 | /* | |
209 | * returns number of cpus not responding | |
210 | */ | |
211 | if (uv_examine_destinations | |
212 | (&bau_desc->distribution) == 0) { | |
213 | __get_cpu_var(ptcstats).d_retry++; | |
214 | return FLUSH_RETRY; | |
215 | } | |
216 | exams++; | |
217 | if (exams >= uv_bau_retry_limit) { | |
218 | printk(KERN_DEBUG | |
219 | "uv_flush_tlb_others"); | |
220 | printk("giving up on cpu %d\n", | |
221 | smp_processor_id()); | |
222 | return FLUSH_GIVEUP; | |
223 | } | |
224 | /* | |
225 | * delays can hang the simulator | |
226 | udelay(1000); | |
227 | */ | |
228 | destination_timeouts = 0; | |
229 | } | |
230 | } | |
18c07cf5 | 231 | cpu_relax(); |
b194b120 CW |
232 | } |
233 | return FLUSH_COMPLETE; | |
234 | } | |
235 | ||
236 | /** | |
237 | * uv_flush_send_and_wait | |
238 | * | |
239 | * Send a broadcast and wait for a broadcast message to complete. | |
240 | * | |
bdbcdd48 | 241 | * The flush_mask contains the cpus the broadcast was sent to. |
b194b120 | 242 | * |
bdbcdd48 TH |
243 | * Returns NULL if all remote flushing was done. The mask is zeroed. |
244 | * Returns @flush_mask if some remote flushing remains to be done. The | |
245 | * mask will have some bits still set. | |
b194b120 | 246 | */ |
9674f35b | 247 | const struct cpumask *uv_flush_send_and_wait(int cpu, int this_pnode, |
bdbcdd48 TH |
248 | struct bau_desc *bau_desc, |
249 | struct cpumask *flush_mask) | |
b194b120 CW |
250 | { |
251 | int completion_status = 0; | |
252 | int right_shift; | |
b194b120 | 253 | int tries = 0; |
9674f35b | 254 | int pnode; |
b4c286e6 | 255 | int bit; |
b194b120 | 256 | unsigned long mmr_offset; |
b4c286e6 | 257 | unsigned long index; |
b194b120 CW |
258 | cycles_t time1; |
259 | cycles_t time2; | |
260 | ||
261 | if (cpu < UV_CPUS_PER_ACT_STATUS) { | |
262 | mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0; | |
263 | right_shift = cpu * UV_ACT_STATUS_SIZE; | |
264 | } else { | |
265 | mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1; | |
266 | right_shift = | |
267 | ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE); | |
268 | } | |
269 | time1 = get_cycles(); | |
270 | do { | |
271 | tries++; | |
dc163a41 IM |
272 | index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | |
273 | cpu; | |
b194b120 CW |
274 | uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); |
275 | completion_status = uv_wait_completion(bau_desc, mmr_offset, | |
276 | right_shift); | |
277 | } while (completion_status == FLUSH_RETRY); | |
278 | time2 = get_cycles(); | |
279 | __get_cpu_var(ptcstats).sflush += (time2 - time1); | |
280 | if (tries > 1) | |
281 | __get_cpu_var(ptcstats).retriesok++; | |
282 | ||
283 | if (completion_status == FLUSH_GIVEUP) { | |
284 | /* | |
285 | * Cause the caller to do an IPI-style TLB shootdown on | |
286 | * the cpu's, all of which are still in the mask. | |
287 | */ | |
288 | __get_cpu_var(ptcstats).ptc_i++; | |
2749ebe3 | 289 | return flush_mask; |
b194b120 CW |
290 | } |
291 | ||
292 | /* | |
293 | * Success, so clear the remote cpu's from the mask so we don't | |
294 | * use the IPI method of shootdown on them. | |
295 | */ | |
bdbcdd48 | 296 | for_each_cpu(bit, flush_mask) { |
9674f35b CW |
297 | pnode = uv_cpu_to_pnode(bit); |
298 | if (pnode == this_pnode) | |
b194b120 | 299 | continue; |
bdbcdd48 | 300 | cpumask_clear_cpu(bit, flush_mask); |
b194b120 | 301 | } |
bdbcdd48 TH |
302 | if (!cpumask_empty(flush_mask)) |
303 | return flush_mask; | |
304 | return NULL; | |
b194b120 CW |
305 | } |
306 | ||
76ba0ecd RR |
307 | static DEFINE_PER_CPU(cpumask_var_t, uv_flush_tlb_mask); |
308 | ||
1812924b CW |
309 | /** |
310 | * uv_flush_tlb_others - globally purge translation cache of a virtual | |
311 | * address or all TLB's | |
bdbcdd48 | 312 | * @cpumask: mask of all cpu's in which the address is to be removed |
1812924b CW |
313 | * @mm: mm_struct containing virtual address range |
314 | * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu) | |
bdbcdd48 | 315 | * @cpu: the current cpu |
1812924b CW |
316 | * |
317 | * This is the entry point for initiating any UV global TLB shootdown. | |
318 | * | |
319 | * Purges the translation caches of all specified processors of the given | |
320 | * virtual address, or purges all TLB's on specified processors. | |
321 | * | |
bdbcdd48 TH |
322 | * The caller has derived the cpumask from the mm_struct. This function |
323 | * is called only if there are bits set in the mask. (e.g. flush_tlb_page()) | |
1812924b | 324 | * |
bdbcdd48 | 325 | * The cpumask is converted into a nodemask of the nodes containing |
1812924b | 326 | * the cpus. |
b194b120 | 327 | * |
bdbcdd48 TH |
328 | * Note that this function should be called with preemption disabled. |
329 | * | |
330 | * Returns NULL if all remote flushing was done. | |
331 | * Returns pointer to cpumask if some remote flushing remains to be | |
332 | * done. The returned pointer is valid till preemption is re-enabled. | |
1812924b | 333 | */ |
bdbcdd48 TH |
334 | const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, |
335 | struct mm_struct *mm, | |
336 | unsigned long va, unsigned int cpu) | |
1812924b | 337 | { |
76ba0ecd | 338 | struct cpumask *flush_mask = __get_cpu_var(uv_flush_tlb_mask); |
1812924b | 339 | int i; |
b194b120 | 340 | int bit; |
9674f35b | 341 | int pnode; |
bdbcdd48 | 342 | int uv_cpu; |
9674f35b | 343 | int this_pnode; |
b194b120 | 344 | int locals = 0; |
dc163a41 | 345 | struct bau_desc *bau_desc; |
bdbcdd48 TH |
346 | |
347 | cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu)); | |
348 | ||
349 | uv_cpu = uv_blade_processor_id(); | |
9674f35b | 350 | this_pnode = uv_hub_info->pnode; |
1812924b | 351 | bau_desc = __get_cpu_var(bau_control).descriptor_base; |
bdbcdd48 | 352 | bau_desc += UV_ITEMS_PER_DESCRIPTOR * uv_cpu; |
1812924b CW |
353 | |
354 | bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); | |
355 | ||
356 | i = 0; | |
bdbcdd48 | 357 | for_each_cpu(bit, flush_mask) { |
9674f35b CW |
358 | pnode = uv_cpu_to_pnode(bit); |
359 | BUG_ON(pnode > (UV_DISTRIBUTION_SIZE - 1)); | |
360 | if (pnode == this_pnode) { | |
b194b120 | 361 | locals++; |
1812924b | 362 | continue; |
b194b120 | 363 | } |
94ca8e48 CW |
364 | bau_node_set(pnode - uv_partition_base_pnode, |
365 | &bau_desc->distribution); | |
1812924b CW |
366 | i++; |
367 | } | |
b194b120 CW |
368 | if (i == 0) { |
369 | /* | |
370 | * no off_node flushing; return status for local node | |
371 | */ | |
372 | if (locals) | |
bdbcdd48 | 373 | return flush_mask; |
b194b120 | 374 | else |
bdbcdd48 | 375 | return NULL; |
b194b120 | 376 | } |
1812924b CW |
377 | __get_cpu_var(ptcstats).requestor++; |
378 | __get_cpu_var(ptcstats).ntargeted += i; | |
379 | ||
380 | bau_desc->payload.address = va; | |
bdbcdd48 | 381 | bau_desc->payload.sending_cpu = cpu; |
1812924b | 382 | |
9674f35b | 383 | return uv_flush_send_and_wait(uv_cpu, this_pnode, bau_desc, flush_mask); |
1812924b CW |
384 | } |
385 | ||
386 | /* | |
387 | * The BAU message interrupt comes here. (registered by set_intr_gate) | |
388 | * See entry_64.S | |
389 | * | |
390 | * We received a broadcast assist message. | |
391 | * | |
392 | * Interrupts may have been disabled; this interrupt could represent | |
393 | * the receipt of several messages. | |
394 | * | |
395 | * All cores/threads on this node get this interrupt. | |
396 | * The last one to see it does the s/w ack. | |
397 | * (the resource will not be freed until noninterruptable cpus see this | |
398 | * interrupt; hardware will timeout the s/w ack and reply ERROR) | |
399 | */ | |
b194b120 | 400 | void uv_bau_message_interrupt(struct pt_regs *regs) |
1812924b | 401 | { |
dc163a41 IM |
402 | struct bau_payload_queue_entry *va_queue_first; |
403 | struct bau_payload_queue_entry *va_queue_last; | |
b4c286e6 | 404 | struct bau_payload_queue_entry *msg; |
1812924b | 405 | struct pt_regs *old_regs = set_irq_regs(regs); |
b4c286e6 IM |
406 | cycles_t time1; |
407 | cycles_t time2; | |
1812924b CW |
408 | int msg_slot; |
409 | int sw_ack_slot; | |
410 | int fw; | |
411 | int count = 0; | |
412 | unsigned long local_pnode; | |
413 | ||
414 | ack_APIC_irq(); | |
415 | exit_idle(); | |
416 | irq_enter(); | |
417 | ||
b194b120 | 418 | time1 = get_cycles(); |
1812924b CW |
419 | |
420 | local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); | |
421 | ||
b4c286e6 | 422 | va_queue_first = __get_cpu_var(bau_control).va_queue_first; |
dc163a41 | 423 | va_queue_last = __get_cpu_var(bau_control).va_queue_last; |
b4c286e6 | 424 | |
1812924b CW |
425 | msg = __get_cpu_var(bau_control).bau_msg_head; |
426 | while (msg->sw_ack_vector) { | |
427 | count++; | |
428 | fw = msg->sw_ack_vector; | |
b4c286e6 | 429 | msg_slot = msg - va_queue_first; |
1812924b CW |
430 | sw_ack_slot = ffs(fw) - 1; |
431 | ||
432 | uv_bau_process_message(msg, msg_slot, sw_ack_slot); | |
433 | ||
434 | msg++; | |
dc163a41 IM |
435 | if (msg > va_queue_last) |
436 | msg = va_queue_first; | |
1812924b CW |
437 | __get_cpu_var(bau_control).bau_msg_head = msg; |
438 | } | |
439 | if (!count) | |
440 | __get_cpu_var(ptcstats).nomsg++; | |
441 | else if (count > 1) | |
442 | __get_cpu_var(ptcstats).multmsg++; | |
443 | ||
b194b120 CW |
444 | time2 = get_cycles(); |
445 | __get_cpu_var(ptcstats).dflush += (time2 - time1); | |
1812924b CW |
446 | |
447 | irq_exit(); | |
448 | set_irq_regs(old_regs); | |
1812924b CW |
449 | } |
450 | ||
c4c4688f CW |
451 | /* |
452 | * uv_enable_timeouts | |
453 | * | |
454 | * Each target blade (i.e. blades that have cpu's) needs to have | |
455 | * shootdown message timeouts enabled. The timeout does not cause | |
456 | * an interrupt, but causes an error message to be returned to | |
457 | * the sender. | |
458 | */ | |
b194b120 | 459 | static void uv_enable_timeouts(void) |
1812924b | 460 | { |
1812924b | 461 | int blade; |
c4c4688f | 462 | int nblades; |
1812924b | 463 | int pnode; |
c4c4688f | 464 | unsigned long mmr_image; |
1812924b | 465 | |
c4c4688f | 466 | nblades = uv_num_possible_blades(); |
1812924b | 467 | |
c4c4688f CW |
468 | for (blade = 0; blade < nblades; blade++) { |
469 | if (!uv_blade_nr_possible_cpus(blade)) | |
1812924b | 470 | continue; |
c4c4688f | 471 | |
1812924b | 472 | pnode = uv_blade_to_pnode(blade); |
c4c4688f CW |
473 | mmr_image = |
474 | uv_read_global_mmr64(pnode, UVH_LB_BAU_MISC_CONTROL); | |
475 | /* | |
476 | * Set the timeout period and then lock it in, in three | |
477 | * steps; captures and locks in the period. | |
478 | * | |
479 | * To program the period, the SOFT_ACK_MODE must be off. | |
480 | */ | |
481 | mmr_image &= ~((unsigned long)1 << | |
482 | UV_ENABLE_INTD_SOFT_ACK_MODE_SHIFT); | |
483 | uv_write_global_mmr64 | |
484 | (pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image); | |
485 | /* | |
486 | * Set the 4-bit period. | |
487 | */ | |
488 | mmr_image &= ~((unsigned long)0xf << | |
489 | UV_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHIFT); | |
490 | mmr_image |= (UV_INTD_SOFT_ACK_TIMEOUT_PERIOD << | |
491 | UV_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHIFT); | |
492 | uv_write_global_mmr64 | |
493 | (pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image); | |
494 | /* | |
495 | * Subsequent reversals of the timebase bit (3) cause an | |
496 | * immediate timeout of one or all INTD resources as | |
497 | * indicated in bits 2:0 (7 causes all of them to timeout). | |
498 | */ | |
499 | mmr_image |= ((unsigned long)1 << | |
500 | UV_ENABLE_INTD_SOFT_ACK_MODE_SHIFT); | |
501 | uv_write_global_mmr64 | |
502 | (pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image); | |
1812924b | 503 | } |
1812924b CW |
504 | } |
505 | ||
b194b120 | 506 | static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset) |
1812924b CW |
507 | { |
508 | if (*offset < num_possible_cpus()) | |
509 | return offset; | |
510 | return NULL; | |
511 | } | |
512 | ||
b194b120 | 513 | static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) |
1812924b CW |
514 | { |
515 | (*offset)++; | |
516 | if (*offset < num_possible_cpus()) | |
517 | return offset; | |
518 | return NULL; | |
519 | } | |
520 | ||
b194b120 | 521 | static void uv_ptc_seq_stop(struct seq_file *file, void *data) |
1812924b CW |
522 | { |
523 | } | |
524 | ||
525 | /* | |
526 | * Display the statistics thru /proc | |
527 | * data points to the cpu number | |
528 | */ | |
b194b120 | 529 | static int uv_ptc_seq_show(struct seq_file *file, void *data) |
1812924b CW |
530 | { |
531 | struct ptc_stats *stat; | |
532 | int cpu; | |
533 | ||
534 | cpu = *(loff_t *)data; | |
535 | ||
536 | if (!cpu) { | |
537 | seq_printf(file, | |
538 | "# cpu requestor requestee one all sretry dretry ptc_i "); | |
539 | seq_printf(file, | |
b194b120 | 540 | "sw_ack sflush dflush sok dnomsg dmult starget\n"); |
1812924b CW |
541 | } |
542 | if (cpu < num_possible_cpus() && cpu_online(cpu)) { | |
543 | stat = &per_cpu(ptcstats, cpu); | |
544 | seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ", | |
545 | cpu, stat->requestor, | |
546 | stat->requestee, stat->onetlb, stat->alltlb, | |
547 | stat->s_retry, stat->d_retry, stat->ptc_i); | |
548 | seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n", | |
9674f35b | 549 | uv_read_global_mmr64(uv_cpu_to_pnode(cpu), |
1812924b | 550 | UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE), |
b194b120 | 551 | stat->sflush, stat->dflush, |
1812924b CW |
552 | stat->retriesok, stat->nomsg, |
553 | stat->multmsg, stat->ntargeted); | |
554 | } | |
555 | ||
556 | return 0; | |
557 | } | |
558 | ||
559 | /* | |
560 | * 0: display meaning of the statistics | |
561 | * >0: retry limit | |
562 | */ | |
b194b120 | 563 | static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, |
b4c286e6 | 564 | size_t count, loff_t *data) |
1812924b CW |
565 | { |
566 | long newmode; | |
567 | char optstr[64]; | |
568 | ||
e7eb8726 | 569 | if (count == 0 || count > sizeof(optstr)) |
cef53278 | 570 | return -EINVAL; |
1812924b CW |
571 | if (copy_from_user(optstr, user, count)) |
572 | return -EFAULT; | |
573 | optstr[count - 1] = '\0'; | |
574 | if (strict_strtoul(optstr, 10, &newmode) < 0) { | |
575 | printk(KERN_DEBUG "%s is invalid\n", optstr); | |
576 | return -EINVAL; | |
577 | } | |
578 | ||
579 | if (newmode == 0) { | |
580 | printk(KERN_DEBUG "# cpu: cpu number\n"); | |
581 | printk(KERN_DEBUG | |
582 | "requestor: times this cpu was the flush requestor\n"); | |
583 | printk(KERN_DEBUG | |
584 | "requestee: times this cpu was requested to flush its TLBs\n"); | |
585 | printk(KERN_DEBUG | |
586 | "one: times requested to flush a single address\n"); | |
587 | printk(KERN_DEBUG | |
588 | "all: times requested to flush all TLB's\n"); | |
589 | printk(KERN_DEBUG | |
590 | "sretry: number of retries of source-side timeouts\n"); | |
591 | printk(KERN_DEBUG | |
592 | "dretry: number of retries of destination-side timeouts\n"); | |
593 | printk(KERN_DEBUG | |
594 | "ptc_i: times UV fell through to IPI-style flushes\n"); | |
595 | printk(KERN_DEBUG | |
596 | "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n"); | |
597 | printk(KERN_DEBUG | |
b194b120 | 598 | "sflush_us: cycles spent in uv_flush_tlb_others()\n"); |
1812924b | 599 | printk(KERN_DEBUG |
b194b120 | 600 | "dflush_us: cycles spent in handling flush requests\n"); |
1812924b CW |
601 | printk(KERN_DEBUG "sok: successes on retry\n"); |
602 | printk(KERN_DEBUG "dnomsg: interrupts with no message\n"); | |
603 | printk(KERN_DEBUG | |
604 | "dmult: interrupts with multiple messages\n"); | |
605 | printk(KERN_DEBUG "starget: nodes targeted\n"); | |
606 | } else { | |
607 | uv_bau_retry_limit = newmode; | |
608 | printk(KERN_DEBUG "timeout retry limit:%d\n", | |
609 | uv_bau_retry_limit); | |
610 | } | |
611 | ||
612 | return count; | |
613 | } | |
614 | ||
615 | static const struct seq_operations uv_ptc_seq_ops = { | |
dc163a41 IM |
616 | .start = uv_ptc_seq_start, |
617 | .next = uv_ptc_seq_next, | |
618 | .stop = uv_ptc_seq_stop, | |
619 | .show = uv_ptc_seq_show | |
1812924b CW |
620 | }; |
621 | ||
b194b120 | 622 | static int uv_ptc_proc_open(struct inode *inode, struct file *file) |
1812924b CW |
623 | { |
624 | return seq_open(file, &uv_ptc_seq_ops); | |
625 | } | |
626 | ||
627 | static const struct file_operations proc_uv_ptc_operations = { | |
b194b120 CW |
628 | .open = uv_ptc_proc_open, |
629 | .read = seq_read, | |
630 | .write = uv_ptc_proc_write, | |
631 | .llseek = seq_lseek, | |
632 | .release = seq_release, | |
1812924b CW |
633 | }; |
634 | ||
b194b120 | 635 | static int __init uv_ptc_init(void) |
1812924b | 636 | { |
b194b120 | 637 | struct proc_dir_entry *proc_uv_ptc; |
1812924b CW |
638 | |
639 | if (!is_uv_system()) | |
640 | return 0; | |
641 | ||
10f02d11 AD |
642 | proc_uv_ptc = proc_create(UV_PTC_BASENAME, 0444, NULL, |
643 | &proc_uv_ptc_operations); | |
1812924b CW |
644 | if (!proc_uv_ptc) { |
645 | printk(KERN_ERR "unable to create %s proc entry\n", | |
646 | UV_PTC_BASENAME); | |
647 | return -EINVAL; | |
648 | } | |
1812924b CW |
649 | return 0; |
650 | } | |
651 | ||
b194b120 CW |
652 | /* |
653 | * begin the initialization of the per-blade control structures | |
654 | */ | |
655 | static struct bau_control * __init uv_table_bases_init(int blade, int node) | |
1812924b | 656 | { |
b194b120 | 657 | int i; |
b194b120 | 658 | struct bau_msg_status *msp; |
dc163a41 | 659 | struct bau_control *bau_tabp; |
b194b120 | 660 | |
dc163a41 | 661 | bau_tabp = |
b194b120 | 662 | kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node); |
dc163a41 | 663 | BUG_ON(!bau_tabp); |
b4c286e6 | 664 | |
dc163a41 | 665 | bau_tabp->msg_statuses = |
b194b120 | 666 | kmalloc_node(sizeof(struct bau_msg_status) * |
dc163a41 IM |
667 | DEST_Q_SIZE, GFP_KERNEL, node); |
668 | BUG_ON(!bau_tabp->msg_statuses); | |
b4c286e6 | 669 | |
dc163a41 | 670 | for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++) |
b194b120 CW |
671 | bau_cpubits_clear(&msp->seen_by, (int) |
672 | uv_blade_nr_possible_cpus(blade)); | |
b4c286e6 | 673 | |
dc163a41 | 674 | uv_bau_table_bases[blade] = bau_tabp; |
b4c286e6 | 675 | |
d400524a | 676 | return bau_tabp; |
1812924b CW |
677 | } |
678 | ||
b194b120 CW |
679 | /* |
680 | * finish the initialization of the per-blade control structures | |
681 | */ | |
b4c286e6 | 682 | static void __init |
9674f35b | 683 | uv_table_bases_finish(int blade, |
b4c286e6 IM |
684 | struct bau_control *bau_tablesp, |
685 | struct bau_desc *adp) | |
b194b120 | 686 | { |
b194b120 | 687 | struct bau_control *bcp; |
9674f35b | 688 | int cpu; |
b194b120 | 689 | |
9674f35b CW |
690 | for_each_present_cpu(cpu) { |
691 | if (blade != uv_cpu_to_blade_id(cpu)) | |
692 | continue; | |
b4c286e6 | 693 | |
9674f35b | 694 | bcp = (struct bau_control *)&per_cpu(bau_control, cpu); |
b4c286e6 IM |
695 | bcp->bau_msg_head = bau_tablesp->va_queue_first; |
696 | bcp->va_queue_first = bau_tablesp->va_queue_first; | |
697 | bcp->va_queue_last = bau_tablesp->va_queue_last; | |
b4c286e6 IM |
698 | bcp->msg_statuses = bau_tablesp->msg_statuses; |
699 | bcp->descriptor_base = adp; | |
b194b120 CW |
700 | } |
701 | } | |
1812924b CW |
702 | |
703 | /* | |
b194b120 | 704 | * initialize the sending side's sending buffers |
1812924b | 705 | */ |
dc163a41 | 706 | static struct bau_desc * __init |
b194b120 | 707 | uv_activation_descriptor_init(int node, int pnode) |
1812924b CW |
708 | { |
709 | int i; | |
1812924b | 710 | unsigned long pa; |
1812924b | 711 | unsigned long m; |
b194b120 | 712 | unsigned long n; |
dc163a41 IM |
713 | struct bau_desc *adp; |
714 | struct bau_desc *ad2; | |
b194b120 | 715 | |
0e2595cd CW |
716 | /* |
717 | * each bau_desc is 64 bytes; there are 8 (UV_ITEMS_PER_DESCRIPTOR) | |
718 | * per cpu; and up to 32 (UV_ADP_SIZE) cpu's per blade | |
719 | */ | |
720 | adp = (struct bau_desc *)kmalloc_node(sizeof(struct bau_desc)* | |
721 | UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR, GFP_KERNEL, node); | |
dc163a41 | 722 | BUG_ON(!adp); |
b4c286e6 | 723 | |
4ea3c51d | 724 | pa = uv_gpa(adp); /* need the real nasid*/ |
1d21e6e3 | 725 | n = uv_gpa_to_pnode(pa); |
b194b120 | 726 | m = pa & uv_mmask; |
b4c286e6 | 727 | |
9c26f52b CW |
728 | uv_write_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE, |
729 | (n << UV_DESC_BASE_PNODE_SHIFT | m)); | |
b4c286e6 | 730 | |
0e2595cd CW |
731 | /* |
732 | * initializing all 8 (UV_ITEMS_PER_DESCRIPTOR) descriptors for each | |
733 | * cpu even though we only use the first one; one descriptor can | |
734 | * describe a broadcast to 256 nodes. | |
735 | */ | |
736 | for (i = 0, ad2 = adp; i < (UV_ADP_SIZE*UV_ITEMS_PER_DESCRIPTOR); | |
737 | i++, ad2++) { | |
dc163a41 | 738 | memset(ad2, 0, sizeof(struct bau_desc)); |
b194b120 | 739 | ad2->header.sw_ack_flag = 1; |
94ca8e48 CW |
740 | /* |
741 | * base_dest_nodeid is the first node in the partition, so | |
742 | * the bit map will indicate partition-relative node numbers. | |
743 | * note that base_dest_nodeid is actually a nasid. | |
744 | */ | |
745 | ad2->header.base_dest_nodeid = uv_partition_base_pnode << 1; | |
3ef12c3c | 746 | ad2->header.dest_subnodeid = 0x10; /* the LB */ |
b194b120 CW |
747 | ad2->header.command = UV_NET_ENDPOINT_INTD; |
748 | ad2->header.int_both = 1; | |
749 | /* | |
750 | * all others need to be set to zero: | |
751 | * fairness chaining multilevel count replied_to | |
752 | */ | |
753 | } | |
754 | return adp; | |
755 | } | |
756 | ||
757 | /* | |
758 | * initialize the destination side's receiving buffers | |
759 | */ | |
b4c286e6 IM |
760 | static struct bau_payload_queue_entry * __init |
761 | uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp) | |
b194b120 | 762 | { |
1812924b | 763 | struct bau_payload_queue_entry *pqp; |
4ea3c51d CW |
764 | unsigned long pa; |
765 | int pn; | |
b4c286e6 | 766 | char *cp; |
1812924b | 767 | |
dc163a41 IM |
768 | pqp = (struct bau_payload_queue_entry *) kmalloc_node( |
769 | (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry), | |
770 | GFP_KERNEL, node); | |
771 | BUG_ON(!pqp); | |
b4c286e6 | 772 | |
b194b120 CW |
773 | cp = (char *)pqp + 31; |
774 | pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5); | |
775 | bau_tablesp->va_queue_first = pqp; | |
4ea3c51d CW |
776 | /* |
777 | * need the pnode of where the memory was really allocated | |
778 | */ | |
779 | pa = uv_gpa(pqp); | |
1d21e6e3 | 780 | pn = uv_gpa_to_pnode(pa); |
b194b120 CW |
781 | uv_write_global_mmr64(pnode, |
782 | UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, | |
4ea3c51d | 783 | ((unsigned long)pn << UV_PAYLOADQ_PNODE_SHIFT) | |
b194b120 CW |
784 | uv_physnodeaddr(pqp)); |
785 | uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, | |
786 | uv_physnodeaddr(pqp)); | |
dc163a41 | 787 | bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1); |
b194b120 CW |
788 | uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, |
789 | (unsigned long) | |
790 | uv_physnodeaddr(bau_tablesp->va_queue_last)); | |
dc163a41 | 791 | memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE); |
b4c286e6 | 792 | |
b194b120 CW |
793 | return pqp; |
794 | } | |
1812924b | 795 | |
b194b120 CW |
796 | /* |
797 | * Initialization of each UV blade's structures | |
798 | */ | |
9674f35b | 799 | static int __init uv_init_blade(int blade) |
b194b120 | 800 | { |
9674f35b | 801 | int node; |
b194b120 CW |
802 | int pnode; |
803 | unsigned long pa; | |
804 | unsigned long apicid; | |
dc163a41 | 805 | struct bau_desc *adp; |
b194b120 CW |
806 | struct bau_payload_queue_entry *pqp; |
807 | struct bau_control *bau_tablesp; | |
1812924b | 808 | |
9674f35b | 809 | node = blade_to_first_node(blade); |
b194b120 CW |
810 | bau_tablesp = uv_table_bases_init(blade, node); |
811 | pnode = uv_blade_to_pnode(blade); | |
812 | adp = uv_activation_descriptor_init(node, pnode); | |
813 | pqp = uv_payload_queue_init(node, pnode, bau_tablesp); | |
9674f35b | 814 | uv_table_bases_finish(blade, bau_tablesp, adp); |
b194b120 CW |
815 | /* |
816 | * the below initialization can't be in firmware because the | |
817 | * messaging IRQ will be determined by the OS | |
818 | */ | |
9674f35b | 819 | apicid = blade_to_first_apicid(blade); |
b194b120 | 820 | pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG); |
e38e2af1 | 821 | uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, |
b194b120 | 822 | ((apicid << 32) | UV_BAU_MESSAGE)); |
b194b120 CW |
823 | return 0; |
824 | } | |
825 | ||
826 | /* | |
827 | * Initialization of BAU-related structures | |
828 | */ | |
829 | static int __init uv_bau_init(void) | |
830 | { | |
831 | int blade; | |
b194b120 | 832 | int nblades; |
2c74d666 | 833 | int cur_cpu; |
b194b120 CW |
834 | |
835 | if (!is_uv_system()) | |
836 | return 0; | |
1812924b | 837 | |
76ba0ecd | 838 | for_each_possible_cpu(cur_cpu) |
eaa95840 | 839 | zalloc_cpumask_var_node(&per_cpu(uv_flush_tlb_mask, cur_cpu), |
76ba0ecd RR |
840 | GFP_KERNEL, cpu_to_node(cur_cpu)); |
841 | ||
b194b120 | 842 | uv_bau_retry_limit = 1; |
036ed8ba | 843 | uv_mmask = (1UL << uv_hub_info->m_val) - 1; |
9674f35b CW |
844 | nblades = uv_num_possible_blades(); |
845 | ||
1812924b CW |
846 | uv_bau_table_bases = (struct bau_control **) |
847 | kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); | |
dc163a41 | 848 | BUG_ON(!uv_bau_table_bases); |
b4c286e6 | 849 | |
94ca8e48 CW |
850 | uv_partition_base_pnode = 0x7fffffff; |
851 | for (blade = 0; blade < nblades; blade++) | |
852 | if (uv_blade_nr_possible_cpus(blade) && | |
853 | (uv_blade_to_pnode(blade) < uv_partition_base_pnode)) | |
854 | uv_partition_base_pnode = uv_blade_to_pnode(blade); | |
9674f35b CW |
855 | for (blade = 0; blade < nblades; blade++) |
856 | if (uv_blade_nr_possible_cpus(blade)) | |
857 | uv_init_blade(blade); | |
858 | ||
99dd8713 | 859 | alloc_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); |
1812924b | 860 | uv_enable_timeouts(); |
b4c286e6 | 861 | |
1812924b CW |
862 | return 0; |
863 | } | |
1812924b | 864 | __initcall(uv_bau_init); |
b194b120 | 865 | __initcall(uv_ptc_init); |