2 * SGI UltraViolet TLB flush routines.
4 * (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI.
6 * This code is released under the GNU General Public License version 2 or
9 #include <linux/seq_file.h>
10 #include <linux/proc_fs.h>
11 #include <linux/kernel.h>
13 #include <asm/mmu_context.h>
14 #include <asm/uv/uv.h>
15 #include <asm/uv/uv_mmrs.h>
16 #include <asm/uv/uv_hub.h>
17 #include <asm/uv/uv_bau.h>
21 #include <asm/irq_vectors.h>
23 static struct bau_control
**uv_bau_table_bases __read_mostly
;
24 static int uv_bau_retry_limit __read_mostly
;
26 /* position of pnode (which is nasid>>1): */
27 static int uv_nshift __read_mostly
;
29 static unsigned long uv_mmask __read_mostly
;
31 static DEFINE_PER_CPU(struct ptc_stats
, ptcstats
);
32 static DEFINE_PER_CPU(struct bau_control
, bau_control
);
35 * Free a software acknowledge hardware resource by clearing its Pending
36 * bit. This will return a reply to the sender.
37 * If the message has timed out, a reply has already been sent by the
38 * hardware but the resource has not been released. In that case our
39 * clear of the Timeout bit (as well) will free the resource. No reply will
40 * be sent (the hardware will only do one reply per message).
42 static void uv_reply_to_message(int resource
,
43 struct bau_payload_queue_entry
*msg
,
44 struct bau_msg_status
*msp
)
48 dw
= (1 << (resource
+ UV_SW_ACK_NPENDING
)) | (1 << resource
);
50 msg
->sw_ack_vector
= 0;
52 msp
->seen_by
.bits
= 0;
53 uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS
, dw
);
57 * Do all the things a cpu should do for a TLB shootdown message.
58 * Other cpu's may come here at the same time for this message.
60 static void uv_bau_process_message(struct bau_payload_queue_entry
*msg
,
61 int msg_slot
, int sw_ack_slot
)
63 unsigned long this_cpu_mask
;
64 struct bau_msg_status
*msp
;
67 msp
= __get_cpu_var(bau_control
).msg_statuses
+ msg_slot
;
68 cpu
= uv_blade_processor_id();
70 uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
71 this_cpu_mask
= 1UL << cpu
;
72 if (msp
->seen_by
.bits
& this_cpu_mask
)
74 atomic_or_long(&msp
->seen_by
.bits
, this_cpu_mask
);
76 if (msg
->replied_to
== 1)
79 if (msg
->address
== TLB_FLUSH_ALL
) {
81 __get_cpu_var(ptcstats
).alltlb
++;
83 __flush_tlb_one(msg
->address
);
84 __get_cpu_var(ptcstats
).onetlb
++;
87 __get_cpu_var(ptcstats
).requestee
++;
89 atomic_inc_short(&msg
->acknowledge_count
);
90 if (msg
->number_of_cpus
== msg
->acknowledge_count
)
91 uv_reply_to_message(sw_ack_slot
, msg
, msp
);
95 * Examine the payload queue on one distribution node to see
96 * which messages have not been seen, and which cpu(s) have not seen them.
98 * Returns the number of cpu's that have not responded.
100 static int uv_examine_destination(struct bau_control
*bau_tablesp
, int sender
)
102 struct bau_payload_queue_entry
*msg
;
103 struct bau_msg_status
*msp
;
108 for (msg
= bau_tablesp
->va_queue_first
, i
= 0; i
< DEST_Q_SIZE
;
110 if ((msg
->sending_cpu
== sender
) && (!msg
->replied_to
)) {
111 msp
= bau_tablesp
->msg_statuses
+ i
;
113 "blade %d: address:%#lx %d of %d, not cpu(s): ",
114 i
, msg
->address
, msg
->acknowledge_count
,
115 msg
->number_of_cpus
);
116 for (j
= 0; j
< msg
->number_of_cpus
; j
++) {
117 if (!((1L << j
) & msp
->seen_by
.bits
)) {
129 * Examine the payload queue on all the distribution nodes to see
130 * which messages have not been seen, and which cpu(s) have not seen them.
132 * Returns the number of cpu's that have not responded.
134 static int uv_examine_destinations(struct bau_target_nodemask
*distribution
)
140 sender
= smp_processor_id();
141 for (i
= 0; i
< sizeof(struct bau_target_nodemask
) * BITSPERBYTE
; i
++) {
142 if (!bau_node_isset(i
, distribution
))
144 count
+= uv_examine_destination(uv_bau_table_bases
[i
], sender
);
150 * wait for completion of a broadcast message
152 * return COMPLETE, RETRY or GIVEUP
154 static int uv_wait_completion(struct bau_desc
*bau_desc
,
155 unsigned long mmr_offset
, int right_shift
)
158 long destination_timeouts
= 0;
159 long source_timeouts
= 0;
160 unsigned long descriptor_status
;
162 while ((descriptor_status
= (((unsigned long)
163 uv_read_local_mmr(mmr_offset
) >>
164 right_shift
) & UV_ACT_STATUS_MASK
)) !=
166 if (descriptor_status
== DESC_STATUS_SOURCE_TIMEOUT
) {
168 if (source_timeouts
> SOURCE_TIMEOUT_LIMIT
)
170 __get_cpu_var(ptcstats
).s_retry
++;
174 * spin here looking for progress at the destinations
176 if (descriptor_status
== DESC_STATUS_DESTINATION_TIMEOUT
) {
177 destination_timeouts
++;
178 if (destination_timeouts
> DESTINATION_TIMEOUT_LIMIT
) {
180 * returns number of cpus not responding
182 if (uv_examine_destinations
183 (&bau_desc
->distribution
) == 0) {
184 __get_cpu_var(ptcstats
).d_retry
++;
188 if (exams
>= uv_bau_retry_limit
) {
190 "uv_flush_tlb_others");
191 printk("giving up on cpu %d\n",
196 * delays can hang the simulator
199 destination_timeouts
= 0;
204 return FLUSH_COMPLETE
;
208 * uv_flush_send_and_wait
210 * Send a broadcast and wait for a broadcast message to complete.
212 * The flush_mask contains the cpus the broadcast was sent to.
214 * Returns NULL if all remote flushing was done. The mask is zeroed.
215 * Returns @flush_mask if some remote flushing remains to be done. The
216 * mask will have some bits still set.
218 const struct cpumask
*uv_flush_send_and_wait(int cpu
, int this_blade
,
219 struct bau_desc
*bau_desc
,
220 struct cpumask
*flush_mask
)
222 int completion_status
= 0;
227 unsigned long mmr_offset
;
232 if (cpu
< UV_CPUS_PER_ACT_STATUS
) {
233 mmr_offset
= UVH_LB_BAU_SB_ACTIVATION_STATUS_0
;
234 right_shift
= cpu
* UV_ACT_STATUS_SIZE
;
236 mmr_offset
= UVH_LB_BAU_SB_ACTIVATION_STATUS_1
;
238 ((cpu
- UV_CPUS_PER_ACT_STATUS
) * UV_ACT_STATUS_SIZE
);
240 time1
= get_cycles();
243 index
= (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT
) |
245 uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL
, index
);
246 completion_status
= uv_wait_completion(bau_desc
, mmr_offset
,
248 } while (completion_status
== FLUSH_RETRY
);
249 time2
= get_cycles();
250 __get_cpu_var(ptcstats
).sflush
+= (time2
- time1
);
252 __get_cpu_var(ptcstats
).retriesok
++;
254 if (completion_status
== FLUSH_GIVEUP
) {
256 * Cause the caller to do an IPI-style TLB shootdown on
257 * the cpu's, all of which are still in the mask.
259 __get_cpu_var(ptcstats
).ptc_i
++;
264 * Success, so clear the remote cpu's from the mask so we don't
265 * use the IPI method of shootdown on them.
267 for_each_cpu(bit
, flush_mask
) {
268 blade
= uv_cpu_to_blade_id(bit
);
269 if (blade
== this_blade
)
271 cpumask_clear_cpu(bit
, flush_mask
);
273 if (!cpumask_empty(flush_mask
))
279 * uv_flush_tlb_others - globally purge translation cache of a virtual
280 * address or all TLB's
281 * @cpumask: mask of all cpu's in which the address is to be removed
282 * @mm: mm_struct containing virtual address range
283 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
284 * @cpu: the current cpu
286 * This is the entry point for initiating any UV global TLB shootdown.
288 * Purges the translation caches of all specified processors of the given
289 * virtual address, or purges all TLB's on specified processors.
291 * The caller has derived the cpumask from the mm_struct. This function
292 * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
294 * The cpumask is converted into a nodemask of the nodes containing
297 * Note that this function should be called with preemption disabled.
299 * Returns NULL if all remote flushing was done.
300 * Returns pointer to cpumask if some remote flushing remains to be
301 * done. The returned pointer is valid till preemption is re-enabled.
303 const struct cpumask
*uv_flush_tlb_others(const struct cpumask
*cpumask
,
304 struct mm_struct
*mm
,
305 unsigned long va
, unsigned int cpu
)
307 static DEFINE_PER_CPU(cpumask_t
, flush_tlb_mask
);
308 struct cpumask
*flush_mask
= &__get_cpu_var(flush_tlb_mask
);
315 struct bau_desc
*bau_desc
;
317 WARN_ON(!in_atomic());
319 cpumask_andnot(flush_mask
, cpumask
, cpumask_of(cpu
));
321 uv_cpu
= uv_blade_processor_id();
322 this_blade
= uv_numa_blade_id();
323 bau_desc
= __get_cpu_var(bau_control
).descriptor_base
;
324 bau_desc
+= UV_ITEMS_PER_DESCRIPTOR
* uv_cpu
;
326 bau_nodes_clear(&bau_desc
->distribution
, UV_DISTRIBUTION_SIZE
);
329 for_each_cpu(bit
, flush_mask
) {
330 blade
= uv_cpu_to_blade_id(bit
);
331 BUG_ON(blade
> (UV_DISTRIBUTION_SIZE
- 1));
332 if (blade
== this_blade
) {
336 bau_node_set(blade
, &bau_desc
->distribution
);
341 * no off_node flushing; return status for local node
348 __get_cpu_var(ptcstats
).requestor
++;
349 __get_cpu_var(ptcstats
).ntargeted
+= i
;
351 bau_desc
->payload
.address
= va
;
352 bau_desc
->payload
.sending_cpu
= cpu
;
354 return uv_flush_send_and_wait(uv_cpu
, this_blade
, bau_desc
, flush_mask
);
358 * The BAU message interrupt comes here. (registered by set_intr_gate)
361 * We received a broadcast assist message.
363 * Interrupts may have been disabled; this interrupt could represent
364 * the receipt of several messages.
366 * All cores/threads on this node get this interrupt.
367 * The last one to see it does the s/w ack.
368 * (the resource will not be freed until noninterruptable cpus see this
369 * interrupt; hardware will timeout the s/w ack and reply ERROR)
371 void uv_bau_message_interrupt(struct pt_regs
*regs
)
373 struct bau_payload_queue_entry
*va_queue_first
;
374 struct bau_payload_queue_entry
*va_queue_last
;
375 struct bau_payload_queue_entry
*msg
;
376 struct pt_regs
*old_regs
= set_irq_regs(regs
);
383 unsigned long local_pnode
;
389 time1
= get_cycles();
391 local_pnode
= uv_blade_to_pnode(uv_numa_blade_id());
393 va_queue_first
= __get_cpu_var(bau_control
).va_queue_first
;
394 va_queue_last
= __get_cpu_var(bau_control
).va_queue_last
;
396 msg
= __get_cpu_var(bau_control
).bau_msg_head
;
397 while (msg
->sw_ack_vector
) {
399 fw
= msg
->sw_ack_vector
;
400 msg_slot
= msg
- va_queue_first
;
401 sw_ack_slot
= ffs(fw
) - 1;
403 uv_bau_process_message(msg
, msg_slot
, sw_ack_slot
);
406 if (msg
> va_queue_last
)
407 msg
= va_queue_first
;
408 __get_cpu_var(bau_control
).bau_msg_head
= msg
;
411 __get_cpu_var(ptcstats
).nomsg
++;
413 __get_cpu_var(ptcstats
).multmsg
++;
415 time2
= get_cycles();
416 __get_cpu_var(ptcstats
).dflush
+= (time2
- time1
);
419 set_irq_regs(old_regs
);
422 static void uv_enable_timeouts(void)
429 unsigned long apicid
;
432 for_each_online_node(i
) {
433 blade
= uv_node_to_blade_id(i
);
434 if (blade
== last_blade
)
437 apicid
= per_cpu(x86_cpu_to_apicid
, cur_cpu
);
438 pnode
= uv_blade_to_pnode(blade
);
439 cur_cpu
+= uv_blade_nr_possible_cpus(i
);
443 static void *uv_ptc_seq_start(struct seq_file
*file
, loff_t
*offset
)
445 if (*offset
< num_possible_cpus())
450 static void *uv_ptc_seq_next(struct seq_file
*file
, void *data
, loff_t
*offset
)
453 if (*offset
< num_possible_cpus())
458 static void uv_ptc_seq_stop(struct seq_file
*file
, void *data
)
463 * Display the statistics thru /proc
464 * data points to the cpu number
466 static int uv_ptc_seq_show(struct seq_file
*file
, void *data
)
468 struct ptc_stats
*stat
;
471 cpu
= *(loff_t
*)data
;
475 "# cpu requestor requestee one all sretry dretry ptc_i ");
477 "sw_ack sflush dflush sok dnomsg dmult starget\n");
479 if (cpu
< num_possible_cpus() && cpu_online(cpu
)) {
480 stat
= &per_cpu(ptcstats
, cpu
);
481 seq_printf(file
, "cpu %d %ld %ld %ld %ld %ld %ld %ld ",
482 cpu
, stat
->requestor
,
483 stat
->requestee
, stat
->onetlb
, stat
->alltlb
,
484 stat
->s_retry
, stat
->d_retry
, stat
->ptc_i
);
485 seq_printf(file
, "%lx %ld %ld %ld %ld %ld %ld\n",
486 uv_read_global_mmr64(uv_blade_to_pnode
487 (uv_cpu_to_blade_id(cpu
)),
488 UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE
),
489 stat
->sflush
, stat
->dflush
,
490 stat
->retriesok
, stat
->nomsg
,
491 stat
->multmsg
, stat
->ntargeted
);
498 * 0: display meaning of the statistics
501 static ssize_t
uv_ptc_proc_write(struct file
*file
, const char __user
*user
,
502 size_t count
, loff_t
*data
)
507 if (count
== 0 || count
> sizeof(optstr
))
509 if (copy_from_user(optstr
, user
, count
))
511 optstr
[count
- 1] = '\0';
512 if (strict_strtoul(optstr
, 10, &newmode
) < 0) {
513 printk(KERN_DEBUG
"%s is invalid\n", optstr
);
518 printk(KERN_DEBUG
"# cpu: cpu number\n");
520 "requestor: times this cpu was the flush requestor\n");
522 "requestee: times this cpu was requested to flush its TLBs\n");
524 "one: times requested to flush a single address\n");
526 "all: times requested to flush all TLB's\n");
528 "sretry: number of retries of source-side timeouts\n");
530 "dretry: number of retries of destination-side timeouts\n");
532 "ptc_i: times UV fell through to IPI-style flushes\n");
534 "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
536 "sflush_us: cycles spent in uv_flush_tlb_others()\n");
538 "dflush_us: cycles spent in handling flush requests\n");
539 printk(KERN_DEBUG
"sok: successes on retry\n");
540 printk(KERN_DEBUG
"dnomsg: interrupts with no message\n");
542 "dmult: interrupts with multiple messages\n");
543 printk(KERN_DEBUG
"starget: nodes targeted\n");
545 uv_bau_retry_limit
= newmode
;
546 printk(KERN_DEBUG
"timeout retry limit:%d\n",
553 static const struct seq_operations uv_ptc_seq_ops
= {
554 .start
= uv_ptc_seq_start
,
555 .next
= uv_ptc_seq_next
,
556 .stop
= uv_ptc_seq_stop
,
557 .show
= uv_ptc_seq_show
560 static int uv_ptc_proc_open(struct inode
*inode
, struct file
*file
)
562 return seq_open(file
, &uv_ptc_seq_ops
);
565 static const struct file_operations proc_uv_ptc_operations
= {
566 .open
= uv_ptc_proc_open
,
568 .write
= uv_ptc_proc_write
,
570 .release
= seq_release
,
573 static int __init
uv_ptc_init(void)
575 struct proc_dir_entry
*proc_uv_ptc
;
580 proc_uv_ptc
= create_proc_entry(UV_PTC_BASENAME
, 0444, NULL
);
582 printk(KERN_ERR
"unable to create %s proc entry\n",
586 proc_uv_ptc
->proc_fops
= &proc_uv_ptc_operations
;
591 * begin the initialization of the per-blade control structures
593 static struct bau_control
* __init
uv_table_bases_init(int blade
, int node
)
596 struct bau_msg_status
*msp
;
597 struct bau_control
*bau_tabp
;
600 kmalloc_node(sizeof(struct bau_control
), GFP_KERNEL
, node
);
603 bau_tabp
->msg_statuses
=
604 kmalloc_node(sizeof(struct bau_msg_status
) *
605 DEST_Q_SIZE
, GFP_KERNEL
, node
);
606 BUG_ON(!bau_tabp
->msg_statuses
);
608 for (i
= 0, msp
= bau_tabp
->msg_statuses
; i
< DEST_Q_SIZE
; i
++, msp
++)
609 bau_cpubits_clear(&msp
->seen_by
, (int)
610 uv_blade_nr_possible_cpus(blade
));
612 uv_bau_table_bases
[blade
] = bau_tabp
;
618 * finish the initialization of the per-blade control structures
621 uv_table_bases_finish(int blade
, int node
, int cur_cpu
,
622 struct bau_control
*bau_tablesp
,
623 struct bau_desc
*adp
)
625 struct bau_control
*bcp
;
628 for (i
= cur_cpu
; i
< cur_cpu
+ uv_blade_nr_possible_cpus(blade
); i
++) {
629 bcp
= (struct bau_control
*)&per_cpu(bau_control
, i
);
631 bcp
->bau_msg_head
= bau_tablesp
->va_queue_first
;
632 bcp
->va_queue_first
= bau_tablesp
->va_queue_first
;
633 bcp
->va_queue_last
= bau_tablesp
->va_queue_last
;
634 bcp
->msg_statuses
= bau_tablesp
->msg_statuses
;
635 bcp
->descriptor_base
= adp
;
640 * initialize the sending side's sending buffers
642 static struct bau_desc
* __init
643 uv_activation_descriptor_init(int node
, int pnode
)
649 unsigned long mmr_image
;
650 struct bau_desc
*adp
;
651 struct bau_desc
*ad2
;
653 adp
= (struct bau_desc
*)
654 kmalloc_node(16384, GFP_KERNEL
, node
);
657 pa
= __pa((unsigned long)adp
);
661 mmr_image
= uv_read_global_mmr64(pnode
, UVH_LB_BAU_SB_DESCRIPTOR_BASE
);
663 uv_write_global_mmr64(pnode
, (unsigned long)
664 UVH_LB_BAU_SB_DESCRIPTOR_BASE
,
665 (n
<< UV_DESC_BASE_PNODE_SHIFT
| m
));
668 for (i
= 0, ad2
= adp
; i
< UV_ACTIVATION_DESCRIPTOR_SIZE
; i
++, ad2
++) {
669 memset(ad2
, 0, sizeof(struct bau_desc
));
670 ad2
->header
.sw_ack_flag
= 1;
671 ad2
->header
.base_dest_nodeid
=
672 uv_blade_to_pnode(uv_cpu_to_blade_id(0));
673 ad2
->header
.command
= UV_NET_ENDPOINT_INTD
;
674 ad2
->header
.int_both
= 1;
676 * all others need to be set to zero:
677 * fairness chaining multilevel count replied_to
684 * initialize the destination side's receiving buffers
686 static struct bau_payload_queue_entry
* __init
687 uv_payload_queue_init(int node
, int pnode
, struct bau_control
*bau_tablesp
)
689 struct bau_payload_queue_entry
*pqp
;
692 pqp
= (struct bau_payload_queue_entry
*) kmalloc_node(
693 (DEST_Q_SIZE
+ 1) * sizeof(struct bau_payload_queue_entry
),
697 cp
= (char *)pqp
+ 31;
698 pqp
= (struct bau_payload_queue_entry
*)(((unsigned long)cp
>> 5) << 5);
699 bau_tablesp
->va_queue_first
= pqp
;
700 uv_write_global_mmr64(pnode
,
701 UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST
,
702 ((unsigned long)pnode
<<
703 UV_PAYLOADQ_PNODE_SHIFT
) |
704 uv_physnodeaddr(pqp
));
705 uv_write_global_mmr64(pnode
, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL
,
706 uv_physnodeaddr(pqp
));
707 bau_tablesp
->va_queue_last
= pqp
+ (DEST_Q_SIZE
- 1);
708 uv_write_global_mmr64(pnode
, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST
,
710 uv_physnodeaddr(bau_tablesp
->va_queue_last
));
711 memset(pqp
, 0, sizeof(struct bau_payload_queue_entry
) * DEST_Q_SIZE
);
717 * Initialization of each UV blade's structures
719 static int __init
uv_init_blade(int blade
, int node
, int cur_cpu
)
723 unsigned long apicid
;
724 struct bau_desc
*adp
;
725 struct bau_payload_queue_entry
*pqp
;
726 struct bau_control
*bau_tablesp
;
728 bau_tablesp
= uv_table_bases_init(blade
, node
);
729 pnode
= uv_blade_to_pnode(blade
);
730 adp
= uv_activation_descriptor_init(node
, pnode
);
731 pqp
= uv_payload_queue_init(node
, pnode
, bau_tablesp
);
732 uv_table_bases_finish(blade
, node
, cur_cpu
, bau_tablesp
, adp
);
734 * the below initialization can't be in firmware because the
735 * messaging IRQ will be determined by the OS
737 apicid
= per_cpu(x86_cpu_to_apicid
, cur_cpu
);
738 pa
= uv_read_global_mmr64(pnode
, UVH_BAU_DATA_CONFIG
);
739 if ((pa
& 0xff) != UV_BAU_MESSAGE
) {
740 uv_write_global_mmr64(pnode
, UVH_BAU_DATA_CONFIG
,
741 ((apicid
<< 32) | UV_BAU_MESSAGE
));
747 * Initialization of BAU-related structures
749 static int __init
uv_bau_init(void)
760 uv_bau_retry_limit
= 1;
761 uv_nshift
= uv_hub_info
->n_val
;
762 uv_mmask
= (1UL << uv_hub_info
->n_val
) - 1;
765 for_each_online_node(node
) {
766 blade
= uv_node_to_blade_id(node
);
767 if (blade
== last_blade
)
772 uv_bau_table_bases
= (struct bau_control
**)
773 kmalloc(nblades
* sizeof(struct bau_control
*), GFP_KERNEL
);
774 BUG_ON(!uv_bau_table_bases
);
777 for_each_online_node(node
) {
778 blade
= uv_node_to_blade_id(node
);
779 if (blade
== last_blade
)
782 uv_init_blade(blade
, node
, cur_cpu
);
783 cur_cpu
+= uv_blade_nr_possible_cpus(blade
);
785 alloc_intr_gate(UV_BAU_MESSAGE
, uv_bau_message_intr1
);
786 uv_enable_timeouts();
790 __initcall(uv_bau_init
);
791 __initcall(uv_ptc_init
);