}
EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
#else
---#define do_trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
+++#define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \
+++ do { } while (0)
#endif
+++
+++ #ifdef CONFIG_RCU_STALL_COMMON
+++
+++ #ifdef CONFIG_PROVE_RCU
+++ #define RCU_STALL_DELAY_DELTA (5 * HZ)
+++ #else
+++ #define RCU_STALL_DELAY_DELTA 0
+++ #endif
+++
+++ int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
+++ int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
+++
+++ module_param(rcu_cpu_stall_suppress, int, 0644);
+++ module_param(rcu_cpu_stall_timeout, int, 0644);
+++
+++ int rcu_jiffies_till_stall_check(void)
+++ {
+++ int till_stall_check = ACCESS_ONCE(rcu_cpu_stall_timeout);
+++
+++ /*
+++ * Limit check must be consistent with the Kconfig limits
+++ * for CONFIG_RCU_CPU_STALL_TIMEOUT.
+++ */
+++ if (till_stall_check < 3) {
+++ ACCESS_ONCE(rcu_cpu_stall_timeout) = 3;
+++ till_stall_check = 3;
+++ } else if (till_stall_check > 300) {
+++ ACCESS_ONCE(rcu_cpu_stall_timeout) = 300;
+++ till_stall_check = 300;
+++ }
+++ return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
+++ }
+++
+++ static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
+++ {
+++ rcu_cpu_stall_suppress = 1;
+++ return NOTIFY_DONE;
+++ }
+++
+++ static struct notifier_block rcu_panic_block = {
+++ .notifier_call = rcu_panic,
+++ };
+++
+++ static int __init check_cpu_stall_init(void)
+++ {
+++ atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
+++ return 0;
+++ }
+++ early_initcall(check_cpu_stall_init);
+++
+++ #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
__rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
{
/* Did another grace period end? */
-- - if (rdp->completed != rnp->completed) {
++ + if (rdp->completed == rnp->completed) {
+ +
- /* Advance callbacks. No harm if list empty. */
- rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL];
- rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL];
- rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
++ + /* No, so just accelerate recent callbacks. */
++ + rcu_accelerate_cbs(rsp, rnp, rdp);
+ +
- /* Advance callbacks. No harm if list empty. */
- rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL];
- rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL];
- rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
++ + } else {
++
- /* Advance callbacks. No harm if list empty. */
- rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL];
- rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL];
- rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
++ + /* Advance callbacks. */
++ + rcu_advance_cbs(rsp, rnp, rdp);
/* Remember that we saw this grace-period completion. */
rdp->completed = rnp->completed;