kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/freezer.h>
  18 #include <linux/kthread.h>
  19 #include <linux/rcupdate.h>
  20 #include <linux/smp.h>
  21 #include <linux/tick.h>
  22
  23 #include <asm/irq.h>
  24 /*
  25    - No shared variables, all the data are CPU local.
  26    - If a softirq needs serialization, let it serialize itself
  27      by its own spinlocks.
  28    - Even if softirq is serialized, only local cpu is marked for
  29      execution. Hence, we get something sort of weak cpu binding.
  30      Though it is still not clear, will it result in better locality
  31      or will not.
  32
  33    Examples:
  34    - NET RX softirq. It is multithreaded and does not require
  35      any global serialization.
  36    - NET TX softirq. It kicks software netdevice queues, hence
  37      it is logically serialized per device, but this serialization
  38      is invisible to common code.
  39    - Tasklets: serialized wrt itself.
  40  */
  41
  42 #ifndef __ARCH_IRQ_STAT
  43 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  44 EXPORT_SYMBOL(irq_stat);
  45 #endif
  46
  47 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  48
  49 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  50
  51 /*
  52  * we cannot loop indefinitely here to avoid userspace starvation,
  53  * but we also don't want to introduce a worst case 1/HZ latency
  54  * to the pending events, so lets the scheduler to balance
  55  * the softirq load for us.
  56  */
  57 static inline void wakeup_softirqd(void)
  58 {
  59         /* Interrupts are disabled: no need to stop preemption */
  60         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  61
  62         if (tsk && tsk->state != TASK_RUNNING)
  63                 wake_up_process(tsk);
  64 }
  65
  66 /*
  67  * This one is for softirq.c-internal use,
  68  * where hardirqs are disabled legitimately:
  69  */
  70 #ifdef CONFIG_TRACE_IRQFLAGS
  71 static void __local_bh_disable(unsigned long ip)
  72 {
  73         unsigned long flags;
  74
  75         WARN_ON_ONCE(in_irq());
  76
  77         raw_local_irq_save(flags);
  78         add_preempt_count(SOFTIRQ_OFFSET);
  79         /*
  80          * Were softirqs turned off above:
  81          */
  82         if (softirq_count() == SOFTIRQ_OFFSET)
  83                 trace_softirqs_off(ip);
  84         raw_local_irq_restore(flags);
  85 }
  86 #else /* !CONFIG_TRACE_IRQFLAGS */
  87 static inline void __local_bh_disable(unsigned long ip)
  88 {
  89         add_preempt_count(SOFTIRQ_OFFSET);
  90         barrier();
  91 }
  92 #endif /* CONFIG_TRACE_IRQFLAGS */
  93
  94 void local_bh_disable(void)
  95 {
  96         __local_bh_disable((unsigned long)__builtin_return_address(0));
  97 }
  98
  99 EXPORT_SYMBOL(local_bh_disable);
 100
 101 void __local_bh_enable(void)
 102 {
 103         WARN_ON_ONCE(in_irq());
 104
 105         /*
 106          * softirqs should never be enabled by __local_bh_enable(),
 107          * it always nests inside local_bh_enable() sections:
 108          */
 109         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 110
 111         sub_preempt_count(SOFTIRQ_OFFSET);
 112 }
 113 EXPORT_SYMBOL_GPL(__local_bh_enable);
 114
 115 /*
 116  * Special-case - softirqs can safely be enabled in
 117  * cond_resched_softirq(), or by __do_softirq(),
 118  * without processing still-pending softirqs:
 119  */
 120 void _local_bh_enable(void)
 121 {
 122         WARN_ON_ONCE(in_irq());
 123         WARN_ON_ONCE(!irqs_disabled());
 124
 125         if (softirq_count() == SOFTIRQ_OFFSET)
 126                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 127         sub_preempt_count(SOFTIRQ_OFFSET);
 128 }
 129
 130 EXPORT_SYMBOL(_local_bh_enable);
 131
 132 void local_bh_enable(void)
 133 {
 134 #ifdef CONFIG_TRACE_IRQFLAGS
 135         unsigned long flags;
 136
 137         WARN_ON_ONCE(in_irq());
 138 #endif
 139         WARN_ON_ONCE(irqs_disabled());
 140
 141 #ifdef CONFIG_TRACE_IRQFLAGS
 142         local_irq_save(flags);
 143 #endif
 144         /*
 145          * Are softirqs going to be turned on now:
 146          */
 147         if (softirq_count() == SOFTIRQ_OFFSET)
 148                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 149         /*
 150          * Keep preemption disabled until we are done with
 151          * softirq processing:
 152          */
 153         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 154
 155         if (unlikely(!in_interrupt() && local_softirq_pending()))
 156                 do_softirq();
 157
 158         dec_preempt_count();
 159 #ifdef CONFIG_TRACE_IRQFLAGS
 160         local_irq_restore(flags);
 161 #endif
 162         preempt_check_resched();
 163 }
 164 EXPORT_SYMBOL(local_bh_enable);
 165
 166 void local_bh_enable_ip(unsigned long ip)
 167 {
 168 #ifdef CONFIG_TRACE_IRQFLAGS
 169         unsigned long flags;
 170
 171         WARN_ON_ONCE(in_irq());
 172
 173         local_irq_save(flags);
 174 #endif
 175         /*
 176          * Are softirqs going to be turned on now:
 177          */
 178         if (softirq_count() == SOFTIRQ_OFFSET)
 179                 trace_softirqs_on(ip);
 180         /*
 181          * Keep preemption disabled until we are done with
 182          * softirq processing:
 183          */
 184         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 185
 186         if (unlikely(!in_interrupt() && local_softirq_pending()))
 187                 do_softirq();
 188
 189         dec_preempt_count();
 190 #ifdef CONFIG_TRACE_IRQFLAGS
 191         local_irq_restore(flags);
 192 #endif
 193         preempt_check_resched();
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 199  * and we fall back to softirqd after that.
 200  *
 201  * This number has been established via experimentation.
 202  * The two things to balance is latency against fairness -
 203  * we want to handle softirqs as soon as possible, but they
 204  * should not be able to lock up the box.
 205  */
 206 #define MAX_SOFTIRQ_RESTART 10
 207
 208 asmlinkage void __do_softirq(void)
 209 {
 210         struct softirq_action *h;
 211         __u32 pending;
 212         int max_restart = MAX_SOFTIRQ_RESTART;
 213         int cpu;
 214
 215         pending = local_softirq_pending();
 216         account_system_vtime(current);
 217
 218         __local_bh_disable((unsigned long)__builtin_return_address(0));
 219         trace_softirq_enter();
 220
 221         cpu = smp_processor_id();
 222 restart:
 223         /* Reset the pending bitmask before enabling irqs */
 224         set_softirq_pending(0);
 225
 226         local_irq_enable();
 227
 228         h = softirq_vec;
 229
 230         do {
 231                 if (pending & 1) {
 232                         h->action(h);
 233                         rcu_bh_qsctr_inc(cpu);
 234                 }
 235                 h++;
 236                 pending >>= 1;
 237         } while (pending);
 238
 239         local_irq_disable();
 240
 241         pending = local_softirq_pending();
 242         if (pending && --max_restart)
 243                 goto restart;
 244
 245         if (pending)
 246                 wakeup_softirqd();
 247
 248         trace_softirq_exit();
 249
 250         account_system_vtime(current);
 251         _local_bh_enable();
 252 }
 253
 254 #ifndef __ARCH_HAS_DO_SOFTIRQ
 255
 256 asmlinkage void do_softirq(void)
 257 {
 258         __u32 pending;
 259         unsigned long flags;
 260
 261         if (in_interrupt())
 262                 return;
 263
 264         local_irq_save(flags);
 265
 266         pending = local_softirq_pending();
 267
 268         if (pending)
 269                 __do_softirq();
 270
 271         local_irq_restore(flags);
 272 }
 273
 274 #endif
 275
 276 /*
 277  * Enter an interrupt context.
 278  */
 279 void irq_enter(void)
 280 {
 281         __irq_enter();
 282 #ifdef CONFIG_NO_HZ
 283         if (idle_cpu(smp_processor_id()))
 284                 tick_nohz_update_jiffies();
 285 #endif
 286 }
 287
 288 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 289 # define invoke_softirq()       __do_softirq()
 290 #else
 291 # define invoke_softirq()       do_softirq()
 292 #endif
 293
 294 /*
 295  * Exit an interrupt context. Process softirqs if needed and possible:
 296  */
 297 void irq_exit(void)
 298 {
 299         account_system_vtime(current);
 300         trace_hardirq_exit();
 301         sub_preempt_count(IRQ_EXIT_OFFSET);
 302         if (!in_interrupt() && local_softirq_pending())
 303                 invoke_softirq();
 304
 305 #ifdef CONFIG_NO_HZ
 306         /* Make sure that timer wheel updates are propagated */
 307         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
 308                 tick_nohz_stop_sched_tick();
 309 #endif
 310         preempt_enable_no_resched();
 311 }
 312
 313 /*
 314  * This function must run with irqs disabled!
 315  */
 316 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 317 {
 318         __raise_softirq_irqoff(nr);
 319
 320         /*
 321          * If we're in an interrupt or softirq, we're done
 322          * (this also catches softirq-disabled code). We will
 323          * actually run the softirq once we return from
 324          * the irq or softirq.
 325          *
 326          * Otherwise we wake up ksoftirqd to make sure we
 327          * schedule the softirq soon.
 328          */
 329         if (!in_interrupt())
 330                 wakeup_softirqd();
 331 }
 332
 333 EXPORT_SYMBOL(raise_softirq_irqoff);
 334
 335 void fastcall raise_softirq(unsigned int nr)
 336 {
 337         unsigned long flags;
 338
 339         local_irq_save(flags);
 340         raise_softirq_irqoff(nr);
 341         local_irq_restore(flags);
 342 }
 343
 344 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 345 {
 346         softirq_vec[nr].data = data;
 347         softirq_vec[nr].action = action;
 348 }
 349
 350 /* Tasklets */
 351 struct tasklet_head
 352 {
 353         struct tasklet_struct *list;
 354 };
 355
 356 /* Some compilers disobey section attribute on statics when not
 357    initialized -- RR */
 358 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 359 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 360
 361 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 362 {
 363         unsigned long flags;
 364
 365         local_irq_save(flags);
 366         t->next = __get_cpu_var(tasklet_vec).list;
 367         __get_cpu_var(tasklet_vec).list = t;
 368         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 369         local_irq_restore(flags);
 370 }
 371
 372 EXPORT_SYMBOL(__tasklet_schedule);
 373
 374 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 375 {
 376         unsigned long flags;
 377
 378         local_irq_save(flags);
 379         t->next = __get_cpu_var(tasklet_hi_vec).list;
 380         __get_cpu_var(tasklet_hi_vec).list = t;
 381         raise_softirq_irqoff(HI_SOFTIRQ);
 382         local_irq_restore(flags);
 383 }
 384
 385 EXPORT_SYMBOL(__tasklet_hi_schedule);
 386
 387 static void tasklet_action(struct softirq_action *a)
 388 {
 389         struct tasklet_struct *list;
 390
 391         local_irq_disable();
 392         list = __get_cpu_var(tasklet_vec).list;
 393         __get_cpu_var(tasklet_vec).list = NULL;
 394         local_irq_enable();
 395
 396         while (list) {
 397                 struct tasklet_struct *t = list;
 398
 399                 list = list->next;
 400
 401                 if (tasklet_trylock(t)) {
 402                         if (!atomic_read(&t->count)) {
 403                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 404                                         BUG();
 405                                 t->func(t->data);
 406                                 tasklet_unlock(t);
 407                                 continue;
 408                         }
 409                         tasklet_unlock(t);
 410                 }
 411
 412                 local_irq_disable();
 413                 t->next = __get_cpu_var(tasklet_vec).list;
 414                 __get_cpu_var(tasklet_vec).list = t;
 415                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 416                 local_irq_enable();
 417         }
 418 }
 419
 420 static void tasklet_hi_action(struct softirq_action *a)
 421 {
 422         struct tasklet_struct *list;
 423
 424         local_irq_disable();
 425         list = __get_cpu_var(tasklet_hi_vec).list;
 426         __get_cpu_var(tasklet_hi_vec).list = NULL;
 427         local_irq_enable();
 428
 429         while (list) {
 430                 struct tasklet_struct *t = list;
 431
 432                 list = list->next;
 433
 434                 if (tasklet_trylock(t)) {
 435                         if (!atomic_read(&t->count)) {
 436                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 437                                         BUG();
 438                                 t->func(t->data);
 439                                 tasklet_unlock(t);
 440                                 continue;
 441                         }
 442                         tasklet_unlock(t);
 443                 }
 444
 445                 local_irq_disable();
 446                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 447                 __get_cpu_var(tasklet_hi_vec).list = t;
 448                 __raise_softirq_irqoff(HI_SOFTIRQ);
 449                 local_irq_enable();
 450         }
 451 }
 452
 453
 454 void tasklet_init(struct tasklet_struct *t,
 455                   void (*func)(unsigned long), unsigned long data)
 456 {
 457         t->next = NULL;
 458         t->state = 0;
 459         atomic_set(&t->count, 0);
 460         t->func = func;
 461         t->data = data;
 462 }
 463
 464 EXPORT_SYMBOL(tasklet_init);
 465
 466 void tasklet_kill(struct tasklet_struct *t)
 467 {
 468         if (in_interrupt())
 469                 printk("Attempt to kill tasklet from interrupt\n");
 470
 471         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 472                 do
 473                         yield();
 474                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 475         }
 476         tasklet_unlock_wait(t);
 477         clear_bit(TASKLET_STATE_SCHED, &t->state);
 478 }
 479
 480 EXPORT_SYMBOL(tasklet_kill);
 481
 482 void __init softirq_init(void)
 483 {
 484         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 485         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 486 }
 487
 488 static int ksoftirqd(void * __bind_cpu)
 489 {
 490         set_current_state(TASK_INTERRUPTIBLE);
 491
 492         while (!kthread_should_stop()) {
 493                 preempt_disable();
 494                 if (!local_softirq_pending()) {
 495                         preempt_enable_no_resched();
 496                         schedule();
 497                         preempt_disable();
 498                 }
 499
 500                 __set_current_state(TASK_RUNNING);
 501
 502                 while (local_softirq_pending()) {
 503                         /* Preempt disable stops cpu going offline.
 504                            If already offline, we'll be on wrong CPU:
 505                            don't process */
 506                         if (cpu_is_offline((long)__bind_cpu))
 507                                 goto wait_to_die;
 508                         do_softirq();
 509                         preempt_enable_no_resched();
 510                         cond_resched();
 511                         preempt_disable();
 512                 }
 513                 preempt_enable();
 514                 set_current_state(TASK_INTERRUPTIBLE);
 515         }
 516         __set_current_state(TASK_RUNNING);
 517         return 0;
 518
 519 wait_to_die:
 520         preempt_enable();
 521         /* Wait for kthread_stop */
 522         set_current_state(TASK_INTERRUPTIBLE);
 523         while (!kthread_should_stop()) {
 524                 schedule();
 525                 set_current_state(TASK_INTERRUPTIBLE);
 526         }
 527         __set_current_state(TASK_RUNNING);
 528         return 0;
 529 }
 530
 531 #ifdef CONFIG_HOTPLUG_CPU
 532 /*
 533  * tasklet_kill_immediate is called to remove a tasklet which can already be
 534  * scheduled for execution on @cpu.
 535  *
 536  * Unlike tasklet_kill, this function removes the tasklet
 537  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 538  *
 539  * When this function is called, @cpu must be in the CPU_DEAD state.
 540  */
 541 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 542 {
 543         struct tasklet_struct **i;
 544
 545         BUG_ON(cpu_online(cpu));
 546         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 547
 548         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 549                 return;
 550
 551         /* CPU is dead, so no lock needed. */
 552         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 553                 if (*i == t) {
 554                         *i = t->next;
 555                         return;
 556                 }
 557         }
 558         BUG();
 559 }
 560
 561 static void takeover_tasklets(unsigned int cpu)
 562 {
 563         struct tasklet_struct **i;
 564
 565         /* CPU is dead, so no lock needed. */
 566         local_irq_disable();
 567
 568         /* Find end, append list for that CPU. */
 569         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 570         *i = per_cpu(tasklet_vec, cpu).list;
 571         per_cpu(tasklet_vec, cpu).list = NULL;
 572         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 573
 574         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 575         *i = per_cpu(tasklet_hi_vec, cpu).list;
 576         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 577         raise_softirq_irqoff(HI_SOFTIRQ);
 578
 579         local_irq_enable();
 580 }
 581 #endif /* CONFIG_HOTPLUG_CPU */
 582
 583 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 584                                   unsigned long action,
 585                                   void *hcpu)
 586 {
 587         int hotcpu = (unsigned long)hcpu;
 588         struct task_struct *p;
 589
 590         switch (action) {
 591         case CPU_UP_PREPARE:
 592         case CPU_UP_PREPARE_FROZEN:
 593                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 594                 if (IS_ERR(p)) {
 595                         printk("ksoftirqd for %i failed\n", hotcpu);
 596                         return NOTIFY_BAD;
 597                 }
 598                 kthread_bind(p, hotcpu);
 599                 per_cpu(ksoftirqd, hotcpu) = p;
 600                 break;
 601         case CPU_ONLINE:
 602         case CPU_ONLINE_FROZEN:
 603                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 604                 break;
 605 #ifdef CONFIG_HOTPLUG_CPU
 606         case CPU_UP_CANCELED:
 607         case CPU_UP_CANCELED_FROZEN:
 608                 if (!per_cpu(ksoftirqd, hotcpu))
 609                         break;
 610                 /* Unbind so it can run.  Fall thru. */
 611                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 612                              any_online_cpu(cpu_online_map));
 613         case CPU_DEAD:
 614         case CPU_DEAD_FROZEN: {
 615                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 616
 617                 p = per_cpu(ksoftirqd, hotcpu);
 618                 per_cpu(ksoftirqd, hotcpu) = NULL;
 619                 sched_setscheduler(p, SCHED_FIFO, &param);
 620                 kthread_stop(p);
 621                 takeover_tasklets(hotcpu);
 622                 break;
 623         }
 624 #endif /* CONFIG_HOTPLUG_CPU */
 625         }
 626         return NOTIFY_OK;
 627 }
 628
 629 static struct notifier_block __cpuinitdata cpu_nfb = {
 630         .notifier_call = cpu_callback
 631 };
 632
 633 __init int spawn_ksoftirqd(void)
 634 {
 635         void *cpu = (void *)(long)smp_processor_id();
 636         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 637
 638         BUG_ON(err == NOTIFY_BAD);
 639         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 640         register_cpu_notifier(&cpu_nfb);
 641         return 0;
 642 }
 643
 644 #ifdef CONFIG_SMP
 645 /*
 646  * Call a function on all processors
 647  */
 648 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 649 {
 650         int ret = 0;
 651
 652         preempt_disable();
 653         ret = smp_call_function(func, info, retry, wait);
 654         local_irq_disable();
 655         func(info);
 656         local_irq_enable();
 657         preempt_enable();
 658         return ret;
 659 }
 660 EXPORT_SYMBOL(on_each_cpu);
 661 #endif