kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 #include <asm/irq.h>
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 char *softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into add_preempt_count and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         preempt_count() += cnt;
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == cnt)
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 #else /* !CONFIG_TRACE_IRQFLAGS */
 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122 {
 123         add_preempt_count(cnt);
 124         barrier();
 125 }
 126 #endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128 void local_bh_disable(void)
 129 {
 130         __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                 SOFTIRQ_DISABLE_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(local_bh_disable);
 135
 136 static void __local_bh_enable(unsigned int cnt)
 137 {
 138         WARN_ON_ONCE(in_irq());
 139         WARN_ON_ONCE(!irqs_disabled());
 140
 141         if (softirq_count() == cnt)
 142                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143         sub_preempt_count(cnt);
 144 }
 145
 146 /*
 147  * Special-case - softirqs can safely be enabled in
 148  * cond_resched_softirq(), or by __do_softirq(),
 149  * without processing still-pending softirqs:
 150  */
 151 void _local_bh_enable(void)
 152 {
 153         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154 }
 155
 156 EXPORT_SYMBOL(_local_bh_enable);
 157
 158 static inline void _local_bh_enable_ip(unsigned long ip)
 159 {
 160         WARN_ON_ONCE(in_irq() || irqs_disabled());
 161 #ifdef CONFIG_TRACE_IRQFLAGS
 162         local_irq_disable();
 163 #endif
 164         /*
 165          * Are softirqs going to be turned on now:
 166          */
 167         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                 trace_softirqs_on(ip);
 169         /*
 170          * Keep preemption disabled until we are done with
 171          * softirq processing:
 172          */
 173         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175         if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                 do_softirq();
 177
 178         dec_preempt_count();
 179 #ifdef CONFIG_TRACE_IRQFLAGS
 180         local_irq_enable();
 181 #endif
 182         preempt_check_resched();
 183 }
 184
 185 void local_bh_enable(void)
 186 {
 187         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188 }
 189 EXPORT_SYMBOL(local_bh_enable);
 190
 191 void local_bh_enable_ip(unsigned long ip)
 192 {
 193         _local_bh_enable_ip(ip);
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 199  * but break the loop if need_resched() is set or after 2 ms.
 200  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 201  * certain cases, such as stop_machine(), jiffies may cease to
 202  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 203  * well to make sure we eventually return from this method.
 204  *
 205  * These limits have been established via experimentation.
 206  * The two things to balance is latency against fairness -
 207  * we want to handle softirqs as soon as possible, but they
 208  * should not be able to lock up the box.
 209  */
 210 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 211 #define MAX_SOFTIRQ_RESTART 10
 212
 213 asmlinkage void __do_softirq(void)
 214 {
 215         struct softirq_action *h;
 216         __u32 pending;
 217         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 218         int cpu;
 219         unsigned long old_flags = current->flags;
 220         int max_restart = MAX_SOFTIRQ_RESTART;
 221
 222         /*
 223          * Mask out PF_MEMALLOC s current task context is borrowed for the
 224          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 225          * again if the socket is related to swap
 226          */
 227         current->flags &= ~PF_MEMALLOC;
 228
 229         pending = local_softirq_pending();
 230         account_irq_enter_time(current);
 231
 232         __local_bh_disable((unsigned long)__builtin_return_address(0),
 233                                 SOFTIRQ_OFFSET);
 234         lockdep_softirq_enter();
 235
 236         cpu = smp_processor_id();
 237 restart:
 238         /* Reset the pending bitmask before enabling irqs */
 239         set_softirq_pending(0);
 240
 241         local_irq_enable();
 242
 243         h = softirq_vec;
 244
 245         do {
 246                 if (pending & 1) {
 247                         unsigned int vec_nr = h - softirq_vec;
 248                         int prev_count = preempt_count();
 249
 250                         kstat_incr_softirqs_this_cpu(vec_nr);
 251
 252                         trace_softirq_entry(vec_nr);
 253                         h->action(h);
 254                         trace_softirq_exit(vec_nr);
 255                         if (unlikely(prev_count != preempt_count())) {
 256                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 257                                        "with preempt_count %08x,"
 258                                        " exited with %08x?\n", vec_nr,
 259                                        softirq_to_name[vec_nr], h->action,
 260                                        prev_count, preempt_count());
 261                                 preempt_count() = prev_count;
 262                         }
 263
 264                         rcu_bh_qs(cpu);
 265                 }
 266                 h++;
 267                 pending >>= 1;
 268         } while (pending);
 269
 270         local_irq_disable();
 271
 272         pending = local_softirq_pending();
 273         if (pending) {
 274                 if (time_before(jiffies, end) && !need_resched() &&
 275                     --max_restart)
 276                         goto restart;
 277
 278                 wakeup_softirqd();
 279         }
 280
 281         lockdep_softirq_exit();
 282
 283         account_irq_exit_time(current);
 284         __local_bh_enable(SOFTIRQ_OFFSET);
 285         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 286 }
 287
 288 #ifndef __ARCH_HAS_DO_SOFTIRQ
 289
 290 asmlinkage void do_softirq(void)
 291 {
 292         __u32 pending;
 293         unsigned long flags;
 294
 295         if (in_interrupt())
 296                 return;
 297
 298         local_irq_save(flags);
 299
 300         pending = local_softirq_pending();
 301
 302         if (pending)
 303                 __do_softirq();
 304
 305         local_irq_restore(flags);
 306 }
 307
 308 #endif
 309
 310 /*
 311  * Enter an interrupt context.
 312  */
 313 void irq_enter(void)
 314 {
 315         int cpu = smp_processor_id();
 316
 317         rcu_irq_enter();
 318         if (is_idle_task(current) && !in_interrupt()) {
 319                 /*
 320                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 321                  * here, as softirq will be serviced on return from interrupt.
 322                  */
 323                 local_bh_disable();
 324                 tick_check_idle(cpu);
 325                 _local_bh_enable();
 326         }
 327
 328         __irq_enter();
 329 }
 330
 331 static inline void invoke_softirq(void)
 332 {
 333         if (!force_irqthreads) {
 334                 /*
 335                  * We can safely execute softirq on the current stack if
 336                  * it is the irq stack, because it should be near empty
 337                  * at this stage. But we have no way to know if the arch
 338                  * calls irq_exit() on the irq stack. So call softirq
 339                  * in its own stack to prevent from any overrun on top
 340                  * of a potentially deep task stack.
 341                  */
 342                 do_softirq();
 343         } else {
 344                 wakeup_softirqd();
 345         }
 346 }
 347
 348 static inline void tick_irq_exit(void)
 349 {
 350 #ifdef CONFIG_NO_HZ_COMMON
 351         int cpu = smp_processor_id();
 352
 353         /* Make sure that timer wheel updates are propagated */
 354         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 355                 if (!in_interrupt())
 356                         tick_nohz_irq_exit();
 357         }
 358 #endif
 359 }
 360
 361 /*
 362  * Exit an interrupt context. Process softirqs if needed and possible:
 363  */
 364 void irq_exit(void)
 365 {
 366 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 367         local_irq_disable();
 368 #else
 369         WARN_ON_ONCE(!irqs_disabled());
 370 #endif
 371
 372         account_irq_exit_time(current);
 373         trace_hardirq_exit();
 374         sub_preempt_count(HARDIRQ_OFFSET);
 375         if (!in_interrupt() && local_softirq_pending())
 376                 invoke_softirq();
 377
 378         tick_irq_exit();
 379         rcu_irq_exit();
 380 }
 381
 382 /*
 383  * This function must run with irqs disabled!
 384  */
 385 inline void raise_softirq_irqoff(unsigned int nr)
 386 {
 387         __raise_softirq_irqoff(nr);
 388
 389         /*
 390          * If we're in an interrupt or softirq, we're done
 391          * (this also catches softirq-disabled code). We will
 392          * actually run the softirq once we return from
 393          * the irq or softirq.
 394          *
 395          * Otherwise we wake up ksoftirqd to make sure we
 396          * schedule the softirq soon.
 397          */
 398         if (!in_interrupt())
 399                 wakeup_softirqd();
 400 }
 401
 402 void raise_softirq(unsigned int nr)
 403 {
 404         unsigned long flags;
 405
 406         local_irq_save(flags);
 407         raise_softirq_irqoff(nr);
 408         local_irq_restore(flags);
 409 }
 410
 411 void __raise_softirq_irqoff(unsigned int nr)
 412 {
 413         trace_softirq_raise(nr);
 414         or_softirq_pending(1UL << nr);
 415 }
 416
 417 void open_softirq(int nr, void (*action)(struct softirq_action *))
 418 {
 419         softirq_vec[nr].action = action;
 420 }
 421
 422 /*
 423  * Tasklets
 424  */
 425 struct tasklet_head
 426 {
 427         struct tasklet_struct *head;
 428         struct tasklet_struct **tail;
 429 };
 430
 431 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 432 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 433
 434 void __tasklet_schedule(struct tasklet_struct *t)
 435 {
 436         unsigned long flags;
 437
 438         local_irq_save(flags);
 439         t->next = NULL;
 440         *__this_cpu_read(tasklet_vec.tail) = t;
 441         __this_cpu_write(tasklet_vec.tail, &(t->next));
 442         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 443         local_irq_restore(flags);
 444 }
 445
 446 EXPORT_SYMBOL(__tasklet_schedule);
 447
 448 void __tasklet_hi_schedule(struct tasklet_struct *t)
 449 {
 450         unsigned long flags;
 451
 452         local_irq_save(flags);
 453         t->next = NULL;
 454         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 455         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 456         raise_softirq_irqoff(HI_SOFTIRQ);
 457         local_irq_restore(flags);
 458 }
 459
 460 EXPORT_SYMBOL(__tasklet_hi_schedule);
 461
 462 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 463 {
 464         BUG_ON(!irqs_disabled());
 465
 466         t->next = __this_cpu_read(tasklet_hi_vec.head);
 467         __this_cpu_write(tasklet_hi_vec.head, t);
 468         __raise_softirq_irqoff(HI_SOFTIRQ);
 469 }
 470
 471 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 472
 473 static void tasklet_action(struct softirq_action *a)
 474 {
 475         struct tasklet_struct *list;
 476
 477         local_irq_disable();
 478         list = __this_cpu_read(tasklet_vec.head);
 479         __this_cpu_write(tasklet_vec.head, NULL);
 480         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 481         local_irq_enable();
 482
 483         while (list) {
 484                 struct tasklet_struct *t = list;
 485
 486                 list = list->next;
 487
 488                 if (tasklet_trylock(t)) {
 489                         if (!atomic_read(&t->count)) {
 490                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 491                                         BUG();
 492                                 t->func(t->data);
 493                                 tasklet_unlock(t);
 494                                 continue;
 495                         }
 496                         tasklet_unlock(t);
 497                 }
 498
 499                 local_irq_disable();
 500                 t->next = NULL;
 501                 *__this_cpu_read(tasklet_vec.tail) = t;
 502                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 503                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 504                 local_irq_enable();
 505         }
 506 }
 507
 508 static void tasklet_hi_action(struct softirq_action *a)
 509 {
 510         struct tasklet_struct *list;
 511
 512         local_irq_disable();
 513         list = __this_cpu_read(tasklet_hi_vec.head);
 514         __this_cpu_write(tasklet_hi_vec.head, NULL);
 515         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 516         local_irq_enable();
 517
 518         while (list) {
 519                 struct tasklet_struct *t = list;
 520
 521                 list = list->next;
 522
 523                 if (tasklet_trylock(t)) {
 524                         if (!atomic_read(&t->count)) {
 525                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 526                                         BUG();
 527                                 t->func(t->data);
 528                                 tasklet_unlock(t);
 529                                 continue;
 530                         }
 531                         tasklet_unlock(t);
 532                 }
 533
 534                 local_irq_disable();
 535                 t->next = NULL;
 536                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 537                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 538                 __raise_softirq_irqoff(HI_SOFTIRQ);
 539                 local_irq_enable();
 540         }
 541 }
 542
 543
 544 void tasklet_init(struct tasklet_struct *t,
 545                   void (*func)(unsigned long), unsigned long data)
 546 {
 547         t->next = NULL;
 548         t->state = 0;
 549         atomic_set(&t->count, 0);
 550         t->func = func;
 551         t->data = data;
 552 }
 553
 554 EXPORT_SYMBOL(tasklet_init);
 555
 556 void tasklet_kill(struct tasklet_struct *t)
 557 {
 558         if (in_interrupt())
 559                 printk("Attempt to kill tasklet from interrupt\n");
 560
 561         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 562                 do {
 563                         yield();
 564                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 565         }
 566         tasklet_unlock_wait(t);
 567         clear_bit(TASKLET_STATE_SCHED, &t->state);
 568 }
 569
 570 EXPORT_SYMBOL(tasklet_kill);
 571
 572 /*
 573  * tasklet_hrtimer
 574  */
 575
 576 /*
 577  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 578  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 579  * hrtimer callback, but from softirq context.
 580  */
 581 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 582 {
 583         struct tasklet_hrtimer *ttimer =
 584                 container_of(timer, struct tasklet_hrtimer, timer);
 585
 586         tasklet_hi_schedule(&ttimer->tasklet);
 587         return HRTIMER_NORESTART;
 588 }
 589
 590 /*
 591  * Helper function which calls the hrtimer callback from
 592  * tasklet/softirq context
 593  */
 594 static void __tasklet_hrtimer_trampoline(unsigned long data)
 595 {
 596         struct tasklet_hrtimer *ttimer = (void *)data;
 597         enum hrtimer_restart restart;
 598
 599         restart = ttimer->function(&ttimer->timer);
 600         if (restart != HRTIMER_NORESTART)
 601                 hrtimer_restart(&ttimer->timer);
 602 }
 603
 604 /**
 605  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 606  * @ttimer:      tasklet_hrtimer which is initialized
 607  * @function:    hrtimer callback function which gets called from softirq context
 608  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 609  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 610  */
 611 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 612                           enum hrtimer_restart (*function)(struct hrtimer *),
 613                           clockid_t which_clock, enum hrtimer_mode mode)
 614 {
 615         hrtimer_init(&ttimer->timer, which_clock, mode);
 616         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 617         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 618                      (unsigned long)ttimer);
 619         ttimer->function = function;
 620 }
 621 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 622
 623 /*
 624  * Remote softirq bits
 625  */
 626
 627 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 628 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 629
 630 static void __local_trigger(struct call_single_data *cp, int softirq)
 631 {
 632         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 633
 634         list_add_tail(&cp->list, head);
 635
 636         /* Trigger the softirq only if the list was previously empty.  */
 637         if (head->next == &cp->list)
 638                 raise_softirq_irqoff(softirq);
 639 }
 640
 641 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 642 static void remote_softirq_receive(void *data)
 643 {
 644         struct call_single_data *cp = data;
 645         unsigned long flags;
 646         int softirq;
 647
 648         softirq = *(int *)cp->info;
 649         local_irq_save(flags);
 650         __local_trigger(cp, softirq);
 651         local_irq_restore(flags);
 652 }
 653
 654 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 655 {
 656         if (cpu_online(cpu)) {
 657                 cp->func = remote_softirq_receive;
 658                 cp->info = &softirq;
 659                 cp->flags = 0;
 660
 661                 __smp_call_function_single(cpu, cp, 0);
 662                 return 0;
 663         }
 664         return 1;
 665 }
 666 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 667 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 668 {
 669         return 1;
 670 }
 671 #endif
 672
 673 /**
 674  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 675  * @cp: private SMP call function data area
 676  * @cpu: the remote cpu
 677  * @this_cpu: the currently executing cpu
 678  * @softirq: the softirq for the work
 679  *
 680  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 681  * done, the work is instead queued up on the local cpu.
 682  *
 683  * Interrupts must be disabled.
 684  */
 685 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 686 {
 687         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 688                 __local_trigger(cp, softirq);
 689 }
 690 EXPORT_SYMBOL(__send_remote_softirq);
 691
 692 /**
 693  * send_remote_softirq - try to schedule softirq work on a remote cpu
 694  * @cp: private SMP call function data area
 695  * @cpu: the remote cpu
 696  * @softirq: the softirq for the work
 697  *
 698  * Like __send_remote_softirq except that disabling interrupts and
 699  * computing the current cpu is done for the caller.
 700  */
 701 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 702 {
 703         unsigned long flags;
 704         int this_cpu;
 705
 706         local_irq_save(flags);
 707         this_cpu = smp_processor_id();
 708         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 709         local_irq_restore(flags);
 710 }
 711 EXPORT_SYMBOL(send_remote_softirq);
 712
 713 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 714                                                unsigned long action, void *hcpu)
 715 {
 716         /*
 717          * If a CPU goes away, splice its entries to the current CPU
 718          * and trigger a run of the softirq
 719          */
 720         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 721                 int cpu = (unsigned long) hcpu;
 722                 int i;
 723
 724                 local_irq_disable();
 725                 for (i = 0; i < NR_SOFTIRQS; i++) {
 726                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 727                         struct list_head *local_head;
 728
 729                         if (list_empty(head))
 730                                 continue;
 731
 732                         local_head = &__get_cpu_var(softirq_work_list[i]);
 733                         list_splice_init(head, local_head);
 734                         raise_softirq_irqoff(i);
 735                 }
 736                 local_irq_enable();
 737         }
 738
 739         return NOTIFY_OK;
 740 }
 741
 742 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 743         .notifier_call  = remote_softirq_cpu_notify,
 744 };
 745
 746 void __init softirq_init(void)
 747 {
 748         int cpu;
 749
 750         for_each_possible_cpu(cpu) {
 751                 int i;
 752
 753                 per_cpu(tasklet_vec, cpu).tail =
 754                         &per_cpu(tasklet_vec, cpu).head;
 755                 per_cpu(tasklet_hi_vec, cpu).tail =
 756                         &per_cpu(tasklet_hi_vec, cpu).head;
 757                 for (i = 0; i < NR_SOFTIRQS; i++)
 758                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 759         }
 760
 761         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 762
 763         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 764         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 765 }
 766
 767 static int ksoftirqd_should_run(unsigned int cpu)
 768 {
 769         return local_softirq_pending();
 770 }
 771
 772 static void run_ksoftirqd(unsigned int cpu)
 773 {
 774         local_irq_disable();
 775         if (local_softirq_pending()) {
 776                 __do_softirq();
 777                 rcu_note_context_switch(cpu);
 778                 local_irq_enable();
 779                 cond_resched();
 780                 return;
 781         }
 782         local_irq_enable();
 783 }
 784
 785 #ifdef CONFIG_HOTPLUG_CPU
 786 /*
 787  * tasklet_kill_immediate is called to remove a tasklet which can already be
 788  * scheduled for execution on @cpu.
 789  *
 790  * Unlike tasklet_kill, this function removes the tasklet
 791  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 792  *
 793  * When this function is called, @cpu must be in the CPU_DEAD state.
 794  */
 795 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 796 {
 797         struct tasklet_struct **i;
 798
 799         BUG_ON(cpu_online(cpu));
 800         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 801
 802         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 803                 return;
 804
 805         /* CPU is dead, so no lock needed. */
 806         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 807                 if (*i == t) {
 808                         *i = t->next;
 809                         /* If this was the tail element, move the tail ptr */
 810                         if (*i == NULL)
 811                                 per_cpu(tasklet_vec, cpu).tail = i;
 812                         return;
 813                 }
 814         }
 815         BUG();
 816 }
 817
 818 static void takeover_tasklets(unsigned int cpu)
 819 {
 820         /* CPU is dead, so no lock needed. */
 821         local_irq_disable();
 822
 823         /* Find end, append list for that CPU. */
 824         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 825                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 826                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 827                 per_cpu(tasklet_vec, cpu).head = NULL;
 828                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 829         }
 830         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 831
 832         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 833                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 834                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 835                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 836                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 837         }
 838         raise_softirq_irqoff(HI_SOFTIRQ);
 839
 840         local_irq_enable();
 841 }
 842 #endif /* CONFIG_HOTPLUG_CPU */
 843
 844 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 845                                   unsigned long action,
 846                                   void *hcpu)
 847 {
 848         switch (action) {
 849 #ifdef CONFIG_HOTPLUG_CPU
 850         case CPU_DEAD:
 851         case CPU_DEAD_FROZEN:
 852                 takeover_tasklets((unsigned long)hcpu);
 853                 break;
 854 #endif /* CONFIG_HOTPLUG_CPU */
 855         }
 856         return NOTIFY_OK;
 857 }
 858
 859 static struct notifier_block __cpuinitdata cpu_nfb = {
 860         .notifier_call = cpu_callback
 861 };
 862
 863 static struct smp_hotplug_thread softirq_threads = {
 864         .store                  = &ksoftirqd,
 865         .thread_should_run      = ksoftirqd_should_run,
 866         .thread_fn              = run_ksoftirqd,
 867         .thread_comm            = "ksoftirqd/%u",
 868 };
 869
 870 static __init int spawn_ksoftirqd(void)
 871 {
 872         register_cpu_notifier(&cpu_nfb);
 873
 874         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 875
 876         return 0;
 877 }
 878 early_initcall(spawn_ksoftirqd);
 879
 880 /*
 881  * [ These __weak aliases are kept in a separate compilation unit, so that
 882  *   GCC does not inline them incorrectly. ]
 883  */
 884
 885 int __init __weak early_irq_init(void)
 886 {
 887         return 0;
 888 }
 889
 890 #ifdef CONFIG_GENERIC_HARDIRQS
 891 int __init __weak arch_probe_nr_irqs(void)
 892 {
 893         return NR_IRQS_LEGACY;
 894 }
 895
 896 int __init __weak arch_early_irq_init(void)
 897 {
 898         return 0;
 899 }
 900 #endif