kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #include <linux/export.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17 #include <linux/percpu.h>
  18 #include <linux/cpu.h>
  19 #include <linux/freezer.h>
  20 #include <linux/kthread.h>
  21 #include <linux/rcupdate.h>
  22 #include <linux/ftrace.h>
  23 #include <linux/smp.h>
  24 #include <linux/smpboot.h>
  25 #include <linux/tick.h>
  26
  27 #define CREATE_TRACE_POINTS
  28 #include <trace/events/irq.h>
  29
  30 #include <asm/irq.h>
  31 /*
  32    - No shared variables, all the data are CPU local.
  33    - If a softirq needs serialization, let it serialize itself
  34      by its own spinlocks.
  35    - Even if softirq is serialized, only local cpu is marked for
  36      execution. Hence, we get something sort of weak cpu binding.
  37      Though it is still not clear, will it result in better locality
  38      or will not.
  39
  40    Examples:
  41    - NET RX softirq. It is multithreaded and does not require
  42      any global serialization.
  43    - NET TX softirq. It kicks software netdevice queues, hence
  44      it is logically serialized per device, but this serialization
  45      is invisible to common code.
  46    - Tasklets: serialized wrt itself.
  47  */
  48
  49 #ifndef __ARCH_IRQ_STAT
  50 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  51 EXPORT_SYMBOL(irq_stat);
  52 #endif
  53
  54 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  55
  56 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  57
  58 char *softirq_to_name[NR_SOFTIRQS] = {
  59         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  60         "TASKLET", "SCHED", "HRTIMER", "RCU"
  61 };
  62
  63 /*
  64  * we cannot loop indefinitely here to avoid userspace starvation,
  65  * but we also don't want to introduce a worst case 1/HZ latency
  66  * to the pending events, so lets the scheduler to balance
  67  * the softirq load for us.
  68  */
  69 static void wakeup_softirqd(void)
  70 {
  71         /* Interrupts are disabled: no need to stop preemption */
  72         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  73
  74         if (tsk && tsk->state != TASK_RUNNING)
  75                 wake_up_process(tsk);
  76 }
  77
  78 /*
  79  * preempt_count and SOFTIRQ_OFFSET usage:
  80  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  81  *   softirq processing.
  82  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  83  *   on local_bh_disable or local_bh_enable.
  84  * This lets us distinguish between whether we are currently processing
  85  * softirq and whether we just have bh disabled.
  86  */
  87
  88 /*
  89  * This one is for softirq.c-internal use,
  90  * where hardirqs are disabled legitimately:
  91  */
  92 #ifdef CONFIG_TRACE_IRQFLAGS
  93 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  94 {
  95         unsigned long flags;
  96
  97         WARN_ON_ONCE(in_irq());
  98
  99         raw_local_irq_save(flags);
 100         /*
 101          * The preempt tracer hooks into add_preempt_count and will break
 102          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 103          * is set and before current->softirq_enabled is cleared.
 104          * We must manually increment preempt_count here and manually
 105          * call the trace_preempt_off later.
 106          */
 107         preempt_count() += cnt;
 108         /*
 109          * Were softirqs turned off above:
 110          */
 111         if (softirq_count() == cnt)
 112                 trace_softirqs_off(ip);
 113         raw_local_irq_restore(flags);
 114
 115         if (preempt_count() == cnt)
 116                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 117 }
 118 #else /* !CONFIG_TRACE_IRQFLAGS */
 119 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 120 {
 121         add_preempt_count(cnt);
 122         barrier();
 123 }
 124 #endif /* CONFIG_TRACE_IRQFLAGS */
 125
 126 void local_bh_disable(void)
 127 {
 128         __local_bh_disable((unsigned long)__builtin_return_address(0),
 129                                 SOFTIRQ_DISABLE_OFFSET);
 130 }
 131
 132 EXPORT_SYMBOL(local_bh_disable);
 133
 134 static void __local_bh_enable(unsigned int cnt)
 135 {
 136         WARN_ON_ONCE(in_irq());
 137         WARN_ON_ONCE(!irqs_disabled());
 138
 139         if (softirq_count() == cnt)
 140                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 141         sub_preempt_count(cnt);
 142 }
 143
 144 /*
 145  * Special-case - softirqs can safely be enabled in
 146  * cond_resched_softirq(), or by __do_softirq(),
 147  * without processing still-pending softirqs:
 148  */
 149 void _local_bh_enable(void)
 150 {
 151         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 152 }
 153
 154 EXPORT_SYMBOL(_local_bh_enable);
 155
 156 static inline void _local_bh_enable_ip(unsigned long ip)
 157 {
 158         WARN_ON_ONCE(in_irq() || irqs_disabled());
 159 #ifdef CONFIG_TRACE_IRQFLAGS
 160         local_irq_disable();
 161 #endif
 162         /*
 163          * Are softirqs going to be turned on now:
 164          */
 165         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 166                 trace_softirqs_on(ip);
 167         /*
 168          * Keep preemption disabled until we are done with
 169          * softirq processing:
 170          */
 171         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 172
 173         if (unlikely(!in_interrupt() && local_softirq_pending()))
 174                 do_softirq();
 175
 176         dec_preempt_count();
 177 #ifdef CONFIG_TRACE_IRQFLAGS
 178         local_irq_enable();
 179 #endif
 180         preempt_check_resched();
 181 }
 182
 183 void local_bh_enable(void)
 184 {
 185         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 186 }
 187 EXPORT_SYMBOL(local_bh_enable);
 188
 189 void local_bh_enable_ip(unsigned long ip)
 190 {
 191         _local_bh_enable_ip(ip);
 192 }
 193 EXPORT_SYMBOL(local_bh_enable_ip);
 194
 195 /*
 196  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 197  * but break the loop if need_resched() is set or after 2 ms.
 198  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 199  * certain cases, such as stop_machine(), jiffies may cease to
 200  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 201  * well to make sure we eventually return from this method.
 202  *
 203  * These limits have been established via experimentation.
 204  * The two things to balance is latency against fairness -
 205  * we want to handle softirqs as soon as possible, but they
 206  * should not be able to lock up the box.
 207  */
 208 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 209 #define MAX_SOFTIRQ_RESTART 10
 210
 211 asmlinkage void __do_softirq(void)
 212 {
 213         struct softirq_action *h;
 214         __u32 pending;
 215         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 216         int cpu;
 217         unsigned long old_flags = current->flags;
 218         int max_restart = MAX_SOFTIRQ_RESTART;
 219
 220         /*
 221          * Mask out PF_MEMALLOC s current task context is borrowed for the
 222          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 223          * again if the socket is related to swap
 224          */
 225         current->flags &= ~PF_MEMALLOC;
 226
 227         pending = local_softirq_pending();
 228         account_irq_enter_time(current);
 229
 230         __local_bh_disable((unsigned long)__builtin_return_address(0),
 231                                 SOFTIRQ_OFFSET);
 232         lockdep_softirq_enter();
 233
 234         cpu = smp_processor_id();
 235 restart:
 236         /* Reset the pending bitmask before enabling irqs */
 237         set_softirq_pending(0);
 238
 239         local_irq_enable();
 240
 241         h = softirq_vec;
 242
 243         do {
 244                 if (pending & 1) {
 245                         unsigned int vec_nr = h - softirq_vec;
 246                         int prev_count = preempt_count();
 247
 248                         kstat_incr_softirqs_this_cpu(vec_nr);
 249
 250                         trace_softirq_entry(vec_nr);
 251                         h->action(h);
 252                         trace_softirq_exit(vec_nr);
 253                         if (unlikely(prev_count != preempt_count())) {
 254                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 255                                        "with preempt_count %08x,"
 256                                        " exited with %08x?\n", vec_nr,
 257                                        softirq_to_name[vec_nr], h->action,
 258                                        prev_count, preempt_count());
 259                                 preempt_count() = prev_count;
 260                         }
 261
 262                         rcu_bh_qs(cpu);
 263                 }
 264                 h++;
 265                 pending >>= 1;
 266         } while (pending);
 267
 268         local_irq_disable();
 269
 270         pending = local_softirq_pending();
 271         if (pending) {
 272                 if (time_before(jiffies, end) && !need_resched() &&
 273                     --max_restart)
 274                         goto restart;
 275
 276                 wakeup_softirqd();
 277         }
 278
 279         lockdep_softirq_exit();
 280
 281         account_irq_exit_time(current);
 282         __local_bh_enable(SOFTIRQ_OFFSET);
 283         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 284 }
 285
 286 #ifndef __ARCH_HAS_DO_SOFTIRQ
 287
 288 asmlinkage void do_softirq(void)
 289 {
 290         __u32 pending;
 291         unsigned long flags;
 292
 293         if (in_interrupt())
 294                 return;
 295
 296         local_irq_save(flags);
 297
 298         pending = local_softirq_pending();
 299
 300         if (pending)
 301                 __do_softirq();
 302
 303         local_irq_restore(flags);
 304 }
 305
 306 #endif
 307
 308 /*
 309  * Enter an interrupt context.
 310  */
 311 void irq_enter(void)
 312 {
 313         int cpu = smp_processor_id();
 314
 315         rcu_irq_enter();
 316         if (is_idle_task(current) && !in_interrupt()) {
 317                 /*
 318                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 319                  * here, as softirq will be serviced on return from interrupt.
 320                  */
 321                 local_bh_disable();
 322                 tick_check_idle(cpu);
 323                 _local_bh_enable();
 324         }
 325
 326         __irq_enter();
 327 }
 328
 329 static inline void invoke_softirq(void)
 330 {
 331         if (!force_irqthreads) {
 332                 /*
 333                  * We can safely execute softirq on the current stack if
 334                  * it is the irq stack, because it should be near empty
 335                  * at this stage. But we have no way to know if the arch
 336                  * calls irq_exit() on the irq stack. So call softirq
 337                  * in its own stack to prevent from any overrun on top
 338                  * of a potentially deep task stack.
 339                  */
 340                 do_softirq();
 341         } else {
 342                 wakeup_softirqd();
 343         }
 344 }
 345
 346 static inline void tick_irq_exit(void)
 347 {
 348 #ifdef CONFIG_NO_HZ_COMMON
 349         int cpu = smp_processor_id();
 350
 351         /* Make sure that timer wheel updates are propagated */
 352         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 353                 if (!in_interrupt())
 354                         tick_nohz_irq_exit();
 355         }
 356 #endif
 357 }
 358
 359 /*
 360  * Exit an interrupt context. Process softirqs if needed and possible:
 361  */
 362 void irq_exit(void)
 363 {
 364 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 365         local_irq_disable();
 366 #else
 367         WARN_ON_ONCE(!irqs_disabled());
 368 #endif
 369
 370         account_irq_exit_time(current);
 371         trace_hardirq_exit();
 372         sub_preempt_count(HARDIRQ_OFFSET);
 373         if (!in_interrupt() && local_softirq_pending())
 374                 invoke_softirq();
 375
 376         tick_irq_exit();
 377         rcu_irq_exit();
 378 }
 379
 380 /*
 381  * This function must run with irqs disabled!
 382  */
 383 inline void raise_softirq_irqoff(unsigned int nr)
 384 {
 385         __raise_softirq_irqoff(nr);
 386
 387         /*
 388          * If we're in an interrupt or softirq, we're done
 389          * (this also catches softirq-disabled code). We will
 390          * actually run the softirq once we return from
 391          * the irq or softirq.
 392          *
 393          * Otherwise we wake up ksoftirqd to make sure we
 394          * schedule the softirq soon.
 395          */
 396         if (!in_interrupt())
 397                 wakeup_softirqd();
 398 }
 399
 400 void raise_softirq(unsigned int nr)
 401 {
 402         unsigned long flags;
 403
 404         local_irq_save(flags);
 405         raise_softirq_irqoff(nr);
 406         local_irq_restore(flags);
 407 }
 408
 409 void __raise_softirq_irqoff(unsigned int nr)
 410 {
 411         trace_softirq_raise(nr);
 412         or_softirq_pending(1UL << nr);
 413 }
 414
 415 void open_softirq(int nr, void (*action)(struct softirq_action *))
 416 {
 417         softirq_vec[nr].action = action;
 418 }
 419
 420 /*
 421  * Tasklets
 422  */
 423 struct tasklet_head
 424 {
 425         struct tasklet_struct *head;
 426         struct tasklet_struct **tail;
 427 };
 428
 429 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 430 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 431
 432 void __tasklet_schedule(struct tasklet_struct *t)
 433 {
 434         unsigned long flags;
 435
 436         local_irq_save(flags);
 437         t->next = NULL;
 438         *__this_cpu_read(tasklet_vec.tail) = t;
 439         __this_cpu_write(tasklet_vec.tail, &(t->next));
 440         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 441         local_irq_restore(flags);
 442 }
 443
 444 EXPORT_SYMBOL(__tasklet_schedule);
 445
 446 void __tasklet_hi_schedule(struct tasklet_struct *t)
 447 {
 448         unsigned long flags;
 449
 450         local_irq_save(flags);
 451         t->next = NULL;
 452         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 453         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 454         raise_softirq_irqoff(HI_SOFTIRQ);
 455         local_irq_restore(flags);
 456 }
 457
 458 EXPORT_SYMBOL(__tasklet_hi_schedule);
 459
 460 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 461 {
 462         BUG_ON(!irqs_disabled());
 463
 464         t->next = __this_cpu_read(tasklet_hi_vec.head);
 465         __this_cpu_write(tasklet_hi_vec.head, t);
 466         __raise_softirq_irqoff(HI_SOFTIRQ);
 467 }
 468
 469 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 470
 471 static void tasklet_action(struct softirq_action *a)
 472 {
 473         struct tasklet_struct *list;
 474
 475         local_irq_disable();
 476         list = __this_cpu_read(tasklet_vec.head);
 477         __this_cpu_write(tasklet_vec.head, NULL);
 478         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 479         local_irq_enable();
 480
 481         while (list) {
 482                 struct tasklet_struct *t = list;
 483
 484                 list = list->next;
 485
 486                 if (tasklet_trylock(t)) {
 487                         if (!atomic_read(&t->count)) {
 488                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 489                                         BUG();
 490                                 t->func(t->data);
 491                                 tasklet_unlock(t);
 492                                 continue;
 493                         }
 494                         tasklet_unlock(t);
 495                 }
 496
 497                 local_irq_disable();
 498                 t->next = NULL;
 499                 *__this_cpu_read(tasklet_vec.tail) = t;
 500                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 501                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 502                 local_irq_enable();
 503         }
 504 }
 505
 506 static void tasklet_hi_action(struct softirq_action *a)
 507 {
 508         struct tasklet_struct *list;
 509
 510         local_irq_disable();
 511         list = __this_cpu_read(tasklet_hi_vec.head);
 512         __this_cpu_write(tasklet_hi_vec.head, NULL);
 513         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 514         local_irq_enable();
 515
 516         while (list) {
 517                 struct tasklet_struct *t = list;
 518
 519                 list = list->next;
 520
 521                 if (tasklet_trylock(t)) {
 522                         if (!atomic_read(&t->count)) {
 523                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 524                                         BUG();
 525                                 t->func(t->data);
 526                                 tasklet_unlock(t);
 527                                 continue;
 528                         }
 529                         tasklet_unlock(t);
 530                 }
 531
 532                 local_irq_disable();
 533                 t->next = NULL;
 534                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 535                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 536                 __raise_softirq_irqoff(HI_SOFTIRQ);
 537                 local_irq_enable();
 538         }
 539 }
 540
 541
 542 void tasklet_init(struct tasklet_struct *t,
 543                   void (*func)(unsigned long), unsigned long data)
 544 {
 545         t->next = NULL;
 546         t->state = 0;
 547         atomic_set(&t->count, 0);
 548         t->func = func;
 549         t->data = data;
 550 }
 551
 552 EXPORT_SYMBOL(tasklet_init);
 553
 554 void tasklet_kill(struct tasklet_struct *t)
 555 {
 556         if (in_interrupt())
 557                 printk("Attempt to kill tasklet from interrupt\n");
 558
 559         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 560                 do {
 561                         yield();
 562                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 563         }
 564         tasklet_unlock_wait(t);
 565         clear_bit(TASKLET_STATE_SCHED, &t->state);
 566 }
 567
 568 EXPORT_SYMBOL(tasklet_kill);
 569
 570 /*
 571  * tasklet_hrtimer
 572  */
 573
 574 /*
 575  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 576  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 577  * hrtimer callback, but from softirq context.
 578  */
 579 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 580 {
 581         struct tasklet_hrtimer *ttimer =
 582                 container_of(timer, struct tasklet_hrtimer, timer);
 583
 584         tasklet_hi_schedule(&ttimer->tasklet);
 585         return HRTIMER_NORESTART;
 586 }
 587
 588 /*
 589  * Helper function which calls the hrtimer callback from
 590  * tasklet/softirq context
 591  */
 592 static void __tasklet_hrtimer_trampoline(unsigned long data)
 593 {
 594         struct tasklet_hrtimer *ttimer = (void *)data;
 595         enum hrtimer_restart restart;
 596
 597         restart = ttimer->function(&ttimer->timer);
 598         if (restart != HRTIMER_NORESTART)
 599                 hrtimer_restart(&ttimer->timer);
 600 }
 601
 602 /**
 603  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 604  * @ttimer:      tasklet_hrtimer which is initialized
 605  * @function:    hrtimer callback function which gets called from softirq context
 606  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 607  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 608  */
 609 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 610                           enum hrtimer_restart (*function)(struct hrtimer *),
 611                           clockid_t which_clock, enum hrtimer_mode mode)
 612 {
 613         hrtimer_init(&ttimer->timer, which_clock, mode);
 614         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 615         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 616                      (unsigned long)ttimer);
 617         ttimer->function = function;
 618 }
 619 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 620
 621 void __init softirq_init(void)
 622 {
 623         int cpu;
 624
 625         for_each_possible_cpu(cpu) {
 626                 per_cpu(tasklet_vec, cpu).tail =
 627                         &per_cpu(tasklet_vec, cpu).head;
 628                 per_cpu(tasklet_hi_vec, cpu).tail =
 629                         &per_cpu(tasklet_hi_vec, cpu).head;
 630         }
 631
 632         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 633         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 634 }
 635
 636 static int ksoftirqd_should_run(unsigned int cpu)
 637 {
 638         return local_softirq_pending();
 639 }
 640
 641 static void run_ksoftirqd(unsigned int cpu)
 642 {
 643         local_irq_disable();
 644         if (local_softirq_pending()) {
 645                 __do_softirq();
 646                 local_irq_enable();
 647                 cond_resched();
 648
 649                 preempt_disable();
 650                 rcu_note_context_switch(cpu);
 651                 preempt_enable();
 652
 653                 return;
 654         }
 655         local_irq_enable();
 656 }
 657
 658 #ifdef CONFIG_HOTPLUG_CPU
 659 /*
 660  * tasklet_kill_immediate is called to remove a tasklet which can already be
 661  * scheduled for execution on @cpu.
 662  *
 663  * Unlike tasklet_kill, this function removes the tasklet
 664  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 665  *
 666  * When this function is called, @cpu must be in the CPU_DEAD state.
 667  */
 668 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 669 {
 670         struct tasklet_struct **i;
 671
 672         BUG_ON(cpu_online(cpu));
 673         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 674
 675         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 676                 return;
 677
 678         /* CPU is dead, so no lock needed. */
 679         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 680                 if (*i == t) {
 681                         *i = t->next;
 682                         /* If this was the tail element, move the tail ptr */
 683                         if (*i == NULL)
 684                                 per_cpu(tasklet_vec, cpu).tail = i;
 685                         return;
 686                 }
 687         }
 688         BUG();
 689 }
 690
 691 static void takeover_tasklets(unsigned int cpu)
 692 {
 693         /* CPU is dead, so no lock needed. */
 694         local_irq_disable();
 695
 696         /* Find end, append list for that CPU. */
 697         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 698                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 699                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 700                 per_cpu(tasklet_vec, cpu).head = NULL;
 701                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 702         }
 703         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 704
 705         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 706                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 707                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 708                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 709                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 710         }
 711         raise_softirq_irqoff(HI_SOFTIRQ);
 712
 713         local_irq_enable();
 714 }
 715 #endif /* CONFIG_HOTPLUG_CPU */
 716
 717 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 718                                   unsigned long action,
 719                                   void *hcpu)
 720 {
 721         switch (action) {
 722 #ifdef CONFIG_HOTPLUG_CPU
 723         case CPU_DEAD:
 724         case CPU_DEAD_FROZEN:
 725                 takeover_tasklets((unsigned long)hcpu);
 726                 break;
 727 #endif /* CONFIG_HOTPLUG_CPU */
 728         }
 729         return NOTIFY_OK;
 730 }
 731
 732 static struct notifier_block __cpuinitdata cpu_nfb = {
 733         .notifier_call = cpu_callback
 734 };
 735
 736 static struct smp_hotplug_thread softirq_threads = {
 737         .store                  = &ksoftirqd,
 738         .thread_should_run      = ksoftirqd_should_run,
 739         .thread_fn              = run_ksoftirqd,
 740         .thread_comm            = "ksoftirqd/%u",
 741 };
 742
 743 static __init int spawn_ksoftirqd(void)
 744 {
 745         register_cpu_notifier(&cpu_nfb);
 746
 747         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 748
 749         return 0;
 750 }
 751 early_initcall(spawn_ksoftirqd);
 752
 753 /*
 754  * [ These __weak aliases are kept in a separate compilation unit, so that
 755  *   GCC does not inline them incorrectly. ]
 756  */
 757
 758 int __init __weak early_irq_init(void)
 759 {
 760         return 0;
 761 }
 762
 763 #ifdef CONFIG_GENERIC_HARDIRQS
 764 int __init __weak arch_probe_nr_irqs(void)
 765 {
 766         return NR_IRQS_LEGACY;
 767 }
 768
 769 int __init __weak arch_early_irq_init(void)
 770 {
 771         return 0;
 772 }
 773 #endif