kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 #include <asm/irq.h>
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 char *softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into add_preempt_count and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         preempt_count() += cnt;
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == cnt)
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 #else /* !CONFIG_TRACE_IRQFLAGS */
 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122 {
 123         add_preempt_count(cnt);
 124         barrier();
 125 }
 126 #endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128 void local_bh_disable(void)
 129 {
 130         __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                 SOFTIRQ_DISABLE_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(local_bh_disable);
 135
 136 static void __local_bh_enable(unsigned int cnt)
 137 {
 138         WARN_ON_ONCE(in_irq());
 139         WARN_ON_ONCE(!irqs_disabled());
 140
 141         if (softirq_count() == cnt)
 142                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143         sub_preempt_count(cnt);
 144 }
 145
 146 /*
 147  * Special-case - softirqs can safely be enabled in
 148  * cond_resched_softirq(), or by __do_softirq(),
 149  * without processing still-pending softirqs:
 150  */
 151 void _local_bh_enable(void)
 152 {
 153         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154 }
 155
 156 EXPORT_SYMBOL(_local_bh_enable);
 157
 158 static inline void _local_bh_enable_ip(unsigned long ip)
 159 {
 160         WARN_ON_ONCE(in_irq() || irqs_disabled());
 161 #ifdef CONFIG_TRACE_IRQFLAGS
 162         local_irq_disable();
 163 #endif
 164         /*
 165          * Are softirqs going to be turned on now:
 166          */
 167         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                 trace_softirqs_on(ip);
 169         /*
 170          * Keep preemption disabled until we are done with
 171          * softirq processing:
 172          */
 173         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175         if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                 do_softirq();
 177
 178         dec_preempt_count();
 179 #ifdef CONFIG_TRACE_IRQFLAGS
 180         local_irq_enable();
 181 #endif
 182         preempt_check_resched();
 183 }
 184
 185 void local_bh_enable(void)
 186 {
 187         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188 }
 189 EXPORT_SYMBOL(local_bh_enable);
 190
 191 void local_bh_enable_ip(unsigned long ip)
 192 {
 193         _local_bh_enable_ip(ip);
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing for at most 2 ms,
 199  * and if need_resched() is not set.
 200  *
 201  * These limits have been established via experimentation.
 202  * The two things to balance is latency against fairness -
 203  * we want to handle softirqs as soon as possible, but they
 204  * should not be able to lock up the box.
 205  */
 206 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 207
 208 asmlinkage void __do_softirq(void)
 209 {
 210         struct softirq_action *h;
 211         __u32 pending;
 212         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 213         int cpu;
 214         unsigned long old_flags = current->flags;
 215
 216         /*
 217          * Mask out PF_MEMALLOC s current task context is borrowed for the
 218          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 219          * again if the socket is related to swap
 220          */
 221         current->flags &= ~PF_MEMALLOC;
 222
 223         pending = local_softirq_pending();
 224         account_irq_enter_time(current);
 225
 226         __local_bh_disable((unsigned long)__builtin_return_address(0),
 227                                 SOFTIRQ_OFFSET);
 228         lockdep_softirq_enter();
 229
 230         cpu = smp_processor_id();
 231 restart:
 232         /* Reset the pending bitmask before enabling irqs */
 233         set_softirq_pending(0);
 234
 235         local_irq_enable();
 236
 237         h = softirq_vec;
 238
 239         do {
 240                 if (pending & 1) {
 241                         unsigned int vec_nr = h - softirq_vec;
 242                         int prev_count = preempt_count();
 243
 244                         kstat_incr_softirqs_this_cpu(vec_nr);
 245
 246                         trace_softirq_entry(vec_nr);
 247                         h->action(h);
 248                         trace_softirq_exit(vec_nr);
 249                         if (unlikely(prev_count != preempt_count())) {
 250                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 251                                        "with preempt_count %08x,"
 252                                        " exited with %08x?\n", vec_nr,
 253                                        softirq_to_name[vec_nr], h->action,
 254                                        prev_count, preempt_count());
 255                                 preempt_count() = prev_count;
 256                         }
 257
 258                         rcu_bh_qs(cpu);
 259                 }
 260                 h++;
 261                 pending >>= 1;
 262         } while (pending);
 263
 264         local_irq_disable();
 265
 266         pending = local_softirq_pending();
 267         if (pending) {
 268                 if (time_before(jiffies, end) && !need_resched())
 269                         goto restart;
 270
 271                 wakeup_softirqd();
 272         }
 273
 274         lockdep_softirq_exit();
 275
 276         account_irq_exit_time(current);
 277         __local_bh_enable(SOFTIRQ_OFFSET);
 278         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 279 }
 280
 281 #ifndef __ARCH_HAS_DO_SOFTIRQ
 282
 283 asmlinkage void do_softirq(void)
 284 {
 285         __u32 pending;
 286         unsigned long flags;
 287
 288         if (in_interrupt())
 289                 return;
 290
 291         local_irq_save(flags);
 292
 293         pending = local_softirq_pending();
 294
 295         if (pending)
 296                 __do_softirq();
 297
 298         local_irq_restore(flags);
 299 }
 300
 301 #endif
 302
 303 /*
 304  * Enter an interrupt context.
 305  */
 306 void irq_enter(void)
 307 {
 308         int cpu = smp_processor_id();
 309
 310         rcu_irq_enter();
 311         if (is_idle_task(current) && !in_interrupt()) {
 312                 /*
 313                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 314                  * here, as softirq will be serviced on return from interrupt.
 315                  */
 316                 local_bh_disable();
 317                 tick_check_idle(cpu);
 318                 _local_bh_enable();
 319         }
 320
 321         __irq_enter();
 322 }
 323
 324 static inline void invoke_softirq(void)
 325 {
 326         if (!force_irqthreads)
 327                 __do_softirq();
 328         else
 329                 wakeup_softirqd();
 330 }
 331
 332 static inline void tick_irq_exit(void)
 333 {
 334 #ifdef CONFIG_NO_HZ_COMMON
 335         int cpu = smp_processor_id();
 336
 337         /* Make sure that timer wheel updates are propagated */
 338         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 339                 if (!in_interrupt())
 340                         tick_nohz_irq_exit();
 341         }
 342 #endif
 343 }
 344
 345 /*
 346  * Exit an interrupt context. Process softirqs if needed and possible:
 347  */
 348 void irq_exit(void)
 349 {
 350 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 351         local_irq_disable();
 352 #else
 353         WARN_ON_ONCE(!irqs_disabled());
 354 #endif
 355
 356         account_irq_exit_time(current);
 357         trace_hardirq_exit();
 358         sub_preempt_count(HARDIRQ_OFFSET);
 359         if (!in_interrupt() && local_softirq_pending())
 360                 invoke_softirq();
 361
 362         tick_irq_exit();
 363         rcu_irq_exit();
 364 }
 365
 366 /*
 367  * This function must run with irqs disabled!
 368  */
 369 inline void raise_softirq_irqoff(unsigned int nr)
 370 {
 371         __raise_softirq_irqoff(nr);
 372
 373         /*
 374          * If we're in an interrupt or softirq, we're done
 375          * (this also catches softirq-disabled code). We will
 376          * actually run the softirq once we return from
 377          * the irq or softirq.
 378          *
 379          * Otherwise we wake up ksoftirqd to make sure we
 380          * schedule the softirq soon.
 381          */
 382         if (!in_interrupt())
 383                 wakeup_softirqd();
 384 }
 385
 386 void raise_softirq(unsigned int nr)
 387 {
 388         unsigned long flags;
 389
 390         local_irq_save(flags);
 391         raise_softirq_irqoff(nr);
 392         local_irq_restore(flags);
 393 }
 394
 395 void __raise_softirq_irqoff(unsigned int nr)
 396 {
 397         trace_softirq_raise(nr);
 398         or_softirq_pending(1UL << nr);
 399 }
 400
 401 void open_softirq(int nr, void (*action)(struct softirq_action *))
 402 {
 403         softirq_vec[nr].action = action;
 404 }
 405
 406 /*
 407  * Tasklets
 408  */
 409 struct tasklet_head
 410 {
 411         struct tasklet_struct *head;
 412         struct tasklet_struct **tail;
 413 };
 414
 415 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 416 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 417
 418 void __tasklet_schedule(struct tasklet_struct *t)
 419 {
 420         unsigned long flags;
 421
 422         local_irq_save(flags);
 423         t->next = NULL;
 424         *__this_cpu_read(tasklet_vec.tail) = t;
 425         __this_cpu_write(tasklet_vec.tail, &(t->next));
 426         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 427         local_irq_restore(flags);
 428 }
 429
 430 EXPORT_SYMBOL(__tasklet_schedule);
 431
 432 void __tasklet_hi_schedule(struct tasklet_struct *t)
 433 {
 434         unsigned long flags;
 435
 436         local_irq_save(flags);
 437         t->next = NULL;
 438         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 439         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 440         raise_softirq_irqoff(HI_SOFTIRQ);
 441         local_irq_restore(flags);
 442 }
 443
 444 EXPORT_SYMBOL(__tasklet_hi_schedule);
 445
 446 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 447 {
 448         BUG_ON(!irqs_disabled());
 449
 450         t->next = __this_cpu_read(tasklet_hi_vec.head);
 451         __this_cpu_write(tasklet_hi_vec.head, t);
 452         __raise_softirq_irqoff(HI_SOFTIRQ);
 453 }
 454
 455 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 456
 457 static void tasklet_action(struct softirq_action *a)
 458 {
 459         struct tasklet_struct *list;
 460
 461         local_irq_disable();
 462         list = __this_cpu_read(tasklet_vec.head);
 463         __this_cpu_write(tasklet_vec.head, NULL);
 464         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 465         local_irq_enable();
 466
 467         while (list) {
 468                 struct tasklet_struct *t = list;
 469
 470                 list = list->next;
 471
 472                 if (tasklet_trylock(t)) {
 473                         if (!atomic_read(&t->count)) {
 474                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 475                                         BUG();
 476                                 t->func(t->data);
 477                                 tasklet_unlock(t);
 478                                 continue;
 479                         }
 480                         tasklet_unlock(t);
 481                 }
 482
 483                 local_irq_disable();
 484                 t->next = NULL;
 485                 *__this_cpu_read(tasklet_vec.tail) = t;
 486                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 487                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 488                 local_irq_enable();
 489         }
 490 }
 491
 492 static void tasklet_hi_action(struct softirq_action *a)
 493 {
 494         struct tasklet_struct *list;
 495
 496         local_irq_disable();
 497         list = __this_cpu_read(tasklet_hi_vec.head);
 498         __this_cpu_write(tasklet_hi_vec.head, NULL);
 499         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 500         local_irq_enable();
 501
 502         while (list) {
 503                 struct tasklet_struct *t = list;
 504
 505                 list = list->next;
 506
 507                 if (tasklet_trylock(t)) {
 508                         if (!atomic_read(&t->count)) {
 509                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 510                                         BUG();
 511                                 t->func(t->data);
 512                                 tasklet_unlock(t);
 513                                 continue;
 514                         }
 515                         tasklet_unlock(t);
 516                 }
 517
 518                 local_irq_disable();
 519                 t->next = NULL;
 520                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 521                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 522                 __raise_softirq_irqoff(HI_SOFTIRQ);
 523                 local_irq_enable();
 524         }
 525 }
 526
 527
 528 void tasklet_init(struct tasklet_struct *t,
 529                   void (*func)(unsigned long), unsigned long data)
 530 {
 531         t->next = NULL;
 532         t->state = 0;
 533         atomic_set(&t->count, 0);
 534         t->func = func;
 535         t->data = data;
 536 }
 537
 538 EXPORT_SYMBOL(tasklet_init);
 539
 540 void tasklet_kill(struct tasklet_struct *t)
 541 {
 542         if (in_interrupt())
 543                 printk("Attempt to kill tasklet from interrupt\n");
 544
 545         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 546                 do {
 547                         yield();
 548                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 549         }
 550         tasklet_unlock_wait(t);
 551         clear_bit(TASKLET_STATE_SCHED, &t->state);
 552 }
 553
 554 EXPORT_SYMBOL(tasklet_kill);
 555
 556 /*
 557  * tasklet_hrtimer
 558  */
 559
 560 /*
 561  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 562  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 563  * hrtimer callback, but from softirq context.
 564  */
 565 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 566 {
 567         struct tasklet_hrtimer *ttimer =
 568                 container_of(timer, struct tasklet_hrtimer, timer);
 569
 570         tasklet_hi_schedule(&ttimer->tasklet);
 571         return HRTIMER_NORESTART;
 572 }
 573
 574 /*
 575  * Helper function which calls the hrtimer callback from
 576  * tasklet/softirq context
 577  */
 578 static void __tasklet_hrtimer_trampoline(unsigned long data)
 579 {
 580         struct tasklet_hrtimer *ttimer = (void *)data;
 581         enum hrtimer_restart restart;
 582
 583         restart = ttimer->function(&ttimer->timer);
 584         if (restart != HRTIMER_NORESTART)
 585                 hrtimer_restart(&ttimer->timer);
 586 }
 587
 588 /**
 589  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 590  * @ttimer:      tasklet_hrtimer which is initialized
 591  * @function:    hrtimer callback function which gets called from softirq context
 592  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 593  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 594  */
 595 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 596                           enum hrtimer_restart (*function)(struct hrtimer *),
 597                           clockid_t which_clock, enum hrtimer_mode mode)
 598 {
 599         hrtimer_init(&ttimer->timer, which_clock, mode);
 600         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 601         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 602                      (unsigned long)ttimer);
 603         ttimer->function = function;
 604 }
 605 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 606
 607 /*
 608  * Remote softirq bits
 609  */
 610
 611 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 612 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 613
 614 static void __local_trigger(struct call_single_data *cp, int softirq)
 615 {
 616         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 617
 618         list_add_tail(&cp->list, head);
 619
 620         /* Trigger the softirq only if the list was previously empty.  */
 621         if (head->next == &cp->list)
 622                 raise_softirq_irqoff(softirq);
 623 }
 624
 625 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 626 static void remote_softirq_receive(void *data)
 627 {
 628         struct call_single_data *cp = data;
 629         unsigned long flags;
 630         int softirq;
 631
 632         softirq = cp->priv;
 633
 634         local_irq_save(flags);
 635         __local_trigger(cp, softirq);
 636         local_irq_restore(flags);
 637 }
 638
 639 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 640 {
 641         if (cpu_online(cpu)) {
 642                 cp->func = remote_softirq_receive;
 643                 cp->info = cp;
 644                 cp->flags = 0;
 645                 cp->priv = softirq;
 646
 647                 __smp_call_function_single(cpu, cp, 0);
 648                 return 0;
 649         }
 650         return 1;
 651 }
 652 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 653 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 654 {
 655         return 1;
 656 }
 657 #endif
 658
 659 /**
 660  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 661  * @cp: private SMP call function data area
 662  * @cpu: the remote cpu
 663  * @this_cpu: the currently executing cpu
 664  * @softirq: the softirq for the work
 665  *
 666  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 667  * done, the work is instead queued up on the local cpu.
 668  *
 669  * Interrupts must be disabled.
 670  */
 671 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 672 {
 673         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 674                 __local_trigger(cp, softirq);
 675 }
 676 EXPORT_SYMBOL(__send_remote_softirq);
 677
 678 /**
 679  * send_remote_softirq - try to schedule softirq work on a remote cpu
 680  * @cp: private SMP call function data area
 681  * @cpu: the remote cpu
 682  * @softirq: the softirq for the work
 683  *
 684  * Like __send_remote_softirq except that disabling interrupts and
 685  * computing the current cpu is done for the caller.
 686  */
 687 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 688 {
 689         unsigned long flags;
 690         int this_cpu;
 691
 692         local_irq_save(flags);
 693         this_cpu = smp_processor_id();
 694         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 695         local_irq_restore(flags);
 696 }
 697 EXPORT_SYMBOL(send_remote_softirq);
 698
 699 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 700                                                unsigned long action, void *hcpu)
 701 {
 702         /*
 703          * If a CPU goes away, splice its entries to the current CPU
 704          * and trigger a run of the softirq
 705          */
 706         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 707                 int cpu = (unsigned long) hcpu;
 708                 int i;
 709
 710                 local_irq_disable();
 711                 for (i = 0; i < NR_SOFTIRQS; i++) {
 712                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 713                         struct list_head *local_head;
 714
 715                         if (list_empty(head))
 716                                 continue;
 717
 718                         local_head = &__get_cpu_var(softirq_work_list[i]);
 719                         list_splice_init(head, local_head);
 720                         raise_softirq_irqoff(i);
 721                 }
 722                 local_irq_enable();
 723         }
 724
 725         return NOTIFY_OK;
 726 }
 727
 728 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 729         .notifier_call  = remote_softirq_cpu_notify,
 730 };
 731
 732 void __init softirq_init(void)
 733 {
 734         int cpu;
 735
 736         for_each_possible_cpu(cpu) {
 737                 int i;
 738
 739                 per_cpu(tasklet_vec, cpu).tail =
 740                         &per_cpu(tasklet_vec, cpu).head;
 741                 per_cpu(tasklet_hi_vec, cpu).tail =
 742                         &per_cpu(tasklet_hi_vec, cpu).head;
 743                 for (i = 0; i < NR_SOFTIRQS; i++)
 744                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 745         }
 746
 747         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 748
 749         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 750         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 751 }
 752
 753 static int ksoftirqd_should_run(unsigned int cpu)
 754 {
 755         return local_softirq_pending();
 756 }
 757
 758 static void run_ksoftirqd(unsigned int cpu)
 759 {
 760         local_irq_disable();
 761         if (local_softirq_pending()) {
 762                 __do_softirq();
 763                 rcu_note_context_switch(cpu);
 764                 local_irq_enable();
 765                 cond_resched();
 766                 return;
 767         }
 768         local_irq_enable();
 769 }
 770
 771 #ifdef CONFIG_HOTPLUG_CPU
 772 /*
 773  * tasklet_kill_immediate is called to remove a tasklet which can already be
 774  * scheduled for execution on @cpu.
 775  *
 776  * Unlike tasklet_kill, this function removes the tasklet
 777  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 778  *
 779  * When this function is called, @cpu must be in the CPU_DEAD state.
 780  */
 781 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 782 {
 783         struct tasklet_struct **i;
 784
 785         BUG_ON(cpu_online(cpu));
 786         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 787
 788         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 789                 return;
 790
 791         /* CPU is dead, so no lock needed. */
 792         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 793                 if (*i == t) {
 794                         *i = t->next;
 795                         /* If this was the tail element, move the tail ptr */
 796                         if (*i == NULL)
 797                                 per_cpu(tasklet_vec, cpu).tail = i;
 798                         return;
 799                 }
 800         }
 801         BUG();
 802 }
 803
 804 static void takeover_tasklets(unsigned int cpu)
 805 {
 806         /* CPU is dead, so no lock needed. */
 807         local_irq_disable();
 808
 809         /* Find end, append list for that CPU. */
 810         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 811                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 812                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 813                 per_cpu(tasklet_vec, cpu).head = NULL;
 814                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 815         }
 816         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 817
 818         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 819                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 820                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 821                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 822                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 823         }
 824         raise_softirq_irqoff(HI_SOFTIRQ);
 825
 826         local_irq_enable();
 827 }
 828 #endif /* CONFIG_HOTPLUG_CPU */
 829
 830 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 831                                   unsigned long action,
 832                                   void *hcpu)
 833 {
 834         switch (action) {
 835 #ifdef CONFIG_HOTPLUG_CPU
 836         case CPU_DEAD:
 837         case CPU_DEAD_FROZEN:
 838                 takeover_tasklets((unsigned long)hcpu);
 839                 break;
 840 #endif /* CONFIG_HOTPLUG_CPU */
 841         }
 842         return NOTIFY_OK;
 843 }
 844
 845 static struct notifier_block __cpuinitdata cpu_nfb = {
 846         .notifier_call = cpu_callback
 847 };
 848
 849 static struct smp_hotplug_thread softirq_threads = {
 850         .store                  = &ksoftirqd,
 851         .thread_should_run      = ksoftirqd_should_run,
 852         .thread_fn              = run_ksoftirqd,
 853         .thread_comm            = "ksoftirqd/%u",
 854 };
 855
 856 static __init int spawn_ksoftirqd(void)
 857 {
 858         register_cpu_notifier(&cpu_nfb);
 859
 860         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 861
 862         return 0;
 863 }
 864 early_initcall(spawn_ksoftirqd);
 865
 866 /*
 867  * [ These __weak aliases are kept in a separate compilation unit, so that
 868  *   GCC does not inline them incorrectly. ]
 869  */
 870
 871 int __init __weak early_irq_init(void)
 872 {
 873         return 0;
 874 }
 875
 876 #ifdef CONFIG_GENERIC_HARDIRQS
 877 int __init __weak arch_probe_nr_irqs(void)
 878 {
 879         return NR_IRQS_LEGACY;
 880 }
 881
 882 int __init __weak arch_early_irq_init(void)
 883 {
 884         return 0;
 885 }
 886 #endif