2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Distribute under GPLv2.
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
10 * Remote softirq infrastructure is by Jens Axboe.
13 #include <linux/export.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/smpboot.h>
27 #include <linux/tick.h>
29 #define CREATE_TRACE_POINTS
30 #include <trace/events/irq.h>
34 - No shared variables, all the data are CPU local.
35 - If a softirq needs serialization, let it serialize itself
37 - Even if softirq is serialized, only local cpu is marked for
38 execution. Hence, we get something sort of weak cpu binding.
39 Though it is still not clear, will it result in better locality
43 - NET RX softirq. It is multithreaded and does not require
44 any global serialization.
45 - NET TX softirq. It kicks software netdevice queues, hence
46 it is logically serialized per device, but this serialization
47 is invisible to common code.
48 - Tasklets: serialized wrt itself.
51 #ifndef __ARCH_IRQ_STAT
52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
53 EXPORT_SYMBOL(irq_stat);
56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
60 char *softirq_to_name[NR_SOFTIRQS] = {
61 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
62 "TASKLET", "SCHED", "HRTIMER", "RCU"
66 * we cannot loop indefinitely here to avoid userspace starvation,
67 * but we also don't want to introduce a worst case 1/HZ latency
68 * to the pending events, so lets the scheduler to balance
69 * the softirq load for us.
71 static void wakeup_softirqd(void)
73 /* Interrupts are disabled: no need to stop preemption */
74 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
76 if (tsk && tsk->state != TASK_RUNNING)
81 * preempt_count and SOFTIRQ_OFFSET usage:
82 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
84 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
85 * on local_bh_disable or local_bh_enable.
86 * This lets us distinguish between whether we are currently processing
87 * softirq and whether we just have bh disabled.
91 * This one is for softirq.c-internal use,
92 * where hardirqs are disabled legitimately:
94 #ifdef CONFIG_TRACE_IRQFLAGS
95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
99 WARN_ON_ONCE(in_irq());
101 raw_local_irq_save(flags);
103 * The preempt tracer hooks into add_preempt_count and will break
104 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
105 * is set and before current->softirq_enabled is cleared.
106 * We must manually increment preempt_count here and manually
107 * call the trace_preempt_off later.
109 preempt_count() += cnt;
111 * Were softirqs turned off above:
113 if (softirq_count() == cnt)
114 trace_softirqs_off(ip);
115 raw_local_irq_restore(flags);
117 if (preempt_count() == cnt)
118 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
120 #else /* !CONFIG_TRACE_IRQFLAGS */
121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
123 add_preempt_count(cnt);
126 #endif /* CONFIG_TRACE_IRQFLAGS */
128 void local_bh_disable(void)
130 __local_bh_disable((unsigned long)__builtin_return_address(0),
131 SOFTIRQ_DISABLE_OFFSET);
134 EXPORT_SYMBOL(local_bh_disable);
136 static void __local_bh_enable(unsigned int cnt)
138 WARN_ON_ONCE(in_irq());
139 WARN_ON_ONCE(!irqs_disabled());
141 if (softirq_count() == cnt)
142 trace_softirqs_on((unsigned long)__builtin_return_address(0));
143 sub_preempt_count(cnt);
147 * Special-case - softirqs can safely be enabled in
148 * cond_resched_softirq(), or by __do_softirq(),
149 * without processing still-pending softirqs:
151 void _local_bh_enable(void)
153 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
156 EXPORT_SYMBOL(_local_bh_enable);
158 static inline void _local_bh_enable_ip(unsigned long ip)
160 WARN_ON_ONCE(in_irq() || irqs_disabled());
161 #ifdef CONFIG_TRACE_IRQFLAGS
165 * Are softirqs going to be turned on now:
167 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
168 trace_softirqs_on(ip);
170 * Keep preemption disabled until we are done with
171 * softirq processing:
173 sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
175 if (unlikely(!in_interrupt() && local_softirq_pending()))
179 #ifdef CONFIG_TRACE_IRQFLAGS
182 preempt_check_resched();
185 void local_bh_enable(void)
187 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
189 EXPORT_SYMBOL(local_bh_enable);
191 void local_bh_enable_ip(unsigned long ip)
193 _local_bh_enable_ip(ip);
195 EXPORT_SYMBOL(local_bh_enable_ip);
198 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
199 * but break the loop if need_resched() is set or after 2 ms.
200 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
201 * certain cases, such as stop_machine(), jiffies may cease to
202 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
203 * well to make sure we eventually return from this method.
205 * These limits have been established via experimentation.
206 * The two things to balance is latency against fairness -
207 * we want to handle softirqs as soon as possible, but they
208 * should not be able to lock up the box.
210 #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
211 #define MAX_SOFTIRQ_RESTART 10
213 asmlinkage void __do_softirq(void)
215 struct softirq_action *h;
217 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
219 unsigned long old_flags = current->flags;
220 int max_restart = MAX_SOFTIRQ_RESTART;
223 * Mask out PF_MEMALLOC s current task context is borrowed for the
224 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
225 * again if the socket is related to swap
227 current->flags &= ~PF_MEMALLOC;
229 pending = local_softirq_pending();
230 account_irq_enter_time(current);
232 __local_bh_disable((unsigned long)__builtin_return_address(0),
234 lockdep_softirq_enter();
236 cpu = smp_processor_id();
238 /* Reset the pending bitmask before enabling irqs */
239 set_softirq_pending(0);
247 unsigned int vec_nr = h - softirq_vec;
248 int prev_count = preempt_count();
250 kstat_incr_softirqs_this_cpu(vec_nr);
252 trace_softirq_entry(vec_nr);
254 trace_softirq_exit(vec_nr);
255 if (unlikely(prev_count != preempt_count())) {
256 printk(KERN_ERR "huh, entered softirq %u %s %p"
257 "with preempt_count %08x,"
258 " exited with %08x?\n", vec_nr,
259 softirq_to_name[vec_nr], h->action,
260 prev_count, preempt_count());
261 preempt_count() = prev_count;
272 pending = local_softirq_pending();
274 if (time_before(jiffies, end) && !need_resched() &&
281 lockdep_softirq_exit();
283 account_irq_exit_time(current);
284 __local_bh_enable(SOFTIRQ_OFFSET);
285 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
288 #ifndef __ARCH_HAS_DO_SOFTIRQ
290 asmlinkage void do_softirq(void)
298 local_irq_save(flags);
300 pending = local_softirq_pending();
305 local_irq_restore(flags);
311 * Enter an interrupt context.
315 int cpu = smp_processor_id();
318 if (is_idle_task(current) && !in_interrupt()) {
320 * Prevent raise_softirq from needlessly waking up ksoftirqd
321 * here, as softirq will be serviced on return from interrupt.
324 tick_check_idle(cpu);
331 static inline void invoke_softirq(void)
333 if (!force_irqthreads) {
335 * We can safely execute softirq on the current stack if
336 * it is the irq stack, because it should be near empty
337 * at this stage. But we have no way to know if the arch
338 * calls irq_exit() on the irq stack. So call softirq
339 * in its own stack to prevent from any overrun on top
340 * of a potentially deep task stack.
348 static inline void tick_irq_exit(void)
350 #ifdef CONFIG_NO_HZ_COMMON
351 int cpu = smp_processor_id();
353 /* Make sure that timer wheel updates are propagated */
354 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
356 tick_nohz_irq_exit();
362 * Exit an interrupt context. Process softirqs if needed and possible:
366 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
369 WARN_ON_ONCE(!irqs_disabled());
372 account_irq_exit_time(current);
373 trace_hardirq_exit();
374 sub_preempt_count(HARDIRQ_OFFSET);
375 if (!in_interrupt() && local_softirq_pending())
383 * This function must run with irqs disabled!
385 inline void raise_softirq_irqoff(unsigned int nr)
387 __raise_softirq_irqoff(nr);
390 * If we're in an interrupt or softirq, we're done
391 * (this also catches softirq-disabled code). We will
392 * actually run the softirq once we return from
393 * the irq or softirq.
395 * Otherwise we wake up ksoftirqd to make sure we
396 * schedule the softirq soon.
402 void raise_softirq(unsigned int nr)
406 local_irq_save(flags);
407 raise_softirq_irqoff(nr);
408 local_irq_restore(flags);
411 void __raise_softirq_irqoff(unsigned int nr)
413 trace_softirq_raise(nr);
414 or_softirq_pending(1UL << nr);
417 void open_softirq(int nr, void (*action)(struct softirq_action *))
419 softirq_vec[nr].action = action;
427 struct tasklet_struct *head;
428 struct tasklet_struct **tail;
431 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
432 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
434 void __tasklet_schedule(struct tasklet_struct *t)
438 local_irq_save(flags);
440 *__this_cpu_read(tasklet_vec.tail) = t;
441 __this_cpu_write(tasklet_vec.tail, &(t->next));
442 raise_softirq_irqoff(TASKLET_SOFTIRQ);
443 local_irq_restore(flags);
446 EXPORT_SYMBOL(__tasklet_schedule);
448 void __tasklet_hi_schedule(struct tasklet_struct *t)
452 local_irq_save(flags);
454 *__this_cpu_read(tasklet_hi_vec.tail) = t;
455 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
456 raise_softirq_irqoff(HI_SOFTIRQ);
457 local_irq_restore(flags);
460 EXPORT_SYMBOL(__tasklet_hi_schedule);
462 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
464 BUG_ON(!irqs_disabled());
466 t->next = __this_cpu_read(tasklet_hi_vec.head);
467 __this_cpu_write(tasklet_hi_vec.head, t);
468 __raise_softirq_irqoff(HI_SOFTIRQ);
471 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
473 static void tasklet_action(struct softirq_action *a)
475 struct tasklet_struct *list;
478 list = __this_cpu_read(tasklet_vec.head);
479 __this_cpu_write(tasklet_vec.head, NULL);
480 __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
484 struct tasklet_struct *t = list;
488 if (tasklet_trylock(t)) {
489 if (!atomic_read(&t->count)) {
490 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
501 *__this_cpu_read(tasklet_vec.tail) = t;
502 __this_cpu_write(tasklet_vec.tail, &(t->next));
503 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
508 static void tasklet_hi_action(struct softirq_action *a)
510 struct tasklet_struct *list;
513 list = __this_cpu_read(tasklet_hi_vec.head);
514 __this_cpu_write(tasklet_hi_vec.head, NULL);
515 __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
519 struct tasklet_struct *t = list;
523 if (tasklet_trylock(t)) {
524 if (!atomic_read(&t->count)) {
525 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
536 *__this_cpu_read(tasklet_hi_vec.tail) = t;
537 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
538 __raise_softirq_irqoff(HI_SOFTIRQ);
544 void tasklet_init(struct tasklet_struct *t,
545 void (*func)(unsigned long), unsigned long data)
549 atomic_set(&t->count, 0);
554 EXPORT_SYMBOL(tasklet_init);
556 void tasklet_kill(struct tasklet_struct *t)
559 printk("Attempt to kill tasklet from interrupt\n");
561 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
564 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
566 tasklet_unlock_wait(t);
567 clear_bit(TASKLET_STATE_SCHED, &t->state);
570 EXPORT_SYMBOL(tasklet_kill);
577 * The trampoline is called when the hrtimer expires. It schedules a tasklet
578 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
579 * hrtimer callback, but from softirq context.
581 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
583 struct tasklet_hrtimer *ttimer =
584 container_of(timer, struct tasklet_hrtimer, timer);
586 tasklet_hi_schedule(&ttimer->tasklet);
587 return HRTIMER_NORESTART;
591 * Helper function which calls the hrtimer callback from
592 * tasklet/softirq context
594 static void __tasklet_hrtimer_trampoline(unsigned long data)
596 struct tasklet_hrtimer *ttimer = (void *)data;
597 enum hrtimer_restart restart;
599 restart = ttimer->function(&ttimer->timer);
600 if (restart != HRTIMER_NORESTART)
601 hrtimer_restart(&ttimer->timer);
605 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
606 * @ttimer: tasklet_hrtimer which is initialized
607 * @function: hrtimer callback function which gets called from softirq context
608 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
609 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
611 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
612 enum hrtimer_restart (*function)(struct hrtimer *),
613 clockid_t which_clock, enum hrtimer_mode mode)
615 hrtimer_init(&ttimer->timer, which_clock, mode);
616 ttimer->timer.function = __hrtimer_tasklet_trampoline;
617 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
618 (unsigned long)ttimer);
619 ttimer->function = function;
621 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
624 * Remote softirq bits
627 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
628 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
630 static void __local_trigger(struct call_single_data *cp, int softirq)
632 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
634 list_add_tail(&cp->list, head);
636 /* Trigger the softirq only if the list was previously empty. */
637 if (head->next == &cp->list)
638 raise_softirq_irqoff(softirq);
641 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
642 static void remote_softirq_receive(void *data)
644 struct call_single_data *cp = data;
648 softirq = *(int *)cp->info;
649 local_irq_save(flags);
650 __local_trigger(cp, softirq);
651 local_irq_restore(flags);
654 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
656 if (cpu_online(cpu)) {
657 cp->func = remote_softirq_receive;
661 __smp_call_function_single(cpu, cp, 0);
666 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
667 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
674 * __send_remote_softirq - try to schedule softirq work on a remote cpu
675 * @cp: private SMP call function data area
676 * @cpu: the remote cpu
677 * @this_cpu: the currently executing cpu
678 * @softirq: the softirq for the work
680 * Attempt to schedule softirq work on a remote cpu. If this cannot be
681 * done, the work is instead queued up on the local cpu.
683 * Interrupts must be disabled.
685 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
687 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
688 __local_trigger(cp, softirq);
690 EXPORT_SYMBOL(__send_remote_softirq);
693 * send_remote_softirq - try to schedule softirq work on a remote cpu
694 * @cp: private SMP call function data area
695 * @cpu: the remote cpu
696 * @softirq: the softirq for the work
698 * Like __send_remote_softirq except that disabling interrupts and
699 * computing the current cpu is done for the caller.
701 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
706 local_irq_save(flags);
707 this_cpu = smp_processor_id();
708 __send_remote_softirq(cp, cpu, this_cpu, softirq);
709 local_irq_restore(flags);
711 EXPORT_SYMBOL(send_remote_softirq);
713 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
714 unsigned long action, void *hcpu)
717 * If a CPU goes away, splice its entries to the current CPU
718 * and trigger a run of the softirq
720 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
721 int cpu = (unsigned long) hcpu;
725 for (i = 0; i < NR_SOFTIRQS; i++) {
726 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
727 struct list_head *local_head;
729 if (list_empty(head))
732 local_head = &__get_cpu_var(softirq_work_list[i]);
733 list_splice_init(head, local_head);
734 raise_softirq_irqoff(i);
742 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
743 .notifier_call = remote_softirq_cpu_notify,
746 void __init softirq_init(void)
750 for_each_possible_cpu(cpu) {
753 per_cpu(tasklet_vec, cpu).tail =
754 &per_cpu(tasklet_vec, cpu).head;
755 per_cpu(tasklet_hi_vec, cpu).tail =
756 &per_cpu(tasklet_hi_vec, cpu).head;
757 for (i = 0; i < NR_SOFTIRQS; i++)
758 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
761 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
763 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
764 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
767 static int ksoftirqd_should_run(unsigned int cpu)
769 return local_softirq_pending();
772 static void run_ksoftirqd(unsigned int cpu)
775 if (local_softirq_pending()) {
777 rcu_note_context_switch(cpu);
785 #ifdef CONFIG_HOTPLUG_CPU
787 * tasklet_kill_immediate is called to remove a tasklet which can already be
788 * scheduled for execution on @cpu.
790 * Unlike tasklet_kill, this function removes the tasklet
791 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
793 * When this function is called, @cpu must be in the CPU_DEAD state.
795 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
797 struct tasklet_struct **i;
799 BUG_ON(cpu_online(cpu));
800 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
802 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
805 /* CPU is dead, so no lock needed. */
806 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
809 /* If this was the tail element, move the tail ptr */
811 per_cpu(tasklet_vec, cpu).tail = i;
818 static void takeover_tasklets(unsigned int cpu)
820 /* CPU is dead, so no lock needed. */
823 /* Find end, append list for that CPU. */
824 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
825 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
826 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
827 per_cpu(tasklet_vec, cpu).head = NULL;
828 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
830 raise_softirq_irqoff(TASKLET_SOFTIRQ);
832 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
833 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
834 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
835 per_cpu(tasklet_hi_vec, cpu).head = NULL;
836 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
838 raise_softirq_irqoff(HI_SOFTIRQ);
842 #endif /* CONFIG_HOTPLUG_CPU */
844 static int __cpuinit cpu_callback(struct notifier_block *nfb,
845 unsigned long action,
849 #ifdef CONFIG_HOTPLUG_CPU
851 case CPU_DEAD_FROZEN:
852 takeover_tasklets((unsigned long)hcpu);
854 #endif /* CONFIG_HOTPLUG_CPU */
859 static struct notifier_block __cpuinitdata cpu_nfb = {
860 .notifier_call = cpu_callback
863 static struct smp_hotplug_thread softirq_threads = {
865 .thread_should_run = ksoftirqd_should_run,
866 .thread_fn = run_ksoftirqd,
867 .thread_comm = "ksoftirqd/%u",
870 static __init int spawn_ksoftirqd(void)
872 register_cpu_notifier(&cpu_nfb);
874 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
878 early_initcall(spawn_ksoftirqd);
881 * [ These __weak aliases are kept in a separate compilation unit, so that
882 * GCC does not inline them incorrectly. ]
885 int __init __weak early_irq_init(void)
890 #ifdef CONFIG_GENERIC_HARDIRQS
891 int __init __weak arch_probe_nr_irqs(void)
893 return NR_IRQS_LEGACY;
896 int __init __weak arch_early_irq_init(void)