Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck...

diff --git a/Documentation/RCU/trace.txt b/Documentation/RCU/trace.txt
index c078ad48f7a187aa8d76e9885d1776a46a4255e7..8173cec473aa0099a185d694bad1b1b6ee48743d 100644 (file)
--- a/Documentation/RCU/trace.txt
+++ b/Documentation/RCU/trace.txt
@@ -99,18 +99,11 @@ o   "qp" indicates that RCU still expects a quiescent state from
 
 o      "dt" is the current value of the dyntick counter that is incremented
        when entering or leaving dynticks idle state, either by the
-       scheduler or by irq.  The number after the "/" is the interrupt
-       nesting depth when in dyntick-idle state, or one greater than
-       the interrupt-nesting depth otherwise.
-
-       This field is displayed only for CONFIG_NO_HZ kernels.
-
-o      "dn" is the current value of the dyntick counter that is incremented
-       when entering or leaving dynticks idle state via NMI.  If both
-       the "dt" and "dn" values are even, then this CPU is in dynticks
-       idle mode and may be ignored by RCU.  If either of these two
-       counters is odd, then RCU must be alert to the possibility of
-       an RCU read-side critical section running on this CPU.
+       scheduler or by irq.  This number is even if the CPU is in
+       dyntick idle mode and odd otherwise.  The number after the first
+       "/" is the interrupt nesting depth when in dyntick-idle state,
+       or one greater than the interrupt-nesting depth otherwise.
+       The number after the second "/" is the NMI nesting depth.
 
        This field is displayed only for CONFIG_NO_HZ kernels.
 

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index f07d2f03181a9b6c296a09e5e2869ea7c5844580..8154a4a3491c0226e825fac340c602be0e422d24 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -163,7 +163,7 @@ EXPORT_SYMBOL_GPL(rcu_note_context_switch);
 #ifdef CONFIG_NO_HZ
 DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
        .dynticks_nesting = 1,
-       .dynticks = 1,
+       .dynticks = ATOMIC_INIT(1),
 };
 #endif /* #ifdef CONFIG_NO_HZ */
 
@@ -322,13 +322,25 @@ void rcu_enter_nohz(void)
        unsigned long flags;
        struct rcu_dynticks *rdtp;
 
-       smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
        local_irq_save(flags);
        rdtp = &__get_cpu_var(rcu_dynticks);
-       rdtp->dynticks++;
-       rdtp->dynticks_nesting--;
-       WARN_ON_ONCE(rdtp->dynticks & 0x1);
+       if (--rdtp->dynticks_nesting) {
+               local_irq_restore(flags);
+               return;
+       }
+       /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
+       smp_mb__before_atomic_inc();  /* See above. */
+       atomic_inc(&rdtp->dynticks);
+       smp_mb__after_atomic_inc();  /* Force ordering with next sojourn. */
+       WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
        local_irq_restore(flags);
+
+       /* If the interrupt queued a callback, get out of dyntick mode. */
+       if (in_irq() &&
+           (__get_cpu_var(rcu_sched_data).nxtlist ||
+            __get_cpu_var(rcu_bh_data).nxtlist ||
+            rcu_preempt_needs_cpu(smp_processor_id())))
+               set_need_resched();
 }
 
 /*
@@ -344,11 +356,16 @@ void rcu_exit_nohz(void)
 
        local_irq_save(flags);
        rdtp = &__get_cpu_var(rcu_dynticks);
-       rdtp->dynticks++;
-       rdtp->dynticks_nesting++;
-       WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
+       if (rdtp->dynticks_nesting++) {
+               local_irq_restore(flags);
+               return;
+       }
+       smp_mb__before_atomic_inc();  /* Force ordering w/previous sojourn. */
+       atomic_inc(&rdtp->dynticks);
+       /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
+       smp_mb__after_atomic_inc();  /* See above. */
+       WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
        local_irq_restore(flags);
-       smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
 }
 
 /**
@@ -362,11 +379,15 @@ void rcu_nmi_enter(void)
 {
        struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
 
-       if (rdtp->dynticks & 0x1)
+       if (rdtp->dynticks_nmi_nesting == 0 &&
+           (atomic_read(&rdtp->dynticks) & 0x1))
                return;
-       rdtp->dynticks_nmi++;
-       WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1));
-       smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
+       rdtp->dynticks_nmi_nesting++;
+       smp_mb__before_atomic_inc();  /* Force delay from prior write. */
+       atomic_inc(&rdtp->dynticks);
+       /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
+       smp_mb__after_atomic_inc();  /* See above. */
+       WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
 }
 
 /**
@@ -380,11 +401,14 @@ void rcu_nmi_exit(void)
 {
        struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
 
-       if (rdtp->dynticks & 0x1)
+       if (rdtp->dynticks_nmi_nesting == 0 ||
+           --rdtp->dynticks_nmi_nesting != 0)
                return;
-       smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
-       rdtp->dynticks_nmi++;
-       WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1);
+       /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
+       smp_mb__before_atomic_inc();  /* See above. */
+       atomic_inc(&rdtp->dynticks);
+       smp_mb__after_atomic_inc();  /* Force delay to next write. */
+       WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
 }
 
 /**
@@ -395,13 +419,7 @@ void rcu_nmi_exit(void)
  */
 void rcu_irq_enter(void)
 {
-       struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
-
-       if (rdtp->dynticks_nesting++)
-               return;
-       rdtp->dynticks++;
-       WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
-       smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
+       rcu_exit_nohz();
 }
 
 /**
@@ -413,18 +431,7 @@ void rcu_irq_enter(void)
  */
 void rcu_irq_exit(void)
 {
-       struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
-
-       if (--rdtp->dynticks_nesting)
-               return;
-       smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
-       rdtp->dynticks++;
-       WARN_ON_ONCE(rdtp->dynticks & 0x1);
-
-       /* If the interrupt queued a callback, get out of dyntick mode. */
-       if (__this_cpu_read(rcu_sched_data.nxtlist) ||
-           __this_cpu_read(rcu_bh_data.nxtlist))
-               set_need_resched();
+       rcu_enter_nohz();
 }
 
 #ifdef CONFIG_SMP
@@ -436,19 +443,8 @@ void rcu_irq_exit(void)
  */
 static int dyntick_save_progress_counter(struct rcu_data *rdp)
 {
-       int ret;
-       int snap;
-       int snap_nmi;
-
-       snap = rdp->dynticks->dynticks;
-       snap_nmi = rdp->dynticks->dynticks_nmi;
-       smp_mb();       /* Order sampling of snap with end of grace period. */
-       rdp->dynticks_snap = snap;
-       rdp->dynticks_nmi_snap = snap_nmi;
-       ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0);
-       if (ret)
-               rdp->dynticks_fqs++;
-       return ret;
+       rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+       return 0;
 }
 
 /*
@@ -459,16 +455,11 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)
  */
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
 {
-       long curr;
-       long curr_nmi;
-       long snap;
-       long snap_nmi;
+       unsigned long curr;
+       unsigned long snap;
 
-       curr = rdp->dynticks->dynticks;
-       snap = rdp->dynticks_snap;
-       curr_nmi = rdp->dynticks->dynticks_nmi;
-       snap_nmi = rdp->dynticks_nmi_snap;
-       smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
+       curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks);
+       snap = (unsigned long)rdp->dynticks_snap;
 
        /*
         * If the CPU passed through or entered a dynticks idle phase with
@@ -478,8 +469,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
         * read-side critical section that started before the beginning
         * of the current RCU grace period.
         */
-       if ((curr != snap || (curr & 0x1) == 0) &&
-           (curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) {
+       if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) {
                rdp->dynticks_fqs++;
                return 1;
        }
@@ -908,6 +898,12 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
        unsigned long gp_duration;
 
        WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
+
+       /*
+        * Ensure that all grace-period and pre-grace-period activity
+        * is seen before the assignment to rsp->completed.
+        */
+       smp_mb(); /* See above block comment. */
        gp_duration = jiffies - rsp->gp_start;
        if (gp_duration > rsp->gp_max)
                rsp->gp_max = gp_duration;
@@ -1455,25 +1451,11 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
  */
 static void rcu_process_callbacks(void)
 {
-       /*
-        * Memory references from any prior RCU read-side critical sections
-        * executed by the interrupted code must be seen before any RCU
-        * grace-period manipulations below.
-        */
-       smp_mb(); /* See above block comment. */
-
        __rcu_process_callbacks(&rcu_sched_state,
                                &__get_cpu_var(rcu_sched_data));
        __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
        rcu_preempt_process_callbacks();
 
-       /*
-        * Memory references from any later RCU read-side critical sections
-        * executed by the interrupted code must be seen after any RCU
-        * grace-period manipulations above.
-        */
-       smp_mb(); /* See above block comment. */
-
        /* If we are last CPU on way to dyntick-idle mode, accelerate it. */
        rcu_needs_cpu_flush();
 }

diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 257664815d5d9cdfd645d08f6bd2e873e10a2cff..93d4a1c2e88b2a46c9f0daae78063c953cc49fa6 100644 (file)
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -84,11 +84,9 @@
  * Dynticks per-CPU state.
  */
 struct rcu_dynticks {
-       int dynticks_nesting;   /* Track nesting level, sort of. */
-       int dynticks;           /* Even value for dynticks-idle, else odd. */
-       int dynticks_nmi;       /* Even value for either dynticks-idle or */
-                               /*  not in nmi handler, else odd.  So this */
-                               /*  remains even for nmi from irq handler. */
+       int dynticks_nesting;   /* Track irq/process nesting level. */
+       int dynticks_nmi_nesting; /* Track NMI nesting level. */
+       atomic_t dynticks;      /* Even value for dynticks-idle, else odd. */
 };
 
 /* RCU's kthread states for tracing. */
@@ -284,7 +282,6 @@ struct rcu_data {
        /* 3) dynticks interface. */
        struct rcu_dynticks *dynticks;  /* Shared per-CPU dynticks state. */
        int dynticks_snap;              /* Per-GP tracking for dynticks. */
-       int dynticks_nmi_snap;          /* Per-GP tracking for dynticks_nmi. */
 #endif /* #ifdef CONFIG_NO_HZ */
 
        /* 4) reasons this CPU needed to be kicked by force_quiescent_state */

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 3f6559a5f5cd7911fac44ac500b670e125e73157..ed339702481dfc3a7746a10a38cc3309fa82e238 100644 (file)
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -1520,7 +1520,6 @@ int rcu_needs_cpu(int cpu)
 {
        int c = 0;
        int snap;
-       int snap_nmi;
        int thatcpu;
 
        /* Check for being in the holdoff period. */
@@ -1531,10 +1530,10 @@ int rcu_needs_cpu(int cpu)
        for_each_online_cpu(thatcpu) {
                if (thatcpu == cpu)
                        continue;
-               snap = per_cpu(rcu_dynticks, thatcpu).dynticks;
-               snap_nmi = per_cpu(rcu_dynticks, thatcpu).dynticks_nmi;
+               snap = atomic_add_return(0, &per_cpu(rcu_dynticks,
+                                                    thatcpu).dynticks);
                smp_mb(); /* Order sampling of snap with end of grace period. */
-               if (((snap & 0x1) != 0) || ((snap_nmi & 0x1) != 0)) {
+               if ((snap & 0x1) != 0) {
                        per_cpu(rcu_dyntick_drain, cpu) = 0;
                        per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
                        return rcu_needs_cpu_quick_check(cpu);

diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
index aa0fd72b4bc714ae3b8f4c41f48f712513e8d6f3..9678cc3650f5e9c8f4e3eb409d26e9c4e784e7d5 100644 (file)
--- a/kernel/rcutree_trace.c
+++ b/kernel/rcutree_trace.c
@@ -69,10 +69,10 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
                   rdp->passed_quiesc, rdp->passed_quiesc_completed,
                   rdp->qs_pending);
 #ifdef CONFIG_NO_HZ
-       seq_printf(m, " dt=%d/%d dn=%d df=%lu",
-                  rdp->dynticks->dynticks,
+       seq_printf(m, " dt=%d/%d/%d df=%lu",
+                  atomic_read(&rdp->dynticks->dynticks),
                   rdp->dynticks->dynticks_nesting,
-                  rdp->dynticks->dynticks_nmi,
+                  rdp->dynticks->dynticks_nmi_nesting,
                   rdp->dynticks_fqs);
 #endif /* #ifdef CONFIG_NO_HZ */
        seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
@@ -141,9 +141,9 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
                   rdp->qs_pending);
 #ifdef CONFIG_NO_HZ
        seq_printf(m, ",%d,%d,%d,%lu",
-                  rdp->dynticks->dynticks,
+                  atomic_read(&rdp->dynticks->dynticks),
                   rdp->dynticks->dynticks_nesting,
-                  rdp->dynticks->dynticks_nmi,
+                  rdp->dynticks->dynticks_nmi_nesting,
                   rdp->dynticks_fqs);
 #endif /* #ifdef CONFIG_NO_HZ */
        seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
@@ -167,7 +167,7 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
 {
        seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pqc\",\"pq\",");
 #ifdef CONFIG_NO_HZ
-       seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
+       seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\",");
 #endif /* #ifdef CONFIG_NO_HZ */
        seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\",\"ci\",\"co\",\"ca\"\n");
 #ifdef CONFIG_TREE_PREEMPT_RCU

diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index 619313ed6c46d78ef7b35a6ab870bfd764376486..507a22fab7380c120bcf45cb683051c2e4adb0da 100644 (file)
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -144,7 +144,7 @@ static void init_shared_classes(void)
 
 #define HARDIRQ_ENTER()                                \
        local_irq_disable();                    \
-       irq_enter();                            \
+       __irq_enter();                          \
        WARN_ON(!in_irq());
 
 #define HARDIRQ_EXIT()                         \