Merge tag 'gfs2-merge-window' of git://git.kernel.org/pub/scm/linux/kernel/git/steve...

[firefly-linux-kernel-4.4.55.git] / kernel / rcu / tree.c

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 30499508c72ea5e2a2d6994203d2522f20d28dd7..b3d116cd072d7bd24803a52c8d6b478930bd6b8b 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -725,7 +725,7 @@ bool rcu_lockdep_current_cpu_online(void)
        bool ret;
 
        if (in_nmi())
-               return 1;
+               return true;
        preempt_disable();
        rdp = this_cpu_ptr(&rcu_sched_data);
        rnp = rdp->mynode;
@@ -1166,8 +1166,10 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp)
         * hold it, acquire the root rcu_node structure's lock in order to
         * start one (if needed).
         */
-       if (rnp != rnp_root)
+       if (rnp != rnp_root) {
                raw_spin_lock(&rnp_root->lock);
+               smp_mb__after_unlock_lock();
+       }
 
        /*
         * Get a new grace-period number.  If there really is no grace
@@ -1387,6 +1389,7 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
                local_irq_restore(flags);
                return;
        }
+       smp_mb__after_unlock_lock();
        __note_gp_changes(rsp, rnp, rdp);
        raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
@@ -1401,6 +1404,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
 
        rcu_bind_gp_kthread();
        raw_spin_lock_irq(&rnp->lock);
+       smp_mb__after_unlock_lock();
        if (rsp->gp_flags == 0) {
                /* Spurious wakeup, tell caller to go back to sleep.  */
                raw_spin_unlock_irq(&rnp->lock);
@@ -1442,6 +1446,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
         */
        rcu_for_each_node_breadth_first(rsp, rnp) {
                raw_spin_lock_irq(&rnp->lock);
+               smp_mb__after_unlock_lock();
                rdp = this_cpu_ptr(rsp->rda);
                rcu_preempt_check_blocked_tasks(rnp);
                rnp->qsmask = rnp->qsmaskinit;
@@ -1496,6 +1501,7 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
        /* Clear flag to prevent immediate re-entry. */
        if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
                raw_spin_lock_irq(&rnp->lock);
+               smp_mb__after_unlock_lock();
                rsp->gp_flags &= ~RCU_GP_FLAG_FQS;
                raw_spin_unlock_irq(&rnp->lock);
        }
@@ -1513,6 +1519,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        struct rcu_node *rnp = rcu_get_root(rsp);
 
        raw_spin_lock_irq(&rnp->lock);
+       smp_mb__after_unlock_lock();
        gp_duration = jiffies - rsp->gp_start;
        if (gp_duration > rsp->gp_max)
                rsp->gp_max = gp_duration;
@@ -1538,6 +1545,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
         */
        rcu_for_each_node_breadth_first(rsp, rnp) {
                raw_spin_lock_irq(&rnp->lock);
+               smp_mb__after_unlock_lock();
                ACCESS_ONCE(rnp->completed) = rsp->gpnum;
                rdp = this_cpu_ptr(rsp->rda);
                if (rnp == rdp->mynode)
@@ -1549,6 +1557,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        }
        rnp = rcu_get_root(rsp);
        raw_spin_lock_irq(&rnp->lock);
+       smp_mb__after_unlock_lock();
        rcu_nocb_gp_set(rnp, nocb);
 
        rsp->completed = rsp->gpnum; /* Declare grace period done. */
@@ -1785,6 +1794,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
                rnp_c = rnp;
                rnp = rnp->parent;
                raw_spin_lock_irqsave(&rnp->lock, flags);
+               smp_mb__after_unlock_lock();
                WARN_ON_ONCE(rnp_c->qsmask);
        }
 
@@ -1814,6 +1824,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
 
        rnp = rdp->mynode;
        raw_spin_lock_irqsave(&rnp->lock, flags);
+       smp_mb__after_unlock_lock();
        if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum ||
            rnp->completed == rnp->gpnum) {
 
@@ -2028,6 +2039,7 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
        mask = rdp->grpmask;    /* rnp->grplo is constant. */
        do {
                raw_spin_lock(&rnp->lock);      /* irqs already disabled. */
+               smp_mb__after_unlock_lock();
                rnp->qsmaskinit &= ~mask;
                if (rnp->qsmaskinit != 0) {
                        if (rnp != rdp->mynode)
@@ -2238,6 +2250,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
                cond_resched();
                mask = 0;
                raw_spin_lock_irqsave(&rnp->lock, flags);
+               smp_mb__after_unlock_lock();
                if (!rcu_gp_in_progress(rsp)) {
                        raw_spin_unlock_irqrestore(&rnp->lock, flags);
                        return;
@@ -2267,6 +2280,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
        rnp = rcu_get_root(rsp);
        if (rnp->qsmask == 0) {
                raw_spin_lock_irqsave(&rnp->lock, flags);
+               smp_mb__after_unlock_lock();
                rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */
        }
 }
@@ -2299,6 +2313,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
 
        /* Reached the root of the rcu_node tree, acquire lock. */
        raw_spin_lock_irqsave(&rnp_old->lock, flags);
+       smp_mb__after_unlock_lock();
        raw_spin_unlock(&rnp_old->fqslock);
        if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
                rsp->n_force_qs_lh++;
@@ -2417,6 +2432,7 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
                        struct rcu_node *rnp_root = rcu_get_root(rsp);
 
                        raw_spin_lock(&rnp_root->lock);
+                       smp_mb__after_unlock_lock();
                        rcu_start_gp(rsp);
                        raw_spin_unlock(&rnp_root->lock);
                } else {
@@ -2796,6 +2812,10 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
        /* Check for CPU stalls, if enabled. */
        check_cpu_stall(rsp, rdp);
 
+       /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
+       if (rcu_nohz_full_cpu(rsp))
+               return 0;
+
        /* Is the RCU core waiting for a quiescent state from this CPU? */
        if (rcu_scheduler_fully_active &&
            rdp->qs_pending && !rdp->passed_quiesce) {
@@ -3259,9 +3279,9 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp)
 {
        int i;
 
-       for (i = rcu_num_lvls - 1; i > 0; i--)
+       rsp->levelspread[rcu_num_lvls - 1] = rcu_fanout_leaf;
+       for (i = rcu_num_lvls - 2; i >= 0; i--)
                rsp->levelspread[i] = CONFIG_RCU_FANOUT;
-       rsp->levelspread[0] = rcu_fanout_leaf;
 }
 #else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
 static void __init rcu_init_levelspread(struct rcu_state *rsp)