mm: fix TLB flush race between migration, and change_protection_range

[firefly-linux-kernel-4.4.55.git] / kernel / stop_machine.c

diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index c530bc5be7cfa9e6be364f83848369d204d6d9f8..84571e09c9079e8887f73a0f24beefe58e00ff77 100644 (file)
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -20,6 +20,7 @@
 #include <linux/kallsyms.h>
 #include <linux/smpboot.h>
 #include <linux/atomic.h>
+#include <linux/lglock.h>
 
 /*
  * Structure to determine completion condition and record errors.  May
@@ -43,6 +44,14 @@ static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
 
+/*
+ * Avoids a race between stop_two_cpus and global stop_cpus, where
+ * the stoppers could get queued up in reverse order, leading to
+ * system deadlock. Using an lglock means stop_two_cpus remains
+ * relatively cheap.
+ */
+DEFINE_STATIC_LGLOCK(stop_cpus_lock);
+
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
        memset(done, 0, sizeof(*done));
@@ -276,6 +285,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
                return -ENOENT;
        }
 
+       lg_local_lock(&stop_cpus_lock);
        /*
         * Queuing needs to be done by the lowest numbered CPU, to ensure
         * that works are always queued in the same order on every CPU.
@@ -284,6 +294,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
        smp_call_function_single(min(cpu1, cpu2),
                                 &irq_cpu_stop_queue_work,
                                 &call_args, 0);
+       lg_local_unlock(&stop_cpus_lock);
        preempt_enable();
 
        wait_for_completion(&done.completion);
@@ -335,10 +346,10 @@ static void queue_stop_cpus_work(const struct cpumask *cpumask,
         * preempted by a stopper which might wait for other stoppers
         * to enter @fn which can lead to deadlock.
         */
-       preempt_disable();
+       lg_global_lock(&stop_cpus_lock);
        for_each_cpu(cpu, cpumask)
                cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
-       preempt_enable();
+       lg_global_unlock(&stop_cpus_lock);
 }
 
 static int __stop_cpus(const struct cpumask *cpumask,