arm64: flush: use local TLB and I-cache invalidation

There are a number of places where a single CPU is running with a
private page-table and we need to perform maintenance on the TLB and
I-cache in order to ensure correctness, but do not require the operation
to be broadcast to other CPUs.

This patch adds local variants of tlb_flush_all and __flush_icache_all
to support these use-cases and updates the callers respectively.
__local_flush_icache_all also implies an isb, since it is intended to be
used synchronously.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: David Daney <david.daney@cavium.com>
Acked-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index c75b8d027eb1e657efaa2ec219e14af27c9fcfa9..54efedaf331fda55478d001d860d6137be5d08e8 100644 (file)
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -115,6 +115,13 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *,
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
 extern void flush_dcache_page(struct page *);
 
+static inline void __local_flush_icache_all(void)
+{
+       asm("ic iallu");
+       dsb(nsh);
+       isb();
+}
+
 static inline void __flush_icache_all(void)
 {
        asm("ic ialluis");

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 7bd2da021658ad765aa6bc93374f443ad0d7b594..96f944e75dc464725f98067f03150548c1038de6 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -63,6 +63,14 @@
  *             only require the D-TLB to be invalidated.
  *             - kaddr - Kernel virtual memory address
  */
+static inline void local_flush_tlb_all(void)
+{
+       dsb(nshst);
+       asm("tlbi       vmalle1");
+       dsb(nsh);
+       isb();
+}
+
 static inline void flush_tlb_all(void)
 {
        dsb(ishst);

diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index 13671a9cf0167057d583f1c2dddca54ee94658f8..4d12926ea40dcbe321ce2fa9bd8439bdd189c871 100644 (file)
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -344,9 +344,9 @@ static void efi_set_pgd(struct mm_struct *mm)
        else
                cpu_switch_mm(mm->pgd, mm);
 
-       flush_tlb_all();
+       local_flush_tlb_all();
        if (icache_is_aivivt())
-               __flush_icache_all();
+               __local_flush_icache_all();
 }
 
 void efi_virtmap_load(void)

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index dbdaacddd9a562bc709193284dbe7fdaf41aed24..fdd4d4dbd64f68b7176018df1f14213db42e4006 100644 (file)
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -152,7 +152,7 @@ asmlinkage void secondary_start_kernel(void)
         * point to zero page to avoid speculatively fetching new entries.
         */
        cpu_set_reserved_ttbr0();
-       flush_tlb_all();
+       local_flush_tlb_all();
        cpu_set_default_tcr_t0sz();
 
        preempt_disable();

diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 8297d502217e13010bc891e7d68b5f7279ea62b9..3c5e4e6dcf687bc57f61c3212bbfd06aad637eb5 100644 (file)
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -90,7 +90,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
                else
                        cpu_switch_mm(mm->pgd, mm);
 
-               flush_tlb_all();
+               local_flush_tlb_all();
 
                /*
                 * Restore per-cpu offset before any kernel

diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index d70ff14dbdbdd33ef4ed65b75caa4fa575c6497b..48b53fb381af0592e20ec238948b90184fce09d6 100644 (file)
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -48,9 +48,9 @@ static void flush_context(void)
 {
        /* set the reserved TTBR0 before flushing the TLB */
        cpu_set_reserved_ttbr0();
-       flush_tlb_all();
+       local_flush_tlb_all();
        if (icache_is_aivivt())
-               __flush_icache_all();
+               __local_flush_icache_all();
 }
 
 static void set_mm_context(struct mm_struct *mm, unsigned int asid)

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9211b8527f2580aeb561b8c7cc3cdc75f728571f..71a310478c9e578f127fc6f54b88f24ca8057b34 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -456,7 +456,7 @@ void __init paging_init(void)
         * point to zero page to avoid speculatively fetching new entries.
         */
        cpu_set_reserved_ttbr0();
-       flush_tlb_all();
+       local_flush_tlb_all();
        cpu_set_default_tcr_t0sz();
 }