s390/mm,tlb: safeguard against speculative TLB creation

The principles of operations states that the CPU is allowed to create
TLB entries for an address space anytime while an ASCE is loaded to
the control register. This is true even if the CPU is running in the
kernel and the user address space is not (actively) accessed.

In theory this can affect two aspects of the TLB flush logic.
For full-mm flushes the ASCE of the dying process is still attached.
The approach to flush first with IDTE and then just free all page
tables can in theory lead to stale TLB entries. Use the batched
free of page tables for the full-mm flushes as well.

For operations that can have a stale ASCE in the control register,
e.g. a delayed update_user_asce in switch_mm, load the kernel ASCE
to prevent invalid TLBs from being created.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 38149b63dc44a360ab3b0db2dfca4f23e64cc63a..7abf318b1522726eb8858226d3f59b211542b799 100644 (file)
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -35,7 +35,7 @@ static inline int init_new_context(struct task_struct *tsk,
 #define LCTL_OPCODE "lctlg"
 #endif
 
-static inline void update_mm(struct mm_struct *mm, struct task_struct *tsk)
+static inline void update_user_asce(struct mm_struct *mm)
 {
        pgd_t *pgd = mm->pgd;
 
@@ -45,6 +45,13 @@ static inline void update_mm(struct mm_struct *mm, struct task_struct *tsk)
        set_fs(current->thread.mm_segment);
 }
 
+static inline void clear_user_asce(struct mm_struct *mm)
+{
+       S390_lowcore.user_asce = S390_lowcore.kernel_asce;
+       asm volatile(LCTL_OPCODE" 1,1,%0\n" : : "m" (S390_lowcore.user_asce));
+       asm volatile(LCTL_OPCODE" 7,7,%0\n" : : "m" (S390_lowcore.user_asce));
+}
+
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                             struct task_struct *tsk)
 {
@@ -53,11 +60,13 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
        if (prev == next)
                return;
        if (atomic_inc_return(&next->context.attach_count) >> 16) {
-               /* Delay update_mm until all TLB flushes are done. */
+               /* Delay update_user_asce until all TLB flushes are done. */
                set_tsk_thread_flag(tsk, TIF_TLB_WAIT);
+               /* Clear old ASCE by loading the kernel ASCE. */
+               clear_user_asce(next);
        } else {
                cpumask_set_cpu(cpu, mm_cpumask(next));
-               update_mm(next, tsk);
+               update_user_asce(next);
                if (next->context.flush_mm)
                        /* Flush pending TLBs */
                        __tlb_flush_mm(next);
@@ -80,7 +89,7 @@ static inline void finish_arch_post_lock_switch(void)
                cpu_relax();
 
        cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
-       update_mm(mm, tsk);
+       update_user_asce(mm);
        if (mm->context.flush_mm)
                __tlb_flush_mm(mm);
        preempt_enable();

diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 2cb846c4b37f1561ac77f2ef687239987c9e6b86..c544b6f05d95e8e6fee2ef5b5f6f24d7acaf19d8 100644 (file)
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -57,8 +57,6 @@ static inline void tlb_gather_mmu(struct mmu_gather *tlb,
        tlb->end = end;
        tlb->fullmm = !(start | (end+1));
        tlb->batch = NULL;
-       if (tlb->fullmm)
-               __tlb_flush_mm(mm);
 }
 
 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
@@ -96,9 +94,7 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
                                unsigned long address)
 {
-       if (!tlb->fullmm)
-               return page_table_free_rcu(tlb, (unsigned long *) pte);
-       page_table_free(tlb->mm, (unsigned long *) pte);
+       page_table_free_rcu(tlb, (unsigned long *) pte);
 }
 
 /*
@@ -114,9 +110,7 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
 #ifdef CONFIG_64BIT
        if (tlb->mm->context.asce_limit <= (1UL << 31))
                return;
-       if (!tlb->fullmm)
-               return tlb_remove_table(tlb, pmd);
-       crst_table_free(tlb->mm, (unsigned long *) pmd);
+       tlb_remove_table(tlb, pmd);
 #endif
 }
 
@@ -133,9 +127,7 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 #ifdef CONFIG_64BIT
        if (tlb->mm->context.asce_limit <= (1UL << 42))
                return;
-       if (!tlb->fullmm)
-               return tlb_remove_table(tlb, pud);
-       crst_table_free(tlb->mm, (unsigned long *) pud);
+       tlb_remove_table(tlb, pud);
 #endif
 }
 

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 796c9320c709f5850bb0679778a28bc488773be2..24c62900b532727c05a6a5e550ebd04b4ff285c1 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -54,7 +54,7 @@ static void __crst_table_upgrade(void *arg)
        struct mm_struct *mm = arg;
 
        if (current->active_mm == mm)
-               update_mm(mm, current);
+               update_user_asce(mm);
        __tlb_flush_local();
 }
 
@@ -107,8 +107,10 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
 {
        pgd_t *pgd;
 
-       if (current->active_mm == mm)
+       if (current->active_mm == mm) {
+               clear_user_asce(mm);
                __tlb_flush_mm(mm);
+       }
        while (mm->context.asce_limit > limit) {
                pgd = mm->pgd;
                switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) {
@@ -132,7 +134,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
                crst_table_free(mm, (unsigned long *) pgd);
        }
        if (current->active_mm == mm)
-               update_mm(mm, current);
+               update_user_asce(mm);
 }
 #endif