Merge branch 'linux-linaro-lsk-v4.4' into linux-linaro-lsk-v4.4-android

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / include / asm / spinlock.h

diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h
index c85e96d174a5fbd4764adb748b9bc11c70479617..43a66881fd57c1b753639aee87958a6b347d4675 100644 (file)
--- a/arch/arm64/include/asm/spinlock.h
+++ b/arch/arm64/include/asm/spinlock.h
@@ -26,9 +26,32 @@
  * The memory barriers are implicit with the load-acquire and store-release
  * instructions.
  */
+static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
+{
+       unsigned int tmp;
+       arch_spinlock_t lockval;
 
-#define arch_spin_unlock_wait(lock) \
-       do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
+       asm volatile(
+"      sevl\n"
+"1:    wfe\n"
+"2:    ldaxr   %w0, %2\n"
+"      eor     %w1, %w0, %w0, ror #16\n"
+"      cbnz    %w1, 1b\n"
+       /* Serialise against any concurrent lockers */
+       ARM64_LSE_ATOMIC_INSN(
+       /* LL/SC */
+"      stxr    %w1, %w0, %2\n"
+"      nop\n"
+"      nop\n",
+       /* LSE atomics */
+"      mov     %w1, %w0\n"
+"      cas     %w0, %w0, %2\n"
+"      eor     %w1, %w1, %w0\n")
+"      cbnz    %w1, 2b\n"
+       : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
+       :
+       : "memory");
+}
 
 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
 
@@ -312,4 +335,14 @@ static inline int arch_read_trylock(arch_rwlock_t *rw)
 #define arch_read_relax(lock)  cpu_relax()
 #define arch_write_relax(lock) cpu_relax()
 
+/*
+ * Accesses appearing in program order before a spin_lock() operation
+ * can be reordered with accesses inside the critical section, by virtue
+ * of arch_spin_lock being constructed using acquire semantics.
+ *
+ * In cases where this is problematic (e.g. try_to_wake_up), an
+ * smp_mb__before_spinlock() can restore the required ordering.
+ */
+#define smp_mb__before_spinlock()      smp_mb()
+
 #endif /* __ASM_SPINLOCK_H */