arch/powerpc/include/asm/simple_spinlock.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 #ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_H
   3 #define _ASM_POWERPC_SIMPLE_SPINLOCK_H
   4
   5 /*
   6  * Simple spin lock operations.
   7  *
   8  * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM
   9  * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
  10  * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM
  11  *      Rework to support virtual processors
  12  *
  13  * Type of int is used as a full 64b word is not necessary.
  14  *
  15  * (the type definitions are in asm/simple_spinlock_types.h)
  16  */
  17 #include <linux/irqflags.h>
  18 #include <asm/paravirt.h>
  19 #include <asm/paca.h>
  20 #include <asm/synch.h>
  21 #include <asm/ppc-opcode.h>
  22
  23 #ifdef CONFIG_PPC64
  24 /* use 0x800000yy when locked, where yy == CPU number */
  25 #ifdef __BIG_ENDIAN__
  26 #define LOCK_TOKEN      (*(u32 *)(&get_paca()->lock_token))
  27 #else
  28 #define LOCK_TOKEN      (*(u32 *)(&get_paca()->paca_index))
  29 #endif
  30 #else
  31 #define LOCK_TOKEN      1
  32 #endif
  33
  34 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
  35 {
  36         return lock.slock == 0;
  37 }
  38
  39 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
  40 {
  41         return !arch_spin_value_unlocked(READ_ONCE(*lock));
  42 }
  43
  44 /*
  45  * This returns the old value in the lock, so we succeeded
  46  * in getting the lock if the return value is 0.
  47  */
  48 static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock)
  49 {
  50         unsigned long tmp, token;
  51         unsigned int eh = IS_ENABLED(CONFIG_PPC64);
  52
  53         token = LOCK_TOKEN;
  54         __asm__ __volatile__(
  55 "1:     lwarx           %0,0,%2,%[eh]\n\
  56         cmpwi           0,%0,0\n\
  57         bne-            2f\n\
  58         stwcx.          %1,0,%2\n\
  59         bne-            1b\n"
  60         PPC_ACQUIRE_BARRIER
  61 "2:"
  62         : "=&r" (tmp)
  63         : "r" (token), "r" (&lock->slock), [eh] "n" (eh)
  64         : "cr0", "memory");
  65
  66         return tmp;
  67 }
  68
  69 static inline int arch_spin_trylock(arch_spinlock_t *lock)
  70 {
  71         return __arch_spin_trylock(lock) == 0;
  72 }
  73
  74 /*
  75  * On a system with shared processors (that is, where a physical
  76  * processor is multiplexed between several virtual processors),
  77  * there is no point spinning on a lock if the holder of the lock
  78  * isn't currently scheduled on a physical processor.  Instead
  79  * we detect this situation and ask the hypervisor to give the
  80  * rest of our timeslice to the lock holder.
  81  *
  82  * So that we can tell which virtual processor is holding a lock,
  83  * we put 0x80000000 | smp_processor_id() in the lock when it is
  84  * held.  Conveniently, we have a word in the paca that holds this
  85  * value.
  86  */
  87
  88 #if defined(CONFIG_PPC_SPLPAR)
  89 /* We only yield to the hypervisor if we are in shared processor mode */
  90 void splpar_spin_yield(arch_spinlock_t *lock);
  91 void splpar_rw_yield(arch_rwlock_t *lock);
  92 #else /* SPLPAR */
  93 static inline void splpar_spin_yield(arch_spinlock_t *lock) {}
  94 static inline void splpar_rw_yield(arch_rwlock_t *lock) {}
  95 #endif
  96
  97 static inline void spin_yield(arch_spinlock_t *lock)
  98 {
  99         if (is_shared_processor())
 100                 splpar_spin_yield(lock);
 101         else
 102                 barrier();
 103 }
 104
 105 static inline void rw_yield(arch_rwlock_t *lock)
 106 {
 107         if (is_shared_processor())
 108                 splpar_rw_yield(lock);
 109         else
 110                 barrier();
 111 }
 112
 113 static inline void arch_spin_lock(arch_spinlock_t *lock)
 114 {
 115         while (1) {
 116                 if (likely(__arch_spin_trylock(lock) == 0))
 117                         break;
 118                 do {
 119                         HMT_low();
 120                         if (is_shared_processor())
 121                                 splpar_spin_yield(lock);
 122                 } while (unlikely(lock->slock != 0));
 123                 HMT_medium();
 124         }
 125 }
 126
 127 static inline
 128 void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags)
 129 {
 130         unsigned long flags_dis;
 131
 132         while (1) {
 133                 if (likely(__arch_spin_trylock(lock) == 0))
 134                         break;
 135                 local_save_flags(flags_dis);
 136                 local_irq_restore(flags);
 137                 do {
 138                         HMT_low();
 139                         if (is_shared_processor())
 140                                 splpar_spin_yield(lock);
 141                 } while (unlikely(lock->slock != 0));
 142                 HMT_medium();
 143                 local_irq_restore(flags_dis);
 144         }
 145 }
 146 #define arch_spin_lock_flags arch_spin_lock_flags
 147
 148 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 149 {
 150         __asm__ __volatile__("# arch_spin_unlock\n\t"
 151                                 PPC_RELEASE_BARRIER: : :"memory");
 152         lock->slock = 0;
 153 }
 154
 155 /*
 156  * Read-write spinlocks, allowing multiple readers
 157  * but only one writer.
 158  *
 159  * NOTE! it is quite common to have readers in interrupts
 160  * but no interrupt writers. For those circumstances we
 161  * can "mix" irq-safe locks - any writer needs to get a
 162  * irq-safe write-lock, but readers can get non-irqsafe
 163  * read-locks.
 164  */
 165
 166 #ifdef CONFIG_PPC64
 167 #define __DO_SIGN_EXTEND        "extsw  %0,%0\n"
 168 #define WRLOCK_TOKEN            LOCK_TOKEN      /* it's negative */
 169 #else
 170 #define __DO_SIGN_EXTEND
 171 #define WRLOCK_TOKEN            (-1)
 172 #endif
 173
 174 /*
 175  * This returns the old value in the lock + 1,
 176  * so we got a read lock if the return value is > 0.
 177  */
 178 static inline long __arch_read_trylock(arch_rwlock_t *rw)
 179 {
 180         long tmp;
 181         unsigned int eh = IS_ENABLED(CONFIG_PPC64);
 182
 183         __asm__ __volatile__(
 184 "1:     lwarx           %0,0,%1,%[eh]\n"
 185         __DO_SIGN_EXTEND
 186 "       addic.          %0,%0,1\n\
 187         ble-            2f\n"
 188 "       stwcx.          %0,0,%1\n\
 189         bne-            1b\n"
 190         PPC_ACQUIRE_BARRIER
 191 "2:"    : "=&r" (tmp)
 192         : "r" (&rw->lock), [eh] "n" (eh)
 193         : "cr0", "xer", "memory");
 194
 195         return tmp;
 196 }
 197
 198 /*
 199  * This returns the old value in the lock,
 200  * so we got the write lock if the return value is 0.
 201  */
 202 static inline long __arch_write_trylock(arch_rwlock_t *rw)
 203 {
 204         long tmp, token;
 205         unsigned int eh = IS_ENABLED(CONFIG_PPC64);
 206
 207         token = WRLOCK_TOKEN;
 208         __asm__ __volatile__(
 209 "1:     lwarx           %0,0,%2,%[eh]\n\
 210         cmpwi           0,%0,0\n\
 211         bne-            2f\n"
 212 "       stwcx.          %1,0,%2\n\
 213         bne-            1b\n"
 214         PPC_ACQUIRE_BARRIER
 215 "2:"    : "=&r" (tmp)
 216         : "r" (token), "r" (&rw->lock), [eh] "n" (eh)
 217         : "cr0", "memory");
 218
 219         return tmp;
 220 }
 221
 222 static inline void arch_read_lock(arch_rwlock_t *rw)
 223 {
 224         while (1) {
 225                 if (likely(__arch_read_trylock(rw) > 0))
 226                         break;
 227                 do {
 228                         HMT_low();
 229                         if (is_shared_processor())
 230                                 splpar_rw_yield(rw);
 231                 } while (unlikely(rw->lock < 0));
 232                 HMT_medium();
 233         }
 234 }
 235
 236 static inline void arch_write_lock(arch_rwlock_t *rw)
 237 {
 238         while (1) {
 239                 if (likely(__arch_write_trylock(rw) == 0))
 240                         break;
 241                 do {
 242                         HMT_low();
 243                         if (is_shared_processor())
 244                                 splpar_rw_yield(rw);
 245                 } while (unlikely(rw->lock != 0));
 246                 HMT_medium();
 247         }
 248 }
 249
 250 static inline int arch_read_trylock(arch_rwlock_t *rw)
 251 {
 252         return __arch_read_trylock(rw) > 0;
 253 }
 254
 255 static inline int arch_write_trylock(arch_rwlock_t *rw)
 256 {
 257         return __arch_write_trylock(rw) == 0;
 258 }
 259
 260 static inline void arch_read_unlock(arch_rwlock_t *rw)
 261 {
 262         long tmp;
 263
 264         __asm__ __volatile__(
 265         "# read_unlock\n\t"
 266         PPC_RELEASE_BARRIER
 267 "1:     lwarx           %0,0,%1\n\
 268         addic           %0,%0,-1\n"
 269 "       stwcx.          %0,0,%1\n\
 270         bne-            1b"
 271         : "=&r"(tmp)
 272         : "r"(&rw->lock)
 273         : "cr0", "xer", "memory");
 274 }
 275
 276 static inline void arch_write_unlock(arch_rwlock_t *rw)
 277 {
 278         __asm__ __volatile__("# write_unlock\n\t"
 279                                 PPC_RELEASE_BARRIER: : :"memory");
 280         rw->lock = 0;
 281 }
 282
 283 #define arch_spin_relax(lock)   spin_yield(lock)
 284 #define arch_read_relax(lock)   rw_yield(lock)
 285 #define arch_write_relax(lock)  rw_yield(lock)
 286
 287 #endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */