arch/arm/include/asm/cmpxchg.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_ARM_CMPXCHG_H
   3 #define __ASM_ARM_CMPXCHG_H
   4
   5 #include <linux/irqflags.h>
   6 #include <linux/prefetch.h>
   7 #include <asm/barrier.h>
   8
   9 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
  10 /*
  11  * On the StrongARM, "swp" is terminally broken since it bypasses the
  12  * cache totally.  This means that the cache becomes inconsistent, and,
  13  * since we use normal loads/stores as well, this is really bad.
  14  * Typically, this causes oopsen in filp_close, but could have other,
  15  * more disastrous effects.  There are two work-arounds:
  16  *  1. Disable interrupts and emulate the atomic swap
  17  *  2. Clean the cache, perform atomic swap, flush the cache
  18  *
  19  * We choose (1) since its the "easiest" to achieve here and is not
  20  * dependent on the processor type.
  21  *
  22  * NOTE that this solution won't work on an SMP system, so explcitly
  23  * forbid it here.
  24  */
  25 #define swp_is_buggy
  26 #endif
  27
  28 static inline unsigned long
  29 __arch_xchg(unsigned long x, volatile void *ptr, int size)
  30 {
  31         extern void __bad_xchg(volatile void *, int);
  32         unsigned long ret;
  33 #ifdef swp_is_buggy
  34         unsigned long flags;
  35 #endif
  36 #if __LINUX_ARM_ARCH__ >= 6
  37         unsigned int tmp;
  38 #endif
  39
  40         prefetchw((const void *)ptr);
  41
  42         switch (size) {
  43 #if __LINUX_ARM_ARCH__ >= 6
  44 #ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */
  45         case 1:
  46                 asm volatile("@ __xchg1\n"
  47                 "1:     ldrexb  %0, [%3]\n"
  48                 "       strexb  %1, %2, [%3]\n"
  49                 "       teq     %1, #0\n"
  50                 "       bne     1b"
  51                         : "=&r" (ret), "=&r" (tmp)
  52                         : "r" (x), "r" (ptr)
  53                         : "memory", "cc");
  54                 break;
  55         case 2:
  56                 asm volatile("@ __xchg2\n"
  57                 "1:     ldrexh  %0, [%3]\n"
  58                 "       strexh  %1, %2, [%3]\n"
  59                 "       teq     %1, #0\n"
  60                 "       bne     1b"
  61                         : "=&r" (ret), "=&r" (tmp)
  62                         : "r" (x), "r" (ptr)
  63                         : "memory", "cc");
  64                 break;
  65 #endif
  66         case 4:
  67                 asm volatile("@ __xchg4\n"
  68                 "1:     ldrex   %0, [%3]\n"
  69                 "       strex   %1, %2, [%3]\n"
  70                 "       teq     %1, #0\n"
  71                 "       bne     1b"
  72                         : "=&r" (ret), "=&r" (tmp)
  73                         : "r" (x), "r" (ptr)
  74                         : "memory", "cc");
  75                 break;
  76 #elif defined(swp_is_buggy)
  77 #ifdef CONFIG_SMP
  78 #error SMP is not supported on this platform
  79 #endif
  80         case 1:
  81                 raw_local_irq_save(flags);
  82                 ret = *(volatile unsigned char *)ptr;
  83                 *(volatile unsigned char *)ptr = x;
  84                 raw_local_irq_restore(flags);
  85                 break;
  86
  87         case 4:
  88                 raw_local_irq_save(flags);
  89                 ret = *(volatile unsigned long *)ptr;
  90                 *(volatile unsigned long *)ptr = x;
  91                 raw_local_irq_restore(flags);
  92                 break;
  93 #else
  94         case 1:
  95                 asm volatile("@ __xchg1\n"
  96                 "       swpb    %0, %1, [%2]"
  97                         : "=&r" (ret)
  98                         : "r" (x), "r" (ptr)
  99                         : "memory", "cc");
 100                 break;
 101         case 4:
 102                 asm volatile("@ __xchg4\n"
 103                 "       swp     %0, %1, [%2]"
 104                         : "=&r" (ret)
 105                         : "r" (x), "r" (ptr)
 106                         : "memory", "cc");
 107                 break;
 108 #endif
 109         default:
 110                 /* Cause a link-time error, the xchg() size is not supported */
 111                 __bad_xchg(ptr, size), ret = 0;
 112                 break;
 113         }
 114
 115         return ret;
 116 }
 117
 118 #define arch_xchg_relaxed(ptr, x) ({                                    \
 119         (__typeof__(*(ptr)))__arch_xchg((unsigned long)(x), (ptr),      \
 120                                         sizeof(*(ptr)));                \
 121 })
 122
 123 #include <asm-generic/cmpxchg-local.h>
 124
 125 #if __LINUX_ARM_ARCH__ < 6
 126 /* min ARCH < ARMv6 */
 127
 128 #ifdef CONFIG_SMP
 129 #error "SMP is not supported on this platform"
 130 #endif
 131
 132 #define arch_xchg arch_xchg_relaxed
 133
 134 /*
 135  * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
 136  * them available.
 137  */
 138 #define arch_cmpxchg_local(ptr, o, n) ({                                \
 139         (__typeof(*ptr))__generic_cmpxchg_local((ptr),                  \
 140                                                 (unsigned long)(o),     \
 141                                                 (unsigned long)(n),     \
 142                                                 sizeof(*(ptr)));        \
 143 })
 144
 145 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
 146
 147 #include <asm-generic/cmpxchg.h>
 148
 149 #else   /* min ARCH >= ARMv6 */
 150
 151 extern void __bad_cmpxchg(volatile void *ptr, int size);
 152
 153 /*
 154  * cmpxchg only support 32-bits operands on ARMv6.
 155  */
 156
 157 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 158                                       unsigned long new, int size)
 159 {
 160         unsigned long oldval, res;
 161
 162         prefetchw((const void *)ptr);
 163
 164         switch (size) {
 165 #ifndef CONFIG_CPU_V6   /* min ARCH >= ARMv6K */
 166         case 1:
 167                 do {
 168                         asm volatile("@ __cmpxchg1\n"
 169                         "       ldrexb  %1, [%2]\n"
 170                         "       mov     %0, #0\n"
 171                         "       teq     %1, %3\n"
 172                         "       strexbeq %0, %4, [%2]\n"
 173                                 : "=&r" (res), "=&r" (oldval)
 174                                 : "r" (ptr), "Ir" (old), "r" (new)
 175                                 : "memory", "cc");
 176                 } while (res);
 177                 break;
 178         case 2:
 179                 do {
 180                         asm volatile("@ __cmpxchg1\n"
 181                         "       ldrexh  %1, [%2]\n"
 182                         "       mov     %0, #0\n"
 183                         "       teq     %1, %3\n"
 184                         "       strexheq %0, %4, [%2]\n"
 185                                 : "=&r" (res), "=&r" (oldval)
 186                                 : "r" (ptr), "Ir" (old), "r" (new)
 187                                 : "memory", "cc");
 188                 } while (res);
 189                 break;
 190 #endif
 191         case 4:
 192                 do {
 193                         asm volatile("@ __cmpxchg4\n"
 194                         "       ldrex   %1, [%2]\n"
 195                         "       mov     %0, #0\n"
 196                         "       teq     %1, %3\n"
 197                         "       strexeq %0, %4, [%2]\n"
 198                                 : "=&r" (res), "=&r" (oldval)
 199                                 : "r" (ptr), "Ir" (old), "r" (new)
 200                                 : "memory", "cc");
 201                 } while (res);
 202                 break;
 203         default:
 204                 __bad_cmpxchg(ptr, size);
 205                 oldval = 0;
 206         }
 207
 208         return oldval;
 209 }
 210
 211 #define arch_cmpxchg_relaxed(ptr,o,n) ({                                \
 212         (__typeof__(*(ptr)))__cmpxchg((ptr),                            \
 213                                       (unsigned long)(o),               \
 214                                       (unsigned long)(n),               \
 215                                       sizeof(*(ptr)));                  \
 216 })
 217
 218 static inline unsigned long __cmpxchg_local(volatile void *ptr,
 219                                             unsigned long old,
 220                                             unsigned long new, int size)
 221 {
 222         unsigned long ret;
 223
 224         switch (size) {
 225 #ifdef CONFIG_CPU_V6    /* min ARCH == ARMv6 */
 226         case 1:
 227         case 2:
 228                 ret = __generic_cmpxchg_local(ptr, old, new, size);
 229                 break;
 230 #endif
 231         default:
 232                 ret = __cmpxchg(ptr, old, new, size);
 233         }
 234
 235         return ret;
 236 }
 237
 238 #define arch_cmpxchg_local(ptr, o, n) ({                                \
 239         (__typeof(*ptr))__cmpxchg_local((ptr),                          \
 240                                         (unsigned long)(o),             \
 241                                         (unsigned long)(n),             \
 242                                         sizeof(*(ptr)));                \
 243 })
 244
 245 static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
 246                                              unsigned long long old,
 247                                              unsigned long long new)
 248 {
 249         unsigned long long oldval;
 250         unsigned long res;
 251
 252         prefetchw(ptr);
 253
 254         __asm__ __volatile__(
 255 "1:     ldrexd          %1, %H1, [%3]\n"
 256 "       teq             %1, %4\n"
 257 "       teqeq           %H1, %H4\n"
 258 "       bne             2f\n"
 259 "       strexd          %0, %5, %H5, [%3]\n"
 260 "       teq             %0, #0\n"
 261 "       bne             1b\n"
 262 "2:"
 263         : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
 264         : "r" (ptr), "r" (old), "r" (new)
 265         : "cc");
 266
 267         return oldval;
 268 }
 269
 270 #define arch_cmpxchg64_relaxed(ptr, o, n) ({                            \
 271         (__typeof__(*(ptr)))__cmpxchg64((ptr),                          \
 272                                         (unsigned long long)(o),        \
 273                                         (unsigned long long)(n));       \
 274 })
 275
 276 #define arch_cmpxchg64_local(ptr, o, n) arch_cmpxchg64_relaxed((ptr), (o), (n))
 277
 278 #endif  /* __LINUX_ARM_ARCH__ >= 6 */
 279
 280 #endif /* __ASM_ARM_CMPXCHG_H */