arch/powerpc/include/asm/cmpxchg.h

   1 #ifndef _ASM_POWERPC_CMPXCHG_H_
   2 #define _ASM_POWERPC_CMPXCHG_H_
   3
   4 #ifdef __KERNEL__
   5 #include <linux/compiler.h>
   6 #include <asm/synch.h>
   7 #include <asm/asm-compat.h>
   8 #include <linux/bug.h>
   9
  10 /*
  11  * Atomic exchange
  12  *
  13  * Changes the memory location '*p' to be val and returns
  14  * the previous value stored there.
  15  */
  16
  17 static __always_inline unsigned long
  18 __xchg_u32_local(volatile void *p, unsigned long val)
  19 {
  20         unsigned long prev;
  21
  22         __asm__ __volatile__(
  23 "1:     lwarx   %0,0,%2 \n"
  24         PPC405_ERR77(0,%2)
  25 "       stwcx.  %3,0,%2 \n\
  26         bne-    1b"
  27         : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
  28         : "r" (p), "r" (val)
  29         : "cc", "memory");
  30
  31         return prev;
  32 }
  33
  34 static __always_inline unsigned long
  35 __xchg_u32_relaxed(u32 *p, unsigned long val)
  36 {
  37         unsigned long prev;
  38
  39         __asm__ __volatile__(
  40 "1:     lwarx   %0,0,%2\n"
  41         PPC405_ERR77(0, %2)
  42 "       stwcx.  %3,0,%2\n"
  43 "       bne-    1b"
  44         : "=&r" (prev), "+m" (*p)
  45         : "r" (p), "r" (val)
  46         : "cc");
  47
  48         return prev;
  49 }
  50
  51 #ifdef CONFIG_PPC64
  52 static __always_inline unsigned long
  53 __xchg_u64_local(volatile void *p, unsigned long val)
  54 {
  55         unsigned long prev;
  56
  57         __asm__ __volatile__(
  58 "1:     ldarx   %0,0,%2 \n"
  59         PPC405_ERR77(0,%2)
  60 "       stdcx.  %3,0,%2 \n\
  61         bne-    1b"
  62         : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
  63         : "r" (p), "r" (val)
  64         : "cc", "memory");
  65
  66         return prev;
  67 }
  68
  69 static __always_inline unsigned long
  70 __xchg_u64_relaxed(u64 *p, unsigned long val)
  71 {
  72         unsigned long prev;
  73
  74         __asm__ __volatile__(
  75 "1:     ldarx   %0,0,%2\n"
  76         PPC405_ERR77(0, %2)
  77 "       stdcx.  %3,0,%2\n"
  78 "       bne-    1b"
  79         : "=&r" (prev), "+m" (*p)
  80         : "r" (p), "r" (val)
  81         : "cc");
  82
  83         return prev;
  84 }
  85 #endif
  86
  87 static __always_inline unsigned long
  88 __xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
  89 {
  90         switch (size) {
  91         case 4:
  92                 return __xchg_u32_local(ptr, x);
  93 #ifdef CONFIG_PPC64
  94         case 8:
  95                 return __xchg_u64_local(ptr, x);
  96 #endif
  97         }
  98         BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
  99         return x;
 100 }
 101
 102 static __always_inline unsigned long
 103 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
 104 {
 105         switch (size) {
 106         case 4:
 107                 return __xchg_u32_relaxed(ptr, x);
 108 #ifdef CONFIG_PPC64
 109         case 8:
 110                 return __xchg_u64_relaxed(ptr, x);
 111 #endif
 112         }
 113         BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
 114         return x;
 115 }
 116 #define xchg_local(ptr,x)                                                    \
 117   ({                                                                         \
 118      __typeof__(*(ptr)) _x_ = (x);                                           \
 119      (__typeof__(*(ptr))) __xchg_local((ptr),                                \
 120                 (unsigned long)_x_, sizeof(*(ptr)));                         \
 121   })
 122
 123 #define xchg_relaxed(ptr, x)                                            \
 124 ({                                                                      \
 125         __typeof__(*(ptr)) _x_ = (x);                                   \
 126         (__typeof__(*(ptr))) __xchg_relaxed((ptr),                      \
 127                         (unsigned long)_x_, sizeof(*(ptr)));            \
 128 })
 129 /*
 130  * Compare and exchange - if *p == old, set it to new,
 131  * and return the old value of *p.
 132  */
 133
 134 static __always_inline unsigned long
 135 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
 136 {
 137         unsigned int prev;
 138
 139         __asm__ __volatile__ (
 140         PPC_ATOMIC_ENTRY_BARRIER
 141 "1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 142         cmpw    0,%0,%3\n\
 143         bne-    2f\n"
 144         PPC405_ERR77(0,%2)
 145 "       stwcx.  %4,0,%2\n\
 146         bne-    1b"
 147         PPC_ATOMIC_EXIT_BARRIER
 148         "\n\
 149 2:"
 150         : "=&r" (prev), "+m" (*p)
 151         : "r" (p), "r" (old), "r" (new)
 152         : "cc", "memory");
 153
 154         return prev;
 155 }
 156
 157 static __always_inline unsigned long
 158 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
 159                         unsigned long new)
 160 {
 161         unsigned int prev;
 162
 163         __asm__ __volatile__ (
 164 "1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 165         cmpw    0,%0,%3\n\
 166         bne-    2f\n"
 167         PPC405_ERR77(0,%2)
 168 "       stwcx.  %4,0,%2\n\
 169         bne-    1b"
 170         "\n\
 171 2:"
 172         : "=&r" (prev), "+m" (*p)
 173         : "r" (p), "r" (old), "r" (new)
 174         : "cc", "memory");
 175
 176         return prev;
 177 }
 178
 179 static __always_inline unsigned long
 180 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
 181 {
 182         unsigned long prev;
 183
 184         __asm__ __volatile__ (
 185 "1:     lwarx   %0,0,%2         # __cmpxchg_u32_relaxed\n"
 186 "       cmpw    0,%0,%3\n"
 187 "       bne-    2f\n"
 188         PPC405_ERR77(0, %2)
 189 "       stwcx.  %4,0,%2\n"
 190 "       bne-    1b\n"
 191 "2:"
 192         : "=&r" (prev), "+m" (*p)
 193         : "r" (p), "r" (old), "r" (new)
 194         : "cc");
 195
 196         return prev;
 197 }
 198
 199 /*
 200  * cmpxchg family don't have order guarantee if cmp part fails, therefore we
 201  * can avoid superfluous barriers if we use assembly code to implement
 202  * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
 203  * cmpxchg_release() because that will result in putting a barrier in the
 204  * middle of a ll/sc loop, which is probably a bad idea. For example, this
 205  * might cause the conditional store more likely to fail.
 206  */
 207 static __always_inline unsigned long
 208 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
 209 {
 210         unsigned long prev;
 211
 212         __asm__ __volatile__ (
 213 "1:     lwarx   %0,0,%2         # __cmpxchg_u32_acquire\n"
 214 "       cmpw    0,%0,%3\n"
 215 "       bne-    2f\n"
 216         PPC405_ERR77(0, %2)
 217 "       stwcx.  %4,0,%2\n"
 218 "       bne-    1b\n"
 219         PPC_ACQUIRE_BARRIER
 220         "\n"
 221 "2:"
 222         : "=&r" (prev), "+m" (*p)
 223         : "r" (p), "r" (old), "r" (new)
 224         : "cc", "memory");
 225
 226         return prev;
 227 }
 228
 229 #ifdef CONFIG_PPC64
 230 static __always_inline unsigned long
 231 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
 232 {
 233         unsigned long prev;
 234
 235         __asm__ __volatile__ (
 236         PPC_ATOMIC_ENTRY_BARRIER
 237 "1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 238         cmpd    0,%0,%3\n\
 239         bne-    2f\n\
 240         stdcx.  %4,0,%2\n\
 241         bne-    1b"
 242         PPC_ATOMIC_EXIT_BARRIER
 243         "\n\
 244 2:"
 245         : "=&r" (prev), "+m" (*p)
 246         : "r" (p), "r" (old), "r" (new)
 247         : "cc", "memory");
 248
 249         return prev;
 250 }
 251
 252 static __always_inline unsigned long
 253 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
 254                         unsigned long new)
 255 {
 256         unsigned long prev;
 257
 258         __asm__ __volatile__ (
 259 "1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 260         cmpd    0,%0,%3\n\
 261         bne-    2f\n\
 262         stdcx.  %4,0,%2\n\
 263         bne-    1b"
 264         "\n\
 265 2:"
 266         : "=&r" (prev), "+m" (*p)
 267         : "r" (p), "r" (old), "r" (new)
 268         : "cc", "memory");
 269
 270         return prev;
 271 }
 272
 273 static __always_inline unsigned long
 274 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
 275 {
 276         unsigned long prev;
 277
 278         __asm__ __volatile__ (
 279 "1:     ldarx   %0,0,%2         # __cmpxchg_u64_relaxed\n"
 280 "       cmpd    0,%0,%3\n"
 281 "       bne-    2f\n"
 282 "       stdcx.  %4,0,%2\n"
 283 "       bne-    1b\n"
 284 "2:"
 285         : "=&r" (prev), "+m" (*p)
 286         : "r" (p), "r" (old), "r" (new)
 287         : "cc");
 288
 289         return prev;
 290 }
 291
 292 static __always_inline unsigned long
 293 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
 294 {
 295         unsigned long prev;
 296
 297         __asm__ __volatile__ (
 298 "1:     ldarx   %0,0,%2         # __cmpxchg_u64_acquire\n"
 299 "       cmpd    0,%0,%3\n"
 300 "       bne-    2f\n"
 301 "       stdcx.  %4,0,%2\n"
 302 "       bne-    1b\n"
 303         PPC_ACQUIRE_BARRIER
 304         "\n"
 305 "2:"
 306         : "=&r" (prev), "+m" (*p)
 307         : "r" (p), "r" (old), "r" (new)
 308         : "cc", "memory");
 309
 310         return prev;
 311 }
 312 #endif
 313
 314 static __always_inline unsigned long
 315 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
 316           unsigned int size)
 317 {
 318         switch (size) {
 319         case 4:
 320                 return __cmpxchg_u32(ptr, old, new);
 321 #ifdef CONFIG_PPC64
 322         case 8:
 323                 return __cmpxchg_u64(ptr, old, new);
 324 #endif
 325         }
 326         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
 327         return old;
 328 }
 329
 330 static __always_inline unsigned long
 331 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
 332           unsigned int size)
 333 {
 334         switch (size) {
 335         case 4:
 336                 return __cmpxchg_u32_local(ptr, old, new);
 337 #ifdef CONFIG_PPC64
 338         case 8:
 339                 return __cmpxchg_u64_local(ptr, old, new);
 340 #endif
 341         }
 342         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
 343         return old;
 344 }
 345
 346 static __always_inline unsigned long
 347 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
 348                   unsigned int size)
 349 {
 350         switch (size) {
 351         case 4:
 352                 return __cmpxchg_u32_relaxed(ptr, old, new);
 353 #ifdef CONFIG_PPC64
 354         case 8:
 355                 return __cmpxchg_u64_relaxed(ptr, old, new);
 356 #endif
 357         }
 358         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
 359         return old;
 360 }
 361
 362 static __always_inline unsigned long
 363 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
 364                   unsigned int size)
 365 {
 366         switch (size) {
 367         case 4:
 368                 return __cmpxchg_u32_acquire(ptr, old, new);
 369 #ifdef CONFIG_PPC64
 370         case 8:
 371                 return __cmpxchg_u64_acquire(ptr, old, new);
 372 #endif
 373         }
 374         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
 375         return old;
 376 }
 377 #define cmpxchg(ptr, o, n)                                               \
 378   ({                                                                     \
 379      __typeof__(*(ptr)) _o_ = (o);                                       \
 380      __typeof__(*(ptr)) _n_ = (n);                                       \
 381      (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
 382                                     (unsigned long)_n_, sizeof(*(ptr))); \
 383   })
 384
 385
 386 #define cmpxchg_local(ptr, o, n)                                         \
 387   ({                                                                     \
 388      __typeof__(*(ptr)) _o_ = (o);                                       \
 389      __typeof__(*(ptr)) _n_ = (n);                                       \
 390      (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
 391                                     (unsigned long)_n_, sizeof(*(ptr))); \
 392   })
 393
 394 #define cmpxchg_relaxed(ptr, o, n)                                      \
 395 ({                                                                      \
 396         __typeof__(*(ptr)) _o_ = (o);                                   \
 397         __typeof__(*(ptr)) _n_ = (n);                                   \
 398         (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),                   \
 399                         (unsigned long)_o_, (unsigned long)_n_,         \
 400                         sizeof(*(ptr)));                                \
 401 })
 402
 403 #define cmpxchg_acquire(ptr, o, n)                                      \
 404 ({                                                                      \
 405         __typeof__(*(ptr)) _o_ = (o);                                   \
 406         __typeof__(*(ptr)) _n_ = (n);                                   \
 407         (__typeof__(*(ptr))) __cmpxchg_acquire((ptr),                   \
 408                         (unsigned long)_o_, (unsigned long)_n_,         \
 409                         sizeof(*(ptr)));                                \
 410 })
 411 #ifdef CONFIG_PPC64
 412 #define cmpxchg64(ptr, o, n)                                            \
 413   ({                                                                    \
 414         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 415         cmpxchg((ptr), (o), (n));                                       \
 416   })
 417 #define cmpxchg64_local(ptr, o, n)                                      \
 418   ({                                                                    \
 419         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 420         cmpxchg_local((ptr), (o), (n));                                 \
 421   })
 422 #define cmpxchg64_relaxed(ptr, o, n)                                    \
 423 ({                                                                      \
 424         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 425         cmpxchg_relaxed((ptr), (o), (n));                               \
 426 })
 427 #define cmpxchg64_acquire(ptr, o, n)                                    \
 428 ({                                                                      \
 429         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 430         cmpxchg_acquire((ptr), (o), (n));                               \
 431 })
 432 #else
 433 #include <asm-generic/cmpxchg-local.h>
 434 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 435 #endif
 436
 437 #endif /* __KERNEL__ */
 438 #endif /* _ASM_POWERPC_CMPXCHG_H_ */