arch/powerpc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_ATOMIC_H_
   3 #define _ASM_POWERPC_ATOMIC_H_
   4
   5 /*
   6  * PowerPC atomic operations
   7  */
   8
   9 #ifdef __KERNEL__
  10 #include <linux/types.h>
  11 #include <asm/cmpxchg.h>
  12 #include <asm/barrier.h>
  13 #include <asm/asm-const.h>
  14
  15 /*
  16  * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
  17  * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
  18  * on the platform without lwsync.
  19  */
  20 #define __atomic_acquire_fence()                                        \
  21         __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory")
  22
  23 #define __atomic_release_fence()                                        \
  24         __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory")
  25
  26 static __inline__ int arch_atomic_read(const atomic_t *v)
  27 {
  28         int t;
  29
  30         __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter));
  31
  32         return t;
  33 }
  34
  35 static __inline__ void arch_atomic_set(atomic_t *v, int i)
  36 {
  37         __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i));
  38 }
  39
  40 #define ATOMIC_OP(op, asm_op, suffix, sign, ...)                        \
  41 static __inline__ void arch_atomic_##op(int a, atomic_t *v)             \
  42 {                                                                       \
  43         int t;                                                          \
  44                                                                         \
  45         __asm__ __volatile__(                                           \
  46 "1:     lwarx   %0,0,%3         # atomic_" #op "\n"                     \
  47         #asm_op "%I2" suffix " %0,%0,%2\n"                              \
  48 "       stwcx.  %0,0,%3 \n"                                             \
  49 "       bne-    1b\n"                                                   \
  50         : "=&r" (t), "+m" (v->counter)                                  \
  51         : "r"#sign (a), "r" (&v->counter)                               \
  52         : "cc", ##__VA_ARGS__);                                         \
  53 }                                                                       \
  54
  55 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ...)         \
  56 static inline int arch_atomic_##op##_return_relaxed(int a, atomic_t *v) \
  57 {                                                                       \
  58         int t;                                                          \
  59                                                                         \
  60         __asm__ __volatile__(                                           \
  61 "1:     lwarx   %0,0,%3         # atomic_" #op "_return_relaxed\n"      \
  62         #asm_op "%I2" suffix " %0,%0,%2\n"                              \
  63 "       stwcx.  %0,0,%3\n"                                              \
  64 "       bne-    1b\n"                                                   \
  65         : "=&r" (t), "+m" (v->counter)                                  \
  66         : "r"#sign (a), "r" (&v->counter)                               \
  67         : "cc", ##__VA_ARGS__);                                         \
  68                                                                         \
  69         return t;                                                       \
  70 }
  71
  72 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ...)          \
  73 static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v)  \
  74 {                                                                       \
  75         int res, t;                                                     \
  76                                                                         \
  77         __asm__ __volatile__(                                           \
  78 "1:     lwarx   %0,0,%4         # atomic_fetch_" #op "_relaxed\n"       \
  79         #asm_op "%I3" suffix " %1,%0,%3\n"                              \
  80 "       stwcx.  %1,0,%4\n"                                              \
  81 "       bne-    1b\n"                                                   \
  82         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
  83         : "r"#sign (a), "r" (&v->counter)                               \
  84         : "cc", ##__VA_ARGS__);                                         \
  85                                                                         \
  86         return res;                                                     \
  87 }
  88
  89 #define ATOMIC_OPS(op, asm_op, suffix, sign, ...)                       \
  90         ATOMIC_OP(op, asm_op, suffix, sign, ##__VA_ARGS__)              \
  91         ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)\
  92         ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)
  93
  94 ATOMIC_OPS(add, add, "c", I, "xer")
  95 ATOMIC_OPS(sub, sub, "c", I, "xer")
  96
  97 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
  98 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
  99
 100 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
 101 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
 102
 103 #undef ATOMIC_OPS
 104 #define ATOMIC_OPS(op, asm_op, suffix, sign)                            \
 105         ATOMIC_OP(op, asm_op, suffix, sign)                             \
 106         ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign)
 107
 108 ATOMIC_OPS(and, and, ".", K)
 109 ATOMIC_OPS(or, or, "", K)
 110 ATOMIC_OPS(xor, xor, "", K)
 111
 112 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
 113 #define arch_atomic_fetch_or_relaxed  arch_atomic_fetch_or_relaxed
 114 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
 115
 116 #undef ATOMIC_OPS
 117 #undef ATOMIC_FETCH_OP_RELAXED
 118 #undef ATOMIC_OP_RETURN_RELAXED
 119 #undef ATOMIC_OP
 120
 121 #define arch_atomic_cmpxchg(v, o, n) \
 122         (arch_cmpxchg(&((v)->counter), (o), (n)))
 123 #define arch_atomic_cmpxchg_relaxed(v, o, n) \
 124         arch_cmpxchg_relaxed(&((v)->counter), (o), (n))
 125 #define arch_atomic_cmpxchg_acquire(v, o, n) \
 126         arch_cmpxchg_acquire(&((v)->counter), (o), (n))
 127
 128 #define arch_atomic_xchg(v, new) \
 129         (arch_xchg(&((v)->counter), new))
 130 #define arch_atomic_xchg_relaxed(v, new) \
 131         arch_xchg_relaxed(&((v)->counter), (new))
 132
 133 /*
 134  * Don't want to override the generic atomic_try_cmpxchg_acquire, because
 135  * we add a lock hint to the lwarx, which may not be wanted for the
 136  * _acquire case (and is not used by the other _acquire variants so it
 137  * would be a surprise).
 138  */
 139 static __always_inline bool
 140 arch_atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new)
 141 {
 142         int r, o = *old;
 143         unsigned int eh = IS_ENABLED(CONFIG_PPC64);
 144
 145         __asm__ __volatile__ (
 146 "1:     lwarx   %0,0,%2,%[eh]   # atomic_try_cmpxchg_acquire            \n"
 147 "       cmpw    0,%0,%3                                                 \n"
 148 "       bne-    2f                                                      \n"
 149 "       stwcx.  %4,0,%2                                                 \n"
 150 "       bne-    1b                                                      \n"
 151 "\t"    PPC_ACQUIRE_BARRIER "                                           \n"
 152 "2:                                                                     \n"
 153         : "=&r" (r), "+m" (v->counter)
 154         : "r" (&v->counter), "r" (o), "r" (new), [eh] "n" (eh)
 155         : "cr0", "memory");
 156
 157         if (unlikely(r != o))
 158                 *old = r;
 159         return likely(r == o);
 160 }
 161
 162 /**
 163  * atomic_fetch_add_unless - add unless the number is a given value
 164  * @v: pointer of type atomic_t
 165  * @a: the amount to add to v...
 166  * @u: ...unless v is equal to u.
 167  *
 168  * Atomically adds @a to @v, so long as it was not @u.
 169  * Returns the old value of @v.
 170  */
 171 static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
 172 {
 173         int t;
 174
 175         __asm__ __volatile__ (
 176         PPC_ATOMIC_ENTRY_BARRIER
 177 "1:     lwarx   %0,0,%1         # atomic_fetch_add_unless\n\
 178         cmpw    0,%0,%3 \n\
 179         beq     2f \n\
 180         add%I2c %0,%0,%2 \n"
 181 "       stwcx.  %0,0,%1 \n\
 182         bne-    1b \n"
 183         PPC_ATOMIC_EXIT_BARRIER
 184 "       sub%I2c %0,%0,%2 \n\
 185 2:"
 186         : "=&r" (t)
 187         : "r" (&v->counter), "rI" (a), "r" (u)
 188         : "cc", "memory", "xer");
 189
 190         return t;
 191 }
 192 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
 193
 194 /*
 195  * Atomically test *v and decrement if it is greater than 0.
 196  * The function returns the old value of *v minus 1, even if
 197  * the atomic variable, v, was not decremented.
 198  */
 199 static __inline__ int arch_atomic_dec_if_positive(atomic_t *v)
 200 {
 201         int t;
 202
 203         __asm__ __volatile__(
 204         PPC_ATOMIC_ENTRY_BARRIER
 205 "1:     lwarx   %0,0,%1         # atomic_dec_if_positive\n\
 206         cmpwi   %0,1\n\
 207         addi    %0,%0,-1\n\
 208         blt-    2f\n"
 209 "       stwcx.  %0,0,%1\n\
 210         bne-    1b"
 211         PPC_ATOMIC_EXIT_BARRIER
 212         "\n\
 213 2:"     : "=&b" (t)
 214         : "r" (&v->counter)
 215         : "cc", "memory");
 216
 217         return t;
 218 }
 219 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
 220
 221 #ifdef __powerpc64__
 222
 223 #define ATOMIC64_INIT(i)        { (i) }
 224
 225 static __inline__ s64 arch_atomic64_read(const atomic64_t *v)
 226 {
 227         s64 t;
 228
 229         __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter));
 230
 231         return t;
 232 }
 233
 234 static __inline__ void arch_atomic64_set(atomic64_t *v, s64 i)
 235 {
 236         __asm__ __volatile__("std%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i));
 237 }
 238
 239 #define ATOMIC64_OP(op, asm_op)                                         \
 240 static __inline__ void arch_atomic64_##op(s64 a, atomic64_t *v)         \
 241 {                                                                       \
 242         s64 t;                                                          \
 243                                                                         \
 244         __asm__ __volatile__(                                           \
 245 "1:     ldarx   %0,0,%3         # atomic64_" #op "\n"                   \
 246         #asm_op " %0,%2,%0\n"                                           \
 247 "       stdcx.  %0,0,%3 \n"                                             \
 248 "       bne-    1b\n"                                                   \
 249         : "=&r" (t), "+m" (v->counter)                                  \
 250         : "r" (a), "r" (&v->counter)                                    \
 251         : "cc");                                                        \
 252 }
 253
 254 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 255 static inline s64                                                       \
 256 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v)               \
 257 {                                                                       \
 258         s64 t;                                                          \
 259                                                                         \
 260         __asm__ __volatile__(                                           \
 261 "1:     ldarx   %0,0,%3         # atomic64_" #op "_return_relaxed\n"    \
 262         #asm_op " %0,%2,%0\n"                                           \
 263 "       stdcx.  %0,0,%3\n"                                              \
 264 "       bne-    1b\n"                                                   \
 265         : "=&r" (t), "+m" (v->counter)                                  \
 266         : "r" (a), "r" (&v->counter)                                    \
 267         : "cc");                                                        \
 268                                                                         \
 269         return t;                                                       \
 270 }
 271
 272 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op)                           \
 273 static inline s64                                                       \
 274 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v)                \
 275 {                                                                       \
 276         s64 res, t;                                                     \
 277                                                                         \
 278         __asm__ __volatile__(                                           \
 279 "1:     ldarx   %0,0,%4         # atomic64_fetch_" #op "_relaxed\n"     \
 280         #asm_op " %1,%3,%0\n"                                           \
 281 "       stdcx.  %1,0,%4\n"                                              \
 282 "       bne-    1b\n"                                                   \
 283         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
 284         : "r" (a), "r" (&v->counter)                                    \
 285         : "cc");                                                        \
 286                                                                         \
 287         return res;                                                     \
 288 }
 289
 290 #define ATOMIC64_OPS(op, asm_op)                                        \
 291         ATOMIC64_OP(op, asm_op)                                         \
 292         ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 293         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 294
 295 ATOMIC64_OPS(add, add)
 296 ATOMIC64_OPS(sub, subf)
 297
 298 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
 299 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
 300
 301 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
 302 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
 303
 304 #undef ATOMIC64_OPS
 305 #define ATOMIC64_OPS(op, asm_op)                                        \
 306         ATOMIC64_OP(op, asm_op)                                         \
 307         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 308
 309 ATOMIC64_OPS(and, and)
 310 ATOMIC64_OPS(or, or)
 311 ATOMIC64_OPS(xor, xor)
 312
 313 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
 314 #define arch_atomic64_fetch_or_relaxed  arch_atomic64_fetch_or_relaxed
 315 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
 316
 317 #undef ATOPIC64_OPS
 318 #undef ATOMIC64_FETCH_OP_RELAXED
 319 #undef ATOMIC64_OP_RETURN_RELAXED
 320 #undef ATOMIC64_OP
 321
 322 static __inline__ void arch_atomic64_inc(atomic64_t *v)
 323 {
 324         s64 t;
 325
 326         __asm__ __volatile__(
 327 "1:     ldarx   %0,0,%2         # atomic64_inc\n\
 328         addic   %0,%0,1\n\
 329         stdcx.  %0,0,%2 \n\
 330         bne-    1b"
 331         : "=&r" (t), "+m" (v->counter)
 332         : "r" (&v->counter)
 333         : "cc", "xer");
 334 }
 335 #define arch_atomic64_inc arch_atomic64_inc
 336
 337 static __inline__ s64 arch_atomic64_inc_return_relaxed(atomic64_t *v)
 338 {
 339         s64 t;
 340
 341         __asm__ __volatile__(
 342 "1:     ldarx   %0,0,%2         # atomic64_inc_return_relaxed\n"
 343 "       addic   %0,%0,1\n"
 344 "       stdcx.  %0,0,%2\n"
 345 "       bne-    1b"
 346         : "=&r" (t), "+m" (v->counter)
 347         : "r" (&v->counter)
 348         : "cc", "xer");
 349
 350         return t;
 351 }
 352
 353 static __inline__ void arch_atomic64_dec(atomic64_t *v)
 354 {
 355         s64 t;
 356
 357         __asm__ __volatile__(
 358 "1:     ldarx   %0,0,%2         # atomic64_dec\n\
 359         addic   %0,%0,-1\n\
 360         stdcx.  %0,0,%2\n\
 361         bne-    1b"
 362         : "=&r" (t), "+m" (v->counter)
 363         : "r" (&v->counter)
 364         : "cc", "xer");
 365 }
 366 #define arch_atomic64_dec arch_atomic64_dec
 367
 368 static __inline__ s64 arch_atomic64_dec_return_relaxed(atomic64_t *v)
 369 {
 370         s64 t;
 371
 372         __asm__ __volatile__(
 373 "1:     ldarx   %0,0,%2         # atomic64_dec_return_relaxed\n"
 374 "       addic   %0,%0,-1\n"
 375 "       stdcx.  %0,0,%2\n"
 376 "       bne-    1b"
 377         : "=&r" (t), "+m" (v->counter)
 378         : "r" (&v->counter)
 379         : "cc", "xer");
 380
 381         return t;
 382 }
 383
 384 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
 385 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
 386
 387 /*
 388  * Atomically test *v and decrement if it is greater than 0.
 389  * The function returns the old value of *v minus 1.
 390  */
 391 static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 392 {
 393         s64 t;
 394
 395         __asm__ __volatile__(
 396         PPC_ATOMIC_ENTRY_BARRIER
 397 "1:     ldarx   %0,0,%1         # atomic64_dec_if_positive\n\
 398         addic.  %0,%0,-1\n\
 399         blt-    2f\n\
 400         stdcx.  %0,0,%1\n\
 401         bne-    1b"
 402         PPC_ATOMIC_EXIT_BARRIER
 403         "\n\
 404 2:"     : "=&r" (t)
 405         : "r" (&v->counter)
 406         : "cc", "xer", "memory");
 407
 408         return t;
 409 }
 410 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 411
 412 #define arch_atomic64_cmpxchg(v, o, n) \
 413         (arch_cmpxchg(&((v)->counter), (o), (n)))
 414 #define arch_atomic64_cmpxchg_relaxed(v, o, n) \
 415         arch_cmpxchg_relaxed(&((v)->counter), (o), (n))
 416 #define arch_atomic64_cmpxchg_acquire(v, o, n) \
 417         arch_cmpxchg_acquire(&((v)->counter), (o), (n))
 418
 419 #define arch_atomic64_xchg(v, new) \
 420         (arch_xchg(&((v)->counter), new))
 421 #define arch_atomic64_xchg_relaxed(v, new) \
 422         arch_xchg_relaxed(&((v)->counter), (new))
 423
 424 /**
 425  * atomic64_fetch_add_unless - add unless the number is a given value
 426  * @v: pointer of type atomic64_t
 427  * @a: the amount to add to v...
 428  * @u: ...unless v is equal to u.
 429  *
 430  * Atomically adds @a to @v, so long as it was not @u.
 431  * Returns the old value of @v.
 432  */
 433 static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 434 {
 435         s64 t;
 436
 437         __asm__ __volatile__ (
 438         PPC_ATOMIC_ENTRY_BARRIER
 439 "1:     ldarx   %0,0,%1         # atomic64_fetch_add_unless\n\
 440         cmpd    0,%0,%3 \n\
 441         beq     2f \n\
 442         add     %0,%2,%0 \n"
 443 "       stdcx.  %0,0,%1 \n\
 444         bne-    1b \n"
 445         PPC_ATOMIC_EXIT_BARRIER
 446 "       subf    %0,%2,%0 \n\
 447 2:"
 448         : "=&r" (t)
 449         : "r" (&v->counter), "r" (a), "r" (u)
 450         : "cc", "memory");
 451
 452         return t;
 453 }
 454 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 455
 456 /**
 457  * atomic_inc64_not_zero - increment unless the number is zero
 458  * @v: pointer of type atomic64_t
 459  *
 460  * Atomically increments @v by 1, so long as @v is non-zero.
 461  * Returns non-zero if @v was non-zero, and zero otherwise.
 462  */
 463 static __inline__ int arch_atomic64_inc_not_zero(atomic64_t *v)
 464 {
 465         s64 t1, t2;
 466
 467         __asm__ __volatile__ (
 468         PPC_ATOMIC_ENTRY_BARRIER
 469 "1:     ldarx   %0,0,%2         # atomic64_inc_not_zero\n\
 470         cmpdi   0,%0,0\n\
 471         beq-    2f\n\
 472         addic   %1,%0,1\n\
 473         stdcx.  %1,0,%2\n\
 474         bne-    1b\n"
 475         PPC_ATOMIC_EXIT_BARRIER
 476         "\n\
 477 2:"
 478         : "=&r" (t1), "=&r" (t2)
 479         : "r" (&v->counter)
 480         : "cc", "xer", "memory");
 481
 482         return t1 != 0;
 483 }
 484 #define arch_atomic64_inc_not_zero(v) arch_atomic64_inc_not_zero((v))
 485
 486 #endif /* __powerpc64__ */
 487
 488 #endif /* __KERNEL__ */
 489 #endif /* _ASM_POWERPC_ATOMIC_H_ */