arch/powerpc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_ATOMIC_H_
   3 #define _ASM_POWERPC_ATOMIC_H_
   4
   5 /*
   6  * PowerPC atomic operations
   7  */
   8
   9 #ifdef __KERNEL__
  10 #include <linux/types.h>
  11 #include <asm/cmpxchg.h>
  12 #include <asm/barrier.h>
  13 #include <asm/asm-const.h>
  14
  15 /*
  16  * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
  17  * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
  18  * on the platform without lwsync.
  19  */
  20 #define __atomic_acquire_fence()                                        \
  21         __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory")
  22
  23 #define __atomic_release_fence()                                        \
  24         __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory")
  25
  26 static __inline__ int arch_atomic_read(const atomic_t *v)
  27 {
  28         int t;
  29
  30         /* -mprefixed can generate offsets beyond range, fall back hack */
  31         if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED))
  32                 __asm__ __volatile__("lwz %0,0(%1)" : "=r"(t) : "b"(&v->counter));
  33         else
  34                 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter));
  35
  36         return t;
  37 }
  38
  39 static __inline__ void arch_atomic_set(atomic_t *v, int i)
  40 {
  41         /* -mprefixed can generate offsets beyond range, fall back hack */
  42         if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED))
  43                 __asm__ __volatile__("stw %1,0(%2)" : "=m"(v->counter) : "r"(i), "b"(&v->counter));
  44         else
  45                 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i));
  46 }
  47
  48 #define ATOMIC_OP(op, asm_op, suffix, sign, ...)                        \
  49 static __inline__ void arch_atomic_##op(int a, atomic_t *v)             \
  50 {                                                                       \
  51         int t;                                                          \
  52                                                                         \
  53         __asm__ __volatile__(                                           \
  54 "1:     lwarx   %0,0,%3         # atomic_" #op "\n"                     \
  55         #asm_op "%I2" suffix " %0,%0,%2\n"                              \
  56 "       stwcx.  %0,0,%3 \n"                                             \
  57 "       bne-    1b\n"                                                   \
  58         : "=&r" (t), "+m" (v->counter)                                  \
  59         : "r"#sign (a), "r" (&v->counter)                               \
  60         : "cc", ##__VA_ARGS__);                                         \
  61 }                                                                       \
  62
  63 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ...)         \
  64 static inline int arch_atomic_##op##_return_relaxed(int a, atomic_t *v) \
  65 {                                                                       \
  66         int t;                                                          \
  67                                                                         \
  68         __asm__ __volatile__(                                           \
  69 "1:     lwarx   %0,0,%3         # atomic_" #op "_return_relaxed\n"      \
  70         #asm_op "%I2" suffix " %0,%0,%2\n"                              \
  71 "       stwcx.  %0,0,%3\n"                                              \
  72 "       bne-    1b\n"                                                   \
  73         : "=&r" (t), "+m" (v->counter)                                  \
  74         : "r"#sign (a), "r" (&v->counter)                               \
  75         : "cc", ##__VA_ARGS__);                                         \
  76                                                                         \
  77         return t;                                                       \
  78 }
  79
  80 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ...)          \
  81 static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v)  \
  82 {                                                                       \
  83         int res, t;                                                     \
  84                                                                         \
  85         __asm__ __volatile__(                                           \
  86 "1:     lwarx   %0,0,%4         # atomic_fetch_" #op "_relaxed\n"       \
  87         #asm_op "%I3" suffix " %1,%0,%3\n"                              \
  88 "       stwcx.  %1,0,%4\n"                                              \
  89 "       bne-    1b\n"                                                   \
  90         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
  91         : "r"#sign (a), "r" (&v->counter)                               \
  92         : "cc", ##__VA_ARGS__);                                         \
  93                                                                         \
  94         return res;                                                     \
  95 }
  96
  97 #define ATOMIC_OPS(op, asm_op, suffix, sign, ...)                       \
  98         ATOMIC_OP(op, asm_op, suffix, sign, ##__VA_ARGS__)              \
  99         ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)\
 100         ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)
 101
 102 ATOMIC_OPS(add, add, "c", I, "xer")
 103 ATOMIC_OPS(sub, sub, "c", I, "xer")
 104
 105 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
 106 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
 107
 108 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
 109 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
 110
 111 #undef ATOMIC_OPS
 112 #define ATOMIC_OPS(op, asm_op, suffix, sign)                            \
 113         ATOMIC_OP(op, asm_op, suffix, sign)                             \
 114         ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign)
 115
 116 ATOMIC_OPS(and, and, ".", K)
 117 ATOMIC_OPS(or, or, "", K)
 118 ATOMIC_OPS(xor, xor, "", K)
 119
 120 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
 121 #define arch_atomic_fetch_or_relaxed  arch_atomic_fetch_or_relaxed
 122 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
 123
 124 #undef ATOMIC_OPS
 125 #undef ATOMIC_FETCH_OP_RELAXED
 126 #undef ATOMIC_OP_RETURN_RELAXED
 127 #undef ATOMIC_OP
 128
 129 /**
 130  * atomic_fetch_add_unless - add unless the number is a given value
 131  * @v: pointer of type atomic_t
 132  * @a: the amount to add to v...
 133  * @u: ...unless v is equal to u.
 134  *
 135  * Atomically adds @a to @v, so long as it was not @u.
 136  * Returns the old value of @v.
 137  */
 138 static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
 139 {
 140         int t;
 141
 142         __asm__ __volatile__ (
 143         PPC_ATOMIC_ENTRY_BARRIER
 144 "1:     lwarx   %0,0,%1         # atomic_fetch_add_unless\n\
 145         cmpw    0,%0,%3 \n\
 146         beq     2f \n\
 147         add%I2c %0,%0,%2 \n"
 148 "       stwcx.  %0,0,%1 \n\
 149         bne-    1b \n"
 150         PPC_ATOMIC_EXIT_BARRIER
 151 "       sub%I2c %0,%0,%2 \n\
 152 2:"
 153         : "=&r" (t)
 154         : "r" (&v->counter), "rI" (a), "r" (u)
 155         : "cc", "memory", "xer");
 156
 157         return t;
 158 }
 159 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
 160
 161 /*
 162  * Atomically test *v and decrement if it is greater than 0.
 163  * The function returns the old value of *v minus 1, even if
 164  * the atomic variable, v, was not decremented.
 165  */
 166 static __inline__ int arch_atomic_dec_if_positive(atomic_t *v)
 167 {
 168         int t;
 169
 170         __asm__ __volatile__(
 171         PPC_ATOMIC_ENTRY_BARRIER
 172 "1:     lwarx   %0,0,%1         # atomic_dec_if_positive\n\
 173         cmpwi   %0,1\n\
 174         addi    %0,%0,-1\n\
 175         blt-    2f\n"
 176 "       stwcx.  %0,0,%1\n\
 177         bne-    1b"
 178         PPC_ATOMIC_EXIT_BARRIER
 179         "\n\
 180 2:"     : "=&b" (t)
 181         : "r" (&v->counter)
 182         : "cc", "memory");
 183
 184         return t;
 185 }
 186 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
 187
 188 #ifdef __powerpc64__
 189
 190 #define ATOMIC64_INIT(i)        { (i) }
 191
 192 static __inline__ s64 arch_atomic64_read(const atomic64_t *v)
 193 {
 194         s64 t;
 195
 196         /* -mprefixed can generate offsets beyond range, fall back hack */
 197         if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED))
 198                 __asm__ __volatile__("ld %0,0(%1)" : "=r"(t) : "b"(&v->counter));
 199         else
 200                 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter));
 201
 202         return t;
 203 }
 204
 205 static __inline__ void arch_atomic64_set(atomic64_t *v, s64 i)
 206 {
 207         /* -mprefixed can generate offsets beyond range, fall back hack */
 208         if (IS_ENABLED(CONFIG_PPC_KERNEL_PREFIXED))
 209                 __asm__ __volatile__("std %1,0(%2)" : "=m"(v->counter) : "r"(i), "b"(&v->counter));
 210         else
 211                 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i));
 212 }
 213
 214 #define ATOMIC64_OP(op, asm_op)                                         \
 215 static __inline__ void arch_atomic64_##op(s64 a, atomic64_t *v)         \
 216 {                                                                       \
 217         s64 t;                                                          \
 218                                                                         \
 219         __asm__ __volatile__(                                           \
 220 "1:     ldarx   %0,0,%3         # atomic64_" #op "\n"                   \
 221         #asm_op " %0,%2,%0\n"                                           \
 222 "       stdcx.  %0,0,%3 \n"                                             \
 223 "       bne-    1b\n"                                                   \
 224         : "=&r" (t), "+m" (v->counter)                                  \
 225         : "r" (a), "r" (&v->counter)                                    \
 226         : "cc");                                                        \
 227 }
 228
 229 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 230 static inline s64                                                       \
 231 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v)               \
 232 {                                                                       \
 233         s64 t;                                                          \
 234                                                                         \
 235         __asm__ __volatile__(                                           \
 236 "1:     ldarx   %0,0,%3         # atomic64_" #op "_return_relaxed\n"    \
 237         #asm_op " %0,%2,%0\n"                                           \
 238 "       stdcx.  %0,0,%3\n"                                              \
 239 "       bne-    1b\n"                                                   \
 240         : "=&r" (t), "+m" (v->counter)                                  \
 241         : "r" (a), "r" (&v->counter)                                    \
 242         : "cc");                                                        \
 243                                                                         \
 244         return t;                                                       \
 245 }
 246
 247 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op)                           \
 248 static inline s64                                                       \
 249 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v)                \
 250 {                                                                       \
 251         s64 res, t;                                                     \
 252                                                                         \
 253         __asm__ __volatile__(                                           \
 254 "1:     ldarx   %0,0,%4         # atomic64_fetch_" #op "_relaxed\n"     \
 255         #asm_op " %1,%3,%0\n"                                           \
 256 "       stdcx.  %1,0,%4\n"                                              \
 257 "       bne-    1b\n"                                                   \
 258         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
 259         : "r" (a), "r" (&v->counter)                                    \
 260         : "cc");                                                        \
 261                                                                         \
 262         return res;                                                     \
 263 }
 264
 265 #define ATOMIC64_OPS(op, asm_op)                                        \
 266         ATOMIC64_OP(op, asm_op)                                         \
 267         ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 268         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 269
 270 ATOMIC64_OPS(add, add)
 271 ATOMIC64_OPS(sub, subf)
 272
 273 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
 274 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
 275
 276 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
 277 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
 278
 279 #undef ATOMIC64_OPS
 280 #define ATOMIC64_OPS(op, asm_op)                                        \
 281         ATOMIC64_OP(op, asm_op)                                         \
 282         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 283
 284 ATOMIC64_OPS(and, and)
 285 ATOMIC64_OPS(or, or)
 286 ATOMIC64_OPS(xor, xor)
 287
 288 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
 289 #define arch_atomic64_fetch_or_relaxed  arch_atomic64_fetch_or_relaxed
 290 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
 291
 292 #undef ATOPIC64_OPS
 293 #undef ATOMIC64_FETCH_OP_RELAXED
 294 #undef ATOMIC64_OP_RETURN_RELAXED
 295 #undef ATOMIC64_OP
 296
 297 static __inline__ void arch_atomic64_inc(atomic64_t *v)
 298 {
 299         s64 t;
 300
 301         __asm__ __volatile__(
 302 "1:     ldarx   %0,0,%2         # atomic64_inc\n\
 303         addic   %0,%0,1\n\
 304         stdcx.  %0,0,%2 \n\
 305         bne-    1b"
 306         : "=&r" (t), "+m" (v->counter)
 307         : "r" (&v->counter)
 308         : "cc", "xer");
 309 }
 310 #define arch_atomic64_inc arch_atomic64_inc
 311
 312 static __inline__ s64 arch_atomic64_inc_return_relaxed(atomic64_t *v)
 313 {
 314         s64 t;
 315
 316         __asm__ __volatile__(
 317 "1:     ldarx   %0,0,%2         # atomic64_inc_return_relaxed\n"
 318 "       addic   %0,%0,1\n"
 319 "       stdcx.  %0,0,%2\n"
 320 "       bne-    1b"
 321         : "=&r" (t), "+m" (v->counter)
 322         : "r" (&v->counter)
 323         : "cc", "xer");
 324
 325         return t;
 326 }
 327
 328 static __inline__ void arch_atomic64_dec(atomic64_t *v)
 329 {
 330         s64 t;
 331
 332         __asm__ __volatile__(
 333 "1:     ldarx   %0,0,%2         # atomic64_dec\n\
 334         addic   %0,%0,-1\n\
 335         stdcx.  %0,0,%2\n\
 336         bne-    1b"
 337         : "=&r" (t), "+m" (v->counter)
 338         : "r" (&v->counter)
 339         : "cc", "xer");
 340 }
 341 #define arch_atomic64_dec arch_atomic64_dec
 342
 343 static __inline__ s64 arch_atomic64_dec_return_relaxed(atomic64_t *v)
 344 {
 345         s64 t;
 346
 347         __asm__ __volatile__(
 348 "1:     ldarx   %0,0,%2         # atomic64_dec_return_relaxed\n"
 349 "       addic   %0,%0,-1\n"
 350 "       stdcx.  %0,0,%2\n"
 351 "       bne-    1b"
 352         : "=&r" (t), "+m" (v->counter)
 353         : "r" (&v->counter)
 354         : "cc", "xer");
 355
 356         return t;
 357 }
 358
 359 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
 360 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
 361
 362 /*
 363  * Atomically test *v and decrement if it is greater than 0.
 364  * The function returns the old value of *v minus 1.
 365  */
 366 static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 367 {
 368         s64 t;
 369
 370         __asm__ __volatile__(
 371         PPC_ATOMIC_ENTRY_BARRIER
 372 "1:     ldarx   %0,0,%1         # atomic64_dec_if_positive\n\
 373         addic.  %0,%0,-1\n\
 374         blt-    2f\n\
 375         stdcx.  %0,0,%1\n\
 376         bne-    1b"
 377         PPC_ATOMIC_EXIT_BARRIER
 378         "\n\
 379 2:"     : "=&r" (t)
 380         : "r" (&v->counter)
 381         : "cc", "xer", "memory");
 382
 383         return t;
 384 }
 385 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 386
 387 /**
 388  * atomic64_fetch_add_unless - add unless the number is a given value
 389  * @v: pointer of type atomic64_t
 390  * @a: the amount to add to v...
 391  * @u: ...unless v is equal to u.
 392  *
 393  * Atomically adds @a to @v, so long as it was not @u.
 394  * Returns the old value of @v.
 395  */
 396 static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 397 {
 398         s64 t;
 399
 400         __asm__ __volatile__ (
 401         PPC_ATOMIC_ENTRY_BARRIER
 402 "1:     ldarx   %0,0,%1         # atomic64_fetch_add_unless\n\
 403         cmpd    0,%0,%3 \n\
 404         beq     2f \n\
 405         add     %0,%2,%0 \n"
 406 "       stdcx.  %0,0,%1 \n\
 407         bne-    1b \n"
 408         PPC_ATOMIC_EXIT_BARRIER
 409 "       subf    %0,%2,%0 \n\
 410 2:"
 411         : "=&r" (t)
 412         : "r" (&v->counter), "r" (a), "r" (u)
 413         : "cc", "memory");
 414
 415         return t;
 416 }
 417 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 418
 419 /**
 420  * atomic_inc64_not_zero - increment unless the number is zero
 421  * @v: pointer of type atomic64_t
 422  *
 423  * Atomically increments @v by 1, so long as @v is non-zero.
 424  * Returns non-zero if @v was non-zero, and zero otherwise.
 425  */
 426 static __inline__ int arch_atomic64_inc_not_zero(atomic64_t *v)
 427 {
 428         s64 t1, t2;
 429
 430         __asm__ __volatile__ (
 431         PPC_ATOMIC_ENTRY_BARRIER
 432 "1:     ldarx   %0,0,%2         # atomic64_inc_not_zero\n\
 433         cmpdi   0,%0,0\n\
 434         beq-    2f\n\
 435         addic   %1,%0,1\n\
 436         stdcx.  %1,0,%2\n\
 437         bne-    1b\n"
 438         PPC_ATOMIC_EXIT_BARRIER
 439         "\n\
 440 2:"
 441         : "=&r" (t1), "=&r" (t2)
 442         : "r" (&v->counter)
 443         : "cc", "xer", "memory");
 444
 445         return t1 != 0;
 446 }
 447 #define arch_atomic64_inc_not_zero(v) arch_atomic64_inc_not_zero((v))
 448
 449 #endif /* __powerpc64__ */
 450
 451 #endif /* __KERNEL__ */
 452 #endif /* _ASM_POWERPC_ATOMIC_H_ */