arch/loongarch/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Atomic operations.
   4  *
   5  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   6  */
   7 #ifndef _ASM_ATOMIC_H
   8 #define _ASM_ATOMIC_H
   9
  10 #include <linux/types.h>
  11 #include <asm/barrier.h>
  12 #include <asm/cmpxchg.h>
  13
  14 #if __SIZEOF_LONG__ == 4
  15 #define __LL            "ll.w   "
  16 #define __SC            "sc.w   "
  17 #define __AMADD         "amadd.w        "
  18 #define __AMAND_DB      "amand_db.w     "
  19 #define __AMOR_DB       "amor_db.w      "
  20 #define __AMXOR_DB      "amxor_db.w     "
  21 #elif __SIZEOF_LONG__ == 8
  22 #define __LL            "ll.d   "
  23 #define __SC            "sc.d   "
  24 #define __AMADD         "amadd.d        "
  25 #define __AMAND_DB      "amand_db.d     "
  26 #define __AMOR_DB       "amor_db.d      "
  27 #define __AMXOR_DB      "amxor_db.d     "
  28 #endif
  29
  30 #define ATOMIC_INIT(i)    { (i) }
  31
  32 #define arch_atomic_read(v)     READ_ONCE((v)->counter)
  33 #define arch_atomic_set(v, i)   WRITE_ONCE((v)->counter, (i))
  34
  35 #define ATOMIC_OP(op, I, asm_op)                                        \
  36 static inline void arch_atomic_##op(int i, atomic_t *v)                 \
  37 {                                                                       \
  38         __asm__ __volatile__(                                           \
  39         "am"#asm_op".w" " $zero, %1, %0 \n"                             \
  40         : "+ZB" (v->counter)                                            \
  41         : "r" (I)                                                       \
  42         : "memory");                                                    \
  43 }
  44
  45 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix)               \
  46 static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
  47 {                                                                       \
  48         int result;                                                     \
  49                                                                         \
  50         __asm__ __volatile__(                                           \
  51         "am"#asm_op#mb".w" " %1, %2, %0         \n"                     \
  52         : "+ZB" (v->counter), "=&r" (result)                            \
  53         : "r" (I)                                                       \
  54         : "memory");                                                    \
  55                                                                         \
  56         return result c_op I;                                           \
  57 }
  58
  59 #define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix)                      \
  60 static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v)    \
  61 {                                                                       \
  62         int result;                                                     \
  63                                                                         \
  64         __asm__ __volatile__(                                           \
  65         "am"#asm_op#mb".w" " %1, %2, %0         \n"                     \
  66         : "+ZB" (v->counter), "=&r" (result)                            \
  67         : "r" (I)                                                       \
  68         : "memory");                                                    \
  69                                                                         \
  70         return result;                                                  \
  71 }
  72
  73 #define ATOMIC_OPS(op, I, asm_op, c_op)                                 \
  74         ATOMIC_OP(op, I, asm_op)                                        \
  75         ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db,         )            \
  76         ATOMIC_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)            \
  77         ATOMIC_FETCH_OP(op, I, asm_op, _db,         )                   \
  78         ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)
  79
  80 ATOMIC_OPS(add, i, add, +)
  81 ATOMIC_OPS(sub, -i, add, +)
  82
  83 #define arch_atomic_add_return          arch_atomic_add_return
  84 #define arch_atomic_add_return_acquire  arch_atomic_add_return
  85 #define arch_atomic_add_return_release  arch_atomic_add_return
  86 #define arch_atomic_add_return_relaxed  arch_atomic_add_return_relaxed
  87 #define arch_atomic_sub_return          arch_atomic_sub_return
  88 #define arch_atomic_sub_return_acquire  arch_atomic_sub_return
  89 #define arch_atomic_sub_return_release  arch_atomic_sub_return
  90 #define arch_atomic_sub_return_relaxed  arch_atomic_sub_return_relaxed
  91 #define arch_atomic_fetch_add           arch_atomic_fetch_add
  92 #define arch_atomic_fetch_add_acquire   arch_atomic_fetch_add
  93 #define arch_atomic_fetch_add_release   arch_atomic_fetch_add
  94 #define arch_atomic_fetch_add_relaxed   arch_atomic_fetch_add_relaxed
  95 #define arch_atomic_fetch_sub           arch_atomic_fetch_sub
  96 #define arch_atomic_fetch_sub_acquire   arch_atomic_fetch_sub
  97 #define arch_atomic_fetch_sub_release   arch_atomic_fetch_sub
  98 #define arch_atomic_fetch_sub_relaxed   arch_atomic_fetch_sub_relaxed
  99
 100 #undef ATOMIC_OPS
 101
 102 #define ATOMIC_OPS(op, I, asm_op)                                       \
 103         ATOMIC_OP(op, I, asm_op)                                        \
 104         ATOMIC_FETCH_OP(op, I, asm_op, _db,         )                   \
 105         ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)
 106
 107 ATOMIC_OPS(and, i, and)
 108 ATOMIC_OPS(or, i, or)
 109 ATOMIC_OPS(xor, i, xor)
 110
 111 #define arch_atomic_fetch_and           arch_atomic_fetch_and
 112 #define arch_atomic_fetch_and_acquire   arch_atomic_fetch_and
 113 #define arch_atomic_fetch_and_release   arch_atomic_fetch_and
 114 #define arch_atomic_fetch_and_relaxed   arch_atomic_fetch_and_relaxed
 115 #define arch_atomic_fetch_or            arch_atomic_fetch_or
 116 #define arch_atomic_fetch_or_acquire    arch_atomic_fetch_or
 117 #define arch_atomic_fetch_or_release    arch_atomic_fetch_or
 118 #define arch_atomic_fetch_or_relaxed    arch_atomic_fetch_or_relaxed
 119 #define arch_atomic_fetch_xor           arch_atomic_fetch_xor
 120 #define arch_atomic_fetch_xor_acquire   arch_atomic_fetch_xor
 121 #define arch_atomic_fetch_xor_release   arch_atomic_fetch_xor
 122 #define arch_atomic_fetch_xor_relaxed   arch_atomic_fetch_xor_relaxed
 123
 124 #undef ATOMIC_OPS
 125 #undef ATOMIC_FETCH_OP
 126 #undef ATOMIC_OP_RETURN
 127 #undef ATOMIC_OP
 128
 129 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
 130 {
 131        int prev, rc;
 132
 133         __asm__ __volatile__ (
 134                 "0:     ll.w    %[p],  %[c]\n"
 135                 "       beq     %[p],  %[u], 1f\n"
 136                 "       add.w   %[rc], %[p], %[a]\n"
 137                 "       sc.w    %[rc], %[c]\n"
 138                 "       beqz    %[rc], 0b\n"
 139                 "       b       2f\n"
 140                 "1:\n"
 141                 __WEAK_LLSC_MB
 142                 "2:\n"
 143                 : [p]"=&r" (prev), [rc]"=&r" (rc),
 144                   [c]"=ZB" (v->counter)
 145                 : [a]"r" (a), [u]"r" (u)
 146                 : "memory");
 147
 148         return prev;
 149 }
 150 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
 151
 152 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
 153 {
 154         int result;
 155         int temp;
 156
 157         if (__builtin_constant_p(i)) {
 158                 __asm__ __volatile__(
 159                 "1:     ll.w    %1, %2          # atomic_sub_if_positive\n"
 160                 "       addi.w  %0, %1, %3                              \n"
 161                 "       move    %1, %0                                  \n"
 162                 "       bltz    %0, 2f                                  \n"
 163                 "       sc.w    %1, %2                                  \n"
 164                 "       beqz    %1, 1b                                  \n"
 165                 "2:                                                     \n"
 166                 __WEAK_LLSC_MB
 167                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 168                 : "I" (-i));
 169         } else {
 170                 __asm__ __volatile__(
 171                 "1:     ll.w    %1, %2          # atomic_sub_if_positive\n"
 172                 "       sub.w   %0, %1, %3                              \n"
 173                 "       move    %1, %0                                  \n"
 174                 "       bltz    %0, 2f                                  \n"
 175                 "       sc.w    %1, %2                                  \n"
 176                 "       beqz    %1, 1b                                  \n"
 177                 "2:                                                     \n"
 178                 __WEAK_LLSC_MB
 179                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 180                 : "r" (i));
 181         }
 182
 183         return result;
 184 }
 185
 186 #define arch_atomic_dec_if_positive(v)  arch_atomic_sub_if_positive(1, v)
 187
 188 #ifdef CONFIG_64BIT
 189
 190 #define ATOMIC64_INIT(i)    { (i) }
 191
 192 #define arch_atomic64_read(v)   READ_ONCE((v)->counter)
 193 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
 194
 195 #define ATOMIC64_OP(op, I, asm_op)                                      \
 196 static inline void arch_atomic64_##op(long i, atomic64_t *v)            \
 197 {                                                                       \
 198         __asm__ __volatile__(                                           \
 199         "am"#asm_op".d " " $zero, %1, %0        \n"                     \
 200         : "+ZB" (v->counter)                                            \
 201         : "r" (I)                                                       \
 202         : "memory");                                                    \
 203 }
 204
 205 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix)                     \
 206 static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v)   \
 207 {                                                                               \
 208         long result;                                                            \
 209         __asm__ __volatile__(                                                   \
 210         "am"#asm_op#mb".d " " %1, %2, %0                \n"                     \
 211         : "+ZB" (v->counter), "=&r" (result)                                    \
 212         : "r" (I)                                                               \
 213         : "memory");                                                            \
 214                                                                                 \
 215         return result c_op I;                                                   \
 216 }
 217
 218 #define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix)                            \
 219 static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v)      \
 220 {                                                                               \
 221         long result;                                                            \
 222                                                                                 \
 223         __asm__ __volatile__(                                                   \
 224         "am"#asm_op#mb".d " " %1, %2, %0                \n"                     \
 225         : "+ZB" (v->counter), "=&r" (result)                                    \
 226         : "r" (I)                                                               \
 227         : "memory");                                                            \
 228                                                                                 \
 229         return result;                                                          \
 230 }
 231
 232 #define ATOMIC64_OPS(op, I, asm_op, c_op)                                     \
 233         ATOMIC64_OP(op, I, asm_op)                                            \
 234         ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db,         )                \
 235         ATOMIC64_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)                \
 236         ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )                       \
 237         ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)
 238
 239 ATOMIC64_OPS(add, i, add, +)
 240 ATOMIC64_OPS(sub, -i, add, +)
 241
 242 #define arch_atomic64_add_return                arch_atomic64_add_return
 243 #define arch_atomic64_add_return_acquire        arch_atomic64_add_return
 244 #define arch_atomic64_add_return_release        arch_atomic64_add_return
 245 #define arch_atomic64_add_return_relaxed        arch_atomic64_add_return_relaxed
 246 #define arch_atomic64_sub_return                arch_atomic64_sub_return
 247 #define arch_atomic64_sub_return_acquire        arch_atomic64_sub_return
 248 #define arch_atomic64_sub_return_release        arch_atomic64_sub_return
 249 #define arch_atomic64_sub_return_relaxed        arch_atomic64_sub_return_relaxed
 250 #define arch_atomic64_fetch_add                 arch_atomic64_fetch_add
 251 #define arch_atomic64_fetch_add_acquire         arch_atomic64_fetch_add
 252 #define arch_atomic64_fetch_add_release         arch_atomic64_fetch_add
 253 #define arch_atomic64_fetch_add_relaxed         arch_atomic64_fetch_add_relaxed
 254 #define arch_atomic64_fetch_sub                 arch_atomic64_fetch_sub
 255 #define arch_atomic64_fetch_sub_acquire         arch_atomic64_fetch_sub
 256 #define arch_atomic64_fetch_sub_release         arch_atomic64_fetch_sub
 257 #define arch_atomic64_fetch_sub_relaxed         arch_atomic64_fetch_sub_relaxed
 258
 259 #undef ATOMIC64_OPS
 260
 261 #define ATOMIC64_OPS(op, I, asm_op)                                           \
 262         ATOMIC64_OP(op, I, asm_op)                                            \
 263         ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )                       \
 264         ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)
 265
 266 ATOMIC64_OPS(and, i, and)
 267 ATOMIC64_OPS(or, i, or)
 268 ATOMIC64_OPS(xor, i, xor)
 269
 270 #define arch_atomic64_fetch_and         arch_atomic64_fetch_and
 271 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
 272 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
 273 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
 274 #define arch_atomic64_fetch_or          arch_atomic64_fetch_or
 275 #define arch_atomic64_fetch_or_acquire  arch_atomic64_fetch_or
 276 #define arch_atomic64_fetch_or_release  arch_atomic64_fetch_or
 277 #define arch_atomic64_fetch_or_relaxed  arch_atomic64_fetch_or_relaxed
 278 #define arch_atomic64_fetch_xor         arch_atomic64_fetch_xor
 279 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
 280 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
 281 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
 282
 283 #undef ATOMIC64_OPS
 284 #undef ATOMIC64_FETCH_OP
 285 #undef ATOMIC64_OP_RETURN
 286 #undef ATOMIC64_OP
 287
 288 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
 289 {
 290        long prev, rc;
 291
 292         __asm__ __volatile__ (
 293                 "0:     ll.d    %[p],  %[c]\n"
 294                 "       beq     %[p],  %[u], 1f\n"
 295                 "       add.d   %[rc], %[p], %[a]\n"
 296                 "       sc.d    %[rc], %[c]\n"
 297                 "       beqz    %[rc], 0b\n"
 298                 "       b       2f\n"
 299                 "1:\n"
 300                 __WEAK_LLSC_MB
 301                 "2:\n"
 302                 : [p]"=&r" (prev), [rc]"=&r" (rc),
 303                   [c] "=ZB" (v->counter)
 304                 : [a]"r" (a), [u]"r" (u)
 305                 : "memory");
 306
 307         return prev;
 308 }
 309 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 310
 311 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
 312 {
 313         long result;
 314         long temp;
 315
 316         if (__builtin_constant_p(i)) {
 317                 __asm__ __volatile__(
 318                 "1:     ll.d    %1, %2  # atomic64_sub_if_positive      \n"
 319                 "       addi.d  %0, %1, %3                              \n"
 320                 "       move    %1, %0                                  \n"
 321                 "       bltz    %0, 2f                                  \n"
 322                 "       sc.d    %1, %2                                  \n"
 323                 "       beqz    %1, 1b                                  \n"
 324                 "2:                                                     \n"
 325                 __WEAK_LLSC_MB
 326                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 327                 : "I" (-i));
 328         } else {
 329                 __asm__ __volatile__(
 330                 "1:     ll.d    %1, %2  # atomic64_sub_if_positive      \n"
 331                 "       sub.d   %0, %1, %3                              \n"
 332                 "       move    %1, %0                                  \n"
 333                 "       bltz    %0, 2f                                  \n"
 334                 "       sc.d    %1, %2                                  \n"
 335                 "       beqz    %1, 1b                                  \n"
 336                 "2:                                                     \n"
 337                 __WEAK_LLSC_MB
 338                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 339                 : "r" (i));
 340         }
 341
 342         return result;
 343 }
 344
 345 #define arch_atomic64_dec_if_positive(v)        arch_atomic64_sub_if_positive(1, v)
 346
 347 #endif /* CONFIG_64BIT */
 348
 349 #endif /* _ASM_ATOMIC_H */