arch/alpha/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ALPHA_ATOMIC_H
   3 #define _ALPHA_ATOMIC_H
   4
   5 #include <linux/types.h>
   6 #include <asm/barrier.h>
   7 #include <asm/cmpxchg.h>
   8
   9 /*
  10  * Atomic operations that C can't guarantee us.  Useful for
  11  * resource counting etc...
  12  *
  13  * But use these as seldom as possible since they are much slower
  14  * than regular operations.
  15  */
  16
  17 /*
  18  * To ensure dependency ordering is preserved for the _relaxed and
  19  * _release atomics, an smp_mb() is unconditionally inserted into the
  20  * _relaxed variants, which are used to build the barriered versions.
  21  * Avoid redundant back-to-back fences in the _acquire and _fence
  22  * versions.
  23  */
  24 #define __atomic_acquire_fence()
  25 #define __atomic_post_full_fence()
  26
  27 #define ATOMIC64_INIT(i)        { (i) }
  28
  29 #define arch_atomic_read(v)     READ_ONCE((v)->counter)
  30 #define arch_atomic64_read(v)   READ_ONCE((v)->counter)
  31
  32 #define arch_atomic_set(v,i)    WRITE_ONCE((v)->counter, (i))
  33 #define arch_atomic64_set(v,i)  WRITE_ONCE((v)->counter, (i))
  34
  35 /*
  36  * To get proper branch prediction for the main line, we must branch
  37  * forward to code at the end of this object's .text section, then
  38  * branch back to restart the operation.
  39  */
  40
  41 #define ATOMIC_OP(op, asm_op)                                           \
  42 static __inline__ void arch_atomic_##op(int i, atomic_t * v)            \
  43 {                                                                       \
  44         unsigned long temp;                                             \
  45         __asm__ __volatile__(                                           \
  46         "1:     ldl_l %0,%1\n"                                          \
  47         "       " #asm_op " %0,%2,%0\n"                                 \
  48         "       stl_c %0,%1\n"                                          \
  49         "       beq %0,2f\n"                                            \
  50         ".subsection 2\n"                                               \
  51         "2:     br 1b\n"                                                \
  52         ".previous"                                                     \
  53         :"=&r" (temp), "=m" (v->counter)                                \
  54         :"Ir" (i), "m" (v->counter));                                   \
  55 }                                                                       \
  56
  57 #define ATOMIC_OP_RETURN(op, asm_op)                                    \
  58 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
  59 {                                                                       \
  60         long temp, result;                                              \
  61         __asm__ __volatile__(                                           \
  62         "1:     ldl_l %0,%1\n"                                          \
  63         "       " #asm_op " %0,%3,%2\n"                                 \
  64         "       " #asm_op " %0,%3,%0\n"                                 \
  65         "       stl_c %0,%1\n"                                          \
  66         "       beq %0,2f\n"                                            \
  67         ".subsection 2\n"                                               \
  68         "2:     br 1b\n"                                                \
  69         ".previous"                                                     \
  70         :"=&r" (temp), "=m" (v->counter), "=&r" (result)                \
  71         :"Ir" (i), "m" (v->counter) : "memory");                        \
  72         smp_mb();                                                       \
  73         return result;                                                  \
  74 }
  75
  76 #define ATOMIC_FETCH_OP(op, asm_op)                                     \
  77 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)  \
  78 {                                                                       \
  79         long temp, result;                                              \
  80         __asm__ __volatile__(                                           \
  81         "1:     ldl_l %2,%1\n"                                          \
  82         "       " #asm_op " %2,%3,%0\n"                                 \
  83         "       stl_c %0,%1\n"                                          \
  84         "       beq %0,2f\n"                                            \
  85         ".subsection 2\n"                                               \
  86         "2:     br 1b\n"                                                \
  87         ".previous"                                                     \
  88         :"=&r" (temp), "=m" (v->counter), "=&r" (result)                \
  89         :"Ir" (i), "m" (v->counter) : "memory");                        \
  90         smp_mb();                                                       \
  91         return result;                                                  \
  92 }
  93
  94 #define ATOMIC64_OP(op, asm_op)                                         \
  95 static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v)        \
  96 {                                                                       \
  97         s64 temp;                                                       \
  98         __asm__ __volatile__(                                           \
  99         "1:     ldq_l %0,%1\n"                                          \
 100         "       " #asm_op " %0,%2,%0\n"                                 \
 101         "       stq_c %0,%1\n"                                          \
 102         "       beq %0,2f\n"                                            \
 103         ".subsection 2\n"                                               \
 104         "2:     br 1b\n"                                                \
 105         ".previous"                                                     \
 106         :"=&r" (temp), "=m" (v->counter)                                \
 107         :"Ir" (i), "m" (v->counter));                                   \
 108 }                                                                       \
 109
 110 #define ATOMIC64_OP_RETURN(op, asm_op)                                  \
 111 static __inline__ s64                                                   \
 112 arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t * v)              \
 113 {                                                                       \
 114         s64 temp, result;                                               \
 115         __asm__ __volatile__(                                           \
 116         "1:     ldq_l %0,%1\n"                                          \
 117         "       " #asm_op " %0,%3,%2\n"                                 \
 118         "       " #asm_op " %0,%3,%0\n"                                 \
 119         "       stq_c %0,%1\n"                                          \
 120         "       beq %0,2f\n"                                            \
 121         ".subsection 2\n"                                               \
 122         "2:     br 1b\n"                                                \
 123         ".previous"                                                     \
 124         :"=&r" (temp), "=m" (v->counter), "=&r" (result)                \
 125         :"Ir" (i), "m" (v->counter) : "memory");                        \
 126         smp_mb();                                                       \
 127         return result;                                                  \
 128 }
 129
 130 #define ATOMIC64_FETCH_OP(op, asm_op)                                   \
 131 static __inline__ s64                                                   \
 132 arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v)               \
 133 {                                                                       \
 134         s64 temp, result;                                               \
 135         __asm__ __volatile__(                                           \
 136         "1:     ldq_l %2,%1\n"                                          \
 137         "       " #asm_op " %2,%3,%0\n"                                 \
 138         "       stq_c %0,%1\n"                                          \
 139         "       beq %0,2f\n"                                            \
 140         ".subsection 2\n"                                               \
 141         "2:     br 1b\n"                                                \
 142         ".previous"                                                     \
 143         :"=&r" (temp), "=m" (v->counter), "=&r" (result)                \
 144         :"Ir" (i), "m" (v->counter) : "memory");                        \
 145         smp_mb();                                                       \
 146         return result;                                                  \
 147 }
 148
 149 #define ATOMIC_OPS(op)                                                  \
 150         ATOMIC_OP(op, op##l)                                            \
 151         ATOMIC_OP_RETURN(op, op##l)                                     \
 152         ATOMIC_FETCH_OP(op, op##l)                                      \
 153         ATOMIC64_OP(op, op##q)                                          \
 154         ATOMIC64_OP_RETURN(op, op##q)                                   \
 155         ATOMIC64_FETCH_OP(op, op##q)
 156
 157 ATOMIC_OPS(add)
 158 ATOMIC_OPS(sub)
 159
 160 #define arch_atomic_add_return_relaxed          arch_atomic_add_return_relaxed
 161 #define arch_atomic_sub_return_relaxed          arch_atomic_sub_return_relaxed
 162 #define arch_atomic_fetch_add_relaxed           arch_atomic_fetch_add_relaxed
 163 #define arch_atomic_fetch_sub_relaxed           arch_atomic_fetch_sub_relaxed
 164
 165 #define arch_atomic64_add_return_relaxed        arch_atomic64_add_return_relaxed
 166 #define arch_atomic64_sub_return_relaxed        arch_atomic64_sub_return_relaxed
 167 #define arch_atomic64_fetch_add_relaxed         arch_atomic64_fetch_add_relaxed
 168 #define arch_atomic64_fetch_sub_relaxed         arch_atomic64_fetch_sub_relaxed
 169
 170 #define arch_atomic_andnot                      arch_atomic_andnot
 171 #define arch_atomic64_andnot                    arch_atomic64_andnot
 172
 173 #undef ATOMIC_OPS
 174 #define ATOMIC_OPS(op, asm)                                             \
 175         ATOMIC_OP(op, asm)                                              \
 176         ATOMIC_FETCH_OP(op, asm)                                        \
 177         ATOMIC64_OP(op, asm)                                            \
 178         ATOMIC64_FETCH_OP(op, asm)
 179
 180 ATOMIC_OPS(and, and)
 181 ATOMIC_OPS(andnot, bic)
 182 ATOMIC_OPS(or, bis)
 183 ATOMIC_OPS(xor, xor)
 184
 185 #define arch_atomic_fetch_and_relaxed           arch_atomic_fetch_and_relaxed
 186 #define arch_atomic_fetch_andnot_relaxed        arch_atomic_fetch_andnot_relaxed
 187 #define arch_atomic_fetch_or_relaxed            arch_atomic_fetch_or_relaxed
 188 #define arch_atomic_fetch_xor_relaxed           arch_atomic_fetch_xor_relaxed
 189
 190 #define arch_atomic64_fetch_and_relaxed         arch_atomic64_fetch_and_relaxed
 191 #define arch_atomic64_fetch_andnot_relaxed      arch_atomic64_fetch_andnot_relaxed
 192 #define arch_atomic64_fetch_or_relaxed          arch_atomic64_fetch_or_relaxed
 193 #define arch_atomic64_fetch_xor_relaxed         arch_atomic64_fetch_xor_relaxed
 194
 195 #undef ATOMIC_OPS
 196 #undef ATOMIC64_FETCH_OP
 197 #undef ATOMIC64_OP_RETURN
 198 #undef ATOMIC64_OP
 199 #undef ATOMIC_FETCH_OP
 200 #undef ATOMIC_OP_RETURN
 201 #undef ATOMIC_OP
 202
 203 static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
 204 {
 205         int c, new, old;
 206         smp_mb();
 207         __asm__ __volatile__(
 208         "1:     ldl_l   %[old],%[mem]\n"
 209         "       cmpeq   %[old],%[u],%[c]\n"
 210         "       addl    %[old],%[a],%[new]\n"
 211         "       bne     %[c],2f\n"
 212         "       stl_c   %[new],%[mem]\n"
 213         "       beq     %[new],3f\n"
 214         "2:\n"
 215         ".subsection 2\n"
 216         "3:     br      1b\n"
 217         ".previous"
 218         : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c)
 219         : [mem] "m"(*v), [a] "rI"(a), [u] "rI"((long)u)
 220         : "memory");
 221         smp_mb();
 222         return old;
 223 }
 224 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
 225
 226 static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 227 {
 228         s64 c, new, old;
 229         smp_mb();
 230         __asm__ __volatile__(
 231         "1:     ldq_l   %[old],%[mem]\n"
 232         "       cmpeq   %[old],%[u],%[c]\n"
 233         "       addq    %[old],%[a],%[new]\n"
 234         "       bne     %[c],2f\n"
 235         "       stq_c   %[new],%[mem]\n"
 236         "       beq     %[new],3f\n"
 237         "2:\n"
 238         ".subsection 2\n"
 239         "3:     br      1b\n"
 240         ".previous"
 241         : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c)
 242         : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u)
 243         : "memory");
 244         smp_mb();
 245         return old;
 246 }
 247 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 248
 249 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 250 {
 251         s64 old, tmp;
 252         smp_mb();
 253         __asm__ __volatile__(
 254         "1:     ldq_l   %[old],%[mem]\n"
 255         "       subq    %[old],1,%[tmp]\n"
 256         "       ble     %[old],2f\n"
 257         "       stq_c   %[tmp],%[mem]\n"
 258         "       beq     %[tmp],3f\n"
 259         "2:\n"
 260         ".subsection 2\n"
 261         "3:     br      1b\n"
 262         ".previous"
 263         : [old] "=&r"(old), [tmp] "=&r"(tmp)
 264         : [mem] "m"(*v)
 265         : "memory");
 266         smp_mb();
 267         return old - 1;
 268 }
 269 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 270
 271 #endif /* _ALPHA_ATOMIC_H */