arch/arc/include/asm/atomic.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef _ASM_ARC_ATOMIC_H
  10 #define _ASM_ARC_ATOMIC_H
  11
  12 #ifndef __ASSEMBLY__
  13
  14 #include <linux/types.h>
  15 #include <linux/compiler.h>
  16 #include <asm/cmpxchg.h>
  17 #include <asm/barrier.h>
  18 #include <asm/smp.h>
  19
  20 #define ATOMIC_INIT(i)  { (i) }
  21
  22 #ifndef CONFIG_ARC_PLAT_EZNPS
  23
  24 #define atomic_read(v)  READ_ONCE((v)->counter)
  25
  26 #ifdef CONFIG_ARC_HAS_LLSC
  27
  28 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  29
  30 #define ATOMIC_OP(op, c_op, asm_op)                                     \
  31 static inline void atomic_##op(int i, atomic_t *v)                      \
  32 {                                                                       \
  33         unsigned int val;                                               \
  34                                                                         \
  35         __asm__ __volatile__(                                           \
  36         "1:     llock   %[val], [%[ctr]]                \n"             \
  37         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  38         "       scond   %[val], [%[ctr]]                \n"             \
  39         "       bnz     1b                              \n"             \
  40         : [val] "=&r"   (val) /* Early clobber to prevent reg reuse */  \
  41         : [ctr] "r"     (&v->counter), /* Not "m": llock only supports reg direct addr mode */  \
  42           [i]   "ir"    (i)                                             \
  43         : "cc");                                                        \
  44 }                                                                       \
  45
  46 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
  47 static inline int atomic_##op##_return(int i, atomic_t *v)              \
  48 {                                                                       \
  49         unsigned int val;                                               \
  50                                                                         \
  51         /*                                                              \
  52          * Explicit full memory barrier needed before/after as          \
  53          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  54          */                                                             \
  55         smp_mb();                                                       \
  56                                                                         \
  57         __asm__ __volatile__(                                           \
  58         "1:     llock   %[val], [%[ctr]]                \n"             \
  59         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  60         "       scond   %[val], [%[ctr]]                \n"             \
  61         "       bnz     1b                              \n"             \
  62         : [val] "=&r"   (val)                                           \
  63         : [ctr] "r"     (&v->counter),                                  \
  64           [i]   "ir"    (i)                                             \
  65         : "cc");                                                        \
  66                                                                         \
  67         smp_mb();                                                       \
  68                                                                         \
  69         return val;                                                     \
  70 }
  71
  72 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
  73 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
  74 {                                                                       \
  75         unsigned int val, orig;                                         \
  76                                                                         \
  77         /*                                                              \
  78          * Explicit full memory barrier needed before/after as          \
  79          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  80          */                                                             \
  81         smp_mb();                                                       \
  82                                                                         \
  83         __asm__ __volatile__(                                           \
  84         "1:     llock   %[orig], [%[ctr]]               \n"             \
  85         "       " #asm_op " %[val], %[orig], %[i]       \n"             \
  86         "       scond   %[val], [%[ctr]]                \n"             \
  87         "       bnz     1b                              \n"             \
  88         : [val] "=&r"   (val),                                          \
  89           [orig] "=&r" (orig)                                           \
  90         : [ctr] "r"     (&v->counter),                                  \
  91           [i]   "ir"    (i)                                             \
  92         : "cc");                                                        \
  93                                                                         \
  94         smp_mb();                                                       \
  95                                                                         \
  96         return orig;                                                    \
  97 }
  98
  99 #else   /* !CONFIG_ARC_HAS_LLSC */
 100
 101 #ifndef CONFIG_SMP
 102
 103  /* violating atomic_xxx API locking protocol in UP for optimization sake */
 104 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
 105
 106 #else
 107
 108 static inline void atomic_set(atomic_t *v, int i)
 109 {
 110         /*
 111          * Independent of hardware support, all of the atomic_xxx() APIs need
 112          * to follow the same locking rules to make sure that a "hardware"
 113          * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
 114          * sequence
 115          *
 116          * Thus atomic_set() despite being 1 insn (and seemingly atomic)
 117          * requires the locking.
 118          */
 119         unsigned long flags;
 120
 121         atomic_ops_lock(flags);
 122         WRITE_ONCE(v->counter, i);
 123         atomic_ops_unlock(flags);
 124 }
 125
 126 #define atomic_set_release(v, i)        atomic_set((v), (i))
 127
 128 #endif
 129
 130 /*
 131  * Non hardware assisted Atomic-R-M-W
 132  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 133  */
 134
 135 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 136 static inline void atomic_##op(int i, atomic_t *v)                      \
 137 {                                                                       \
 138         unsigned long flags;                                            \
 139                                                                         \
 140         atomic_ops_lock(flags);                                         \
 141         v->counter c_op i;                                              \
 142         atomic_ops_unlock(flags);                                       \
 143 }
 144
 145 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 146 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 147 {                                                                       \
 148         unsigned long flags;                                            \
 149         unsigned long temp;                                             \
 150                                                                         \
 151         /*                                                              \
 152          * spin lock/unlock provides the needed smp_mb() before/after   \
 153          */                                                             \
 154         atomic_ops_lock(flags);                                         \
 155         temp = v->counter;                                              \
 156         temp c_op i;                                                    \
 157         v->counter = temp;                                              \
 158         atomic_ops_unlock(flags);                                       \
 159                                                                         \
 160         return temp;                                                    \
 161 }
 162
 163 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 164 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 165 {                                                                       \
 166         unsigned long flags;                                            \
 167         unsigned long orig;                                             \
 168                                                                         \
 169         /*                                                              \
 170          * spin lock/unlock provides the needed smp_mb() before/after   \
 171          */                                                             \
 172         atomic_ops_lock(flags);                                         \
 173         orig = v->counter;                                              \
 174         v->counter c_op i;                                              \
 175         atomic_ops_unlock(flags);                                       \
 176                                                                         \
 177         return orig;                                                    \
 178 }
 179
 180 #endif /* !CONFIG_ARC_HAS_LLSC */
 181
 182 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 183         ATOMIC_OP(op, c_op, asm_op)                                     \
 184         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 185         ATOMIC_FETCH_OP(op, c_op, asm_op)
 186
 187 ATOMIC_OPS(add, +=, add)
 188 ATOMIC_OPS(sub, -=, sub)
 189
 190 #define atomic_andnot           atomic_andnot
 191 #define atomic_fetch_andnot     atomic_fetch_andnot
 192
 193 #undef ATOMIC_OPS
 194 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 195         ATOMIC_OP(op, c_op, asm_op)                                     \
 196         ATOMIC_FETCH_OP(op, c_op, asm_op)
 197
 198 ATOMIC_OPS(and, &=, and)
 199 ATOMIC_OPS(andnot, &= ~, bic)
 200 ATOMIC_OPS(or, |=, or)
 201 ATOMIC_OPS(xor, ^=, xor)
 202
 203 #else /* CONFIG_ARC_PLAT_EZNPS */
 204
 205 static inline int atomic_read(const atomic_t *v)
 206 {
 207         int temp;
 208
 209         __asm__ __volatile__(
 210         "       ld.di %0, [%1]"
 211         : "=r"(temp)
 212         : "r"(&v->counter)
 213         : "memory");
 214         return temp;
 215 }
 216
 217 static inline void atomic_set(atomic_t *v, int i)
 218 {
 219         __asm__ __volatile__(
 220         "       st.di %0,[%1]"
 221         :
 222         : "r"(i), "r"(&v->counter)
 223         : "memory");
 224 }
 225
 226 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 227 static inline void atomic_##op(int i, atomic_t *v)                      \
 228 {                                                                       \
 229         __asm__ __volatile__(                                           \
 230         "       mov r2, %0\n"                                           \
 231         "       mov r3, %1\n"                                           \
 232         "       .word %2\n"                                             \
 233         :                                                               \
 234         : "r"(i), "r"(&v->counter), "i"(asm_op)                         \
 235         : "r2", "r3", "memory");                                        \
 236 }                                                                       \
 237
 238 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 239 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 240 {                                                                       \
 241         unsigned int temp = i;                                          \
 242                                                                         \
 243         /* Explicit full memory barrier needed before/after */          \
 244         smp_mb();                                                       \
 245                                                                         \
 246         __asm__ __volatile__(                                           \
 247         "       mov r2, %0\n"                                           \
 248         "       mov r3, %1\n"                                           \
 249         "       .word %2\n"                                             \
 250         "       mov %0, r2"                                             \
 251         : "+r"(temp)                                                    \
 252         : "r"(&v->counter), "i"(asm_op)                                 \
 253         : "r2", "r3", "memory");                                        \
 254                                                                         \
 255         smp_mb();                                                       \
 256                                                                         \
 257         temp c_op i;                                                    \
 258                                                                         \
 259         return temp;                                                    \
 260 }
 261
 262 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 263 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 264 {                                                                       \
 265         unsigned int temp = i;                                          \
 266                                                                         \
 267         /* Explicit full memory barrier needed before/after */          \
 268         smp_mb();                                                       \
 269                                                                         \
 270         __asm__ __volatile__(                                           \
 271         "       mov r2, %0\n"                                           \
 272         "       mov r3, %1\n"                                           \
 273         "       .word %2\n"                                             \
 274         "       mov %0, r2"                                             \
 275         : "+r"(temp)                                                    \
 276         : "r"(&v->counter), "i"(asm_op)                                 \
 277         : "r2", "r3", "memory");                                        \
 278                                                                         \
 279         smp_mb();                                                       \
 280                                                                         \
 281         return temp;                                                    \
 282 }
 283
 284 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 285         ATOMIC_OP(op, c_op, asm_op)                                     \
 286         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 287         ATOMIC_FETCH_OP(op, c_op, asm_op)
 288
 289 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
 290 #define atomic_sub(i, v) atomic_add(-(i), (v))
 291 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
 292 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
 293
 294 #undef ATOMIC_OPS
 295 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 296         ATOMIC_OP(op, c_op, asm_op)                                     \
 297         ATOMIC_FETCH_OP(op, c_op, asm_op)
 298
 299 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
 300 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
 301 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
 302
 303 #endif /* CONFIG_ARC_PLAT_EZNPS */
 304
 305 #undef ATOMIC_OPS
 306 #undef ATOMIC_FETCH_OP
 307 #undef ATOMIC_OP_RETURN
 308 #undef ATOMIC_OP
 309
 310 #ifdef CONFIG_GENERIC_ATOMIC64
 311
 312 #include <asm-generic/atomic64.h>
 313
 314 #else   /* Kconfig ensures this is only enabled with needed h/w assist */
 315
 316 /*
 317  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 318  *  - The address HAS to be 64-bit aligned
 319  *  - There are 2 semantics involved here:
 320  *    = exclusive implies no interim update between load/store to same addr
 321  *    = both words are observed/updated together: this is guaranteed even
 322  *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
 323  *      is NOT required to use LLOCKD+SCONDD, STD suffices
 324  */
 325
 326 typedef struct {
 327         aligned_u64 counter;
 328 } atomic64_t;
 329
 330 #define ATOMIC64_INIT(a) { (a) }
 331
 332 static inline long long atomic64_read(const atomic64_t *v)
 333 {
 334         unsigned long long val;
 335
 336         __asm__ __volatile__(
 337         "       ldd   %0, [%1]  \n"
 338         : "=r"(val)
 339         : "r"(&v->counter));
 340
 341         return val;
 342 }
 343
 344 static inline void atomic64_set(atomic64_t *v, long long a)
 345 {
 346         /*
 347          * This could have been a simple assignment in "C" but would need
 348          * explicit volatile. Otherwise gcc optimizers could elide the store
 349          * which borked atomic64 self-test
 350          * In the inline asm version, memory clobber needed for exact same
 351          * reason, to tell gcc about the store.
 352          *
 353          * This however is not needed for sibling atomic64_add() etc since both
 354          * load/store are explicitly done in inline asm. As long as API is used
 355          * for each access, gcc has no way to optimize away any load/store
 356          */
 357         __asm__ __volatile__(
 358         "       std   %0, [%1]  \n"
 359         :
 360         : "r"(a), "r"(&v->counter)
 361         : "memory");
 362 }
 363
 364 #define ATOMIC64_OP(op, op1, op2)                                       \
 365 static inline void atomic64_##op(long long a, atomic64_t *v)            \
 366 {                                                                       \
 367         unsigned long long val;                                         \
 368                                                                         \
 369         __asm__ __volatile__(                                           \
 370         "1:                             \n"                             \
 371         "       llockd  %0, [%1]        \n"                             \
 372         "       " #op1 " %L0, %L0, %L2  \n"                             \
 373         "       " #op2 " %H0, %H0, %H2  \n"                             \
 374         "       scondd   %0, [%1]       \n"                             \
 375         "       bnz     1b              \n"                             \
 376         : "=&r"(val)                                                    \
 377         : "r"(&v->counter), "ir"(a)                                     \
 378         : "cc");                                                \
 379 }                                                                       \
 380
 381 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
 382 static inline long long atomic64_##op##_return(long long a, atomic64_t *v)      \
 383 {                                                                       \
 384         unsigned long long val;                                         \
 385                                                                         \
 386         smp_mb();                                                       \
 387                                                                         \
 388         __asm__ __volatile__(                                           \
 389         "1:                             \n"                             \
 390         "       llockd   %0, [%1]       \n"                             \
 391         "       " #op1 " %L0, %L0, %L2  \n"                             \
 392         "       " #op2 " %H0, %H0, %H2  \n"                             \
 393         "       scondd   %0, [%1]       \n"                             \
 394         "       bnz     1b              \n"                             \
 395         : [val] "=&r"(val)                                              \
 396         : "r"(&v->counter), "ir"(a)                                     \
 397         : "cc");        /* memory clobber comes from smp_mb() */        \
 398                                                                         \
 399         smp_mb();                                                       \
 400                                                                         \
 401         return val;                                                     \
 402 }
 403
 404 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
 405 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
 406 {                                                                       \
 407         unsigned long long val, orig;                                   \
 408                                                                         \
 409         smp_mb();                                                       \
 410                                                                         \
 411         __asm__ __volatile__(                                           \
 412         "1:                             \n"                             \
 413         "       llockd   %0, [%2]       \n"                             \
 414         "       " #op1 " %L1, %L0, %L3  \n"                             \
 415         "       " #op2 " %H1, %H0, %H3  \n"                             \
 416         "       scondd   %1, [%2]       \n"                             \
 417         "       bnz     1b              \n"                             \
 418         : "=&r"(orig), "=&r"(val)                                       \
 419         : "r"(&v->counter), "ir"(a)                                     \
 420         : "cc");        /* memory clobber comes from smp_mb() */        \
 421                                                                         \
 422         smp_mb();                                                       \
 423                                                                         \
 424         return orig;                                                    \
 425 }
 426
 427 #define ATOMIC64_OPS(op, op1, op2)                                      \
 428         ATOMIC64_OP(op, op1, op2)                                       \
 429         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 430         ATOMIC64_FETCH_OP(op, op1, op2)
 431
 432 #define atomic64_andnot         atomic64_andnot
 433 #define atomic64_fetch_andnot   atomic64_fetch_andnot
 434
 435 ATOMIC64_OPS(add, add.f, adc)
 436 ATOMIC64_OPS(sub, sub.f, sbc)
 437 ATOMIC64_OPS(and, and, and)
 438 ATOMIC64_OPS(andnot, bic, bic)
 439 ATOMIC64_OPS(or, or, or)
 440 ATOMIC64_OPS(xor, xor, xor)
 441
 442 #undef ATOMIC64_OPS
 443 #undef ATOMIC64_FETCH_OP
 444 #undef ATOMIC64_OP_RETURN
 445 #undef ATOMIC64_OP
 446
 447 static inline long long
 448 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
 449 {
 450         long long prev;
 451
 452         smp_mb();
 453
 454         __asm__ __volatile__(
 455         "1:     llockd  %0, [%1]        \n"
 456         "       brne    %L0, %L2, 2f    \n"
 457         "       brne    %H0, %H2, 2f    \n"
 458         "       scondd  %3, [%1]        \n"
 459         "       bnz     1b              \n"
 460         "2:                             \n"
 461         : "=&r"(prev)
 462         : "r"(ptr), "ir"(expected), "r"(new)
 463         : "cc");        /* memory clobber comes from smp_mb() */
 464
 465         smp_mb();
 466
 467         return prev;
 468 }
 469
 470 static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
 471 {
 472         long long prev;
 473
 474         smp_mb();
 475
 476         __asm__ __volatile__(
 477         "1:     llockd  %0, [%1]        \n"
 478         "       scondd  %2, [%1]        \n"
 479         "       bnz     1b              \n"
 480         "2:                             \n"
 481         : "=&r"(prev)
 482         : "r"(ptr), "r"(new)
 483         : "cc");        /* memory clobber comes from smp_mb() */
 484
 485         smp_mb();
 486
 487         return prev;
 488 }
 489
 490 /**
 491  * atomic64_dec_if_positive - decrement by 1 if old value positive
 492  * @v: pointer of type atomic64_t
 493  *
 494  * The function returns the old value of *v minus 1, even if
 495  * the atomic variable, v, was not decremented.
 496  */
 497
 498 static inline long long atomic64_dec_if_positive(atomic64_t *v)
 499 {
 500         long long val;
 501
 502         smp_mb();
 503
 504         __asm__ __volatile__(
 505         "1:     llockd  %0, [%1]        \n"
 506         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 507         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 508         "       brlt    %H0, 0, 2f      \n"
 509         "       scondd  %0, [%1]        \n"
 510         "       bnz     1b              \n"
 511         "2:                             \n"
 512         : "=&r"(val)
 513         : "r"(&v->counter)
 514         : "cc");        /* memory clobber comes from smp_mb() */
 515
 516         smp_mb();
 517
 518         return val;
 519 }
 520 #define atomic64_dec_if_positive atomic64_dec_if_positive
 521
 522 /**
 523  * atomic64_fetch_add_unless - add unless the number is a given value
 524  * @v: pointer of type atomic64_t
 525  * @a: the amount to add to v...
 526  * @u: ...unless v is equal to u.
 527  *
 528  * Atomically adds @a to @v, if it was not @u.
 529  * Returns the old value of @v
 530  */
 531 static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a,
 532                                                   long long u)
 533 {
 534         long long old, temp;
 535
 536         smp_mb();
 537
 538         __asm__ __volatile__(
 539         "1:     llockd  %0, [%2]        \n"
 540         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 541         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 542         "2:                             \n"
 543         "       add.f   %L1, %L0, %L3   \n"
 544         "       adc     %H1, %H0, %H3   \n"
 545         "       scondd  %1, [%2]        \n"
 546         "       bnz     1b              \n"
 547         "3:                             \n"
 548         : "=&r"(old), "=&r" (temp)
 549         : "r"(&v->counter), "r"(a), "r"(u)
 550         : "cc");        /* memory clobber comes from smp_mb() */
 551
 552         smp_mb();
 553
 554         return old;
 555 }
 556 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
 557
 558 #endif  /* !CONFIG_GENERIC_ATOMIC64 */
 559
 560 #endif  /* !__ASSEMBLY__ */
 561
 562 #endif