include/linux/atomic/atomic-arch-fallback.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-fallback.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 #ifndef _LINUX_ATOMIC_FALLBACK_H
   7 #define _LINUX_ATOMIC_FALLBACK_H
   8
   9 #include <linux/compiler.h>
  10
  11 #if defined(arch_xchg)
  12 #define raw_xchg arch_xchg
  13 #elif defined(arch_xchg_relaxed)
  14 #define raw_xchg(...) \
  15         __atomic_op_fence(arch_xchg, __VA_ARGS__)
  16 #else
  17 extern void raw_xchg_not_implemented(void);
  18 #define raw_xchg(...) raw_xchg_not_implemented()
  19 #endif
  20
  21 #if defined(arch_xchg_acquire)
  22 #define raw_xchg_acquire arch_xchg_acquire
  23 #elif defined(arch_xchg_relaxed)
  24 #define raw_xchg_acquire(...) \
  25         __atomic_op_acquire(arch_xchg, __VA_ARGS__)
  26 #elif defined(arch_xchg)
  27 #define raw_xchg_acquire arch_xchg
  28 #else
  29 extern void raw_xchg_acquire_not_implemented(void);
  30 #define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
  31 #endif
  32
  33 #if defined(arch_xchg_release)
  34 #define raw_xchg_release arch_xchg_release
  35 #elif defined(arch_xchg_relaxed)
  36 #define raw_xchg_release(...) \
  37         __atomic_op_release(arch_xchg, __VA_ARGS__)
  38 #elif defined(arch_xchg)
  39 #define raw_xchg_release arch_xchg
  40 #else
  41 extern void raw_xchg_release_not_implemented(void);
  42 #define raw_xchg_release(...) raw_xchg_release_not_implemented()
  43 #endif
  44
  45 #if defined(arch_xchg_relaxed)
  46 #define raw_xchg_relaxed arch_xchg_relaxed
  47 #elif defined(arch_xchg)
  48 #define raw_xchg_relaxed arch_xchg
  49 #else
  50 extern void raw_xchg_relaxed_not_implemented(void);
  51 #define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
  52 #endif
  53
  54 #if defined(arch_cmpxchg)
  55 #define raw_cmpxchg arch_cmpxchg
  56 #elif defined(arch_cmpxchg_relaxed)
  57 #define raw_cmpxchg(...) \
  58         __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
  59 #else
  60 extern void raw_cmpxchg_not_implemented(void);
  61 #define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
  62 #endif
  63
  64 #if defined(arch_cmpxchg_acquire)
  65 #define raw_cmpxchg_acquire arch_cmpxchg_acquire
  66 #elif defined(arch_cmpxchg_relaxed)
  67 #define raw_cmpxchg_acquire(...) \
  68         __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
  69 #elif defined(arch_cmpxchg)
  70 #define raw_cmpxchg_acquire arch_cmpxchg
  71 #else
  72 extern void raw_cmpxchg_acquire_not_implemented(void);
  73 #define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
  74 #endif
  75
  76 #if defined(arch_cmpxchg_release)
  77 #define raw_cmpxchg_release arch_cmpxchg_release
  78 #elif defined(arch_cmpxchg_relaxed)
  79 #define raw_cmpxchg_release(...) \
  80         __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
  81 #elif defined(arch_cmpxchg)
  82 #define raw_cmpxchg_release arch_cmpxchg
  83 #else
  84 extern void raw_cmpxchg_release_not_implemented(void);
  85 #define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
  86 #endif
  87
  88 #if defined(arch_cmpxchg_relaxed)
  89 #define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
  90 #elif defined(arch_cmpxchg)
  91 #define raw_cmpxchg_relaxed arch_cmpxchg
  92 #else
  93 extern void raw_cmpxchg_relaxed_not_implemented(void);
  94 #define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
  95 #endif
  96
  97 #if defined(arch_cmpxchg64)
  98 #define raw_cmpxchg64 arch_cmpxchg64
  99 #elif defined(arch_cmpxchg64_relaxed)
 100 #define raw_cmpxchg64(...) \
 101         __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
 102 #else
 103 extern void raw_cmpxchg64_not_implemented(void);
 104 #define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
 105 #endif
 106
 107 #if defined(arch_cmpxchg64_acquire)
 108 #define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
 109 #elif defined(arch_cmpxchg64_relaxed)
 110 #define raw_cmpxchg64_acquire(...) \
 111         __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
 112 #elif defined(arch_cmpxchg64)
 113 #define raw_cmpxchg64_acquire arch_cmpxchg64
 114 #else
 115 extern void raw_cmpxchg64_acquire_not_implemented(void);
 116 #define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
 117 #endif
 118
 119 #if defined(arch_cmpxchg64_release)
 120 #define raw_cmpxchg64_release arch_cmpxchg64_release
 121 #elif defined(arch_cmpxchg64_relaxed)
 122 #define raw_cmpxchg64_release(...) \
 123         __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
 124 #elif defined(arch_cmpxchg64)
 125 #define raw_cmpxchg64_release arch_cmpxchg64
 126 #else
 127 extern void raw_cmpxchg64_release_not_implemented(void);
 128 #define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
 129 #endif
 130
 131 #if defined(arch_cmpxchg64_relaxed)
 132 #define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
 133 #elif defined(arch_cmpxchg64)
 134 #define raw_cmpxchg64_relaxed arch_cmpxchg64
 135 #else
 136 extern void raw_cmpxchg64_relaxed_not_implemented(void);
 137 #define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
 138 #endif
 139
 140 #if defined(arch_cmpxchg128)
 141 #define raw_cmpxchg128 arch_cmpxchg128
 142 #elif defined(arch_cmpxchg128_relaxed)
 143 #define raw_cmpxchg128(...) \
 144         __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
 145 #else
 146 extern void raw_cmpxchg128_not_implemented(void);
 147 #define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
 148 #endif
 149
 150 #if defined(arch_cmpxchg128_acquire)
 151 #define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
 152 #elif defined(arch_cmpxchg128_relaxed)
 153 #define raw_cmpxchg128_acquire(...) \
 154         __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
 155 #elif defined(arch_cmpxchg128)
 156 #define raw_cmpxchg128_acquire arch_cmpxchg128
 157 #else
 158 extern void raw_cmpxchg128_acquire_not_implemented(void);
 159 #define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
 160 #endif
 161
 162 #if defined(arch_cmpxchg128_release)
 163 #define raw_cmpxchg128_release arch_cmpxchg128_release
 164 #elif defined(arch_cmpxchg128_relaxed)
 165 #define raw_cmpxchg128_release(...) \
 166         __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
 167 #elif defined(arch_cmpxchg128)
 168 #define raw_cmpxchg128_release arch_cmpxchg128
 169 #else
 170 extern void raw_cmpxchg128_release_not_implemented(void);
 171 #define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
 172 #endif
 173
 174 #if defined(arch_cmpxchg128_relaxed)
 175 #define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
 176 #elif defined(arch_cmpxchg128)
 177 #define raw_cmpxchg128_relaxed arch_cmpxchg128
 178 #else
 179 extern void raw_cmpxchg128_relaxed_not_implemented(void);
 180 #define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
 181 #endif
 182
 183 #if defined(arch_try_cmpxchg)
 184 #define raw_try_cmpxchg arch_try_cmpxchg
 185 #elif defined(arch_try_cmpxchg_relaxed)
 186 #define raw_try_cmpxchg(...) \
 187         __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
 188 #else
 189 #define raw_try_cmpxchg(_ptr, _oldp, _new) \
 190 ({ \
 191         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 192         ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
 193         if (unlikely(___r != ___o)) \
 194                 *___op = ___r; \
 195         likely(___r == ___o); \
 196 })
 197 #endif
 198
 199 #if defined(arch_try_cmpxchg_acquire)
 200 #define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
 201 #elif defined(arch_try_cmpxchg_relaxed)
 202 #define raw_try_cmpxchg_acquire(...) \
 203         __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
 204 #elif defined(arch_try_cmpxchg)
 205 #define raw_try_cmpxchg_acquire arch_try_cmpxchg
 206 #else
 207 #define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
 208 ({ \
 209         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 210         ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
 211         if (unlikely(___r != ___o)) \
 212                 *___op = ___r; \
 213         likely(___r == ___o); \
 214 })
 215 #endif
 216
 217 #if defined(arch_try_cmpxchg_release)
 218 #define raw_try_cmpxchg_release arch_try_cmpxchg_release
 219 #elif defined(arch_try_cmpxchg_relaxed)
 220 #define raw_try_cmpxchg_release(...) \
 221         __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
 222 #elif defined(arch_try_cmpxchg)
 223 #define raw_try_cmpxchg_release arch_try_cmpxchg
 224 #else
 225 #define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
 226 ({ \
 227         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 228         ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
 229         if (unlikely(___r != ___o)) \
 230                 *___op = ___r; \
 231         likely(___r == ___o); \
 232 })
 233 #endif
 234
 235 #if defined(arch_try_cmpxchg_relaxed)
 236 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
 237 #elif defined(arch_try_cmpxchg)
 238 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg
 239 #else
 240 #define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
 241 ({ \
 242         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 243         ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
 244         if (unlikely(___r != ___o)) \
 245                 *___op = ___r; \
 246         likely(___r == ___o); \
 247 })
 248 #endif
 249
 250 #if defined(arch_try_cmpxchg64)
 251 #define raw_try_cmpxchg64 arch_try_cmpxchg64
 252 #elif defined(arch_try_cmpxchg64_relaxed)
 253 #define raw_try_cmpxchg64(...) \
 254         __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
 255 #else
 256 #define raw_try_cmpxchg64(_ptr, _oldp, _new) \
 257 ({ \
 258         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 259         ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
 260         if (unlikely(___r != ___o)) \
 261                 *___op = ___r; \
 262         likely(___r == ___o); \
 263 })
 264 #endif
 265
 266 #if defined(arch_try_cmpxchg64_acquire)
 267 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
 268 #elif defined(arch_try_cmpxchg64_relaxed)
 269 #define raw_try_cmpxchg64_acquire(...) \
 270         __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
 271 #elif defined(arch_try_cmpxchg64)
 272 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
 273 #else
 274 #define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
 275 ({ \
 276         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 277         ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
 278         if (unlikely(___r != ___o)) \
 279                 *___op = ___r; \
 280         likely(___r == ___o); \
 281 })
 282 #endif
 283
 284 #if defined(arch_try_cmpxchg64_release)
 285 #define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
 286 #elif defined(arch_try_cmpxchg64_relaxed)
 287 #define raw_try_cmpxchg64_release(...) \
 288         __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
 289 #elif defined(arch_try_cmpxchg64)
 290 #define raw_try_cmpxchg64_release arch_try_cmpxchg64
 291 #else
 292 #define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
 293 ({ \
 294         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 295         ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
 296         if (unlikely(___r != ___o)) \
 297                 *___op = ___r; \
 298         likely(___r == ___o); \
 299 })
 300 #endif
 301
 302 #if defined(arch_try_cmpxchg64_relaxed)
 303 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
 304 #elif defined(arch_try_cmpxchg64)
 305 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
 306 #else
 307 #define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
 308 ({ \
 309         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 310         ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
 311         if (unlikely(___r != ___o)) \
 312                 *___op = ___r; \
 313         likely(___r == ___o); \
 314 })
 315 #endif
 316
 317 #if defined(arch_try_cmpxchg128)
 318 #define raw_try_cmpxchg128 arch_try_cmpxchg128
 319 #elif defined(arch_try_cmpxchg128_relaxed)
 320 #define raw_try_cmpxchg128(...) \
 321         __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
 322 #else
 323 #define raw_try_cmpxchg128(_ptr, _oldp, _new) \
 324 ({ \
 325         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 326         ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
 327         if (unlikely(___r != ___o)) \
 328                 *___op = ___r; \
 329         likely(___r == ___o); \
 330 })
 331 #endif
 332
 333 #if defined(arch_try_cmpxchg128_acquire)
 334 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
 335 #elif defined(arch_try_cmpxchg128_relaxed)
 336 #define raw_try_cmpxchg128_acquire(...) \
 337         __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
 338 #elif defined(arch_try_cmpxchg128)
 339 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
 340 #else
 341 #define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
 342 ({ \
 343         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 344         ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
 345         if (unlikely(___r != ___o)) \
 346                 *___op = ___r; \
 347         likely(___r == ___o); \
 348 })
 349 #endif
 350
 351 #if defined(arch_try_cmpxchg128_release)
 352 #define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
 353 #elif defined(arch_try_cmpxchg128_relaxed)
 354 #define raw_try_cmpxchg128_release(...) \
 355         __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
 356 #elif defined(arch_try_cmpxchg128)
 357 #define raw_try_cmpxchg128_release arch_try_cmpxchg128
 358 #else
 359 #define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
 360 ({ \
 361         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 362         ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
 363         if (unlikely(___r != ___o)) \
 364                 *___op = ___r; \
 365         likely(___r == ___o); \
 366 })
 367 #endif
 368
 369 #if defined(arch_try_cmpxchg128_relaxed)
 370 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
 371 #elif defined(arch_try_cmpxchg128)
 372 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
 373 #else
 374 #define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
 375 ({ \
 376         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 377         ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
 378         if (unlikely(___r != ___o)) \
 379                 *___op = ___r; \
 380         likely(___r == ___o); \
 381 })
 382 #endif
 383
 384 #define raw_cmpxchg_local arch_cmpxchg_local
 385
 386 #ifdef arch_try_cmpxchg_local
 387 #define raw_try_cmpxchg_local arch_try_cmpxchg_local
 388 #else
 389 #define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
 390 ({ \
 391         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 392         ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
 393         if (unlikely(___r != ___o)) \
 394                 *___op = ___r; \
 395         likely(___r == ___o); \
 396 })
 397 #endif
 398
 399 #define raw_cmpxchg64_local arch_cmpxchg64_local
 400
 401 #ifdef arch_try_cmpxchg64_local
 402 #define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
 403 #else
 404 #define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
 405 ({ \
 406         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 407         ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
 408         if (unlikely(___r != ___o)) \
 409                 *___op = ___r; \
 410         likely(___r == ___o); \
 411 })
 412 #endif
 413
 414 #define raw_cmpxchg128_local arch_cmpxchg128_local
 415
 416 #ifdef arch_try_cmpxchg128_local
 417 #define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
 418 #else
 419 #define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
 420 ({ \
 421         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 422         ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
 423         if (unlikely(___r != ___o)) \
 424                 *___op = ___r; \
 425         likely(___r == ___o); \
 426 })
 427 #endif
 428
 429 #define raw_sync_cmpxchg arch_sync_cmpxchg
 430
 431 #ifdef arch_sync_try_cmpxchg
 432 #define raw_sync_try_cmpxchg arch_sync_try_cmpxchg
 433 #else
 434 #define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \
 435 ({ \
 436         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 437         ___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \
 438         if (unlikely(___r != ___o)) \
 439                 *___op = ___r; \
 440         likely(___r == ___o); \
 441 })
 442 #endif
 443
 444 /**
 445  * raw_atomic_read() - atomic load with relaxed ordering
 446  * @v: pointer to atomic_t
 447  *
 448  * Atomically loads the value of @v with relaxed ordering.
 449  *
 450  * Safe to use in noinstr code; prefer atomic_read() elsewhere.
 451  *
 452  * Return: The value loaded from @v.
 453  */
 454 static __always_inline int
 455 raw_atomic_read(const atomic_t *v)
 456 {
 457         return arch_atomic_read(v);
 458 }
 459
 460 /**
 461  * raw_atomic_read_acquire() - atomic load with acquire ordering
 462  * @v: pointer to atomic_t
 463  *
 464  * Atomically loads the value of @v with acquire ordering.
 465  *
 466  * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
 467  *
 468  * Return: The value loaded from @v.
 469  */
 470 static __always_inline int
 471 raw_atomic_read_acquire(const atomic_t *v)
 472 {
 473 #if defined(arch_atomic_read_acquire)
 474         return arch_atomic_read_acquire(v);
 475 #else
 476         int ret;
 477
 478         if (__native_word(atomic_t)) {
 479                 ret = smp_load_acquire(&(v)->counter);
 480         } else {
 481                 ret = raw_atomic_read(v);
 482                 __atomic_acquire_fence();
 483         }
 484
 485         return ret;
 486 #endif
 487 }
 488
 489 /**
 490  * raw_atomic_set() - atomic set with relaxed ordering
 491  * @v: pointer to atomic_t
 492  * @i: int value to assign
 493  *
 494  * Atomically sets @v to @i with relaxed ordering.
 495  *
 496  * Safe to use in noinstr code; prefer atomic_set() elsewhere.
 497  *
 498  * Return: Nothing.
 499  */
 500 static __always_inline void
 501 raw_atomic_set(atomic_t *v, int i)
 502 {
 503         arch_atomic_set(v, i);
 504 }
 505
 506 /**
 507  * raw_atomic_set_release() - atomic set with release ordering
 508  * @v: pointer to atomic_t
 509  * @i: int value to assign
 510  *
 511  * Atomically sets @v to @i with release ordering.
 512  *
 513  * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
 514  *
 515  * Return: Nothing.
 516  */
 517 static __always_inline void
 518 raw_atomic_set_release(atomic_t *v, int i)
 519 {
 520 #if defined(arch_atomic_set_release)
 521         arch_atomic_set_release(v, i);
 522 #else
 523         if (__native_word(atomic_t)) {
 524                 smp_store_release(&(v)->counter, i);
 525         } else {
 526                 __atomic_release_fence();
 527                 raw_atomic_set(v, i);
 528         }
 529 #endif
 530 }
 531
 532 /**
 533  * raw_atomic_add() - atomic add with relaxed ordering
 534  * @i: int value to add
 535  * @v: pointer to atomic_t
 536  *
 537  * Atomically updates @v to (@v + @i) with relaxed ordering.
 538  *
 539  * Safe to use in noinstr code; prefer atomic_add() elsewhere.
 540  *
 541  * Return: Nothing.
 542  */
 543 static __always_inline void
 544 raw_atomic_add(int i, atomic_t *v)
 545 {
 546         arch_atomic_add(i, v);
 547 }
 548
 549 /**
 550  * raw_atomic_add_return() - atomic add with full ordering
 551  * @i: int value to add
 552  * @v: pointer to atomic_t
 553  *
 554  * Atomically updates @v to (@v + @i) with full ordering.
 555  *
 556  * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
 557  *
 558  * Return: The updated value of @v.
 559  */
 560 static __always_inline int
 561 raw_atomic_add_return(int i, atomic_t *v)
 562 {
 563 #if defined(arch_atomic_add_return)
 564         return arch_atomic_add_return(i, v);
 565 #elif defined(arch_atomic_add_return_relaxed)
 566         int ret;
 567         __atomic_pre_full_fence();
 568         ret = arch_atomic_add_return_relaxed(i, v);
 569         __atomic_post_full_fence();
 570         return ret;
 571 #else
 572 #error "Unable to define raw_atomic_add_return"
 573 #endif
 574 }
 575
 576 /**
 577  * raw_atomic_add_return_acquire() - atomic add with acquire ordering
 578  * @i: int value to add
 579  * @v: pointer to atomic_t
 580  *
 581  * Atomically updates @v to (@v + @i) with acquire ordering.
 582  *
 583  * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
 584  *
 585  * Return: The updated value of @v.
 586  */
 587 static __always_inline int
 588 raw_atomic_add_return_acquire(int i, atomic_t *v)
 589 {
 590 #if defined(arch_atomic_add_return_acquire)
 591         return arch_atomic_add_return_acquire(i, v);
 592 #elif defined(arch_atomic_add_return_relaxed)
 593         int ret = arch_atomic_add_return_relaxed(i, v);
 594         __atomic_acquire_fence();
 595         return ret;
 596 #elif defined(arch_atomic_add_return)
 597         return arch_atomic_add_return(i, v);
 598 #else
 599 #error "Unable to define raw_atomic_add_return_acquire"
 600 #endif
 601 }
 602
 603 /**
 604  * raw_atomic_add_return_release() - atomic add with release ordering
 605  * @i: int value to add
 606  * @v: pointer to atomic_t
 607  *
 608  * Atomically updates @v to (@v + @i) with release ordering.
 609  *
 610  * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
 611  *
 612  * Return: The updated value of @v.
 613  */
 614 static __always_inline int
 615 raw_atomic_add_return_release(int i, atomic_t *v)
 616 {
 617 #if defined(arch_atomic_add_return_release)
 618         return arch_atomic_add_return_release(i, v);
 619 #elif defined(arch_atomic_add_return_relaxed)
 620         __atomic_release_fence();
 621         return arch_atomic_add_return_relaxed(i, v);
 622 #elif defined(arch_atomic_add_return)
 623         return arch_atomic_add_return(i, v);
 624 #else
 625 #error "Unable to define raw_atomic_add_return_release"
 626 #endif
 627 }
 628
 629 /**
 630  * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
 631  * @i: int value to add
 632  * @v: pointer to atomic_t
 633  *
 634  * Atomically updates @v to (@v + @i) with relaxed ordering.
 635  *
 636  * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
 637  *
 638  * Return: The updated value of @v.
 639  */
 640 static __always_inline int
 641 raw_atomic_add_return_relaxed(int i, atomic_t *v)
 642 {
 643 #if defined(arch_atomic_add_return_relaxed)
 644         return arch_atomic_add_return_relaxed(i, v);
 645 #elif defined(arch_atomic_add_return)
 646         return arch_atomic_add_return(i, v);
 647 #else
 648 #error "Unable to define raw_atomic_add_return_relaxed"
 649 #endif
 650 }
 651
 652 /**
 653  * raw_atomic_fetch_add() - atomic add with full ordering
 654  * @i: int value to add
 655  * @v: pointer to atomic_t
 656  *
 657  * Atomically updates @v to (@v + @i) with full ordering.
 658  *
 659  * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
 660  *
 661  * Return: The original value of @v.
 662  */
 663 static __always_inline int
 664 raw_atomic_fetch_add(int i, atomic_t *v)
 665 {
 666 #if defined(arch_atomic_fetch_add)
 667         return arch_atomic_fetch_add(i, v);
 668 #elif defined(arch_atomic_fetch_add_relaxed)
 669         int ret;
 670         __atomic_pre_full_fence();
 671         ret = arch_atomic_fetch_add_relaxed(i, v);
 672         __atomic_post_full_fence();
 673         return ret;
 674 #else
 675 #error "Unable to define raw_atomic_fetch_add"
 676 #endif
 677 }
 678
 679 /**
 680  * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
 681  * @i: int value to add
 682  * @v: pointer to atomic_t
 683  *
 684  * Atomically updates @v to (@v + @i) with acquire ordering.
 685  *
 686  * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
 687  *
 688  * Return: The original value of @v.
 689  */
 690 static __always_inline int
 691 raw_atomic_fetch_add_acquire(int i, atomic_t *v)
 692 {
 693 #if defined(arch_atomic_fetch_add_acquire)
 694         return arch_atomic_fetch_add_acquire(i, v);
 695 #elif defined(arch_atomic_fetch_add_relaxed)
 696         int ret = arch_atomic_fetch_add_relaxed(i, v);
 697         __atomic_acquire_fence();
 698         return ret;
 699 #elif defined(arch_atomic_fetch_add)
 700         return arch_atomic_fetch_add(i, v);
 701 #else
 702 #error "Unable to define raw_atomic_fetch_add_acquire"
 703 #endif
 704 }
 705
 706 /**
 707  * raw_atomic_fetch_add_release() - atomic add with release ordering
 708  * @i: int value to add
 709  * @v: pointer to atomic_t
 710  *
 711  * Atomically updates @v to (@v + @i) with release ordering.
 712  *
 713  * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
 714  *
 715  * Return: The original value of @v.
 716  */
 717 static __always_inline int
 718 raw_atomic_fetch_add_release(int i, atomic_t *v)
 719 {
 720 #if defined(arch_atomic_fetch_add_release)
 721         return arch_atomic_fetch_add_release(i, v);
 722 #elif defined(arch_atomic_fetch_add_relaxed)
 723         __atomic_release_fence();
 724         return arch_atomic_fetch_add_relaxed(i, v);
 725 #elif defined(arch_atomic_fetch_add)
 726         return arch_atomic_fetch_add(i, v);
 727 #else
 728 #error "Unable to define raw_atomic_fetch_add_release"
 729 #endif
 730 }
 731
 732 /**
 733  * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
 734  * @i: int value to add
 735  * @v: pointer to atomic_t
 736  *
 737  * Atomically updates @v to (@v + @i) with relaxed ordering.
 738  *
 739  * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
 740  *
 741  * Return: The original value of @v.
 742  */
 743 static __always_inline int
 744 raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
 745 {
 746 #if defined(arch_atomic_fetch_add_relaxed)
 747         return arch_atomic_fetch_add_relaxed(i, v);
 748 #elif defined(arch_atomic_fetch_add)
 749         return arch_atomic_fetch_add(i, v);
 750 #else
 751 #error "Unable to define raw_atomic_fetch_add_relaxed"
 752 #endif
 753 }
 754
 755 /**
 756  * raw_atomic_sub() - atomic subtract with relaxed ordering
 757  * @i: int value to subtract
 758  * @v: pointer to atomic_t
 759  *
 760  * Atomically updates @v to (@v - @i) with relaxed ordering.
 761  *
 762  * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
 763  *
 764  * Return: Nothing.
 765  */
 766 static __always_inline void
 767 raw_atomic_sub(int i, atomic_t *v)
 768 {
 769         arch_atomic_sub(i, v);
 770 }
 771
 772 /**
 773  * raw_atomic_sub_return() - atomic subtract with full ordering
 774  * @i: int value to subtract
 775  * @v: pointer to atomic_t
 776  *
 777  * Atomically updates @v to (@v - @i) with full ordering.
 778  *
 779  * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
 780  *
 781  * Return: The updated value of @v.
 782  */
 783 static __always_inline int
 784 raw_atomic_sub_return(int i, atomic_t *v)
 785 {
 786 #if defined(arch_atomic_sub_return)
 787         return arch_atomic_sub_return(i, v);
 788 #elif defined(arch_atomic_sub_return_relaxed)
 789         int ret;
 790         __atomic_pre_full_fence();
 791         ret = arch_atomic_sub_return_relaxed(i, v);
 792         __atomic_post_full_fence();
 793         return ret;
 794 #else
 795 #error "Unable to define raw_atomic_sub_return"
 796 #endif
 797 }
 798
 799 /**
 800  * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
 801  * @i: int value to subtract
 802  * @v: pointer to atomic_t
 803  *
 804  * Atomically updates @v to (@v - @i) with acquire ordering.
 805  *
 806  * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
 807  *
 808  * Return: The updated value of @v.
 809  */
 810 static __always_inline int
 811 raw_atomic_sub_return_acquire(int i, atomic_t *v)
 812 {
 813 #if defined(arch_atomic_sub_return_acquire)
 814         return arch_atomic_sub_return_acquire(i, v);
 815 #elif defined(arch_atomic_sub_return_relaxed)
 816         int ret = arch_atomic_sub_return_relaxed(i, v);
 817         __atomic_acquire_fence();
 818         return ret;
 819 #elif defined(arch_atomic_sub_return)
 820         return arch_atomic_sub_return(i, v);
 821 #else
 822 #error "Unable to define raw_atomic_sub_return_acquire"
 823 #endif
 824 }
 825
 826 /**
 827  * raw_atomic_sub_return_release() - atomic subtract with release ordering
 828  * @i: int value to subtract
 829  * @v: pointer to atomic_t
 830  *
 831  * Atomically updates @v to (@v - @i) with release ordering.
 832  *
 833  * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
 834  *
 835  * Return: The updated value of @v.
 836  */
 837 static __always_inline int
 838 raw_atomic_sub_return_release(int i, atomic_t *v)
 839 {
 840 #if defined(arch_atomic_sub_return_release)
 841         return arch_atomic_sub_return_release(i, v);
 842 #elif defined(arch_atomic_sub_return_relaxed)
 843         __atomic_release_fence();
 844         return arch_atomic_sub_return_relaxed(i, v);
 845 #elif defined(arch_atomic_sub_return)
 846         return arch_atomic_sub_return(i, v);
 847 #else
 848 #error "Unable to define raw_atomic_sub_return_release"
 849 #endif
 850 }
 851
 852 /**
 853  * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
 854  * @i: int value to subtract
 855  * @v: pointer to atomic_t
 856  *
 857  * Atomically updates @v to (@v - @i) with relaxed ordering.
 858  *
 859  * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
 860  *
 861  * Return: The updated value of @v.
 862  */
 863 static __always_inline int
 864 raw_atomic_sub_return_relaxed(int i, atomic_t *v)
 865 {
 866 #if defined(arch_atomic_sub_return_relaxed)
 867         return arch_atomic_sub_return_relaxed(i, v);
 868 #elif defined(arch_atomic_sub_return)
 869         return arch_atomic_sub_return(i, v);
 870 #else
 871 #error "Unable to define raw_atomic_sub_return_relaxed"
 872 #endif
 873 }
 874
 875 /**
 876  * raw_atomic_fetch_sub() - atomic subtract with full ordering
 877  * @i: int value to subtract
 878  * @v: pointer to atomic_t
 879  *
 880  * Atomically updates @v to (@v - @i) with full ordering.
 881  *
 882  * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
 883  *
 884  * Return: The original value of @v.
 885  */
 886 static __always_inline int
 887 raw_atomic_fetch_sub(int i, atomic_t *v)
 888 {
 889 #if defined(arch_atomic_fetch_sub)
 890         return arch_atomic_fetch_sub(i, v);
 891 #elif defined(arch_atomic_fetch_sub_relaxed)
 892         int ret;
 893         __atomic_pre_full_fence();
 894         ret = arch_atomic_fetch_sub_relaxed(i, v);
 895         __atomic_post_full_fence();
 896         return ret;
 897 #else
 898 #error "Unable to define raw_atomic_fetch_sub"
 899 #endif
 900 }
 901
 902 /**
 903  * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
 904  * @i: int value to subtract
 905  * @v: pointer to atomic_t
 906  *
 907  * Atomically updates @v to (@v - @i) with acquire ordering.
 908  *
 909  * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
 910  *
 911  * Return: The original value of @v.
 912  */
 913 static __always_inline int
 914 raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
 915 {
 916 #if defined(arch_atomic_fetch_sub_acquire)
 917         return arch_atomic_fetch_sub_acquire(i, v);
 918 #elif defined(arch_atomic_fetch_sub_relaxed)
 919         int ret = arch_atomic_fetch_sub_relaxed(i, v);
 920         __atomic_acquire_fence();
 921         return ret;
 922 #elif defined(arch_atomic_fetch_sub)
 923         return arch_atomic_fetch_sub(i, v);
 924 #else
 925 #error "Unable to define raw_atomic_fetch_sub_acquire"
 926 #endif
 927 }
 928
 929 /**
 930  * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
 931  * @i: int value to subtract
 932  * @v: pointer to atomic_t
 933  *
 934  * Atomically updates @v to (@v - @i) with release ordering.
 935  *
 936  * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
 937  *
 938  * Return: The original value of @v.
 939  */
 940 static __always_inline int
 941 raw_atomic_fetch_sub_release(int i, atomic_t *v)
 942 {
 943 #if defined(arch_atomic_fetch_sub_release)
 944         return arch_atomic_fetch_sub_release(i, v);
 945 #elif defined(arch_atomic_fetch_sub_relaxed)
 946         __atomic_release_fence();
 947         return arch_atomic_fetch_sub_relaxed(i, v);
 948 #elif defined(arch_atomic_fetch_sub)
 949         return arch_atomic_fetch_sub(i, v);
 950 #else
 951 #error "Unable to define raw_atomic_fetch_sub_release"
 952 #endif
 953 }
 954
 955 /**
 956  * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 957  * @i: int value to subtract
 958  * @v: pointer to atomic_t
 959  *
 960  * Atomically updates @v to (@v - @i) with relaxed ordering.
 961  *
 962  * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
 963  *
 964  * Return: The original value of @v.
 965  */
 966 static __always_inline int
 967 raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
 968 {
 969 #if defined(arch_atomic_fetch_sub_relaxed)
 970         return arch_atomic_fetch_sub_relaxed(i, v);
 971 #elif defined(arch_atomic_fetch_sub)
 972         return arch_atomic_fetch_sub(i, v);
 973 #else
 974 #error "Unable to define raw_atomic_fetch_sub_relaxed"
 975 #endif
 976 }
 977
 978 /**
 979  * raw_atomic_inc() - atomic increment with relaxed ordering
 980  * @v: pointer to atomic_t
 981  *
 982  * Atomically updates @v to (@v + 1) with relaxed ordering.
 983  *
 984  * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
 985  *
 986  * Return: Nothing.
 987  */
 988 static __always_inline void
 989 raw_atomic_inc(atomic_t *v)
 990 {
 991 #if defined(arch_atomic_inc)
 992         arch_atomic_inc(v);
 993 #else
 994         raw_atomic_add(1, v);
 995 #endif
 996 }
 997
 998 /**
 999  * raw_atomic_inc_return() - atomic increment with full ordering
1000  * @v: pointer to atomic_t
1001  *
1002  * Atomically updates @v to (@v + 1) with full ordering.
1003  *
1004  * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
1005  *
1006  * Return: The updated value of @v.
1007  */
1008 static __always_inline int
1009 raw_atomic_inc_return(atomic_t *v)
1010 {
1011 #if defined(arch_atomic_inc_return)
1012         return arch_atomic_inc_return(v);
1013 #elif defined(arch_atomic_inc_return_relaxed)
1014         int ret;
1015         __atomic_pre_full_fence();
1016         ret = arch_atomic_inc_return_relaxed(v);
1017         __atomic_post_full_fence();
1018         return ret;
1019 #else
1020         return raw_atomic_add_return(1, v);
1021 #endif
1022 }
1023
1024 /**
1025  * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
1026  * @v: pointer to atomic_t
1027  *
1028  * Atomically updates @v to (@v + 1) with acquire ordering.
1029  *
1030  * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1031  *
1032  * Return: The updated value of @v.
1033  */
1034 static __always_inline int
1035 raw_atomic_inc_return_acquire(atomic_t *v)
1036 {
1037 #if defined(arch_atomic_inc_return_acquire)
1038         return arch_atomic_inc_return_acquire(v);
1039 #elif defined(arch_atomic_inc_return_relaxed)
1040         int ret = arch_atomic_inc_return_relaxed(v);
1041         __atomic_acquire_fence();
1042         return ret;
1043 #elif defined(arch_atomic_inc_return)
1044         return arch_atomic_inc_return(v);
1045 #else
1046         return raw_atomic_add_return_acquire(1, v);
1047 #endif
1048 }
1049
1050 /**
1051  * raw_atomic_inc_return_release() - atomic increment with release ordering
1052  * @v: pointer to atomic_t
1053  *
1054  * Atomically updates @v to (@v + 1) with release ordering.
1055  *
1056  * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1057  *
1058  * Return: The updated value of @v.
1059  */
1060 static __always_inline int
1061 raw_atomic_inc_return_release(atomic_t *v)
1062 {
1063 #if defined(arch_atomic_inc_return_release)
1064         return arch_atomic_inc_return_release(v);
1065 #elif defined(arch_atomic_inc_return_relaxed)
1066         __atomic_release_fence();
1067         return arch_atomic_inc_return_relaxed(v);
1068 #elif defined(arch_atomic_inc_return)
1069         return arch_atomic_inc_return(v);
1070 #else
1071         return raw_atomic_add_return_release(1, v);
1072 #endif
1073 }
1074
1075 /**
1076  * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
1077  * @v: pointer to atomic_t
1078  *
1079  * Atomically updates @v to (@v + 1) with relaxed ordering.
1080  *
1081  * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
1082  *
1083  * Return: The updated value of @v.
1084  */
1085 static __always_inline int
1086 raw_atomic_inc_return_relaxed(atomic_t *v)
1087 {
1088 #if defined(arch_atomic_inc_return_relaxed)
1089         return arch_atomic_inc_return_relaxed(v);
1090 #elif defined(arch_atomic_inc_return)
1091         return arch_atomic_inc_return(v);
1092 #else
1093         return raw_atomic_add_return_relaxed(1, v);
1094 #endif
1095 }
1096
1097 /**
1098  * raw_atomic_fetch_inc() - atomic increment with full ordering
1099  * @v: pointer to atomic_t
1100  *
1101  * Atomically updates @v to (@v + 1) with full ordering.
1102  *
1103  * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1104  *
1105  * Return: The original value of @v.
1106  */
1107 static __always_inline int
1108 raw_atomic_fetch_inc(atomic_t *v)
1109 {
1110 #if defined(arch_atomic_fetch_inc)
1111         return arch_atomic_fetch_inc(v);
1112 #elif defined(arch_atomic_fetch_inc_relaxed)
1113         int ret;
1114         __atomic_pre_full_fence();
1115         ret = arch_atomic_fetch_inc_relaxed(v);
1116         __atomic_post_full_fence();
1117         return ret;
1118 #else
1119         return raw_atomic_fetch_add(1, v);
1120 #endif
1121 }
1122
1123 /**
1124  * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
1125  * @v: pointer to atomic_t
1126  *
1127  * Atomically updates @v to (@v + 1) with acquire ordering.
1128  *
1129  * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1130  *
1131  * Return: The original value of @v.
1132  */
1133 static __always_inline int
1134 raw_atomic_fetch_inc_acquire(atomic_t *v)
1135 {
1136 #if defined(arch_atomic_fetch_inc_acquire)
1137         return arch_atomic_fetch_inc_acquire(v);
1138 #elif defined(arch_atomic_fetch_inc_relaxed)
1139         int ret = arch_atomic_fetch_inc_relaxed(v);
1140         __atomic_acquire_fence();
1141         return ret;
1142 #elif defined(arch_atomic_fetch_inc)
1143         return arch_atomic_fetch_inc(v);
1144 #else
1145         return raw_atomic_fetch_add_acquire(1, v);
1146 #endif
1147 }
1148
1149 /**
1150  * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1151  * @v: pointer to atomic_t
1152  *
1153  * Atomically updates @v to (@v + 1) with release ordering.
1154  *
1155  * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1156  *
1157  * Return: The original value of @v.
1158  */
1159 static __always_inline int
1160 raw_atomic_fetch_inc_release(atomic_t *v)
1161 {
1162 #if defined(arch_atomic_fetch_inc_release)
1163         return arch_atomic_fetch_inc_release(v);
1164 #elif defined(arch_atomic_fetch_inc_relaxed)
1165         __atomic_release_fence();
1166         return arch_atomic_fetch_inc_relaxed(v);
1167 #elif defined(arch_atomic_fetch_inc)
1168         return arch_atomic_fetch_inc(v);
1169 #else
1170         return raw_atomic_fetch_add_release(1, v);
1171 #endif
1172 }
1173
1174 /**
1175  * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1176  * @v: pointer to atomic_t
1177  *
1178  * Atomically updates @v to (@v + 1) with relaxed ordering.
1179  *
1180  * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1181  *
1182  * Return: The original value of @v.
1183  */
1184 static __always_inline int
1185 raw_atomic_fetch_inc_relaxed(atomic_t *v)
1186 {
1187 #if defined(arch_atomic_fetch_inc_relaxed)
1188         return arch_atomic_fetch_inc_relaxed(v);
1189 #elif defined(arch_atomic_fetch_inc)
1190         return arch_atomic_fetch_inc(v);
1191 #else
1192         return raw_atomic_fetch_add_relaxed(1, v);
1193 #endif
1194 }
1195
1196 /**
1197  * raw_atomic_dec() - atomic decrement with relaxed ordering
1198  * @v: pointer to atomic_t
1199  *
1200  * Atomically updates @v to (@v - 1) with relaxed ordering.
1201  *
1202  * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1203  *
1204  * Return: Nothing.
1205  */
1206 static __always_inline void
1207 raw_atomic_dec(atomic_t *v)
1208 {
1209 #if defined(arch_atomic_dec)
1210         arch_atomic_dec(v);
1211 #else
1212         raw_atomic_sub(1, v);
1213 #endif
1214 }
1215
1216 /**
1217  * raw_atomic_dec_return() - atomic decrement with full ordering
1218  * @v: pointer to atomic_t
1219  *
1220  * Atomically updates @v to (@v - 1) with full ordering.
1221  *
1222  * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1223  *
1224  * Return: The updated value of @v.
1225  */
1226 static __always_inline int
1227 raw_atomic_dec_return(atomic_t *v)
1228 {
1229 #if defined(arch_atomic_dec_return)
1230         return arch_atomic_dec_return(v);
1231 #elif defined(arch_atomic_dec_return_relaxed)
1232         int ret;
1233         __atomic_pre_full_fence();
1234         ret = arch_atomic_dec_return_relaxed(v);
1235         __atomic_post_full_fence();
1236         return ret;
1237 #else
1238         return raw_atomic_sub_return(1, v);
1239 #endif
1240 }
1241
1242 /**
1243  * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
1244  * @v: pointer to atomic_t
1245  *
1246  * Atomically updates @v to (@v - 1) with acquire ordering.
1247  *
1248  * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1249  *
1250  * Return: The updated value of @v.
1251  */
1252 static __always_inline int
1253 raw_atomic_dec_return_acquire(atomic_t *v)
1254 {
1255 #if defined(arch_atomic_dec_return_acquire)
1256         return arch_atomic_dec_return_acquire(v);
1257 #elif defined(arch_atomic_dec_return_relaxed)
1258         int ret = arch_atomic_dec_return_relaxed(v);
1259         __atomic_acquire_fence();
1260         return ret;
1261 #elif defined(arch_atomic_dec_return)
1262         return arch_atomic_dec_return(v);
1263 #else
1264         return raw_atomic_sub_return_acquire(1, v);
1265 #endif
1266 }
1267
1268 /**
1269  * raw_atomic_dec_return_release() - atomic decrement with release ordering
1270  * @v: pointer to atomic_t
1271  *
1272  * Atomically updates @v to (@v - 1) with release ordering.
1273  *
1274  * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1275  *
1276  * Return: The updated value of @v.
1277  */
1278 static __always_inline int
1279 raw_atomic_dec_return_release(atomic_t *v)
1280 {
1281 #if defined(arch_atomic_dec_return_release)
1282         return arch_atomic_dec_return_release(v);
1283 #elif defined(arch_atomic_dec_return_relaxed)
1284         __atomic_release_fence();
1285         return arch_atomic_dec_return_relaxed(v);
1286 #elif defined(arch_atomic_dec_return)
1287         return arch_atomic_dec_return(v);
1288 #else
1289         return raw_atomic_sub_return_release(1, v);
1290 #endif
1291 }
1292
1293 /**
1294  * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1295  * @v: pointer to atomic_t
1296  *
1297  * Atomically updates @v to (@v - 1) with relaxed ordering.
1298  *
1299  * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1300  *
1301  * Return: The updated value of @v.
1302  */
1303 static __always_inline int
1304 raw_atomic_dec_return_relaxed(atomic_t *v)
1305 {
1306 #if defined(arch_atomic_dec_return_relaxed)
1307         return arch_atomic_dec_return_relaxed(v);
1308 #elif defined(arch_atomic_dec_return)
1309         return arch_atomic_dec_return(v);
1310 #else
1311         return raw_atomic_sub_return_relaxed(1, v);
1312 #endif
1313 }
1314
1315 /**
1316  * raw_atomic_fetch_dec() - atomic decrement with full ordering
1317  * @v: pointer to atomic_t
1318  *
1319  * Atomically updates @v to (@v - 1) with full ordering.
1320  *
1321  * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1322  *
1323  * Return: The original value of @v.
1324  */
1325 static __always_inline int
1326 raw_atomic_fetch_dec(atomic_t *v)
1327 {
1328 #if defined(arch_atomic_fetch_dec)
1329         return arch_atomic_fetch_dec(v);
1330 #elif defined(arch_atomic_fetch_dec_relaxed)
1331         int ret;
1332         __atomic_pre_full_fence();
1333         ret = arch_atomic_fetch_dec_relaxed(v);
1334         __atomic_post_full_fence();
1335         return ret;
1336 #else
1337         return raw_atomic_fetch_sub(1, v);
1338 #endif
1339 }
1340
1341 /**
1342  * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1343  * @v: pointer to atomic_t
1344  *
1345  * Atomically updates @v to (@v - 1) with acquire ordering.
1346  *
1347  * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1348  *
1349  * Return: The original value of @v.
1350  */
1351 static __always_inline int
1352 raw_atomic_fetch_dec_acquire(atomic_t *v)
1353 {
1354 #if defined(arch_atomic_fetch_dec_acquire)
1355         return arch_atomic_fetch_dec_acquire(v);
1356 #elif defined(arch_atomic_fetch_dec_relaxed)
1357         int ret = arch_atomic_fetch_dec_relaxed(v);
1358         __atomic_acquire_fence();
1359         return ret;
1360 #elif defined(arch_atomic_fetch_dec)
1361         return arch_atomic_fetch_dec(v);
1362 #else
1363         return raw_atomic_fetch_sub_acquire(1, v);
1364 #endif
1365 }
1366
1367 /**
1368  * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1369  * @v: pointer to atomic_t
1370  *
1371  * Atomically updates @v to (@v - 1) with release ordering.
1372  *
1373  * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1374  *
1375  * Return: The original value of @v.
1376  */
1377 static __always_inline int
1378 raw_atomic_fetch_dec_release(atomic_t *v)
1379 {
1380 #if defined(arch_atomic_fetch_dec_release)
1381         return arch_atomic_fetch_dec_release(v);
1382 #elif defined(arch_atomic_fetch_dec_relaxed)
1383         __atomic_release_fence();
1384         return arch_atomic_fetch_dec_relaxed(v);
1385 #elif defined(arch_atomic_fetch_dec)
1386         return arch_atomic_fetch_dec(v);
1387 #else
1388         return raw_atomic_fetch_sub_release(1, v);
1389 #endif
1390 }
1391
1392 /**
1393  * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1394  * @v: pointer to atomic_t
1395  *
1396  * Atomically updates @v to (@v - 1) with relaxed ordering.
1397  *
1398  * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1399  *
1400  * Return: The original value of @v.
1401  */
1402 static __always_inline int
1403 raw_atomic_fetch_dec_relaxed(atomic_t *v)
1404 {
1405 #if defined(arch_atomic_fetch_dec_relaxed)
1406         return arch_atomic_fetch_dec_relaxed(v);
1407 #elif defined(arch_atomic_fetch_dec)
1408         return arch_atomic_fetch_dec(v);
1409 #else
1410         return raw_atomic_fetch_sub_relaxed(1, v);
1411 #endif
1412 }
1413
1414 /**
1415  * raw_atomic_and() - atomic bitwise AND with relaxed ordering
1416  * @i: int value
1417  * @v: pointer to atomic_t
1418  *
1419  * Atomically updates @v to (@v & @i) with relaxed ordering.
1420  *
1421  * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1422  *
1423  * Return: Nothing.
1424  */
1425 static __always_inline void
1426 raw_atomic_and(int i, atomic_t *v)
1427 {
1428         arch_atomic_and(i, v);
1429 }
1430
1431 /**
1432  * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1433  * @i: int value
1434  * @v: pointer to atomic_t
1435  *
1436  * Atomically updates @v to (@v & @i) with full ordering.
1437  *
1438  * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1439  *
1440  * Return: The original value of @v.
1441  */
1442 static __always_inline int
1443 raw_atomic_fetch_and(int i, atomic_t *v)
1444 {
1445 #if defined(arch_atomic_fetch_and)
1446         return arch_atomic_fetch_and(i, v);
1447 #elif defined(arch_atomic_fetch_and_relaxed)
1448         int ret;
1449         __atomic_pre_full_fence();
1450         ret = arch_atomic_fetch_and_relaxed(i, v);
1451         __atomic_post_full_fence();
1452         return ret;
1453 #else
1454 #error "Unable to define raw_atomic_fetch_and"
1455 #endif
1456 }
1457
1458 /**
1459  * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
1460  * @i: int value
1461  * @v: pointer to atomic_t
1462  *
1463  * Atomically updates @v to (@v & @i) with acquire ordering.
1464  *
1465  * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
1466  *
1467  * Return: The original value of @v.
1468  */
1469 static __always_inline int
1470 raw_atomic_fetch_and_acquire(int i, atomic_t *v)
1471 {
1472 #if defined(arch_atomic_fetch_and_acquire)
1473         return arch_atomic_fetch_and_acquire(i, v);
1474 #elif defined(arch_atomic_fetch_and_relaxed)
1475         int ret = arch_atomic_fetch_and_relaxed(i, v);
1476         __atomic_acquire_fence();
1477         return ret;
1478 #elif defined(arch_atomic_fetch_and)
1479         return arch_atomic_fetch_and(i, v);
1480 #else
1481 #error "Unable to define raw_atomic_fetch_and_acquire"
1482 #endif
1483 }
1484
1485 /**
1486  * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1487  * @i: int value
1488  * @v: pointer to atomic_t
1489  *
1490  * Atomically updates @v to (@v & @i) with release ordering.
1491  *
1492  * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1493  *
1494  * Return: The original value of @v.
1495  */
1496 static __always_inline int
1497 raw_atomic_fetch_and_release(int i, atomic_t *v)
1498 {
1499 #if defined(arch_atomic_fetch_and_release)
1500         return arch_atomic_fetch_and_release(i, v);
1501 #elif defined(arch_atomic_fetch_and_relaxed)
1502         __atomic_release_fence();
1503         return arch_atomic_fetch_and_relaxed(i, v);
1504 #elif defined(arch_atomic_fetch_and)
1505         return arch_atomic_fetch_and(i, v);
1506 #else
1507 #error "Unable to define raw_atomic_fetch_and_release"
1508 #endif
1509 }
1510
1511 /**
1512  * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1513  * @i: int value
1514  * @v: pointer to atomic_t
1515  *
1516  * Atomically updates @v to (@v & @i) with relaxed ordering.
1517  *
1518  * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1519  *
1520  * Return: The original value of @v.
1521  */
1522 static __always_inline int
1523 raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
1524 {
1525 #if defined(arch_atomic_fetch_and_relaxed)
1526         return arch_atomic_fetch_and_relaxed(i, v);
1527 #elif defined(arch_atomic_fetch_and)
1528         return arch_atomic_fetch_and(i, v);
1529 #else
1530 #error "Unable to define raw_atomic_fetch_and_relaxed"
1531 #endif
1532 }
1533
1534 /**
1535  * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
1536  * @i: int value
1537  * @v: pointer to atomic_t
1538  *
1539  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1540  *
1541  * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1542  *
1543  * Return: Nothing.
1544  */
1545 static __always_inline void
1546 raw_atomic_andnot(int i, atomic_t *v)
1547 {
1548 #if defined(arch_atomic_andnot)
1549         arch_atomic_andnot(i, v);
1550 #else
1551         raw_atomic_and(~i, v);
1552 #endif
1553 }
1554
1555 /**
1556  * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
1557  * @i: int value
1558  * @v: pointer to atomic_t
1559  *
1560  * Atomically updates @v to (@v & ~@i) with full ordering.
1561  *
1562  * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
1563  *
1564  * Return: The original value of @v.
1565  */
1566 static __always_inline int
1567 raw_atomic_fetch_andnot(int i, atomic_t *v)
1568 {
1569 #if defined(arch_atomic_fetch_andnot)
1570         return arch_atomic_fetch_andnot(i, v);
1571 #elif defined(arch_atomic_fetch_andnot_relaxed)
1572         int ret;
1573         __atomic_pre_full_fence();
1574         ret = arch_atomic_fetch_andnot_relaxed(i, v);
1575         __atomic_post_full_fence();
1576         return ret;
1577 #else
1578         return raw_atomic_fetch_and(~i, v);
1579 #endif
1580 }
1581
1582 /**
1583  * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1584  * @i: int value
1585  * @v: pointer to atomic_t
1586  *
1587  * Atomically updates @v to (@v & ~@i) with acquire ordering.
1588  *
1589  * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1590  *
1591  * Return: The original value of @v.
1592  */
1593 static __always_inline int
1594 raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
1595 {
1596 #if defined(arch_atomic_fetch_andnot_acquire)
1597         return arch_atomic_fetch_andnot_acquire(i, v);
1598 #elif defined(arch_atomic_fetch_andnot_relaxed)
1599         int ret = arch_atomic_fetch_andnot_relaxed(i, v);
1600         __atomic_acquire_fence();
1601         return ret;
1602 #elif defined(arch_atomic_fetch_andnot)
1603         return arch_atomic_fetch_andnot(i, v);
1604 #else
1605         return raw_atomic_fetch_and_acquire(~i, v);
1606 #endif
1607 }
1608
1609 /**
1610  * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1611  * @i: int value
1612  * @v: pointer to atomic_t
1613  *
1614  * Atomically updates @v to (@v & ~@i) with release ordering.
1615  *
1616  * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1617  *
1618  * Return: The original value of @v.
1619  */
1620 static __always_inline int
1621 raw_atomic_fetch_andnot_release(int i, atomic_t *v)
1622 {
1623 #if defined(arch_atomic_fetch_andnot_release)
1624         return arch_atomic_fetch_andnot_release(i, v);
1625 #elif defined(arch_atomic_fetch_andnot_relaxed)
1626         __atomic_release_fence();
1627         return arch_atomic_fetch_andnot_relaxed(i, v);
1628 #elif defined(arch_atomic_fetch_andnot)
1629         return arch_atomic_fetch_andnot(i, v);
1630 #else
1631         return raw_atomic_fetch_and_release(~i, v);
1632 #endif
1633 }
1634
1635 /**
1636  * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1637  * @i: int value
1638  * @v: pointer to atomic_t
1639  *
1640  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1641  *
1642  * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1643  *
1644  * Return: The original value of @v.
1645  */
1646 static __always_inline int
1647 raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1648 {
1649 #if defined(arch_atomic_fetch_andnot_relaxed)
1650         return arch_atomic_fetch_andnot_relaxed(i, v);
1651 #elif defined(arch_atomic_fetch_andnot)
1652         return arch_atomic_fetch_andnot(i, v);
1653 #else
1654         return raw_atomic_fetch_and_relaxed(~i, v);
1655 #endif
1656 }
1657
1658 /**
1659  * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1660  * @i: int value
1661  * @v: pointer to atomic_t
1662  *
1663  * Atomically updates @v to (@v | @i) with relaxed ordering.
1664  *
1665  * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1666  *
1667  * Return: Nothing.
1668  */
1669 static __always_inline void
1670 raw_atomic_or(int i, atomic_t *v)
1671 {
1672         arch_atomic_or(i, v);
1673 }
1674
1675 /**
1676  * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1677  * @i: int value
1678  * @v: pointer to atomic_t
1679  *
1680  * Atomically updates @v to (@v | @i) with full ordering.
1681  *
1682  * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1683  *
1684  * Return: The original value of @v.
1685  */
1686 static __always_inline int
1687 raw_atomic_fetch_or(int i, atomic_t *v)
1688 {
1689 #if defined(arch_atomic_fetch_or)
1690         return arch_atomic_fetch_or(i, v);
1691 #elif defined(arch_atomic_fetch_or_relaxed)
1692         int ret;
1693         __atomic_pre_full_fence();
1694         ret = arch_atomic_fetch_or_relaxed(i, v);
1695         __atomic_post_full_fence();
1696         return ret;
1697 #else
1698 #error "Unable to define raw_atomic_fetch_or"
1699 #endif
1700 }
1701
1702 /**
1703  * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1704  * @i: int value
1705  * @v: pointer to atomic_t
1706  *
1707  * Atomically updates @v to (@v | @i) with acquire ordering.
1708  *
1709  * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
1710  *
1711  * Return: The original value of @v.
1712  */
1713 static __always_inline int
1714 raw_atomic_fetch_or_acquire(int i, atomic_t *v)
1715 {
1716 #if defined(arch_atomic_fetch_or_acquire)
1717         return arch_atomic_fetch_or_acquire(i, v);
1718 #elif defined(arch_atomic_fetch_or_relaxed)
1719         int ret = arch_atomic_fetch_or_relaxed(i, v);
1720         __atomic_acquire_fence();
1721         return ret;
1722 #elif defined(arch_atomic_fetch_or)
1723         return arch_atomic_fetch_or(i, v);
1724 #else
1725 #error "Unable to define raw_atomic_fetch_or_acquire"
1726 #endif
1727 }
1728
1729 /**
1730  * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1731  * @i: int value
1732  * @v: pointer to atomic_t
1733  *
1734  * Atomically updates @v to (@v | @i) with release ordering.
1735  *
1736  * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1737  *
1738  * Return: The original value of @v.
1739  */
1740 static __always_inline int
1741 raw_atomic_fetch_or_release(int i, atomic_t *v)
1742 {
1743 #if defined(arch_atomic_fetch_or_release)
1744         return arch_atomic_fetch_or_release(i, v);
1745 #elif defined(arch_atomic_fetch_or_relaxed)
1746         __atomic_release_fence();
1747         return arch_atomic_fetch_or_relaxed(i, v);
1748 #elif defined(arch_atomic_fetch_or)
1749         return arch_atomic_fetch_or(i, v);
1750 #else
1751 #error "Unable to define raw_atomic_fetch_or_release"
1752 #endif
1753 }
1754
1755 /**
1756  * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1757  * @i: int value
1758  * @v: pointer to atomic_t
1759  *
1760  * Atomically updates @v to (@v | @i) with relaxed ordering.
1761  *
1762  * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1763  *
1764  * Return: The original value of @v.
1765  */
1766 static __always_inline int
1767 raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1768 {
1769 #if defined(arch_atomic_fetch_or_relaxed)
1770         return arch_atomic_fetch_or_relaxed(i, v);
1771 #elif defined(arch_atomic_fetch_or)
1772         return arch_atomic_fetch_or(i, v);
1773 #else
1774 #error "Unable to define raw_atomic_fetch_or_relaxed"
1775 #endif
1776 }
1777
1778 /**
1779  * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1780  * @i: int value
1781  * @v: pointer to atomic_t
1782  *
1783  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1784  *
1785  * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1786  *
1787  * Return: Nothing.
1788  */
1789 static __always_inline void
1790 raw_atomic_xor(int i, atomic_t *v)
1791 {
1792         arch_atomic_xor(i, v);
1793 }
1794
1795 /**
1796  * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1797  * @i: int value
1798  * @v: pointer to atomic_t
1799  *
1800  * Atomically updates @v to (@v ^ @i) with full ordering.
1801  *
1802  * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1803  *
1804  * Return: The original value of @v.
1805  */
1806 static __always_inline int
1807 raw_atomic_fetch_xor(int i, atomic_t *v)
1808 {
1809 #if defined(arch_atomic_fetch_xor)
1810         return arch_atomic_fetch_xor(i, v);
1811 #elif defined(arch_atomic_fetch_xor_relaxed)
1812         int ret;
1813         __atomic_pre_full_fence();
1814         ret = arch_atomic_fetch_xor_relaxed(i, v);
1815         __atomic_post_full_fence();
1816         return ret;
1817 #else
1818 #error "Unable to define raw_atomic_fetch_xor"
1819 #endif
1820 }
1821
1822 /**
1823  * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1824  * @i: int value
1825  * @v: pointer to atomic_t
1826  *
1827  * Atomically updates @v to (@v ^ @i) with acquire ordering.
1828  *
1829  * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
1830  *
1831  * Return: The original value of @v.
1832  */
1833 static __always_inline int
1834 raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
1835 {
1836 #if defined(arch_atomic_fetch_xor_acquire)
1837         return arch_atomic_fetch_xor_acquire(i, v);
1838 #elif defined(arch_atomic_fetch_xor_relaxed)
1839         int ret = arch_atomic_fetch_xor_relaxed(i, v);
1840         __atomic_acquire_fence();
1841         return ret;
1842 #elif defined(arch_atomic_fetch_xor)
1843         return arch_atomic_fetch_xor(i, v);
1844 #else
1845 #error "Unable to define raw_atomic_fetch_xor_acquire"
1846 #endif
1847 }
1848
1849 /**
1850  * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1851  * @i: int value
1852  * @v: pointer to atomic_t
1853  *
1854  * Atomically updates @v to (@v ^ @i) with release ordering.
1855  *
1856  * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1857  *
1858  * Return: The original value of @v.
1859  */
1860 static __always_inline int
1861 raw_atomic_fetch_xor_release(int i, atomic_t *v)
1862 {
1863 #if defined(arch_atomic_fetch_xor_release)
1864         return arch_atomic_fetch_xor_release(i, v);
1865 #elif defined(arch_atomic_fetch_xor_relaxed)
1866         __atomic_release_fence();
1867         return arch_atomic_fetch_xor_relaxed(i, v);
1868 #elif defined(arch_atomic_fetch_xor)
1869         return arch_atomic_fetch_xor(i, v);
1870 #else
1871 #error "Unable to define raw_atomic_fetch_xor_release"
1872 #endif
1873 }
1874
1875 /**
1876  * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1877  * @i: int value
1878  * @v: pointer to atomic_t
1879  *
1880  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1881  *
1882  * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1883  *
1884  * Return: The original value of @v.
1885  */
1886 static __always_inline int
1887 raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1888 {
1889 #if defined(arch_atomic_fetch_xor_relaxed)
1890         return arch_atomic_fetch_xor_relaxed(i, v);
1891 #elif defined(arch_atomic_fetch_xor)
1892         return arch_atomic_fetch_xor(i, v);
1893 #else
1894 #error "Unable to define raw_atomic_fetch_xor_relaxed"
1895 #endif
1896 }
1897
1898 /**
1899  * raw_atomic_xchg() - atomic exchange with full ordering
1900  * @v: pointer to atomic_t
1901  * @new: int value to assign
1902  *
1903  * Atomically updates @v to @new with full ordering.
1904  *
1905  * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1906  *
1907  * Return: The original value of @v.
1908  */
1909 static __always_inline int
1910 raw_atomic_xchg(atomic_t *v, int new)
1911 {
1912 #if defined(arch_atomic_xchg)
1913         return arch_atomic_xchg(v, new);
1914 #elif defined(arch_atomic_xchg_relaxed)
1915         int ret;
1916         __atomic_pre_full_fence();
1917         ret = arch_atomic_xchg_relaxed(v, new);
1918         __atomic_post_full_fence();
1919         return ret;
1920 #else
1921         return raw_xchg(&v->counter, new);
1922 #endif
1923 }
1924
1925 /**
1926  * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1927  * @v: pointer to atomic_t
1928  * @new: int value to assign
1929  *
1930  * Atomically updates @v to @new with acquire ordering.
1931  *
1932  * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1933  *
1934  * Return: The original value of @v.
1935  */
1936 static __always_inline int
1937 raw_atomic_xchg_acquire(atomic_t *v, int new)
1938 {
1939 #if defined(arch_atomic_xchg_acquire)
1940         return arch_atomic_xchg_acquire(v, new);
1941 #elif defined(arch_atomic_xchg_relaxed)
1942         int ret = arch_atomic_xchg_relaxed(v, new);
1943         __atomic_acquire_fence();
1944         return ret;
1945 #elif defined(arch_atomic_xchg)
1946         return arch_atomic_xchg(v, new);
1947 #else
1948         return raw_xchg_acquire(&v->counter, new);
1949 #endif
1950 }
1951
1952 /**
1953  * raw_atomic_xchg_release() - atomic exchange with release ordering
1954  * @v: pointer to atomic_t
1955  * @new: int value to assign
1956  *
1957  * Atomically updates @v to @new with release ordering.
1958  *
1959  * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1960  *
1961  * Return: The original value of @v.
1962  */
1963 static __always_inline int
1964 raw_atomic_xchg_release(atomic_t *v, int new)
1965 {
1966 #if defined(arch_atomic_xchg_release)
1967         return arch_atomic_xchg_release(v, new);
1968 #elif defined(arch_atomic_xchg_relaxed)
1969         __atomic_release_fence();
1970         return arch_atomic_xchg_relaxed(v, new);
1971 #elif defined(arch_atomic_xchg)
1972         return arch_atomic_xchg(v, new);
1973 #else
1974         return raw_xchg_release(&v->counter, new);
1975 #endif
1976 }
1977
1978 /**
1979  * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1980  * @v: pointer to atomic_t
1981  * @new: int value to assign
1982  *
1983  * Atomically updates @v to @new with relaxed ordering.
1984  *
1985  * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1986  *
1987  * Return: The original value of @v.
1988  */
1989 static __always_inline int
1990 raw_atomic_xchg_relaxed(atomic_t *v, int new)
1991 {
1992 #if defined(arch_atomic_xchg_relaxed)
1993         return arch_atomic_xchg_relaxed(v, new);
1994 #elif defined(arch_atomic_xchg)
1995         return arch_atomic_xchg(v, new);
1996 #else
1997         return raw_xchg_relaxed(&v->counter, new);
1998 #endif
1999 }
2000
2001 /**
2002  * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
2003  * @v: pointer to atomic_t
2004  * @old: int value to compare with
2005  * @new: int value to assign
2006  *
2007  * If (@v == @old), atomically updates @v to @new with full ordering.
2008  *
2009  * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
2010  *
2011  * Return: The original value of @v.
2012  */
2013 static __always_inline int
2014 raw_atomic_cmpxchg(atomic_t *v, int old, int new)
2015 {
2016 #if defined(arch_atomic_cmpxchg)
2017         return arch_atomic_cmpxchg(v, old, new);
2018 #elif defined(arch_atomic_cmpxchg_relaxed)
2019         int ret;
2020         __atomic_pre_full_fence();
2021         ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2022         __atomic_post_full_fence();
2023         return ret;
2024 #else
2025         return raw_cmpxchg(&v->counter, old, new);
2026 #endif
2027 }
2028
2029 /**
2030  * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2031  * @v: pointer to atomic_t
2032  * @old: int value to compare with
2033  * @new: int value to assign
2034  *
2035  * If (@v == @old), atomically updates @v to @new with acquire ordering.
2036  *
2037  * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2038  *
2039  * Return: The original value of @v.
2040  */
2041 static __always_inline int
2042 raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
2043 {
2044 #if defined(arch_atomic_cmpxchg_acquire)
2045         return arch_atomic_cmpxchg_acquire(v, old, new);
2046 #elif defined(arch_atomic_cmpxchg_relaxed)
2047         int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2048         __atomic_acquire_fence();
2049         return ret;
2050 #elif defined(arch_atomic_cmpxchg)
2051         return arch_atomic_cmpxchg(v, old, new);
2052 #else
2053         return raw_cmpxchg_acquire(&v->counter, old, new);
2054 #endif
2055 }
2056
2057 /**
2058  * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2059  * @v: pointer to atomic_t
2060  * @old: int value to compare with
2061  * @new: int value to assign
2062  *
2063  * If (@v == @old), atomically updates @v to @new with release ordering.
2064  *
2065  * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2066  *
2067  * Return: The original value of @v.
2068  */
2069 static __always_inline int
2070 raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
2071 {
2072 #if defined(arch_atomic_cmpxchg_release)
2073         return arch_atomic_cmpxchg_release(v, old, new);
2074 #elif defined(arch_atomic_cmpxchg_relaxed)
2075         __atomic_release_fence();
2076         return arch_atomic_cmpxchg_relaxed(v, old, new);
2077 #elif defined(arch_atomic_cmpxchg)
2078         return arch_atomic_cmpxchg(v, old, new);
2079 #else
2080         return raw_cmpxchg_release(&v->counter, old, new);
2081 #endif
2082 }
2083
2084 /**
2085  * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2086  * @v: pointer to atomic_t
2087  * @old: int value to compare with
2088  * @new: int value to assign
2089  *
2090  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2091  *
2092  * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2093  *
2094  * Return: The original value of @v.
2095  */
2096 static __always_inline int
2097 raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
2098 {
2099 #if defined(arch_atomic_cmpxchg_relaxed)
2100         return arch_atomic_cmpxchg_relaxed(v, old, new);
2101 #elif defined(arch_atomic_cmpxchg)
2102         return arch_atomic_cmpxchg(v, old, new);
2103 #else
2104         return raw_cmpxchg_relaxed(&v->counter, old, new);
2105 #endif
2106 }
2107
2108 /**
2109  * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
2110  * @v: pointer to atomic_t
2111  * @old: pointer to int value to compare with
2112  * @new: int value to assign
2113  *
2114  * If (@v == @old), atomically updates @v to @new with full ordering.
2115  * Otherwise, updates @old to the current value of @v.
2116  *
2117  * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2118  *
2119  * Return: @true if the exchange occured, @false otherwise.
2120  */
2121 static __always_inline bool
2122 raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
2123 {
2124 #if defined(arch_atomic_try_cmpxchg)
2125         return arch_atomic_try_cmpxchg(v, old, new);
2126 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2127         bool ret;
2128         __atomic_pre_full_fence();
2129         ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2130         __atomic_post_full_fence();
2131         return ret;
2132 #else
2133         int r, o = *old;
2134         r = raw_atomic_cmpxchg(v, o, new);
2135         if (unlikely(r != o))
2136                 *old = r;
2137         return likely(r == o);
2138 #endif
2139 }
2140
2141 /**
2142  * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2143  * @v: pointer to atomic_t
2144  * @old: pointer to int value to compare with
2145  * @new: int value to assign
2146  *
2147  * If (@v == @old), atomically updates @v to @new with acquire ordering.
2148  * Otherwise, updates @old to the current value of @v.
2149  *
2150  * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2151  *
2152  * Return: @true if the exchange occured, @false otherwise.
2153  */
2154 static __always_inline bool
2155 raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
2156 {
2157 #if defined(arch_atomic_try_cmpxchg_acquire)
2158         return arch_atomic_try_cmpxchg_acquire(v, old, new);
2159 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2160         bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2161         __atomic_acquire_fence();
2162         return ret;
2163 #elif defined(arch_atomic_try_cmpxchg)
2164         return arch_atomic_try_cmpxchg(v, old, new);
2165 #else
2166         int r, o = *old;
2167         r = raw_atomic_cmpxchg_acquire(v, o, new);
2168         if (unlikely(r != o))
2169                 *old = r;
2170         return likely(r == o);
2171 #endif
2172 }
2173
2174 /**
2175  * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2176  * @v: pointer to atomic_t
2177  * @old: pointer to int value to compare with
2178  * @new: int value to assign
2179  *
2180  * If (@v == @old), atomically updates @v to @new with release ordering.
2181  * Otherwise, updates @old to the current value of @v.
2182  *
2183  * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2184  *
2185  * Return: @true if the exchange occured, @false otherwise.
2186  */
2187 static __always_inline bool
2188 raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
2189 {
2190 #if defined(arch_atomic_try_cmpxchg_release)
2191         return arch_atomic_try_cmpxchg_release(v, old, new);
2192 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2193         __atomic_release_fence();
2194         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2195 #elif defined(arch_atomic_try_cmpxchg)
2196         return arch_atomic_try_cmpxchg(v, old, new);
2197 #else
2198         int r, o = *old;
2199         r = raw_atomic_cmpxchg_release(v, o, new);
2200         if (unlikely(r != o))
2201                 *old = r;
2202         return likely(r == o);
2203 #endif
2204 }
2205
2206 /**
2207  * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2208  * @v: pointer to atomic_t
2209  * @old: pointer to int value to compare with
2210  * @new: int value to assign
2211  *
2212  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2213  * Otherwise, updates @old to the current value of @v.
2214  *
2215  * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2216  *
2217  * Return: @true if the exchange occured, @false otherwise.
2218  */
2219 static __always_inline bool
2220 raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
2221 {
2222 #if defined(arch_atomic_try_cmpxchg_relaxed)
2223         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2224 #elif defined(arch_atomic_try_cmpxchg)
2225         return arch_atomic_try_cmpxchg(v, old, new);
2226 #else
2227         int r, o = *old;
2228         r = raw_atomic_cmpxchg_relaxed(v, o, new);
2229         if (unlikely(r != o))
2230                 *old = r;
2231         return likely(r == o);
2232 #endif
2233 }
2234
2235 /**
2236  * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
2237  * @i: int value to add
2238  * @v: pointer to atomic_t
2239  *
2240  * Atomically updates @v to (@v - @i) with full ordering.
2241  *
2242  * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
2243  *
2244  * Return: @true if the resulting value of @v is zero, @false otherwise.
2245  */
2246 static __always_inline bool
2247 raw_atomic_sub_and_test(int i, atomic_t *v)
2248 {
2249 #if defined(arch_atomic_sub_and_test)
2250         return arch_atomic_sub_and_test(i, v);
2251 #else
2252         return raw_atomic_sub_return(i, v) == 0;
2253 #endif
2254 }
2255
2256 /**
2257  * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
2258  * @v: pointer to atomic_t
2259  *
2260  * Atomically updates @v to (@v - 1) with full ordering.
2261  *
2262  * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
2263  *
2264  * Return: @true if the resulting value of @v is zero, @false otherwise.
2265  */
2266 static __always_inline bool
2267 raw_atomic_dec_and_test(atomic_t *v)
2268 {
2269 #if defined(arch_atomic_dec_and_test)
2270         return arch_atomic_dec_and_test(v);
2271 #else
2272         return raw_atomic_dec_return(v) == 0;
2273 #endif
2274 }
2275
2276 /**
2277  * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
2278  * @v: pointer to atomic_t
2279  *
2280  * Atomically updates @v to (@v + 1) with full ordering.
2281  *
2282  * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
2283  *
2284  * Return: @true if the resulting value of @v is zero, @false otherwise.
2285  */
2286 static __always_inline bool
2287 raw_atomic_inc_and_test(atomic_t *v)
2288 {
2289 #if defined(arch_atomic_inc_and_test)
2290         return arch_atomic_inc_and_test(v);
2291 #else
2292         return raw_atomic_inc_return(v) == 0;
2293 #endif
2294 }
2295
2296 /**
2297  * raw_atomic_add_negative() - atomic add and test if negative with full ordering
2298  * @i: int value to add
2299  * @v: pointer to atomic_t
2300  *
2301  * Atomically updates @v to (@v + @i) with full ordering.
2302  *
2303  * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2304  *
2305  * Return: @true if the resulting value of @v is negative, @false otherwise.
2306  */
2307 static __always_inline bool
2308 raw_atomic_add_negative(int i, atomic_t *v)
2309 {
2310 #if defined(arch_atomic_add_negative)
2311         return arch_atomic_add_negative(i, v);
2312 #elif defined(arch_atomic_add_negative_relaxed)
2313         bool ret;
2314         __atomic_pre_full_fence();
2315         ret = arch_atomic_add_negative_relaxed(i, v);
2316         __atomic_post_full_fence();
2317         return ret;
2318 #else
2319         return raw_atomic_add_return(i, v) < 0;
2320 #endif
2321 }
2322
2323 /**
2324  * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
2325  * @i: int value to add
2326  * @v: pointer to atomic_t
2327  *
2328  * Atomically updates @v to (@v + @i) with acquire ordering.
2329  *
2330  * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
2331  *
2332  * Return: @true if the resulting value of @v is negative, @false otherwise.
2333  */
2334 static __always_inline bool
2335 raw_atomic_add_negative_acquire(int i, atomic_t *v)
2336 {
2337 #if defined(arch_atomic_add_negative_acquire)
2338         return arch_atomic_add_negative_acquire(i, v);
2339 #elif defined(arch_atomic_add_negative_relaxed)
2340         bool ret = arch_atomic_add_negative_relaxed(i, v);
2341         __atomic_acquire_fence();
2342         return ret;
2343 #elif defined(arch_atomic_add_negative)
2344         return arch_atomic_add_negative(i, v);
2345 #else
2346         return raw_atomic_add_return_acquire(i, v) < 0;
2347 #endif
2348 }
2349
2350 /**
2351  * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
2352  * @i: int value to add
2353  * @v: pointer to atomic_t
2354  *
2355  * Atomically updates @v to (@v + @i) with release ordering.
2356  *
2357  * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2358  *
2359  * Return: @true if the resulting value of @v is negative, @false otherwise.
2360  */
2361 static __always_inline bool
2362 raw_atomic_add_negative_release(int i, atomic_t *v)
2363 {
2364 #if defined(arch_atomic_add_negative_release)
2365         return arch_atomic_add_negative_release(i, v);
2366 #elif defined(arch_atomic_add_negative_relaxed)
2367         __atomic_release_fence();
2368         return arch_atomic_add_negative_relaxed(i, v);
2369 #elif defined(arch_atomic_add_negative)
2370         return arch_atomic_add_negative(i, v);
2371 #else
2372         return raw_atomic_add_return_release(i, v) < 0;
2373 #endif
2374 }
2375
2376 /**
2377  * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
2378  * @i: int value to add
2379  * @v: pointer to atomic_t
2380  *
2381  * Atomically updates @v to (@v + @i) with relaxed ordering.
2382  *
2383  * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
2384  *
2385  * Return: @true if the resulting value of @v is negative, @false otherwise.
2386  */
2387 static __always_inline bool
2388 raw_atomic_add_negative_relaxed(int i, atomic_t *v)
2389 {
2390 #if defined(arch_atomic_add_negative_relaxed)
2391         return arch_atomic_add_negative_relaxed(i, v);
2392 #elif defined(arch_atomic_add_negative)
2393         return arch_atomic_add_negative(i, v);
2394 #else
2395         return raw_atomic_add_return_relaxed(i, v) < 0;
2396 #endif
2397 }
2398
2399 /**
2400  * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
2401  * @v: pointer to atomic_t
2402  * @a: int value to add
2403  * @u: int value to compare with
2404  *
2405  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2406  *
2407  * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2408  *
2409  * Return: The original value of @v.
2410  */
2411 static __always_inline int
2412 raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
2413 {
2414 #if defined(arch_atomic_fetch_add_unless)
2415         return arch_atomic_fetch_add_unless(v, a, u);
2416 #else
2417         int c = raw_atomic_read(v);
2418
2419         do {
2420                 if (unlikely(c == u))
2421                         break;
2422         } while (!raw_atomic_try_cmpxchg(v, &c, c + a));
2423
2424         return c;
2425 #endif
2426 }
2427
2428 /**
2429  * raw_atomic_add_unless() - atomic add unless value with full ordering
2430  * @v: pointer to atomic_t
2431  * @a: int value to add
2432  * @u: int value to compare with
2433  *
2434  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2435  *
2436  * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2437  *
2438  * Return: @true if @v was updated, @false otherwise.
2439  */
2440 static __always_inline bool
2441 raw_atomic_add_unless(atomic_t *v, int a, int u)
2442 {
2443 #if defined(arch_atomic_add_unless)
2444         return arch_atomic_add_unless(v, a, u);
2445 #else
2446         return raw_atomic_fetch_add_unless(v, a, u) != u;
2447 #endif
2448 }
2449
2450 /**
2451  * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
2452  * @v: pointer to atomic_t
2453  *
2454  * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2455  *
2456  * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2457  *
2458  * Return: @true if @v was updated, @false otherwise.
2459  */
2460 static __always_inline bool
2461 raw_atomic_inc_not_zero(atomic_t *v)
2462 {
2463 #if defined(arch_atomic_inc_not_zero)
2464         return arch_atomic_inc_not_zero(v);
2465 #else
2466         return raw_atomic_add_unless(v, 1, 0);
2467 #endif
2468 }
2469
2470 /**
2471  * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2472  * @v: pointer to atomic_t
2473  *
2474  * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2475  *
2476  * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2477  *
2478  * Return: @true if @v was updated, @false otherwise.
2479  */
2480 static __always_inline bool
2481 raw_atomic_inc_unless_negative(atomic_t *v)
2482 {
2483 #if defined(arch_atomic_inc_unless_negative)
2484         return arch_atomic_inc_unless_negative(v);
2485 #else
2486         int c = raw_atomic_read(v);
2487
2488         do {
2489                 if (unlikely(c < 0))
2490                         return false;
2491         } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
2492
2493         return true;
2494 #endif
2495 }
2496
2497 /**
2498  * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2499  * @v: pointer to atomic_t
2500  *
2501  * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2502  *
2503  * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2504  *
2505  * Return: @true if @v was updated, @false otherwise.
2506  */
2507 static __always_inline bool
2508 raw_atomic_dec_unless_positive(atomic_t *v)
2509 {
2510 #if defined(arch_atomic_dec_unless_positive)
2511         return arch_atomic_dec_unless_positive(v);
2512 #else
2513         int c = raw_atomic_read(v);
2514
2515         do {
2516                 if (unlikely(c > 0))
2517                         return false;
2518         } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
2519
2520         return true;
2521 #endif
2522 }
2523
2524 /**
2525  * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2526  * @v: pointer to atomic_t
2527  *
2528  * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2529  *
2530  * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2531  *
2532  * Return: The old value of (@v - 1), regardless of whether @v was updated.
2533  */
2534 static __always_inline int
2535 raw_atomic_dec_if_positive(atomic_t *v)
2536 {
2537 #if defined(arch_atomic_dec_if_positive)
2538         return arch_atomic_dec_if_positive(v);
2539 #else
2540         int dec, c = raw_atomic_read(v);
2541
2542         do {
2543                 dec = c - 1;
2544                 if (unlikely(dec < 0))
2545                         break;
2546         } while (!raw_atomic_try_cmpxchg(v, &c, dec));
2547
2548         return dec;
2549 #endif
2550 }
2551
2552 #ifdef CONFIG_GENERIC_ATOMIC64
2553 #include <asm-generic/atomic64.h>
2554 #endif
2555
2556 /**
2557  * raw_atomic64_read() - atomic load with relaxed ordering
2558  * @v: pointer to atomic64_t
2559  *
2560  * Atomically loads the value of @v with relaxed ordering.
2561  *
2562  * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2563  *
2564  * Return: The value loaded from @v.
2565  */
2566 static __always_inline s64
2567 raw_atomic64_read(const atomic64_t *v)
2568 {
2569         return arch_atomic64_read(v);
2570 }
2571
2572 /**
2573  * raw_atomic64_read_acquire() - atomic load with acquire ordering
2574  * @v: pointer to atomic64_t
2575  *
2576  * Atomically loads the value of @v with acquire ordering.
2577  *
2578  * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2579  *
2580  * Return: The value loaded from @v.
2581  */
2582 static __always_inline s64
2583 raw_atomic64_read_acquire(const atomic64_t *v)
2584 {
2585 #if defined(arch_atomic64_read_acquire)
2586         return arch_atomic64_read_acquire(v);
2587 #else
2588         s64 ret;
2589
2590         if (__native_word(atomic64_t)) {
2591                 ret = smp_load_acquire(&(v)->counter);
2592         } else {
2593                 ret = raw_atomic64_read(v);
2594                 __atomic_acquire_fence();
2595         }
2596
2597         return ret;
2598 #endif
2599 }
2600
2601 /**
2602  * raw_atomic64_set() - atomic set with relaxed ordering
2603  * @v: pointer to atomic64_t
2604  * @i: s64 value to assign
2605  *
2606  * Atomically sets @v to @i with relaxed ordering.
2607  *
2608  * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2609  *
2610  * Return: Nothing.
2611  */
2612 static __always_inline void
2613 raw_atomic64_set(atomic64_t *v, s64 i)
2614 {
2615         arch_atomic64_set(v, i);
2616 }
2617
2618 /**
2619  * raw_atomic64_set_release() - atomic set with release ordering
2620  * @v: pointer to atomic64_t
2621  * @i: s64 value to assign
2622  *
2623  * Atomically sets @v to @i with release ordering.
2624  *
2625  * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2626  *
2627  * Return: Nothing.
2628  */
2629 static __always_inline void
2630 raw_atomic64_set_release(atomic64_t *v, s64 i)
2631 {
2632 #if defined(arch_atomic64_set_release)
2633         arch_atomic64_set_release(v, i);
2634 #else
2635         if (__native_word(atomic64_t)) {
2636                 smp_store_release(&(v)->counter, i);
2637         } else {
2638                 __atomic_release_fence();
2639                 raw_atomic64_set(v, i);
2640         }
2641 #endif
2642 }
2643
2644 /**
2645  * raw_atomic64_add() - atomic add with relaxed ordering
2646  * @i: s64 value to add
2647  * @v: pointer to atomic64_t
2648  *
2649  * Atomically updates @v to (@v + @i) with relaxed ordering.
2650  *
2651  * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2652  *
2653  * Return: Nothing.
2654  */
2655 static __always_inline void
2656 raw_atomic64_add(s64 i, atomic64_t *v)
2657 {
2658         arch_atomic64_add(i, v);
2659 }
2660
2661 /**
2662  * raw_atomic64_add_return() - atomic add with full ordering
2663  * @i: s64 value to add
2664  * @v: pointer to atomic64_t
2665  *
2666  * Atomically updates @v to (@v + @i) with full ordering.
2667  *
2668  * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2669  *
2670  * Return: The updated value of @v.
2671  */
2672 static __always_inline s64
2673 raw_atomic64_add_return(s64 i, atomic64_t *v)
2674 {
2675 #if defined(arch_atomic64_add_return)
2676         return arch_atomic64_add_return(i, v);
2677 #elif defined(arch_atomic64_add_return_relaxed)
2678         s64 ret;
2679         __atomic_pre_full_fence();
2680         ret = arch_atomic64_add_return_relaxed(i, v);
2681         __atomic_post_full_fence();
2682         return ret;
2683 #else
2684 #error "Unable to define raw_atomic64_add_return"
2685 #endif
2686 }
2687
2688 /**
2689  * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2690  * @i: s64 value to add
2691  * @v: pointer to atomic64_t
2692  *
2693  * Atomically updates @v to (@v + @i) with acquire ordering.
2694  *
2695  * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2696  *
2697  * Return: The updated value of @v.
2698  */
2699 static __always_inline s64
2700 raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
2701 {
2702 #if defined(arch_atomic64_add_return_acquire)
2703         return arch_atomic64_add_return_acquire(i, v);
2704 #elif defined(arch_atomic64_add_return_relaxed)
2705         s64 ret = arch_atomic64_add_return_relaxed(i, v);
2706         __atomic_acquire_fence();
2707         return ret;
2708 #elif defined(arch_atomic64_add_return)
2709         return arch_atomic64_add_return(i, v);
2710 #else
2711 #error "Unable to define raw_atomic64_add_return_acquire"
2712 #endif
2713 }
2714
2715 /**
2716  * raw_atomic64_add_return_release() - atomic add with release ordering
2717  * @i: s64 value to add
2718  * @v: pointer to atomic64_t
2719  *
2720  * Atomically updates @v to (@v + @i) with release ordering.
2721  *
2722  * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2723  *
2724  * Return: The updated value of @v.
2725  */
2726 static __always_inline s64
2727 raw_atomic64_add_return_release(s64 i, atomic64_t *v)
2728 {
2729 #if defined(arch_atomic64_add_return_release)
2730         return arch_atomic64_add_return_release(i, v);
2731 #elif defined(arch_atomic64_add_return_relaxed)
2732         __atomic_release_fence();
2733         return arch_atomic64_add_return_relaxed(i, v);
2734 #elif defined(arch_atomic64_add_return)
2735         return arch_atomic64_add_return(i, v);
2736 #else
2737 #error "Unable to define raw_atomic64_add_return_release"
2738 #endif
2739 }
2740
2741 /**
2742  * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2743  * @i: s64 value to add
2744  * @v: pointer to atomic64_t
2745  *
2746  * Atomically updates @v to (@v + @i) with relaxed ordering.
2747  *
2748  * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2749  *
2750  * Return: The updated value of @v.
2751  */
2752 static __always_inline s64
2753 raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2754 {
2755 #if defined(arch_atomic64_add_return_relaxed)
2756         return arch_atomic64_add_return_relaxed(i, v);
2757 #elif defined(arch_atomic64_add_return)
2758         return arch_atomic64_add_return(i, v);
2759 #else
2760 #error "Unable to define raw_atomic64_add_return_relaxed"
2761 #endif
2762 }
2763
2764 /**
2765  * raw_atomic64_fetch_add() - atomic add with full ordering
2766  * @i: s64 value to add
2767  * @v: pointer to atomic64_t
2768  *
2769  * Atomically updates @v to (@v + @i) with full ordering.
2770  *
2771  * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2772  *
2773  * Return: The original value of @v.
2774  */
2775 static __always_inline s64
2776 raw_atomic64_fetch_add(s64 i, atomic64_t *v)
2777 {
2778 #if defined(arch_atomic64_fetch_add)
2779         return arch_atomic64_fetch_add(i, v);
2780 #elif defined(arch_atomic64_fetch_add_relaxed)
2781         s64 ret;
2782         __atomic_pre_full_fence();
2783         ret = arch_atomic64_fetch_add_relaxed(i, v);
2784         __atomic_post_full_fence();
2785         return ret;
2786 #else
2787 #error "Unable to define raw_atomic64_fetch_add"
2788 #endif
2789 }
2790
2791 /**
2792  * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
2793  * @i: s64 value to add
2794  * @v: pointer to atomic64_t
2795  *
2796  * Atomically updates @v to (@v + @i) with acquire ordering.
2797  *
2798  * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
2799  *
2800  * Return: The original value of @v.
2801  */
2802 static __always_inline s64
2803 raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
2804 {
2805 #if defined(arch_atomic64_fetch_add_acquire)
2806         return arch_atomic64_fetch_add_acquire(i, v);
2807 #elif defined(arch_atomic64_fetch_add_relaxed)
2808         s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
2809         __atomic_acquire_fence();
2810         return ret;
2811 #elif defined(arch_atomic64_fetch_add)
2812         return arch_atomic64_fetch_add(i, v);
2813 #else
2814 #error "Unable to define raw_atomic64_fetch_add_acquire"
2815 #endif
2816 }
2817
2818 /**
2819  * raw_atomic64_fetch_add_release() - atomic add with release ordering
2820  * @i: s64 value to add
2821  * @v: pointer to atomic64_t
2822  *
2823  * Atomically updates @v to (@v + @i) with release ordering.
2824  *
2825  * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2826  *
2827  * Return: The original value of @v.
2828  */
2829 static __always_inline s64
2830 raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
2831 {
2832 #if defined(arch_atomic64_fetch_add_release)
2833         return arch_atomic64_fetch_add_release(i, v);
2834 #elif defined(arch_atomic64_fetch_add_relaxed)
2835         __atomic_release_fence();
2836         return arch_atomic64_fetch_add_relaxed(i, v);
2837 #elif defined(arch_atomic64_fetch_add)
2838         return arch_atomic64_fetch_add(i, v);
2839 #else
2840 #error "Unable to define raw_atomic64_fetch_add_release"
2841 #endif
2842 }
2843
2844 /**
2845  * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2846  * @i: s64 value to add
2847  * @v: pointer to atomic64_t
2848  *
2849  * Atomically updates @v to (@v + @i) with relaxed ordering.
2850  *
2851  * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2852  *
2853  * Return: The original value of @v.
2854  */
2855 static __always_inline s64
2856 raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2857 {
2858 #if defined(arch_atomic64_fetch_add_relaxed)
2859         return arch_atomic64_fetch_add_relaxed(i, v);
2860 #elif defined(arch_atomic64_fetch_add)
2861         return arch_atomic64_fetch_add(i, v);
2862 #else
2863 #error "Unable to define raw_atomic64_fetch_add_relaxed"
2864 #endif
2865 }
2866
2867 /**
2868  * raw_atomic64_sub() - atomic subtract with relaxed ordering
2869  * @i: s64 value to subtract
2870  * @v: pointer to atomic64_t
2871  *
2872  * Atomically updates @v to (@v - @i) with relaxed ordering.
2873  *
2874  * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2875  *
2876  * Return: Nothing.
2877  */
2878 static __always_inline void
2879 raw_atomic64_sub(s64 i, atomic64_t *v)
2880 {
2881         arch_atomic64_sub(i, v);
2882 }
2883
2884 /**
2885  * raw_atomic64_sub_return() - atomic subtract with full ordering
2886  * @i: s64 value to subtract
2887  * @v: pointer to atomic64_t
2888  *
2889  * Atomically updates @v to (@v - @i) with full ordering.
2890  *
2891  * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2892  *
2893  * Return: The updated value of @v.
2894  */
2895 static __always_inline s64
2896 raw_atomic64_sub_return(s64 i, atomic64_t *v)
2897 {
2898 #if defined(arch_atomic64_sub_return)
2899         return arch_atomic64_sub_return(i, v);
2900 #elif defined(arch_atomic64_sub_return_relaxed)
2901         s64 ret;
2902         __atomic_pre_full_fence();
2903         ret = arch_atomic64_sub_return_relaxed(i, v);
2904         __atomic_post_full_fence();
2905         return ret;
2906 #else
2907 #error "Unable to define raw_atomic64_sub_return"
2908 #endif
2909 }
2910
2911 /**
2912  * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
2913  * @i: s64 value to subtract
2914  * @v: pointer to atomic64_t
2915  *
2916  * Atomically updates @v to (@v - @i) with acquire ordering.
2917  *
2918  * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
2919  *
2920  * Return: The updated value of @v.
2921  */
2922 static __always_inline s64
2923 raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
2924 {
2925 #if defined(arch_atomic64_sub_return_acquire)
2926         return arch_atomic64_sub_return_acquire(i, v);
2927 #elif defined(arch_atomic64_sub_return_relaxed)
2928         s64 ret = arch_atomic64_sub_return_relaxed(i, v);
2929         __atomic_acquire_fence();
2930         return ret;
2931 #elif defined(arch_atomic64_sub_return)
2932         return arch_atomic64_sub_return(i, v);
2933 #else
2934 #error "Unable to define raw_atomic64_sub_return_acquire"
2935 #endif
2936 }
2937
2938 /**
2939  * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2940  * @i: s64 value to subtract
2941  * @v: pointer to atomic64_t
2942  *
2943  * Atomically updates @v to (@v - @i) with release ordering.
2944  *
2945  * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2946  *
2947  * Return: The updated value of @v.
2948  */
2949 static __always_inline s64
2950 raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
2951 {
2952 #if defined(arch_atomic64_sub_return_release)
2953         return arch_atomic64_sub_return_release(i, v);
2954 #elif defined(arch_atomic64_sub_return_relaxed)
2955         __atomic_release_fence();
2956         return arch_atomic64_sub_return_relaxed(i, v);
2957 #elif defined(arch_atomic64_sub_return)
2958         return arch_atomic64_sub_return(i, v);
2959 #else
2960 #error "Unable to define raw_atomic64_sub_return_release"
2961 #endif
2962 }
2963
2964 /**
2965  * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2966  * @i: s64 value to subtract
2967  * @v: pointer to atomic64_t
2968  *
2969  * Atomically updates @v to (@v - @i) with relaxed ordering.
2970  *
2971  * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2972  *
2973  * Return: The updated value of @v.
2974  */
2975 static __always_inline s64
2976 raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2977 {
2978 #if defined(arch_atomic64_sub_return_relaxed)
2979         return arch_atomic64_sub_return_relaxed(i, v);
2980 #elif defined(arch_atomic64_sub_return)
2981         return arch_atomic64_sub_return(i, v);
2982 #else
2983 #error "Unable to define raw_atomic64_sub_return_relaxed"
2984 #endif
2985 }
2986
2987 /**
2988  * raw_atomic64_fetch_sub() - atomic subtract with full ordering
2989  * @i: s64 value to subtract
2990  * @v: pointer to atomic64_t
2991  *
2992  * Atomically updates @v to (@v - @i) with full ordering.
2993  *
2994  * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
2995  *
2996  * Return: The original value of @v.
2997  */
2998 static __always_inline s64
2999 raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
3000 {
3001 #if defined(arch_atomic64_fetch_sub)
3002         return arch_atomic64_fetch_sub(i, v);
3003 #elif defined(arch_atomic64_fetch_sub_relaxed)
3004         s64 ret;
3005         __atomic_pre_full_fence();
3006         ret = arch_atomic64_fetch_sub_relaxed(i, v);
3007         __atomic_post_full_fence();
3008         return ret;
3009 #else
3010 #error "Unable to define raw_atomic64_fetch_sub"
3011 #endif
3012 }
3013
3014 /**
3015  * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
3016  * @i: s64 value to subtract
3017  * @v: pointer to atomic64_t
3018  *
3019  * Atomically updates @v to (@v - @i) with acquire ordering.
3020  *
3021  * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
3022  *
3023  * Return: The original value of @v.
3024  */
3025 static __always_inline s64
3026 raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
3027 {
3028 #if defined(arch_atomic64_fetch_sub_acquire)
3029         return arch_atomic64_fetch_sub_acquire(i, v);
3030 #elif defined(arch_atomic64_fetch_sub_relaxed)
3031         s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3032         __atomic_acquire_fence();
3033         return ret;
3034 #elif defined(arch_atomic64_fetch_sub)
3035         return arch_atomic64_fetch_sub(i, v);
3036 #else
3037 #error "Unable to define raw_atomic64_fetch_sub_acquire"
3038 #endif
3039 }
3040
3041 /**
3042  * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3043  * @i: s64 value to subtract
3044  * @v: pointer to atomic64_t
3045  *
3046  * Atomically updates @v to (@v - @i) with release ordering.
3047  *
3048  * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3049  *
3050  * Return: The original value of @v.
3051  */
3052 static __always_inline s64
3053 raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
3054 {
3055 #if defined(arch_atomic64_fetch_sub_release)
3056         return arch_atomic64_fetch_sub_release(i, v);
3057 #elif defined(arch_atomic64_fetch_sub_relaxed)
3058         __atomic_release_fence();
3059         return arch_atomic64_fetch_sub_relaxed(i, v);
3060 #elif defined(arch_atomic64_fetch_sub)
3061         return arch_atomic64_fetch_sub(i, v);
3062 #else
3063 #error "Unable to define raw_atomic64_fetch_sub_release"
3064 #endif
3065 }
3066
3067 /**
3068  * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3069  * @i: s64 value to subtract
3070  * @v: pointer to atomic64_t
3071  *
3072  * Atomically updates @v to (@v - @i) with relaxed ordering.
3073  *
3074  * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3075  *
3076  * Return: The original value of @v.
3077  */
3078 static __always_inline s64
3079 raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
3080 {
3081 #if defined(arch_atomic64_fetch_sub_relaxed)
3082         return arch_atomic64_fetch_sub_relaxed(i, v);
3083 #elif defined(arch_atomic64_fetch_sub)
3084         return arch_atomic64_fetch_sub(i, v);
3085 #else
3086 #error "Unable to define raw_atomic64_fetch_sub_relaxed"
3087 #endif
3088 }
3089
3090 /**
3091  * raw_atomic64_inc() - atomic increment with relaxed ordering
3092  * @v: pointer to atomic64_t
3093  *
3094  * Atomically updates @v to (@v + 1) with relaxed ordering.
3095  *
3096  * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
3097  *
3098  * Return: Nothing.
3099  */
3100 static __always_inline void
3101 raw_atomic64_inc(atomic64_t *v)
3102 {
3103 #if defined(arch_atomic64_inc)
3104         arch_atomic64_inc(v);
3105 #else
3106         raw_atomic64_add(1, v);
3107 #endif
3108 }
3109
3110 /**
3111  * raw_atomic64_inc_return() - atomic increment with full ordering
3112  * @v: pointer to atomic64_t
3113  *
3114  * Atomically updates @v to (@v + 1) with full ordering.
3115  *
3116  * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3117  *
3118  * Return: The updated value of @v.
3119  */
3120 static __always_inline s64
3121 raw_atomic64_inc_return(atomic64_t *v)
3122 {
3123 #if defined(arch_atomic64_inc_return)
3124         return arch_atomic64_inc_return(v);
3125 #elif defined(arch_atomic64_inc_return_relaxed)
3126         s64 ret;
3127         __atomic_pre_full_fence();
3128         ret = arch_atomic64_inc_return_relaxed(v);
3129         __atomic_post_full_fence();
3130         return ret;
3131 #else
3132         return raw_atomic64_add_return(1, v);
3133 #endif
3134 }
3135
3136 /**
3137  * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
3138  * @v: pointer to atomic64_t
3139  *
3140  * Atomically updates @v to (@v + 1) with acquire ordering.
3141  *
3142  * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3143  *
3144  * Return: The updated value of @v.
3145  */
3146 static __always_inline s64
3147 raw_atomic64_inc_return_acquire(atomic64_t *v)
3148 {
3149 #if defined(arch_atomic64_inc_return_acquire)
3150         return arch_atomic64_inc_return_acquire(v);
3151 #elif defined(arch_atomic64_inc_return_relaxed)
3152         s64 ret = arch_atomic64_inc_return_relaxed(v);
3153         __atomic_acquire_fence();
3154         return ret;
3155 #elif defined(arch_atomic64_inc_return)
3156         return arch_atomic64_inc_return(v);
3157 #else
3158         return raw_atomic64_add_return_acquire(1, v);
3159 #endif
3160 }
3161
3162 /**
3163  * raw_atomic64_inc_return_release() - atomic increment with release ordering
3164  * @v: pointer to atomic64_t
3165  *
3166  * Atomically updates @v to (@v + 1) with release ordering.
3167  *
3168  * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3169  *
3170  * Return: The updated value of @v.
3171  */
3172 static __always_inline s64
3173 raw_atomic64_inc_return_release(atomic64_t *v)
3174 {
3175 #if defined(arch_atomic64_inc_return_release)
3176         return arch_atomic64_inc_return_release(v);
3177 #elif defined(arch_atomic64_inc_return_relaxed)
3178         __atomic_release_fence();
3179         return arch_atomic64_inc_return_relaxed(v);
3180 #elif defined(arch_atomic64_inc_return)
3181         return arch_atomic64_inc_return(v);
3182 #else
3183         return raw_atomic64_add_return_release(1, v);
3184 #endif
3185 }
3186
3187 /**
3188  * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
3189  * @v: pointer to atomic64_t
3190  *
3191  * Atomically updates @v to (@v + 1) with relaxed ordering.
3192  *
3193  * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
3194  *
3195  * Return: The updated value of @v.
3196  */
3197 static __always_inline s64
3198 raw_atomic64_inc_return_relaxed(atomic64_t *v)
3199 {
3200 #if defined(arch_atomic64_inc_return_relaxed)
3201         return arch_atomic64_inc_return_relaxed(v);
3202 #elif defined(arch_atomic64_inc_return)
3203         return arch_atomic64_inc_return(v);
3204 #else
3205         return raw_atomic64_add_return_relaxed(1, v);
3206 #endif
3207 }
3208
3209 /**
3210  * raw_atomic64_fetch_inc() - atomic increment with full ordering
3211  * @v: pointer to atomic64_t
3212  *
3213  * Atomically updates @v to (@v + 1) with full ordering.
3214  *
3215  * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3216  *
3217  * Return: The original value of @v.
3218  */
3219 static __always_inline s64
3220 raw_atomic64_fetch_inc(atomic64_t *v)
3221 {
3222 #if defined(arch_atomic64_fetch_inc)
3223         return arch_atomic64_fetch_inc(v);
3224 #elif defined(arch_atomic64_fetch_inc_relaxed)
3225         s64 ret;
3226         __atomic_pre_full_fence();
3227         ret = arch_atomic64_fetch_inc_relaxed(v);
3228         __atomic_post_full_fence();
3229         return ret;
3230 #else
3231         return raw_atomic64_fetch_add(1, v);
3232 #endif
3233 }
3234
3235 /**
3236  * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3237  * @v: pointer to atomic64_t
3238  *
3239  * Atomically updates @v to (@v + 1) with acquire ordering.
3240  *
3241  * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3242  *
3243  * Return: The original value of @v.
3244  */
3245 static __always_inline s64
3246 raw_atomic64_fetch_inc_acquire(atomic64_t *v)
3247 {
3248 #if defined(arch_atomic64_fetch_inc_acquire)
3249         return arch_atomic64_fetch_inc_acquire(v);
3250 #elif defined(arch_atomic64_fetch_inc_relaxed)
3251         s64 ret = arch_atomic64_fetch_inc_relaxed(v);
3252         __atomic_acquire_fence();
3253         return ret;
3254 #elif defined(arch_atomic64_fetch_inc)
3255         return arch_atomic64_fetch_inc(v);
3256 #else
3257         return raw_atomic64_fetch_add_acquire(1, v);
3258 #endif
3259 }
3260
3261 /**
3262  * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3263  * @v: pointer to atomic64_t
3264  *
3265  * Atomically updates @v to (@v + 1) with release ordering.
3266  *
3267  * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3268  *
3269  * Return: The original value of @v.
3270  */
3271 static __always_inline s64
3272 raw_atomic64_fetch_inc_release(atomic64_t *v)
3273 {
3274 #if defined(arch_atomic64_fetch_inc_release)
3275         return arch_atomic64_fetch_inc_release(v);
3276 #elif defined(arch_atomic64_fetch_inc_relaxed)
3277         __atomic_release_fence();
3278         return arch_atomic64_fetch_inc_relaxed(v);
3279 #elif defined(arch_atomic64_fetch_inc)
3280         return arch_atomic64_fetch_inc(v);
3281 #else
3282         return raw_atomic64_fetch_add_release(1, v);
3283 #endif
3284 }
3285
3286 /**
3287  * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
3288  * @v: pointer to atomic64_t
3289  *
3290  * Atomically updates @v to (@v + 1) with relaxed ordering.
3291  *
3292  * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
3293  *
3294  * Return: The original value of @v.
3295  */
3296 static __always_inline s64
3297 raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
3298 {
3299 #if defined(arch_atomic64_fetch_inc_relaxed)
3300         return arch_atomic64_fetch_inc_relaxed(v);
3301 #elif defined(arch_atomic64_fetch_inc)
3302         return arch_atomic64_fetch_inc(v);
3303 #else
3304         return raw_atomic64_fetch_add_relaxed(1, v);
3305 #endif
3306 }
3307
3308 /**
3309  * raw_atomic64_dec() - atomic decrement with relaxed ordering
3310  * @v: pointer to atomic64_t
3311  *
3312  * Atomically updates @v to (@v - 1) with relaxed ordering.
3313  *
3314  * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3315  *
3316  * Return: Nothing.
3317  */
3318 static __always_inline void
3319 raw_atomic64_dec(atomic64_t *v)
3320 {
3321 #if defined(arch_atomic64_dec)
3322         arch_atomic64_dec(v);
3323 #else
3324         raw_atomic64_sub(1, v);
3325 #endif
3326 }
3327
3328 /**
3329  * raw_atomic64_dec_return() - atomic decrement with full ordering
3330  * @v: pointer to atomic64_t
3331  *
3332  * Atomically updates @v to (@v - 1) with full ordering.
3333  *
3334  * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3335  *
3336  * Return: The updated value of @v.
3337  */
3338 static __always_inline s64
3339 raw_atomic64_dec_return(atomic64_t *v)
3340 {
3341 #if defined(arch_atomic64_dec_return)
3342         return arch_atomic64_dec_return(v);
3343 #elif defined(arch_atomic64_dec_return_relaxed)
3344         s64 ret;
3345         __atomic_pre_full_fence();
3346         ret = arch_atomic64_dec_return_relaxed(v);
3347         __atomic_post_full_fence();
3348         return ret;
3349 #else
3350         return raw_atomic64_sub_return(1, v);
3351 #endif
3352 }
3353
3354 /**
3355  * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
3356  * @v: pointer to atomic64_t
3357  *
3358  * Atomically updates @v to (@v - 1) with acquire ordering.
3359  *
3360  * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3361  *
3362  * Return: The updated value of @v.
3363  */
3364 static __always_inline s64
3365 raw_atomic64_dec_return_acquire(atomic64_t *v)
3366 {
3367 #if defined(arch_atomic64_dec_return_acquire)
3368         return arch_atomic64_dec_return_acquire(v);
3369 #elif defined(arch_atomic64_dec_return_relaxed)
3370         s64 ret = arch_atomic64_dec_return_relaxed(v);
3371         __atomic_acquire_fence();
3372         return ret;
3373 #elif defined(arch_atomic64_dec_return)
3374         return arch_atomic64_dec_return(v);
3375 #else
3376         return raw_atomic64_sub_return_acquire(1, v);
3377 #endif
3378 }
3379
3380 /**
3381  * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3382  * @v: pointer to atomic64_t
3383  *
3384  * Atomically updates @v to (@v - 1) with release ordering.
3385  *
3386  * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3387  *
3388  * Return: The updated value of @v.
3389  */
3390 static __always_inline s64
3391 raw_atomic64_dec_return_release(atomic64_t *v)
3392 {
3393 #if defined(arch_atomic64_dec_return_release)
3394         return arch_atomic64_dec_return_release(v);
3395 #elif defined(arch_atomic64_dec_return_relaxed)
3396         __atomic_release_fence();
3397         return arch_atomic64_dec_return_relaxed(v);
3398 #elif defined(arch_atomic64_dec_return)
3399         return arch_atomic64_dec_return(v);
3400 #else
3401         return raw_atomic64_sub_return_release(1, v);
3402 #endif
3403 }
3404
3405 /**
3406  * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3407  * @v: pointer to atomic64_t
3408  *
3409  * Atomically updates @v to (@v - 1) with relaxed ordering.
3410  *
3411  * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3412  *
3413  * Return: The updated value of @v.
3414  */
3415 static __always_inline s64
3416 raw_atomic64_dec_return_relaxed(atomic64_t *v)
3417 {
3418 #if defined(arch_atomic64_dec_return_relaxed)
3419         return arch_atomic64_dec_return_relaxed(v);
3420 #elif defined(arch_atomic64_dec_return)
3421         return arch_atomic64_dec_return(v);
3422 #else
3423         return raw_atomic64_sub_return_relaxed(1, v);
3424 #endif
3425 }
3426
3427 /**
3428  * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3429  * @v: pointer to atomic64_t
3430  *
3431  * Atomically updates @v to (@v - 1) with full ordering.
3432  *
3433  * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3434  *
3435  * Return: The original value of @v.
3436  */
3437 static __always_inline s64
3438 raw_atomic64_fetch_dec(atomic64_t *v)
3439 {
3440 #if defined(arch_atomic64_fetch_dec)
3441         return arch_atomic64_fetch_dec(v);
3442 #elif defined(arch_atomic64_fetch_dec_relaxed)
3443         s64 ret;
3444         __atomic_pre_full_fence();
3445         ret = arch_atomic64_fetch_dec_relaxed(v);
3446         __atomic_post_full_fence();
3447         return ret;
3448 #else
3449         return raw_atomic64_fetch_sub(1, v);
3450 #endif
3451 }
3452
3453 /**
3454  * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3455  * @v: pointer to atomic64_t
3456  *
3457  * Atomically updates @v to (@v - 1) with acquire ordering.
3458  *
3459  * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3460  *
3461  * Return: The original value of @v.
3462  */
3463 static __always_inline s64
3464 raw_atomic64_fetch_dec_acquire(atomic64_t *v)
3465 {
3466 #if defined(arch_atomic64_fetch_dec_acquire)
3467         return arch_atomic64_fetch_dec_acquire(v);
3468 #elif defined(arch_atomic64_fetch_dec_relaxed)
3469         s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3470         __atomic_acquire_fence();
3471         return ret;
3472 #elif defined(arch_atomic64_fetch_dec)
3473         return arch_atomic64_fetch_dec(v);
3474 #else
3475         return raw_atomic64_fetch_sub_acquire(1, v);
3476 #endif
3477 }
3478
3479 /**
3480  * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3481  * @v: pointer to atomic64_t
3482  *
3483  * Atomically updates @v to (@v - 1) with release ordering.
3484  *
3485  * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3486  *
3487  * Return: The original value of @v.
3488  */
3489 static __always_inline s64
3490 raw_atomic64_fetch_dec_release(atomic64_t *v)
3491 {
3492 #if defined(arch_atomic64_fetch_dec_release)
3493         return arch_atomic64_fetch_dec_release(v);
3494 #elif defined(arch_atomic64_fetch_dec_relaxed)
3495         __atomic_release_fence();
3496         return arch_atomic64_fetch_dec_relaxed(v);
3497 #elif defined(arch_atomic64_fetch_dec)
3498         return arch_atomic64_fetch_dec(v);
3499 #else
3500         return raw_atomic64_fetch_sub_release(1, v);
3501 #endif
3502 }
3503
3504 /**
3505  * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3506  * @v: pointer to atomic64_t
3507  *
3508  * Atomically updates @v to (@v - 1) with relaxed ordering.
3509  *
3510  * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3511  *
3512  * Return: The original value of @v.
3513  */
3514 static __always_inline s64
3515 raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
3516 {
3517 #if defined(arch_atomic64_fetch_dec_relaxed)
3518         return arch_atomic64_fetch_dec_relaxed(v);
3519 #elif defined(arch_atomic64_fetch_dec)
3520         return arch_atomic64_fetch_dec(v);
3521 #else
3522         return raw_atomic64_fetch_sub_relaxed(1, v);
3523 #endif
3524 }
3525
3526 /**
3527  * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
3528  * @i: s64 value
3529  * @v: pointer to atomic64_t
3530  *
3531  * Atomically updates @v to (@v & @i) with relaxed ordering.
3532  *
3533  * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3534  *
3535  * Return: Nothing.
3536  */
3537 static __always_inline void
3538 raw_atomic64_and(s64 i, atomic64_t *v)
3539 {
3540         arch_atomic64_and(i, v);
3541 }
3542
3543 /**
3544  * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3545  * @i: s64 value
3546  * @v: pointer to atomic64_t
3547  *
3548  * Atomically updates @v to (@v & @i) with full ordering.
3549  *
3550  * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3551  *
3552  * Return: The original value of @v.
3553  */
3554 static __always_inline s64
3555 raw_atomic64_fetch_and(s64 i, atomic64_t *v)
3556 {
3557 #if defined(arch_atomic64_fetch_and)
3558         return arch_atomic64_fetch_and(i, v);
3559 #elif defined(arch_atomic64_fetch_and_relaxed)
3560         s64 ret;
3561         __atomic_pre_full_fence();
3562         ret = arch_atomic64_fetch_and_relaxed(i, v);
3563         __atomic_post_full_fence();
3564         return ret;
3565 #else
3566 #error "Unable to define raw_atomic64_fetch_and"
3567 #endif
3568 }
3569
3570 /**
3571  * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3572  * @i: s64 value
3573  * @v: pointer to atomic64_t
3574  *
3575  * Atomically updates @v to (@v & @i) with acquire ordering.
3576  *
3577  * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
3578  *
3579  * Return: The original value of @v.
3580  */
3581 static __always_inline s64
3582 raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
3583 {
3584 #if defined(arch_atomic64_fetch_and_acquire)
3585         return arch_atomic64_fetch_and_acquire(i, v);
3586 #elif defined(arch_atomic64_fetch_and_relaxed)
3587         s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
3588         __atomic_acquire_fence();
3589         return ret;
3590 #elif defined(arch_atomic64_fetch_and)
3591         return arch_atomic64_fetch_and(i, v);
3592 #else
3593 #error "Unable to define raw_atomic64_fetch_and_acquire"
3594 #endif
3595 }
3596
3597 /**
3598  * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3599  * @i: s64 value
3600  * @v: pointer to atomic64_t
3601  *
3602  * Atomically updates @v to (@v & @i) with release ordering.
3603  *
3604  * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3605  *
3606  * Return: The original value of @v.
3607  */
3608 static __always_inline s64
3609 raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
3610 {
3611 #if defined(arch_atomic64_fetch_and_release)
3612         return arch_atomic64_fetch_and_release(i, v);
3613 #elif defined(arch_atomic64_fetch_and_relaxed)
3614         __atomic_release_fence();
3615         return arch_atomic64_fetch_and_relaxed(i, v);
3616 #elif defined(arch_atomic64_fetch_and)
3617         return arch_atomic64_fetch_and(i, v);
3618 #else
3619 #error "Unable to define raw_atomic64_fetch_and_release"
3620 #endif
3621 }
3622
3623 /**
3624  * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3625  * @i: s64 value
3626  * @v: pointer to atomic64_t
3627  *
3628  * Atomically updates @v to (@v & @i) with relaxed ordering.
3629  *
3630  * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3631  *
3632  * Return: The original value of @v.
3633  */
3634 static __always_inline s64
3635 raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
3636 {
3637 #if defined(arch_atomic64_fetch_and_relaxed)
3638         return arch_atomic64_fetch_and_relaxed(i, v);
3639 #elif defined(arch_atomic64_fetch_and)
3640         return arch_atomic64_fetch_and(i, v);
3641 #else
3642 #error "Unable to define raw_atomic64_fetch_and_relaxed"
3643 #endif
3644 }
3645
3646 /**
3647  * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
3648  * @i: s64 value
3649  * @v: pointer to atomic64_t
3650  *
3651  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3652  *
3653  * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3654  *
3655  * Return: Nothing.
3656  */
3657 static __always_inline void
3658 raw_atomic64_andnot(s64 i, atomic64_t *v)
3659 {
3660 #if defined(arch_atomic64_andnot)
3661         arch_atomic64_andnot(i, v);
3662 #else
3663         raw_atomic64_and(~i, v);
3664 #endif
3665 }
3666
3667 /**
3668  * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3669  * @i: s64 value
3670  * @v: pointer to atomic64_t
3671  *
3672  * Atomically updates @v to (@v & ~@i) with full ordering.
3673  *
3674  * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3675  *
3676  * Return: The original value of @v.
3677  */
3678 static __always_inline s64
3679 raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
3680 {
3681 #if defined(arch_atomic64_fetch_andnot)
3682         return arch_atomic64_fetch_andnot(i, v);
3683 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3684         s64 ret;
3685         __atomic_pre_full_fence();
3686         ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3687         __atomic_post_full_fence();
3688         return ret;
3689 #else
3690         return raw_atomic64_fetch_and(~i, v);
3691 #endif
3692 }
3693
3694 /**
3695  * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3696  * @i: s64 value
3697  * @v: pointer to atomic64_t
3698  *
3699  * Atomically updates @v to (@v & ~@i) with acquire ordering.
3700  *
3701  * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3702  *
3703  * Return: The original value of @v.
3704  */
3705 static __always_inline s64
3706 raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
3707 {
3708 #if defined(arch_atomic64_fetch_andnot_acquire)
3709         return arch_atomic64_fetch_andnot_acquire(i, v);
3710 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3711         s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3712         __atomic_acquire_fence();
3713         return ret;
3714 #elif defined(arch_atomic64_fetch_andnot)
3715         return arch_atomic64_fetch_andnot(i, v);
3716 #else
3717         return raw_atomic64_fetch_and_acquire(~i, v);
3718 #endif
3719 }
3720
3721 /**
3722  * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3723  * @i: s64 value
3724  * @v: pointer to atomic64_t
3725  *
3726  * Atomically updates @v to (@v & ~@i) with release ordering.
3727  *
3728  * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3729  *
3730  * Return: The original value of @v.
3731  */
3732 static __always_inline s64
3733 raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
3734 {
3735 #if defined(arch_atomic64_fetch_andnot_release)
3736         return arch_atomic64_fetch_andnot_release(i, v);
3737 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3738         __atomic_release_fence();
3739         return arch_atomic64_fetch_andnot_relaxed(i, v);
3740 #elif defined(arch_atomic64_fetch_andnot)
3741         return arch_atomic64_fetch_andnot(i, v);
3742 #else
3743         return raw_atomic64_fetch_and_release(~i, v);
3744 #endif
3745 }
3746
3747 /**
3748  * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
3749  * @i: s64 value
3750  * @v: pointer to atomic64_t
3751  *
3752  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3753  *
3754  * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
3755  *
3756  * Return: The original value of @v.
3757  */
3758 static __always_inline s64
3759 raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
3760 {
3761 #if defined(arch_atomic64_fetch_andnot_relaxed)
3762         return arch_atomic64_fetch_andnot_relaxed(i, v);
3763 #elif defined(arch_atomic64_fetch_andnot)
3764         return arch_atomic64_fetch_andnot(i, v);
3765 #else
3766         return raw_atomic64_fetch_and_relaxed(~i, v);
3767 #endif
3768 }
3769
3770 /**
3771  * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
3772  * @i: s64 value
3773  * @v: pointer to atomic64_t
3774  *
3775  * Atomically updates @v to (@v | @i) with relaxed ordering.
3776  *
3777  * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
3778  *
3779  * Return: Nothing.
3780  */
3781 static __always_inline void
3782 raw_atomic64_or(s64 i, atomic64_t *v)
3783 {
3784         arch_atomic64_or(i, v);
3785 }
3786
3787 /**
3788  * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
3789  * @i: s64 value
3790  * @v: pointer to atomic64_t
3791  *
3792  * Atomically updates @v to (@v | @i) with full ordering.
3793  *
3794  * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
3795  *
3796  * Return: The original value of @v.
3797  */
3798 static __always_inline s64
3799 raw_atomic64_fetch_or(s64 i, atomic64_t *v)
3800 {
3801 #if defined(arch_atomic64_fetch_or)
3802         return arch_atomic64_fetch_or(i, v);
3803 #elif defined(arch_atomic64_fetch_or_relaxed)
3804         s64 ret;
3805         __atomic_pre_full_fence();
3806         ret = arch_atomic64_fetch_or_relaxed(i, v);
3807         __atomic_post_full_fence();
3808         return ret;
3809 #else
3810 #error "Unable to define raw_atomic64_fetch_or"
3811 #endif
3812 }
3813
3814 /**
3815  * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
3816  * @i: s64 value
3817  * @v: pointer to atomic64_t
3818  *
3819  * Atomically updates @v to (@v | @i) with acquire ordering.
3820  *
3821  * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
3822  *
3823  * Return: The original value of @v.
3824  */
3825 static __always_inline s64
3826 raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
3827 {
3828 #if defined(arch_atomic64_fetch_or_acquire)
3829         return arch_atomic64_fetch_or_acquire(i, v);
3830 #elif defined(arch_atomic64_fetch_or_relaxed)
3831         s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
3832         __atomic_acquire_fence();
3833         return ret;
3834 #elif defined(arch_atomic64_fetch_or)
3835         return arch_atomic64_fetch_or(i, v);
3836 #else
3837 #error "Unable to define raw_atomic64_fetch_or_acquire"
3838 #endif
3839 }
3840
3841 /**
3842  * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
3843  * @i: s64 value
3844  * @v: pointer to atomic64_t
3845  *
3846  * Atomically updates @v to (@v | @i) with release ordering.
3847  *
3848  * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
3849  *
3850  * Return: The original value of @v.
3851  */
3852 static __always_inline s64
3853 raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
3854 {
3855 #if defined(arch_atomic64_fetch_or_release)
3856         return arch_atomic64_fetch_or_release(i, v);
3857 #elif defined(arch_atomic64_fetch_or_relaxed)
3858         __atomic_release_fence();
3859         return arch_atomic64_fetch_or_relaxed(i, v);
3860 #elif defined(arch_atomic64_fetch_or)
3861         return arch_atomic64_fetch_or(i, v);
3862 #else
3863 #error "Unable to define raw_atomic64_fetch_or_release"
3864 #endif
3865 }
3866
3867 /**
3868  * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
3869  * @i: s64 value
3870  * @v: pointer to atomic64_t
3871  *
3872  * Atomically updates @v to (@v | @i) with relaxed ordering.
3873  *
3874  * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
3875  *
3876  * Return: The original value of @v.
3877  */
3878 static __always_inline s64
3879 raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
3880 {
3881 #if defined(arch_atomic64_fetch_or_relaxed)
3882         return arch_atomic64_fetch_or_relaxed(i, v);
3883 #elif defined(arch_atomic64_fetch_or)
3884         return arch_atomic64_fetch_or(i, v);
3885 #else
3886 #error "Unable to define raw_atomic64_fetch_or_relaxed"
3887 #endif
3888 }
3889
3890 /**
3891  * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
3892  * @i: s64 value
3893  * @v: pointer to atomic64_t
3894  *
3895  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3896  *
3897  * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
3898  *
3899  * Return: Nothing.
3900  */
3901 static __always_inline void
3902 raw_atomic64_xor(s64 i, atomic64_t *v)
3903 {
3904         arch_atomic64_xor(i, v);
3905 }
3906
3907 /**
3908  * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
3909  * @i: s64 value
3910  * @v: pointer to atomic64_t
3911  *
3912  * Atomically updates @v to (@v ^ @i) with full ordering.
3913  *
3914  * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
3915  *
3916  * Return: The original value of @v.
3917  */
3918 static __always_inline s64
3919 raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
3920 {
3921 #if defined(arch_atomic64_fetch_xor)
3922         return arch_atomic64_fetch_xor(i, v);
3923 #elif defined(arch_atomic64_fetch_xor_relaxed)
3924         s64 ret;
3925         __atomic_pre_full_fence();
3926         ret = arch_atomic64_fetch_xor_relaxed(i, v);
3927         __atomic_post_full_fence();
3928         return ret;
3929 #else
3930 #error "Unable to define raw_atomic64_fetch_xor"
3931 #endif
3932 }
3933
3934 /**
3935  * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
3936  * @i: s64 value
3937  * @v: pointer to atomic64_t
3938  *
3939  * Atomically updates @v to (@v ^ @i) with acquire ordering.
3940  *
3941  * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
3942  *
3943  * Return: The original value of @v.
3944  */
3945 static __always_inline s64
3946 raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
3947 {
3948 #if defined(arch_atomic64_fetch_xor_acquire)
3949         return arch_atomic64_fetch_xor_acquire(i, v);
3950 #elif defined(arch_atomic64_fetch_xor_relaxed)
3951         s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3952         __atomic_acquire_fence();
3953         return ret;
3954 #elif defined(arch_atomic64_fetch_xor)
3955         return arch_atomic64_fetch_xor(i, v);
3956 #else
3957 #error "Unable to define raw_atomic64_fetch_xor_acquire"
3958 #endif
3959 }
3960
3961 /**
3962  * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
3963  * @i: s64 value
3964  * @v: pointer to atomic64_t
3965  *
3966  * Atomically updates @v to (@v ^ @i) with release ordering.
3967  *
3968  * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
3969  *
3970  * Return: The original value of @v.
3971  */
3972 static __always_inline s64
3973 raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
3974 {
3975 #if defined(arch_atomic64_fetch_xor_release)
3976         return arch_atomic64_fetch_xor_release(i, v);
3977 #elif defined(arch_atomic64_fetch_xor_relaxed)
3978         __atomic_release_fence();
3979         return arch_atomic64_fetch_xor_relaxed(i, v);
3980 #elif defined(arch_atomic64_fetch_xor)
3981         return arch_atomic64_fetch_xor(i, v);
3982 #else
3983 #error "Unable to define raw_atomic64_fetch_xor_release"
3984 #endif
3985 }
3986
3987 /**
3988  * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
3989  * @i: s64 value
3990  * @v: pointer to atomic64_t
3991  *
3992  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3993  *
3994  * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
3995  *
3996  * Return: The original value of @v.
3997  */
3998 static __always_inline s64
3999 raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
4000 {
4001 #if defined(arch_atomic64_fetch_xor_relaxed)
4002         return arch_atomic64_fetch_xor_relaxed(i, v);
4003 #elif defined(arch_atomic64_fetch_xor)
4004         return arch_atomic64_fetch_xor(i, v);
4005 #else
4006 #error "Unable to define raw_atomic64_fetch_xor_relaxed"
4007 #endif
4008 }
4009
4010 /**
4011  * raw_atomic64_xchg() - atomic exchange with full ordering
4012  * @v: pointer to atomic64_t
4013  * @new: s64 value to assign
4014  *
4015  * Atomically updates @v to @new with full ordering.
4016  *
4017  * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
4018  *
4019  * Return: The original value of @v.
4020  */
4021 static __always_inline s64
4022 raw_atomic64_xchg(atomic64_t *v, s64 new)
4023 {
4024 #if defined(arch_atomic64_xchg)
4025         return arch_atomic64_xchg(v, new);
4026 #elif defined(arch_atomic64_xchg_relaxed)
4027         s64 ret;
4028         __atomic_pre_full_fence();
4029         ret = arch_atomic64_xchg_relaxed(v, new);
4030         __atomic_post_full_fence();
4031         return ret;
4032 #else
4033         return raw_xchg(&v->counter, new);
4034 #endif
4035 }
4036
4037 /**
4038  * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
4039  * @v: pointer to atomic64_t
4040  * @new: s64 value to assign
4041  *
4042  * Atomically updates @v to @new with acquire ordering.
4043  *
4044  * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
4045  *
4046  * Return: The original value of @v.
4047  */
4048 static __always_inline s64
4049 raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
4050 {
4051 #if defined(arch_atomic64_xchg_acquire)
4052         return arch_atomic64_xchg_acquire(v, new);
4053 #elif defined(arch_atomic64_xchg_relaxed)
4054         s64 ret = arch_atomic64_xchg_relaxed(v, new);
4055         __atomic_acquire_fence();
4056         return ret;
4057 #elif defined(arch_atomic64_xchg)
4058         return arch_atomic64_xchg(v, new);
4059 #else
4060         return raw_xchg_acquire(&v->counter, new);
4061 #endif
4062 }
4063
4064 /**
4065  * raw_atomic64_xchg_release() - atomic exchange with release ordering
4066  * @v: pointer to atomic64_t
4067  * @new: s64 value to assign
4068  *
4069  * Atomically updates @v to @new with release ordering.
4070  *
4071  * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
4072  *
4073  * Return: The original value of @v.
4074  */
4075 static __always_inline s64
4076 raw_atomic64_xchg_release(atomic64_t *v, s64 new)
4077 {
4078 #if defined(arch_atomic64_xchg_release)
4079         return arch_atomic64_xchg_release(v, new);
4080 #elif defined(arch_atomic64_xchg_relaxed)
4081         __atomic_release_fence();
4082         return arch_atomic64_xchg_relaxed(v, new);
4083 #elif defined(arch_atomic64_xchg)
4084         return arch_atomic64_xchg(v, new);
4085 #else
4086         return raw_xchg_release(&v->counter, new);
4087 #endif
4088 }
4089
4090 /**
4091  * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
4092  * @v: pointer to atomic64_t
4093  * @new: s64 value to assign
4094  *
4095  * Atomically updates @v to @new with relaxed ordering.
4096  *
4097  * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
4098  *
4099  * Return: The original value of @v.
4100  */
4101 static __always_inline s64
4102 raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
4103 {
4104 #if defined(arch_atomic64_xchg_relaxed)
4105         return arch_atomic64_xchg_relaxed(v, new);
4106 #elif defined(arch_atomic64_xchg)
4107         return arch_atomic64_xchg(v, new);
4108 #else
4109         return raw_xchg_relaxed(&v->counter, new);
4110 #endif
4111 }
4112
4113 /**
4114  * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
4115  * @v: pointer to atomic64_t
4116  * @old: s64 value to compare with
4117  * @new: s64 value to assign
4118  *
4119  * If (@v == @old), atomically updates @v to @new with full ordering.
4120  *
4121  * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4122  *
4123  * Return: The original value of @v.
4124  */
4125 static __always_inline s64
4126 raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
4127 {
4128 #if defined(arch_atomic64_cmpxchg)
4129         return arch_atomic64_cmpxchg(v, old, new);
4130 #elif defined(arch_atomic64_cmpxchg_relaxed)
4131         s64 ret;
4132         __atomic_pre_full_fence();
4133         ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4134         __atomic_post_full_fence();
4135         return ret;
4136 #else
4137         return raw_cmpxchg(&v->counter, old, new);
4138 #endif
4139 }
4140
4141 /**
4142  * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4143  * @v: pointer to atomic64_t
4144  * @old: s64 value to compare with
4145  * @new: s64 value to assign
4146  *
4147  * If (@v == @old), atomically updates @v to @new with acquire ordering.
4148  *
4149  * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4150  *
4151  * Return: The original value of @v.
4152  */
4153 static __always_inline s64
4154 raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
4155 {
4156 #if defined(arch_atomic64_cmpxchg_acquire)
4157         return arch_atomic64_cmpxchg_acquire(v, old, new);
4158 #elif defined(arch_atomic64_cmpxchg_relaxed)
4159         s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4160         __atomic_acquire_fence();
4161         return ret;
4162 #elif defined(arch_atomic64_cmpxchg)
4163         return arch_atomic64_cmpxchg(v, old, new);
4164 #else
4165         return raw_cmpxchg_acquire(&v->counter, old, new);
4166 #endif
4167 }
4168
4169 /**
4170  * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
4171  * @v: pointer to atomic64_t
4172  * @old: s64 value to compare with
4173  * @new: s64 value to assign
4174  *
4175  * If (@v == @old), atomically updates @v to @new with release ordering.
4176  *
4177  * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4178  *
4179  * Return: The original value of @v.
4180  */
4181 static __always_inline s64
4182 raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
4183 {
4184 #if defined(arch_atomic64_cmpxchg_release)
4185         return arch_atomic64_cmpxchg_release(v, old, new);
4186 #elif defined(arch_atomic64_cmpxchg_relaxed)
4187         __atomic_release_fence();
4188         return arch_atomic64_cmpxchg_relaxed(v, old, new);
4189 #elif defined(arch_atomic64_cmpxchg)
4190         return arch_atomic64_cmpxchg(v, old, new);
4191 #else
4192         return raw_cmpxchg_release(&v->counter, old, new);
4193 #endif
4194 }
4195
4196 /**
4197  * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4198  * @v: pointer to atomic64_t
4199  * @old: s64 value to compare with
4200  * @new: s64 value to assign
4201  *
4202  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4203  *
4204  * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4205  *
4206  * Return: The original value of @v.
4207  */
4208 static __always_inline s64
4209 raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
4210 {
4211 #if defined(arch_atomic64_cmpxchg_relaxed)
4212         return arch_atomic64_cmpxchg_relaxed(v, old, new);
4213 #elif defined(arch_atomic64_cmpxchg)
4214         return arch_atomic64_cmpxchg(v, old, new);
4215 #else
4216         return raw_cmpxchg_relaxed(&v->counter, old, new);
4217 #endif
4218 }
4219
4220 /**
4221  * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
4222  * @v: pointer to atomic64_t
4223  * @old: pointer to s64 value to compare with
4224  * @new: s64 value to assign
4225  *
4226  * If (@v == @old), atomically updates @v to @new with full ordering.
4227  * Otherwise, updates @old to the current value of @v.
4228  *
4229  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4230  *
4231  * Return: @true if the exchange occured, @false otherwise.
4232  */
4233 static __always_inline bool
4234 raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
4235 {
4236 #if defined(arch_atomic64_try_cmpxchg)
4237         return arch_atomic64_try_cmpxchg(v, old, new);
4238 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4239         bool ret;
4240         __atomic_pre_full_fence();
4241         ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4242         __atomic_post_full_fence();
4243         return ret;
4244 #else
4245         s64 r, o = *old;
4246         r = raw_atomic64_cmpxchg(v, o, new);
4247         if (unlikely(r != o))
4248                 *old = r;
4249         return likely(r == o);
4250 #endif
4251 }
4252
4253 /**
4254  * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4255  * @v: pointer to atomic64_t
4256  * @old: pointer to s64 value to compare with
4257  * @new: s64 value to assign
4258  *
4259  * If (@v == @old), atomically updates @v to @new with acquire ordering.
4260  * Otherwise, updates @old to the current value of @v.
4261  *
4262  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4263  *
4264  * Return: @true if the exchange occured, @false otherwise.
4265  */
4266 static __always_inline bool
4267 raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
4268 {
4269 #if defined(arch_atomic64_try_cmpxchg_acquire)
4270         return arch_atomic64_try_cmpxchg_acquire(v, old, new);
4271 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4272         bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4273         __atomic_acquire_fence();
4274         return ret;
4275 #elif defined(arch_atomic64_try_cmpxchg)
4276         return arch_atomic64_try_cmpxchg(v, old, new);
4277 #else
4278         s64 r, o = *old;
4279         r = raw_atomic64_cmpxchg_acquire(v, o, new);
4280         if (unlikely(r != o))
4281                 *old = r;
4282         return likely(r == o);
4283 #endif
4284 }
4285
4286 /**
4287  * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
4288  * @v: pointer to atomic64_t
4289  * @old: pointer to s64 value to compare with
4290  * @new: s64 value to assign
4291  *
4292  * If (@v == @old), atomically updates @v to @new with release ordering.
4293  * Otherwise, updates @old to the current value of @v.
4294  *
4295  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4296  *
4297  * Return: @true if the exchange occured, @false otherwise.
4298  */
4299 static __always_inline bool
4300 raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
4301 {
4302 #if defined(arch_atomic64_try_cmpxchg_release)
4303         return arch_atomic64_try_cmpxchg_release(v, old, new);
4304 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4305         __atomic_release_fence();
4306         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4307 #elif defined(arch_atomic64_try_cmpxchg)
4308         return arch_atomic64_try_cmpxchg(v, old, new);
4309 #else
4310         s64 r, o = *old;
4311         r = raw_atomic64_cmpxchg_release(v, o, new);
4312         if (unlikely(r != o))
4313                 *old = r;
4314         return likely(r == o);
4315 #endif
4316 }
4317
4318 /**
4319  * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4320  * @v: pointer to atomic64_t
4321  * @old: pointer to s64 value to compare with
4322  * @new: s64 value to assign
4323  *
4324  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4325  * Otherwise, updates @old to the current value of @v.
4326  *
4327  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4328  *
4329  * Return: @true if the exchange occured, @false otherwise.
4330  */
4331 static __always_inline bool
4332 raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
4333 {
4334 #if defined(arch_atomic64_try_cmpxchg_relaxed)
4335         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4336 #elif defined(arch_atomic64_try_cmpxchg)
4337         return arch_atomic64_try_cmpxchg(v, old, new);
4338 #else
4339         s64 r, o = *old;
4340         r = raw_atomic64_cmpxchg_relaxed(v, o, new);
4341         if (unlikely(r != o))
4342                 *old = r;
4343         return likely(r == o);
4344 #endif
4345 }
4346
4347 /**
4348  * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
4349  * @i: s64 value to add
4350  * @v: pointer to atomic64_t
4351  *
4352  * Atomically updates @v to (@v - @i) with full ordering.
4353  *
4354  * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
4355  *
4356  * Return: @true if the resulting value of @v is zero, @false otherwise.
4357  */
4358 static __always_inline bool
4359 raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
4360 {
4361 #if defined(arch_atomic64_sub_and_test)
4362         return arch_atomic64_sub_and_test(i, v);
4363 #else
4364         return raw_atomic64_sub_return(i, v) == 0;
4365 #endif
4366 }
4367
4368 /**
4369  * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
4370  * @v: pointer to atomic64_t
4371  *
4372  * Atomically updates @v to (@v - 1) with full ordering.
4373  *
4374  * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
4375  *
4376  * Return: @true if the resulting value of @v is zero, @false otherwise.
4377  */
4378 static __always_inline bool
4379 raw_atomic64_dec_and_test(atomic64_t *v)
4380 {
4381 #if defined(arch_atomic64_dec_and_test)
4382         return arch_atomic64_dec_and_test(v);
4383 #else
4384         return raw_atomic64_dec_return(v) == 0;
4385 #endif
4386 }
4387
4388 /**
4389  * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
4390  * @v: pointer to atomic64_t
4391  *
4392  * Atomically updates @v to (@v + 1) with full ordering.
4393  *
4394  * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
4395  *
4396  * Return: @true if the resulting value of @v is zero, @false otherwise.
4397  */
4398 static __always_inline bool
4399 raw_atomic64_inc_and_test(atomic64_t *v)
4400 {
4401 #if defined(arch_atomic64_inc_and_test)
4402         return arch_atomic64_inc_and_test(v);
4403 #else
4404         return raw_atomic64_inc_return(v) == 0;
4405 #endif
4406 }
4407
4408 /**
4409  * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
4410  * @i: s64 value to add
4411  * @v: pointer to atomic64_t
4412  *
4413  * Atomically updates @v to (@v + @i) with full ordering.
4414  *
4415  * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
4416  *
4417  * Return: @true if the resulting value of @v is negative, @false otherwise.
4418  */
4419 static __always_inline bool
4420 raw_atomic64_add_negative(s64 i, atomic64_t *v)
4421 {
4422 #if defined(arch_atomic64_add_negative)
4423         return arch_atomic64_add_negative(i, v);
4424 #elif defined(arch_atomic64_add_negative_relaxed)
4425         bool ret;
4426         __atomic_pre_full_fence();
4427         ret = arch_atomic64_add_negative_relaxed(i, v);
4428         __atomic_post_full_fence();
4429         return ret;
4430 #else
4431         return raw_atomic64_add_return(i, v) < 0;
4432 #endif
4433 }
4434
4435 /**
4436  * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
4437  * @i: s64 value to add
4438  * @v: pointer to atomic64_t
4439  *
4440  * Atomically updates @v to (@v + @i) with acquire ordering.
4441  *
4442  * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
4443  *
4444  * Return: @true if the resulting value of @v is negative, @false otherwise.
4445  */
4446 static __always_inline bool
4447 raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
4448 {
4449 #if defined(arch_atomic64_add_negative_acquire)
4450         return arch_atomic64_add_negative_acquire(i, v);
4451 #elif defined(arch_atomic64_add_negative_relaxed)
4452         bool ret = arch_atomic64_add_negative_relaxed(i, v);
4453         __atomic_acquire_fence();
4454         return ret;
4455 #elif defined(arch_atomic64_add_negative)
4456         return arch_atomic64_add_negative(i, v);
4457 #else
4458         return raw_atomic64_add_return_acquire(i, v) < 0;
4459 #endif
4460 }
4461
4462 /**
4463  * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
4464  * @i: s64 value to add
4465  * @v: pointer to atomic64_t
4466  *
4467  * Atomically updates @v to (@v + @i) with release ordering.
4468  *
4469  * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
4470  *
4471  * Return: @true if the resulting value of @v is negative, @false otherwise.
4472  */
4473 static __always_inline bool
4474 raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
4475 {
4476 #if defined(arch_atomic64_add_negative_release)
4477         return arch_atomic64_add_negative_release(i, v);
4478 #elif defined(arch_atomic64_add_negative_relaxed)
4479         __atomic_release_fence();
4480         return arch_atomic64_add_negative_relaxed(i, v);
4481 #elif defined(arch_atomic64_add_negative)
4482         return arch_atomic64_add_negative(i, v);
4483 #else
4484         return raw_atomic64_add_return_release(i, v) < 0;
4485 #endif
4486 }
4487
4488 /**
4489  * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4490  * @i: s64 value to add
4491  * @v: pointer to atomic64_t
4492  *
4493  * Atomically updates @v to (@v + @i) with relaxed ordering.
4494  *
4495  * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
4496  *
4497  * Return: @true if the resulting value of @v is negative, @false otherwise.
4498  */
4499 static __always_inline bool
4500 raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
4501 {
4502 #if defined(arch_atomic64_add_negative_relaxed)
4503         return arch_atomic64_add_negative_relaxed(i, v);
4504 #elif defined(arch_atomic64_add_negative)
4505         return arch_atomic64_add_negative(i, v);
4506 #else
4507         return raw_atomic64_add_return_relaxed(i, v) < 0;
4508 #endif
4509 }
4510
4511 /**
4512  * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
4513  * @v: pointer to atomic64_t
4514  * @a: s64 value to add
4515  * @u: s64 value to compare with
4516  *
4517  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4518  *
4519  * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4520  *
4521  * Return: The original value of @v.
4522  */
4523 static __always_inline s64
4524 raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
4525 {
4526 #if defined(arch_atomic64_fetch_add_unless)
4527         return arch_atomic64_fetch_add_unless(v, a, u);
4528 #else
4529         s64 c = raw_atomic64_read(v);
4530
4531         do {
4532                 if (unlikely(c == u))
4533                         break;
4534         } while (!raw_atomic64_try_cmpxchg(v, &c, c + a));
4535
4536         return c;
4537 #endif
4538 }
4539
4540 /**
4541  * raw_atomic64_add_unless() - atomic add unless value with full ordering
4542  * @v: pointer to atomic64_t
4543  * @a: s64 value to add
4544  * @u: s64 value to compare with
4545  *
4546  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4547  *
4548  * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4549  *
4550  * Return: @true if @v was updated, @false otherwise.
4551  */
4552 static __always_inline bool
4553 raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
4554 {
4555 #if defined(arch_atomic64_add_unless)
4556         return arch_atomic64_add_unless(v, a, u);
4557 #else
4558         return raw_atomic64_fetch_add_unless(v, a, u) != u;
4559 #endif
4560 }
4561
4562 /**
4563  * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
4564  * @v: pointer to atomic64_t
4565  *
4566  * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4567  *
4568  * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4569  *
4570  * Return: @true if @v was updated, @false otherwise.
4571  */
4572 static __always_inline bool
4573 raw_atomic64_inc_not_zero(atomic64_t *v)
4574 {
4575 #if defined(arch_atomic64_inc_not_zero)
4576         return arch_atomic64_inc_not_zero(v);
4577 #else
4578         return raw_atomic64_add_unless(v, 1, 0);
4579 #endif
4580 }
4581
4582 /**
4583  * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
4584  * @v: pointer to atomic64_t
4585  *
4586  * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4587  *
4588  * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4589  *
4590  * Return: @true if @v was updated, @false otherwise.
4591  */
4592 static __always_inline bool
4593 raw_atomic64_inc_unless_negative(atomic64_t *v)
4594 {
4595 #if defined(arch_atomic64_inc_unless_negative)
4596         return arch_atomic64_inc_unless_negative(v);
4597 #else
4598         s64 c = raw_atomic64_read(v);
4599
4600         do {
4601                 if (unlikely(c < 0))
4602                         return false;
4603         } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));
4604
4605         return true;
4606 #endif
4607 }
4608
4609 /**
4610  * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
4611  * @v: pointer to atomic64_t
4612  *
4613  * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4614  *
4615  * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4616  *
4617  * Return: @true if @v was updated, @false otherwise.
4618  */
4619 static __always_inline bool
4620 raw_atomic64_dec_unless_positive(atomic64_t *v)
4621 {
4622 #if defined(arch_atomic64_dec_unless_positive)
4623         return arch_atomic64_dec_unless_positive(v);
4624 #else
4625         s64 c = raw_atomic64_read(v);
4626
4627         do {
4628                 if (unlikely(c > 0))
4629                         return false;
4630         } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));
4631
4632         return true;
4633 #endif
4634 }
4635
4636 /**
4637  * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
4638  * @v: pointer to atomic64_t
4639  *
4640  * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4641  *
4642  * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4643  *
4644  * Return: The old value of (@v - 1), regardless of whether @v was updated.
4645  */
4646 static __always_inline s64
4647 raw_atomic64_dec_if_positive(atomic64_t *v)
4648 {
4649 #if defined(arch_atomic64_dec_if_positive)
4650         return arch_atomic64_dec_if_positive(v);
4651 #else
4652         s64 dec, c = raw_atomic64_read(v);
4653
4654         do {
4655                 dec = c - 1;
4656                 if (unlikely(dec < 0))
4657                         break;
4658         } while (!raw_atomic64_try_cmpxchg(v, &c, dec));
4659
4660         return dec;
4661 #endif
4662 }
4663
4664 #endif /* _LINUX_ATOMIC_FALLBACK_H */
4665 // eec048affea735b8464f58e6d96992101f8f85f1