1 // SPDX-License-Identifier: GPL-2.0
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
7 * This file provoides atomic operations with explicit instrumentation (e.g.
8 * KASAN, KCSAN), which should be used unless it is necessary to avoid
9 * instrumentation. Where it is necessary to aovid instrumenation, the
10 * raw_atomic*() operations should be used.
12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
13 #define _LINUX_ATOMIC_INSTRUMENTED_H
15 #include <linux/build_bug.h>
16 #include <linux/compiler.h>
17 #include <linux/instrumented.h>
20 * atomic_read() - atomic load with relaxed ordering
21 * @v: pointer to atomic_t
23 * Atomically loads the value of @v with relaxed ordering.
25 * Unsafe to use in noinstr code; use raw_atomic_read() there.
27 * Return: The value loaded from @v.
29 static __always_inline int
30 atomic_read(const atomic_t *v)
32 instrument_atomic_read(v, sizeof(*v));
33 return raw_atomic_read(v);
37 * atomic_read_acquire() - atomic load with acquire ordering
38 * @v: pointer to atomic_t
40 * Atomically loads the value of @v with acquire ordering.
42 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
44 * Return: The value loaded from @v.
46 static __always_inline int
47 atomic_read_acquire(const atomic_t *v)
49 instrument_atomic_read(v, sizeof(*v));
50 return raw_atomic_read_acquire(v);
54 * atomic_set() - atomic set with relaxed ordering
55 * @v: pointer to atomic_t
56 * @i: int value to assign
58 * Atomically sets @v to @i with relaxed ordering.
60 * Unsafe to use in noinstr code; use raw_atomic_set() there.
64 static __always_inline void
65 atomic_set(atomic_t *v, int i)
67 instrument_atomic_write(v, sizeof(*v));
72 * atomic_set_release() - atomic set with release ordering
73 * @v: pointer to atomic_t
74 * @i: int value to assign
76 * Atomically sets @v to @i with release ordering.
78 * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
82 static __always_inline void
83 atomic_set_release(atomic_t *v, int i)
86 instrument_atomic_write(v, sizeof(*v));
87 raw_atomic_set_release(v, i);
91 * atomic_add() - atomic add with relaxed ordering
92 * @i: int value to add
93 * @v: pointer to atomic_t
95 * Atomically updates @v to (@v + @i) with relaxed ordering.
97 * Unsafe to use in noinstr code; use raw_atomic_add() there.
101 static __always_inline void
102 atomic_add(int i, atomic_t *v)
104 instrument_atomic_read_write(v, sizeof(*v));
105 raw_atomic_add(i, v);
109 * atomic_add_return() - atomic add with full ordering
110 * @i: int value to add
111 * @v: pointer to atomic_t
113 * Atomically updates @v to (@v + @i) with full ordering.
115 * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
117 * Return: The updated value of @v.
119 static __always_inline int
120 atomic_add_return(int i, atomic_t *v)
123 instrument_atomic_read_write(v, sizeof(*v));
124 return raw_atomic_add_return(i, v);
128 * atomic_add_return_acquire() - atomic add with acquire ordering
129 * @i: int value to add
130 * @v: pointer to atomic_t
132 * Atomically updates @v to (@v + @i) with acquire ordering.
134 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
136 * Return: The updated value of @v.
138 static __always_inline int
139 atomic_add_return_acquire(int i, atomic_t *v)
141 instrument_atomic_read_write(v, sizeof(*v));
142 return raw_atomic_add_return_acquire(i, v);
146 * atomic_add_return_release() - atomic add with release ordering
147 * @i: int value to add
148 * @v: pointer to atomic_t
150 * Atomically updates @v to (@v + @i) with release ordering.
152 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
154 * Return: The updated value of @v.
156 static __always_inline int
157 atomic_add_return_release(int i, atomic_t *v)
160 instrument_atomic_read_write(v, sizeof(*v));
161 return raw_atomic_add_return_release(i, v);
165 * atomic_add_return_relaxed() - atomic add with relaxed ordering
166 * @i: int value to add
167 * @v: pointer to atomic_t
169 * Atomically updates @v to (@v + @i) with relaxed ordering.
171 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
173 * Return: The updated value of @v.
175 static __always_inline int
176 atomic_add_return_relaxed(int i, atomic_t *v)
178 instrument_atomic_read_write(v, sizeof(*v));
179 return raw_atomic_add_return_relaxed(i, v);
183 * atomic_fetch_add() - atomic add with full ordering
184 * @i: int value to add
185 * @v: pointer to atomic_t
187 * Atomically updates @v to (@v + @i) with full ordering.
189 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
191 * Return: The original value of @v.
193 static __always_inline int
194 atomic_fetch_add(int i, atomic_t *v)
197 instrument_atomic_read_write(v, sizeof(*v));
198 return raw_atomic_fetch_add(i, v);
202 * atomic_fetch_add_acquire() - atomic add with acquire ordering
203 * @i: int value to add
204 * @v: pointer to atomic_t
206 * Atomically updates @v to (@v + @i) with acquire ordering.
208 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
210 * Return: The original value of @v.
212 static __always_inline int
213 atomic_fetch_add_acquire(int i, atomic_t *v)
215 instrument_atomic_read_write(v, sizeof(*v));
216 return raw_atomic_fetch_add_acquire(i, v);
220 * atomic_fetch_add_release() - atomic add with release ordering
221 * @i: int value to add
222 * @v: pointer to atomic_t
224 * Atomically updates @v to (@v + @i) with release ordering.
226 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
228 * Return: The original value of @v.
230 static __always_inline int
231 atomic_fetch_add_release(int i, atomic_t *v)
234 instrument_atomic_read_write(v, sizeof(*v));
235 return raw_atomic_fetch_add_release(i, v);
239 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
240 * @i: int value to add
241 * @v: pointer to atomic_t
243 * Atomically updates @v to (@v + @i) with relaxed ordering.
245 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
247 * Return: The original value of @v.
249 static __always_inline int
250 atomic_fetch_add_relaxed(int i, atomic_t *v)
252 instrument_atomic_read_write(v, sizeof(*v));
253 return raw_atomic_fetch_add_relaxed(i, v);
257 * atomic_sub() - atomic subtract with relaxed ordering
258 * @i: int value to subtract
259 * @v: pointer to atomic_t
261 * Atomically updates @v to (@v - @i) with relaxed ordering.
263 * Unsafe to use in noinstr code; use raw_atomic_sub() there.
267 static __always_inline void
268 atomic_sub(int i, atomic_t *v)
270 instrument_atomic_read_write(v, sizeof(*v));
271 raw_atomic_sub(i, v);
275 * atomic_sub_return() - atomic subtract with full ordering
276 * @i: int value to subtract
277 * @v: pointer to atomic_t
279 * Atomically updates @v to (@v - @i) with full ordering.
281 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
283 * Return: The updated value of @v.
285 static __always_inline int
286 atomic_sub_return(int i, atomic_t *v)
289 instrument_atomic_read_write(v, sizeof(*v));
290 return raw_atomic_sub_return(i, v);
294 * atomic_sub_return_acquire() - atomic subtract with acquire ordering
295 * @i: int value to subtract
296 * @v: pointer to atomic_t
298 * Atomically updates @v to (@v - @i) with acquire ordering.
300 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
302 * Return: The updated value of @v.
304 static __always_inline int
305 atomic_sub_return_acquire(int i, atomic_t *v)
307 instrument_atomic_read_write(v, sizeof(*v));
308 return raw_atomic_sub_return_acquire(i, v);
312 * atomic_sub_return_release() - atomic subtract with release ordering
313 * @i: int value to subtract
314 * @v: pointer to atomic_t
316 * Atomically updates @v to (@v - @i) with release ordering.
318 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
320 * Return: The updated value of @v.
322 static __always_inline int
323 atomic_sub_return_release(int i, atomic_t *v)
326 instrument_atomic_read_write(v, sizeof(*v));
327 return raw_atomic_sub_return_release(i, v);
331 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
332 * @i: int value to subtract
333 * @v: pointer to atomic_t
335 * Atomically updates @v to (@v - @i) with relaxed ordering.
337 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
339 * Return: The updated value of @v.
341 static __always_inline int
342 atomic_sub_return_relaxed(int i, atomic_t *v)
344 instrument_atomic_read_write(v, sizeof(*v));
345 return raw_atomic_sub_return_relaxed(i, v);
349 * atomic_fetch_sub() - atomic subtract with full ordering
350 * @i: int value to subtract
351 * @v: pointer to atomic_t
353 * Atomically updates @v to (@v - @i) with full ordering.
355 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
357 * Return: The original value of @v.
359 static __always_inline int
360 atomic_fetch_sub(int i, atomic_t *v)
363 instrument_atomic_read_write(v, sizeof(*v));
364 return raw_atomic_fetch_sub(i, v);
368 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
369 * @i: int value to subtract
370 * @v: pointer to atomic_t
372 * Atomically updates @v to (@v - @i) with acquire ordering.
374 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
376 * Return: The original value of @v.
378 static __always_inline int
379 atomic_fetch_sub_acquire(int i, atomic_t *v)
381 instrument_atomic_read_write(v, sizeof(*v));
382 return raw_atomic_fetch_sub_acquire(i, v);
386 * atomic_fetch_sub_release() - atomic subtract with release ordering
387 * @i: int value to subtract
388 * @v: pointer to atomic_t
390 * Atomically updates @v to (@v - @i) with release ordering.
392 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
394 * Return: The original value of @v.
396 static __always_inline int
397 atomic_fetch_sub_release(int i, atomic_t *v)
400 instrument_atomic_read_write(v, sizeof(*v));
401 return raw_atomic_fetch_sub_release(i, v);
405 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
406 * @i: int value to subtract
407 * @v: pointer to atomic_t
409 * Atomically updates @v to (@v - @i) with relaxed ordering.
411 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
413 * Return: The original value of @v.
415 static __always_inline int
416 atomic_fetch_sub_relaxed(int i, atomic_t *v)
418 instrument_atomic_read_write(v, sizeof(*v));
419 return raw_atomic_fetch_sub_relaxed(i, v);
423 * atomic_inc() - atomic increment with relaxed ordering
424 * @v: pointer to atomic_t
426 * Atomically updates @v to (@v + 1) with relaxed ordering.
428 * Unsafe to use in noinstr code; use raw_atomic_inc() there.
432 static __always_inline void
433 atomic_inc(atomic_t *v)
435 instrument_atomic_read_write(v, sizeof(*v));
440 * atomic_inc_return() - atomic increment with full ordering
441 * @v: pointer to atomic_t
443 * Atomically updates @v to (@v + 1) with full ordering.
445 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
447 * Return: The updated value of @v.
449 static __always_inline int
450 atomic_inc_return(atomic_t *v)
453 instrument_atomic_read_write(v, sizeof(*v));
454 return raw_atomic_inc_return(v);
458 * atomic_inc_return_acquire() - atomic increment with acquire ordering
459 * @v: pointer to atomic_t
461 * Atomically updates @v to (@v + 1) with acquire ordering.
463 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
465 * Return: The updated value of @v.
467 static __always_inline int
468 atomic_inc_return_acquire(atomic_t *v)
470 instrument_atomic_read_write(v, sizeof(*v));
471 return raw_atomic_inc_return_acquire(v);
475 * atomic_inc_return_release() - atomic increment with release ordering
476 * @v: pointer to atomic_t
478 * Atomically updates @v to (@v + 1) with release ordering.
480 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
482 * Return: The updated value of @v.
484 static __always_inline int
485 atomic_inc_return_release(atomic_t *v)
488 instrument_atomic_read_write(v, sizeof(*v));
489 return raw_atomic_inc_return_release(v);
493 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
494 * @v: pointer to atomic_t
496 * Atomically updates @v to (@v + 1) with relaxed ordering.
498 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
500 * Return: The updated value of @v.
502 static __always_inline int
503 atomic_inc_return_relaxed(atomic_t *v)
505 instrument_atomic_read_write(v, sizeof(*v));
506 return raw_atomic_inc_return_relaxed(v);
510 * atomic_fetch_inc() - atomic increment with full ordering
511 * @v: pointer to atomic_t
513 * Atomically updates @v to (@v + 1) with full ordering.
515 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
517 * Return: The original value of @v.
519 static __always_inline int
520 atomic_fetch_inc(atomic_t *v)
523 instrument_atomic_read_write(v, sizeof(*v));
524 return raw_atomic_fetch_inc(v);
528 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
529 * @v: pointer to atomic_t
531 * Atomically updates @v to (@v + 1) with acquire ordering.
533 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
535 * Return: The original value of @v.
537 static __always_inline int
538 atomic_fetch_inc_acquire(atomic_t *v)
540 instrument_atomic_read_write(v, sizeof(*v));
541 return raw_atomic_fetch_inc_acquire(v);
545 * atomic_fetch_inc_release() - atomic increment with release ordering
546 * @v: pointer to atomic_t
548 * Atomically updates @v to (@v + 1) with release ordering.
550 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
552 * Return: The original value of @v.
554 static __always_inline int
555 atomic_fetch_inc_release(atomic_t *v)
558 instrument_atomic_read_write(v, sizeof(*v));
559 return raw_atomic_fetch_inc_release(v);
563 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
564 * @v: pointer to atomic_t
566 * Atomically updates @v to (@v + 1) with relaxed ordering.
568 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
570 * Return: The original value of @v.
572 static __always_inline int
573 atomic_fetch_inc_relaxed(atomic_t *v)
575 instrument_atomic_read_write(v, sizeof(*v));
576 return raw_atomic_fetch_inc_relaxed(v);
580 * atomic_dec() - atomic decrement with relaxed ordering
581 * @v: pointer to atomic_t
583 * Atomically updates @v to (@v - 1) with relaxed ordering.
585 * Unsafe to use in noinstr code; use raw_atomic_dec() there.
589 static __always_inline void
590 atomic_dec(atomic_t *v)
592 instrument_atomic_read_write(v, sizeof(*v));
597 * atomic_dec_return() - atomic decrement with full ordering
598 * @v: pointer to atomic_t
600 * Atomically updates @v to (@v - 1) with full ordering.
602 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
604 * Return: The updated value of @v.
606 static __always_inline int
607 atomic_dec_return(atomic_t *v)
610 instrument_atomic_read_write(v, sizeof(*v));
611 return raw_atomic_dec_return(v);
615 * atomic_dec_return_acquire() - atomic decrement with acquire ordering
616 * @v: pointer to atomic_t
618 * Atomically updates @v to (@v - 1) with acquire ordering.
620 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
622 * Return: The updated value of @v.
624 static __always_inline int
625 atomic_dec_return_acquire(atomic_t *v)
627 instrument_atomic_read_write(v, sizeof(*v));
628 return raw_atomic_dec_return_acquire(v);
632 * atomic_dec_return_release() - atomic decrement with release ordering
633 * @v: pointer to atomic_t
635 * Atomically updates @v to (@v - 1) with release ordering.
637 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
639 * Return: The updated value of @v.
641 static __always_inline int
642 atomic_dec_return_release(atomic_t *v)
645 instrument_atomic_read_write(v, sizeof(*v));
646 return raw_atomic_dec_return_release(v);
650 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
651 * @v: pointer to atomic_t
653 * Atomically updates @v to (@v - 1) with relaxed ordering.
655 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
657 * Return: The updated value of @v.
659 static __always_inline int
660 atomic_dec_return_relaxed(atomic_t *v)
662 instrument_atomic_read_write(v, sizeof(*v));
663 return raw_atomic_dec_return_relaxed(v);
667 * atomic_fetch_dec() - atomic decrement with full ordering
668 * @v: pointer to atomic_t
670 * Atomically updates @v to (@v - 1) with full ordering.
672 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
674 * Return: The original value of @v.
676 static __always_inline int
677 atomic_fetch_dec(atomic_t *v)
680 instrument_atomic_read_write(v, sizeof(*v));
681 return raw_atomic_fetch_dec(v);
685 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
686 * @v: pointer to atomic_t
688 * Atomically updates @v to (@v - 1) with acquire ordering.
690 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
692 * Return: The original value of @v.
694 static __always_inline int
695 atomic_fetch_dec_acquire(atomic_t *v)
697 instrument_atomic_read_write(v, sizeof(*v));
698 return raw_atomic_fetch_dec_acquire(v);
702 * atomic_fetch_dec_release() - atomic decrement with release ordering
703 * @v: pointer to atomic_t
705 * Atomically updates @v to (@v - 1) with release ordering.
707 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
709 * Return: The original value of @v.
711 static __always_inline int
712 atomic_fetch_dec_release(atomic_t *v)
715 instrument_atomic_read_write(v, sizeof(*v));
716 return raw_atomic_fetch_dec_release(v);
720 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
721 * @v: pointer to atomic_t
723 * Atomically updates @v to (@v - 1) with relaxed ordering.
725 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
727 * Return: The original value of @v.
729 static __always_inline int
730 atomic_fetch_dec_relaxed(atomic_t *v)
732 instrument_atomic_read_write(v, sizeof(*v));
733 return raw_atomic_fetch_dec_relaxed(v);
737 * atomic_and() - atomic bitwise AND with relaxed ordering
739 * @v: pointer to atomic_t
741 * Atomically updates @v to (@v & @i) with relaxed ordering.
743 * Unsafe to use in noinstr code; use raw_atomic_and() there.
747 static __always_inline void
748 atomic_and(int i, atomic_t *v)
750 instrument_atomic_read_write(v, sizeof(*v));
751 raw_atomic_and(i, v);
755 * atomic_fetch_and() - atomic bitwise AND with full ordering
757 * @v: pointer to atomic_t
759 * Atomically updates @v to (@v & @i) with full ordering.
761 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
763 * Return: The original value of @v.
765 static __always_inline int
766 atomic_fetch_and(int i, atomic_t *v)
769 instrument_atomic_read_write(v, sizeof(*v));
770 return raw_atomic_fetch_and(i, v);
774 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
776 * @v: pointer to atomic_t
778 * Atomically updates @v to (@v & @i) with acquire ordering.
780 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
782 * Return: The original value of @v.
784 static __always_inline int
785 atomic_fetch_and_acquire(int i, atomic_t *v)
787 instrument_atomic_read_write(v, sizeof(*v));
788 return raw_atomic_fetch_and_acquire(i, v);
792 * atomic_fetch_and_release() - atomic bitwise AND with release ordering
794 * @v: pointer to atomic_t
796 * Atomically updates @v to (@v & @i) with release ordering.
798 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
800 * Return: The original value of @v.
802 static __always_inline int
803 atomic_fetch_and_release(int i, atomic_t *v)
806 instrument_atomic_read_write(v, sizeof(*v));
807 return raw_atomic_fetch_and_release(i, v);
811 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
813 * @v: pointer to atomic_t
815 * Atomically updates @v to (@v & @i) with relaxed ordering.
817 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
819 * Return: The original value of @v.
821 static __always_inline int
822 atomic_fetch_and_relaxed(int i, atomic_t *v)
824 instrument_atomic_read_write(v, sizeof(*v));
825 return raw_atomic_fetch_and_relaxed(i, v);
829 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
831 * @v: pointer to atomic_t
833 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
835 * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
839 static __always_inline void
840 atomic_andnot(int i, atomic_t *v)
842 instrument_atomic_read_write(v, sizeof(*v));
843 raw_atomic_andnot(i, v);
847 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
849 * @v: pointer to atomic_t
851 * Atomically updates @v to (@v & ~@i) with full ordering.
853 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
855 * Return: The original value of @v.
857 static __always_inline int
858 atomic_fetch_andnot(int i, atomic_t *v)
861 instrument_atomic_read_write(v, sizeof(*v));
862 return raw_atomic_fetch_andnot(i, v);
866 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
868 * @v: pointer to atomic_t
870 * Atomically updates @v to (@v & ~@i) with acquire ordering.
872 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
874 * Return: The original value of @v.
876 static __always_inline int
877 atomic_fetch_andnot_acquire(int i, atomic_t *v)
879 instrument_atomic_read_write(v, sizeof(*v));
880 return raw_atomic_fetch_andnot_acquire(i, v);
884 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
886 * @v: pointer to atomic_t
888 * Atomically updates @v to (@v & ~@i) with release ordering.
890 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
892 * Return: The original value of @v.
894 static __always_inline int
895 atomic_fetch_andnot_release(int i, atomic_t *v)
898 instrument_atomic_read_write(v, sizeof(*v));
899 return raw_atomic_fetch_andnot_release(i, v);
903 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
905 * @v: pointer to atomic_t
907 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
909 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
911 * Return: The original value of @v.
913 static __always_inline int
914 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
916 instrument_atomic_read_write(v, sizeof(*v));
917 return raw_atomic_fetch_andnot_relaxed(i, v);
921 * atomic_or() - atomic bitwise OR with relaxed ordering
923 * @v: pointer to atomic_t
925 * Atomically updates @v to (@v | @i) with relaxed ordering.
927 * Unsafe to use in noinstr code; use raw_atomic_or() there.
931 static __always_inline void
932 atomic_or(int i, atomic_t *v)
934 instrument_atomic_read_write(v, sizeof(*v));
939 * atomic_fetch_or() - atomic bitwise OR with full ordering
941 * @v: pointer to atomic_t
943 * Atomically updates @v to (@v | @i) with full ordering.
945 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
947 * Return: The original value of @v.
949 static __always_inline int
950 atomic_fetch_or(int i, atomic_t *v)
953 instrument_atomic_read_write(v, sizeof(*v));
954 return raw_atomic_fetch_or(i, v);
958 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
960 * @v: pointer to atomic_t
962 * Atomically updates @v to (@v | @i) with acquire ordering.
964 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
966 * Return: The original value of @v.
968 static __always_inline int
969 atomic_fetch_or_acquire(int i, atomic_t *v)
971 instrument_atomic_read_write(v, sizeof(*v));
972 return raw_atomic_fetch_or_acquire(i, v);
976 * atomic_fetch_or_release() - atomic bitwise OR with release ordering
978 * @v: pointer to atomic_t
980 * Atomically updates @v to (@v | @i) with release ordering.
982 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
984 * Return: The original value of @v.
986 static __always_inline int
987 atomic_fetch_or_release(int i, atomic_t *v)
990 instrument_atomic_read_write(v, sizeof(*v));
991 return raw_atomic_fetch_or_release(i, v);
995 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
997 * @v: pointer to atomic_t
999 * Atomically updates @v to (@v | @i) with relaxed ordering.
1001 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
1003 * Return: The original value of @v.
1005 static __always_inline int
1006 atomic_fetch_or_relaxed(int i, atomic_t *v)
1008 instrument_atomic_read_write(v, sizeof(*v));
1009 return raw_atomic_fetch_or_relaxed(i, v);
1013 * atomic_xor() - atomic bitwise XOR with relaxed ordering
1015 * @v: pointer to atomic_t
1017 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1019 * Unsafe to use in noinstr code; use raw_atomic_xor() there.
1023 static __always_inline void
1024 atomic_xor(int i, atomic_t *v)
1026 instrument_atomic_read_write(v, sizeof(*v));
1027 raw_atomic_xor(i, v);
1031 * atomic_fetch_xor() - atomic bitwise XOR with full ordering
1033 * @v: pointer to atomic_t
1035 * Atomically updates @v to (@v ^ @i) with full ordering.
1037 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
1039 * Return: The original value of @v.
1041 static __always_inline int
1042 atomic_fetch_xor(int i, atomic_t *v)
1045 instrument_atomic_read_write(v, sizeof(*v));
1046 return raw_atomic_fetch_xor(i, v);
1050 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1052 * @v: pointer to atomic_t
1054 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1056 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
1058 * Return: The original value of @v.
1060 static __always_inline int
1061 atomic_fetch_xor_acquire(int i, atomic_t *v)
1063 instrument_atomic_read_write(v, sizeof(*v));
1064 return raw_atomic_fetch_xor_acquire(i, v);
1068 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1070 * @v: pointer to atomic_t
1072 * Atomically updates @v to (@v ^ @i) with release ordering.
1074 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
1076 * Return: The original value of @v.
1078 static __always_inline int
1079 atomic_fetch_xor_release(int i, atomic_t *v)
1082 instrument_atomic_read_write(v, sizeof(*v));
1083 return raw_atomic_fetch_xor_release(i, v);
1087 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1089 * @v: pointer to atomic_t
1091 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1093 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
1095 * Return: The original value of @v.
1097 static __always_inline int
1098 atomic_fetch_xor_relaxed(int i, atomic_t *v)
1100 instrument_atomic_read_write(v, sizeof(*v));
1101 return raw_atomic_fetch_xor_relaxed(i, v);
1105 * atomic_xchg() - atomic exchange with full ordering
1106 * @v: pointer to atomic_t
1107 * @new: int value to assign
1109 * Atomically updates @v to @new with full ordering.
1111 * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
1113 * Return: The original value of @v.
1115 static __always_inline int
1116 atomic_xchg(atomic_t *v, int new)
1119 instrument_atomic_read_write(v, sizeof(*v));
1120 return raw_atomic_xchg(v, new);
1124 * atomic_xchg_acquire() - atomic exchange with acquire ordering
1125 * @v: pointer to atomic_t
1126 * @new: int value to assign
1128 * Atomically updates @v to @new with acquire ordering.
1130 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
1132 * Return: The original value of @v.
1134 static __always_inline int
1135 atomic_xchg_acquire(atomic_t *v, int new)
1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return raw_atomic_xchg_acquire(v, new);
1142 * atomic_xchg_release() - atomic exchange with release ordering
1143 * @v: pointer to atomic_t
1144 * @new: int value to assign
1146 * Atomically updates @v to @new with release ordering.
1148 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
1150 * Return: The original value of @v.
1152 static __always_inline int
1153 atomic_xchg_release(atomic_t *v, int new)
1156 instrument_atomic_read_write(v, sizeof(*v));
1157 return raw_atomic_xchg_release(v, new);
1161 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1162 * @v: pointer to atomic_t
1163 * @new: int value to assign
1165 * Atomically updates @v to @new with relaxed ordering.
1167 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
1169 * Return: The original value of @v.
1171 static __always_inline int
1172 atomic_xchg_relaxed(atomic_t *v, int new)
1174 instrument_atomic_read_write(v, sizeof(*v));
1175 return raw_atomic_xchg_relaxed(v, new);
1179 * atomic_cmpxchg() - atomic compare and exchange with full ordering
1180 * @v: pointer to atomic_t
1181 * @old: int value to compare with
1182 * @new: int value to assign
1184 * If (@v == @old), atomically updates @v to @new with full ordering.
1186 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
1188 * Return: The original value of @v.
1190 static __always_inline int
1191 atomic_cmpxchg(atomic_t *v, int old, int new)
1194 instrument_atomic_read_write(v, sizeof(*v));
1195 return raw_atomic_cmpxchg(v, old, new);
1199 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1200 * @v: pointer to atomic_t
1201 * @old: int value to compare with
1202 * @new: int value to assign
1204 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1206 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
1208 * Return: The original value of @v.
1210 static __always_inline int
1211 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1213 instrument_atomic_read_write(v, sizeof(*v));
1214 return raw_atomic_cmpxchg_acquire(v, old, new);
1218 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
1219 * @v: pointer to atomic_t
1220 * @old: int value to compare with
1221 * @new: int value to assign
1223 * If (@v == @old), atomically updates @v to @new with release ordering.
1225 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
1227 * Return: The original value of @v.
1229 static __always_inline int
1230 atomic_cmpxchg_release(atomic_t *v, int old, int new)
1233 instrument_atomic_read_write(v, sizeof(*v));
1234 return raw_atomic_cmpxchg_release(v, old, new);
1238 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1239 * @v: pointer to atomic_t
1240 * @old: int value to compare with
1241 * @new: int value to assign
1243 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1245 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
1247 * Return: The original value of @v.
1249 static __always_inline int
1250 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1252 instrument_atomic_read_write(v, sizeof(*v));
1253 return raw_atomic_cmpxchg_relaxed(v, old, new);
1257 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
1258 * @v: pointer to atomic_t
1259 * @old: pointer to int value to compare with
1260 * @new: int value to assign
1262 * If (@v == @old), atomically updates @v to @new with full ordering.
1263 * Otherwise, updates @old to the current value of @v.
1265 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
1267 * Return: @true if the exchange occured, @false otherwise.
1269 static __always_inline bool
1270 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1273 instrument_atomic_read_write(v, sizeof(*v));
1274 instrument_atomic_read_write(old, sizeof(*old));
1275 return raw_atomic_try_cmpxchg(v, old, new);
1279 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1280 * @v: pointer to atomic_t
1281 * @old: pointer to int value to compare with
1282 * @new: int value to assign
1284 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1285 * Otherwise, updates @old to the current value of @v.
1287 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
1289 * Return: @true if the exchange occured, @false otherwise.
1291 static __always_inline bool
1292 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1294 instrument_atomic_read_write(v, sizeof(*v));
1295 instrument_atomic_read_write(old, sizeof(*old));
1296 return raw_atomic_try_cmpxchg_acquire(v, old, new);
1300 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
1301 * @v: pointer to atomic_t
1302 * @old: pointer to int value to compare with
1303 * @new: int value to assign
1305 * If (@v == @old), atomically updates @v to @new with release ordering.
1306 * Otherwise, updates @old to the current value of @v.
1308 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
1310 * Return: @true if the exchange occured, @false otherwise.
1312 static __always_inline bool
1313 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1316 instrument_atomic_read_write(v, sizeof(*v));
1317 instrument_atomic_read_write(old, sizeof(*old));
1318 return raw_atomic_try_cmpxchg_release(v, old, new);
1322 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1323 * @v: pointer to atomic_t
1324 * @old: pointer to int value to compare with
1325 * @new: int value to assign
1327 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1328 * Otherwise, updates @old to the current value of @v.
1330 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
1332 * Return: @true if the exchange occured, @false otherwise.
1334 static __always_inline bool
1335 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1337 instrument_atomic_read_write(v, sizeof(*v));
1338 instrument_atomic_read_write(old, sizeof(*old));
1339 return raw_atomic_try_cmpxchg_relaxed(v, old, new);
1343 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
1344 * @i: int value to add
1345 * @v: pointer to atomic_t
1347 * Atomically updates @v to (@v - @i) with full ordering.
1349 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
1351 * Return: @true if the resulting value of @v is zero, @false otherwise.
1353 static __always_inline bool
1354 atomic_sub_and_test(int i, atomic_t *v)
1357 instrument_atomic_read_write(v, sizeof(*v));
1358 return raw_atomic_sub_and_test(i, v);
1362 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
1363 * @v: pointer to atomic_t
1365 * Atomically updates @v to (@v - 1) with full ordering.
1367 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
1369 * Return: @true if the resulting value of @v is zero, @false otherwise.
1371 static __always_inline bool
1372 atomic_dec_and_test(atomic_t *v)
1375 instrument_atomic_read_write(v, sizeof(*v));
1376 return raw_atomic_dec_and_test(v);
1380 * atomic_inc_and_test() - atomic increment and test if zero with full ordering
1381 * @v: pointer to atomic_t
1383 * Atomically updates @v to (@v + 1) with full ordering.
1385 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
1387 * Return: @true if the resulting value of @v is zero, @false otherwise.
1389 static __always_inline bool
1390 atomic_inc_and_test(atomic_t *v)
1393 instrument_atomic_read_write(v, sizeof(*v));
1394 return raw_atomic_inc_and_test(v);
1398 * atomic_add_negative() - atomic add and test if negative with full ordering
1399 * @i: int value to add
1400 * @v: pointer to atomic_t
1402 * Atomically updates @v to (@v + @i) with full ordering.
1404 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
1406 * Return: @true if the resulting value of @v is negative, @false otherwise.
1408 static __always_inline bool
1409 atomic_add_negative(int i, atomic_t *v)
1412 instrument_atomic_read_write(v, sizeof(*v));
1413 return raw_atomic_add_negative(i, v);
1417 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
1418 * @i: int value to add
1419 * @v: pointer to atomic_t
1421 * Atomically updates @v to (@v + @i) with acquire ordering.
1423 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
1425 * Return: @true if the resulting value of @v is negative, @false otherwise.
1427 static __always_inline bool
1428 atomic_add_negative_acquire(int i, atomic_t *v)
1430 instrument_atomic_read_write(v, sizeof(*v));
1431 return raw_atomic_add_negative_acquire(i, v);
1435 * atomic_add_negative_release() - atomic add and test if negative with release ordering
1436 * @i: int value to add
1437 * @v: pointer to atomic_t
1439 * Atomically updates @v to (@v + @i) with release ordering.
1441 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
1443 * Return: @true if the resulting value of @v is negative, @false otherwise.
1445 static __always_inline bool
1446 atomic_add_negative_release(int i, atomic_t *v)
1449 instrument_atomic_read_write(v, sizeof(*v));
1450 return raw_atomic_add_negative_release(i, v);
1454 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1455 * @i: int value to add
1456 * @v: pointer to atomic_t
1458 * Atomically updates @v to (@v + @i) with relaxed ordering.
1460 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
1462 * Return: @true if the resulting value of @v is negative, @false otherwise.
1464 static __always_inline bool
1465 atomic_add_negative_relaxed(int i, atomic_t *v)
1467 instrument_atomic_read_write(v, sizeof(*v));
1468 return raw_atomic_add_negative_relaxed(i, v);
1472 * atomic_fetch_add_unless() - atomic add unless value with full ordering
1473 * @v: pointer to atomic_t
1474 * @a: int value to add
1475 * @u: int value to compare with
1477 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1479 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
1481 * Return: The original value of @v.
1483 static __always_inline int
1484 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1487 instrument_atomic_read_write(v, sizeof(*v));
1488 return raw_atomic_fetch_add_unless(v, a, u);
1492 * atomic_add_unless() - atomic add unless value with full ordering
1493 * @v: pointer to atomic_t
1494 * @a: int value to add
1495 * @u: int value to compare with
1497 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1499 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
1501 * Return: @true if @v was updated, @false otherwise.
1503 static __always_inline bool
1504 atomic_add_unless(atomic_t *v, int a, int u)
1507 instrument_atomic_read_write(v, sizeof(*v));
1508 return raw_atomic_add_unless(v, a, u);
1512 * atomic_inc_not_zero() - atomic increment unless zero with full ordering
1513 * @v: pointer to atomic_t
1515 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
1517 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
1519 * Return: @true if @v was updated, @false otherwise.
1521 static __always_inline bool
1522 atomic_inc_not_zero(atomic_t *v)
1525 instrument_atomic_read_write(v, sizeof(*v));
1526 return raw_atomic_inc_not_zero(v);
1530 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
1531 * @v: pointer to atomic_t
1533 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
1535 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
1537 * Return: @true if @v was updated, @false otherwise.
1539 static __always_inline bool
1540 atomic_inc_unless_negative(atomic_t *v)
1543 instrument_atomic_read_write(v, sizeof(*v));
1544 return raw_atomic_inc_unless_negative(v);
1548 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
1549 * @v: pointer to atomic_t
1551 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
1553 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
1555 * Return: @true if @v was updated, @false otherwise.
1557 static __always_inline bool
1558 atomic_dec_unless_positive(atomic_t *v)
1561 instrument_atomic_read_write(v, sizeof(*v));
1562 return raw_atomic_dec_unless_positive(v);
1566 * atomic_dec_if_positive() - atomic decrement if positive with full ordering
1567 * @v: pointer to atomic_t
1569 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
1571 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
1573 * Return: The old value of (@v - 1), regardless of whether @v was updated.
1575 static __always_inline int
1576 atomic_dec_if_positive(atomic_t *v)
1579 instrument_atomic_read_write(v, sizeof(*v));
1580 return raw_atomic_dec_if_positive(v);
1584 * atomic64_read() - atomic load with relaxed ordering
1585 * @v: pointer to atomic64_t
1587 * Atomically loads the value of @v with relaxed ordering.
1589 * Unsafe to use in noinstr code; use raw_atomic64_read() there.
1591 * Return: The value loaded from @v.
1593 static __always_inline s64
1594 atomic64_read(const atomic64_t *v)
1596 instrument_atomic_read(v, sizeof(*v));
1597 return raw_atomic64_read(v);
1601 * atomic64_read_acquire() - atomic load with acquire ordering
1602 * @v: pointer to atomic64_t
1604 * Atomically loads the value of @v with acquire ordering.
1606 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
1608 * Return: The value loaded from @v.
1610 static __always_inline s64
1611 atomic64_read_acquire(const atomic64_t *v)
1613 instrument_atomic_read(v, sizeof(*v));
1614 return raw_atomic64_read_acquire(v);
1618 * atomic64_set() - atomic set with relaxed ordering
1619 * @v: pointer to atomic64_t
1620 * @i: s64 value to assign
1622 * Atomically sets @v to @i with relaxed ordering.
1624 * Unsafe to use in noinstr code; use raw_atomic64_set() there.
1628 static __always_inline void
1629 atomic64_set(atomic64_t *v, s64 i)
1631 instrument_atomic_write(v, sizeof(*v));
1632 raw_atomic64_set(v, i);
1636 * atomic64_set_release() - atomic set with release ordering
1637 * @v: pointer to atomic64_t
1638 * @i: s64 value to assign
1640 * Atomically sets @v to @i with release ordering.
1642 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
1646 static __always_inline void
1647 atomic64_set_release(atomic64_t *v, s64 i)
1650 instrument_atomic_write(v, sizeof(*v));
1651 raw_atomic64_set_release(v, i);
1655 * atomic64_add() - atomic add with relaxed ordering
1656 * @i: s64 value to add
1657 * @v: pointer to atomic64_t
1659 * Atomically updates @v to (@v + @i) with relaxed ordering.
1661 * Unsafe to use in noinstr code; use raw_atomic64_add() there.
1665 static __always_inline void
1666 atomic64_add(s64 i, atomic64_t *v)
1668 instrument_atomic_read_write(v, sizeof(*v));
1669 raw_atomic64_add(i, v);
1673 * atomic64_add_return() - atomic add with full ordering
1674 * @i: s64 value to add
1675 * @v: pointer to atomic64_t
1677 * Atomically updates @v to (@v + @i) with full ordering.
1679 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
1681 * Return: The updated value of @v.
1683 static __always_inline s64
1684 atomic64_add_return(s64 i, atomic64_t *v)
1687 instrument_atomic_read_write(v, sizeof(*v));
1688 return raw_atomic64_add_return(i, v);
1692 * atomic64_add_return_acquire() - atomic add with acquire ordering
1693 * @i: s64 value to add
1694 * @v: pointer to atomic64_t
1696 * Atomically updates @v to (@v + @i) with acquire ordering.
1698 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
1700 * Return: The updated value of @v.
1702 static __always_inline s64
1703 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1705 instrument_atomic_read_write(v, sizeof(*v));
1706 return raw_atomic64_add_return_acquire(i, v);
1710 * atomic64_add_return_release() - atomic add with release ordering
1711 * @i: s64 value to add
1712 * @v: pointer to atomic64_t
1714 * Atomically updates @v to (@v + @i) with release ordering.
1716 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
1718 * Return: The updated value of @v.
1720 static __always_inline s64
1721 atomic64_add_return_release(s64 i, atomic64_t *v)
1724 instrument_atomic_read_write(v, sizeof(*v));
1725 return raw_atomic64_add_return_release(i, v);
1729 * atomic64_add_return_relaxed() - atomic add with relaxed ordering
1730 * @i: s64 value to add
1731 * @v: pointer to atomic64_t
1733 * Atomically updates @v to (@v + @i) with relaxed ordering.
1735 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
1737 * Return: The updated value of @v.
1739 static __always_inline s64
1740 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
1742 instrument_atomic_read_write(v, sizeof(*v));
1743 return raw_atomic64_add_return_relaxed(i, v);
1747 * atomic64_fetch_add() - atomic add with full ordering
1748 * @i: s64 value to add
1749 * @v: pointer to atomic64_t
1751 * Atomically updates @v to (@v + @i) with full ordering.
1753 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
1755 * Return: The original value of @v.
1757 static __always_inline s64
1758 atomic64_fetch_add(s64 i, atomic64_t *v)
1761 instrument_atomic_read_write(v, sizeof(*v));
1762 return raw_atomic64_fetch_add(i, v);
1766 * atomic64_fetch_add_acquire() - atomic add with acquire ordering
1767 * @i: s64 value to add
1768 * @v: pointer to atomic64_t
1770 * Atomically updates @v to (@v + @i) with acquire ordering.
1772 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
1774 * Return: The original value of @v.
1776 static __always_inline s64
1777 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1779 instrument_atomic_read_write(v, sizeof(*v));
1780 return raw_atomic64_fetch_add_acquire(i, v);
1784 * atomic64_fetch_add_release() - atomic add with release ordering
1785 * @i: s64 value to add
1786 * @v: pointer to atomic64_t
1788 * Atomically updates @v to (@v + @i) with release ordering.
1790 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
1792 * Return: The original value of @v.
1794 static __always_inline s64
1795 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1798 instrument_atomic_read_write(v, sizeof(*v));
1799 return raw_atomic64_fetch_add_release(i, v);
1803 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
1804 * @i: s64 value to add
1805 * @v: pointer to atomic64_t
1807 * Atomically updates @v to (@v + @i) with relaxed ordering.
1809 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
1811 * Return: The original value of @v.
1813 static __always_inline s64
1814 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
1816 instrument_atomic_read_write(v, sizeof(*v));
1817 return raw_atomic64_fetch_add_relaxed(i, v);
1821 * atomic64_sub() - atomic subtract with relaxed ordering
1822 * @i: s64 value to subtract
1823 * @v: pointer to atomic64_t
1825 * Atomically updates @v to (@v - @i) with relaxed ordering.
1827 * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
1831 static __always_inline void
1832 atomic64_sub(s64 i, atomic64_t *v)
1834 instrument_atomic_read_write(v, sizeof(*v));
1835 raw_atomic64_sub(i, v);
1839 * atomic64_sub_return() - atomic subtract with full ordering
1840 * @i: s64 value to subtract
1841 * @v: pointer to atomic64_t
1843 * Atomically updates @v to (@v - @i) with full ordering.
1845 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
1847 * Return: The updated value of @v.
1849 static __always_inline s64
1850 atomic64_sub_return(s64 i, atomic64_t *v)
1853 instrument_atomic_read_write(v, sizeof(*v));
1854 return raw_atomic64_sub_return(i, v);
1858 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
1859 * @i: s64 value to subtract
1860 * @v: pointer to atomic64_t
1862 * Atomically updates @v to (@v - @i) with acquire ordering.
1864 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
1866 * Return: The updated value of @v.
1868 static __always_inline s64
1869 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1871 instrument_atomic_read_write(v, sizeof(*v));
1872 return raw_atomic64_sub_return_acquire(i, v);
1876 * atomic64_sub_return_release() - atomic subtract with release ordering
1877 * @i: s64 value to subtract
1878 * @v: pointer to atomic64_t
1880 * Atomically updates @v to (@v - @i) with release ordering.
1882 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
1884 * Return: The updated value of @v.
1886 static __always_inline s64
1887 atomic64_sub_return_release(s64 i, atomic64_t *v)
1890 instrument_atomic_read_write(v, sizeof(*v));
1891 return raw_atomic64_sub_return_release(i, v);
1895 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
1896 * @i: s64 value to subtract
1897 * @v: pointer to atomic64_t
1899 * Atomically updates @v to (@v - @i) with relaxed ordering.
1901 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
1903 * Return: The updated value of @v.
1905 static __always_inline s64
1906 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
1908 instrument_atomic_read_write(v, sizeof(*v));
1909 return raw_atomic64_sub_return_relaxed(i, v);
1913 * atomic64_fetch_sub() - atomic subtract with full ordering
1914 * @i: s64 value to subtract
1915 * @v: pointer to atomic64_t
1917 * Atomically updates @v to (@v - @i) with full ordering.
1919 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
1921 * Return: The original value of @v.
1923 static __always_inline s64
1924 atomic64_fetch_sub(s64 i, atomic64_t *v)
1927 instrument_atomic_read_write(v, sizeof(*v));
1928 return raw_atomic64_fetch_sub(i, v);
1932 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
1933 * @i: s64 value to subtract
1934 * @v: pointer to atomic64_t
1936 * Atomically updates @v to (@v - @i) with acquire ordering.
1938 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
1940 * Return: The original value of @v.
1942 static __always_inline s64
1943 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1945 instrument_atomic_read_write(v, sizeof(*v));
1946 return raw_atomic64_fetch_sub_acquire(i, v);
1950 * atomic64_fetch_sub_release() - atomic subtract with release ordering
1951 * @i: s64 value to subtract
1952 * @v: pointer to atomic64_t
1954 * Atomically updates @v to (@v - @i) with release ordering.
1956 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
1958 * Return: The original value of @v.
1960 static __always_inline s64
1961 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1964 instrument_atomic_read_write(v, sizeof(*v));
1965 return raw_atomic64_fetch_sub_release(i, v);
1969 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
1970 * @i: s64 value to subtract
1971 * @v: pointer to atomic64_t
1973 * Atomically updates @v to (@v - @i) with relaxed ordering.
1975 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
1977 * Return: The original value of @v.
1979 static __always_inline s64
1980 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1982 instrument_atomic_read_write(v, sizeof(*v));
1983 return raw_atomic64_fetch_sub_relaxed(i, v);
1987 * atomic64_inc() - atomic increment with relaxed ordering
1988 * @v: pointer to atomic64_t
1990 * Atomically updates @v to (@v + 1) with relaxed ordering.
1992 * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
1996 static __always_inline void
1997 atomic64_inc(atomic64_t *v)
1999 instrument_atomic_read_write(v, sizeof(*v));
2000 raw_atomic64_inc(v);
2004 * atomic64_inc_return() - atomic increment with full ordering
2005 * @v: pointer to atomic64_t
2007 * Atomically updates @v to (@v + 1) with full ordering.
2009 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
2011 * Return: The updated value of @v.
2013 static __always_inline s64
2014 atomic64_inc_return(atomic64_t *v)
2017 instrument_atomic_read_write(v, sizeof(*v));
2018 return raw_atomic64_inc_return(v);
2022 * atomic64_inc_return_acquire() - atomic increment with acquire ordering
2023 * @v: pointer to atomic64_t
2025 * Atomically updates @v to (@v + 1) with acquire ordering.
2027 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
2029 * Return: The updated value of @v.
2031 static __always_inline s64
2032 atomic64_inc_return_acquire(atomic64_t *v)
2034 instrument_atomic_read_write(v, sizeof(*v));
2035 return raw_atomic64_inc_return_acquire(v);
2039 * atomic64_inc_return_release() - atomic increment with release ordering
2040 * @v: pointer to atomic64_t
2042 * Atomically updates @v to (@v + 1) with release ordering.
2044 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
2046 * Return: The updated value of @v.
2048 static __always_inline s64
2049 atomic64_inc_return_release(atomic64_t *v)
2052 instrument_atomic_read_write(v, sizeof(*v));
2053 return raw_atomic64_inc_return_release(v);
2057 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
2058 * @v: pointer to atomic64_t
2060 * Atomically updates @v to (@v + 1) with relaxed ordering.
2062 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
2064 * Return: The updated value of @v.
2066 static __always_inline s64
2067 atomic64_inc_return_relaxed(atomic64_t *v)
2069 instrument_atomic_read_write(v, sizeof(*v));
2070 return raw_atomic64_inc_return_relaxed(v);
2074 * atomic64_fetch_inc() - atomic increment with full ordering
2075 * @v: pointer to atomic64_t
2077 * Atomically updates @v to (@v + 1) with full ordering.
2079 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
2081 * Return: The original value of @v.
2083 static __always_inline s64
2084 atomic64_fetch_inc(atomic64_t *v)
2087 instrument_atomic_read_write(v, sizeof(*v));
2088 return raw_atomic64_fetch_inc(v);
2092 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
2093 * @v: pointer to atomic64_t
2095 * Atomically updates @v to (@v + 1) with acquire ordering.
2097 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
2099 * Return: The original value of @v.
2101 static __always_inline s64
2102 atomic64_fetch_inc_acquire(atomic64_t *v)
2104 instrument_atomic_read_write(v, sizeof(*v));
2105 return raw_atomic64_fetch_inc_acquire(v);
2109 * atomic64_fetch_inc_release() - atomic increment with release ordering
2110 * @v: pointer to atomic64_t
2112 * Atomically updates @v to (@v + 1) with release ordering.
2114 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
2116 * Return: The original value of @v.
2118 static __always_inline s64
2119 atomic64_fetch_inc_release(atomic64_t *v)
2122 instrument_atomic_read_write(v, sizeof(*v));
2123 return raw_atomic64_fetch_inc_release(v);
2127 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
2128 * @v: pointer to atomic64_t
2130 * Atomically updates @v to (@v + 1) with relaxed ordering.
2132 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
2134 * Return: The original value of @v.
2136 static __always_inline s64
2137 atomic64_fetch_inc_relaxed(atomic64_t *v)
2139 instrument_atomic_read_write(v, sizeof(*v));
2140 return raw_atomic64_fetch_inc_relaxed(v);
2144 * atomic64_dec() - atomic decrement with relaxed ordering
2145 * @v: pointer to atomic64_t
2147 * Atomically updates @v to (@v - 1) with relaxed ordering.
2149 * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
2153 static __always_inline void
2154 atomic64_dec(atomic64_t *v)
2156 instrument_atomic_read_write(v, sizeof(*v));
2157 raw_atomic64_dec(v);
2161 * atomic64_dec_return() - atomic decrement with full ordering
2162 * @v: pointer to atomic64_t
2164 * Atomically updates @v to (@v - 1) with full ordering.
2166 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
2168 * Return: The updated value of @v.
2170 static __always_inline s64
2171 atomic64_dec_return(atomic64_t *v)
2174 instrument_atomic_read_write(v, sizeof(*v));
2175 return raw_atomic64_dec_return(v);
2179 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
2180 * @v: pointer to atomic64_t
2182 * Atomically updates @v to (@v - 1) with acquire ordering.
2184 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
2186 * Return: The updated value of @v.
2188 static __always_inline s64
2189 atomic64_dec_return_acquire(atomic64_t *v)
2191 instrument_atomic_read_write(v, sizeof(*v));
2192 return raw_atomic64_dec_return_acquire(v);
2196 * atomic64_dec_return_release() - atomic decrement with release ordering
2197 * @v: pointer to atomic64_t
2199 * Atomically updates @v to (@v - 1) with release ordering.
2201 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
2203 * Return: The updated value of @v.
2205 static __always_inline s64
2206 atomic64_dec_return_release(atomic64_t *v)
2209 instrument_atomic_read_write(v, sizeof(*v));
2210 return raw_atomic64_dec_return_release(v);
2214 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
2215 * @v: pointer to atomic64_t
2217 * Atomically updates @v to (@v - 1) with relaxed ordering.
2219 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
2221 * Return: The updated value of @v.
2223 static __always_inline s64
2224 atomic64_dec_return_relaxed(atomic64_t *v)
2226 instrument_atomic_read_write(v, sizeof(*v));
2227 return raw_atomic64_dec_return_relaxed(v);
2231 * atomic64_fetch_dec() - atomic decrement with full ordering
2232 * @v: pointer to atomic64_t
2234 * Atomically updates @v to (@v - 1) with full ordering.
2236 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
2238 * Return: The original value of @v.
2240 static __always_inline s64
2241 atomic64_fetch_dec(atomic64_t *v)
2244 instrument_atomic_read_write(v, sizeof(*v));
2245 return raw_atomic64_fetch_dec(v);
2249 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
2250 * @v: pointer to atomic64_t
2252 * Atomically updates @v to (@v - 1) with acquire ordering.
2254 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
2256 * Return: The original value of @v.
2258 static __always_inline s64
2259 atomic64_fetch_dec_acquire(atomic64_t *v)
2261 instrument_atomic_read_write(v, sizeof(*v));
2262 return raw_atomic64_fetch_dec_acquire(v);
2266 * atomic64_fetch_dec_release() - atomic decrement with release ordering
2267 * @v: pointer to atomic64_t
2269 * Atomically updates @v to (@v - 1) with release ordering.
2271 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
2273 * Return: The original value of @v.
2275 static __always_inline s64
2276 atomic64_fetch_dec_release(atomic64_t *v)
2279 instrument_atomic_read_write(v, sizeof(*v));
2280 return raw_atomic64_fetch_dec_release(v);
2284 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
2285 * @v: pointer to atomic64_t
2287 * Atomically updates @v to (@v - 1) with relaxed ordering.
2289 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
2291 * Return: The original value of @v.
2293 static __always_inline s64
2294 atomic64_fetch_dec_relaxed(atomic64_t *v)
2296 instrument_atomic_read_write(v, sizeof(*v));
2297 return raw_atomic64_fetch_dec_relaxed(v);
2301 * atomic64_and() - atomic bitwise AND with relaxed ordering
2303 * @v: pointer to atomic64_t
2305 * Atomically updates @v to (@v & @i) with relaxed ordering.
2307 * Unsafe to use in noinstr code; use raw_atomic64_and() there.
2311 static __always_inline void
2312 atomic64_and(s64 i, atomic64_t *v)
2314 instrument_atomic_read_write(v, sizeof(*v));
2315 raw_atomic64_and(i, v);
2319 * atomic64_fetch_and() - atomic bitwise AND with full ordering
2321 * @v: pointer to atomic64_t
2323 * Atomically updates @v to (@v & @i) with full ordering.
2325 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
2327 * Return: The original value of @v.
2329 static __always_inline s64
2330 atomic64_fetch_and(s64 i, atomic64_t *v)
2333 instrument_atomic_read_write(v, sizeof(*v));
2334 return raw_atomic64_fetch_and(i, v);
2338 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
2340 * @v: pointer to atomic64_t
2342 * Atomically updates @v to (@v & @i) with acquire ordering.
2344 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
2346 * Return: The original value of @v.
2348 static __always_inline s64
2349 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2351 instrument_atomic_read_write(v, sizeof(*v));
2352 return raw_atomic64_fetch_and_acquire(i, v);
2356 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
2358 * @v: pointer to atomic64_t
2360 * Atomically updates @v to (@v & @i) with release ordering.
2362 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
2364 * Return: The original value of @v.
2366 static __always_inline s64
2367 atomic64_fetch_and_release(s64 i, atomic64_t *v)
2370 instrument_atomic_read_write(v, sizeof(*v));
2371 return raw_atomic64_fetch_and_release(i, v);
2375 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
2377 * @v: pointer to atomic64_t
2379 * Atomically updates @v to (@v & @i) with relaxed ordering.
2381 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
2383 * Return: The original value of @v.
2385 static __always_inline s64
2386 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2388 instrument_atomic_read_write(v, sizeof(*v));
2389 return raw_atomic64_fetch_and_relaxed(i, v);
2393 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
2395 * @v: pointer to atomic64_t
2397 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2399 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
2403 static __always_inline void
2404 atomic64_andnot(s64 i, atomic64_t *v)
2406 instrument_atomic_read_write(v, sizeof(*v));
2407 raw_atomic64_andnot(i, v);
2411 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
2413 * @v: pointer to atomic64_t
2415 * Atomically updates @v to (@v & ~@i) with full ordering.
2417 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
2419 * Return: The original value of @v.
2421 static __always_inline s64
2422 atomic64_fetch_andnot(s64 i, atomic64_t *v)
2425 instrument_atomic_read_write(v, sizeof(*v));
2426 return raw_atomic64_fetch_andnot(i, v);
2430 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
2432 * @v: pointer to atomic64_t
2434 * Atomically updates @v to (@v & ~@i) with acquire ordering.
2436 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
2438 * Return: The original value of @v.
2440 static __always_inline s64
2441 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2443 instrument_atomic_read_write(v, sizeof(*v));
2444 return raw_atomic64_fetch_andnot_acquire(i, v);
2448 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
2450 * @v: pointer to atomic64_t
2452 * Atomically updates @v to (@v & ~@i) with release ordering.
2454 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
2456 * Return: The original value of @v.
2458 static __always_inline s64
2459 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2462 instrument_atomic_read_write(v, sizeof(*v));
2463 return raw_atomic64_fetch_andnot_release(i, v);
2467 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
2469 * @v: pointer to atomic64_t
2471 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2473 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
2475 * Return: The original value of @v.
2477 static __always_inline s64
2478 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2480 instrument_atomic_read_write(v, sizeof(*v));
2481 return raw_atomic64_fetch_andnot_relaxed(i, v);
2485 * atomic64_or() - atomic bitwise OR with relaxed ordering
2487 * @v: pointer to atomic64_t
2489 * Atomically updates @v to (@v | @i) with relaxed ordering.
2491 * Unsafe to use in noinstr code; use raw_atomic64_or() there.
2495 static __always_inline void
2496 atomic64_or(s64 i, atomic64_t *v)
2498 instrument_atomic_read_write(v, sizeof(*v));
2499 raw_atomic64_or(i, v);
2503 * atomic64_fetch_or() - atomic bitwise OR with full ordering
2505 * @v: pointer to atomic64_t
2507 * Atomically updates @v to (@v | @i) with full ordering.
2509 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
2511 * Return: The original value of @v.
2513 static __always_inline s64
2514 atomic64_fetch_or(s64 i, atomic64_t *v)
2517 instrument_atomic_read_write(v, sizeof(*v));
2518 return raw_atomic64_fetch_or(i, v);
2522 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
2524 * @v: pointer to atomic64_t
2526 * Atomically updates @v to (@v | @i) with acquire ordering.
2528 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
2530 * Return: The original value of @v.
2532 static __always_inline s64
2533 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2535 instrument_atomic_read_write(v, sizeof(*v));
2536 return raw_atomic64_fetch_or_acquire(i, v);
2540 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
2542 * @v: pointer to atomic64_t
2544 * Atomically updates @v to (@v | @i) with release ordering.
2546 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
2548 * Return: The original value of @v.
2550 static __always_inline s64
2551 atomic64_fetch_or_release(s64 i, atomic64_t *v)
2554 instrument_atomic_read_write(v, sizeof(*v));
2555 return raw_atomic64_fetch_or_release(i, v);
2559 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
2561 * @v: pointer to atomic64_t
2563 * Atomically updates @v to (@v | @i) with relaxed ordering.
2565 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
2567 * Return: The original value of @v.
2569 static __always_inline s64
2570 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
2572 instrument_atomic_read_write(v, sizeof(*v));
2573 return raw_atomic64_fetch_or_relaxed(i, v);
2577 * atomic64_xor() - atomic bitwise XOR with relaxed ordering
2579 * @v: pointer to atomic64_t
2581 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2583 * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
2587 static __always_inline void
2588 atomic64_xor(s64 i, atomic64_t *v)
2590 instrument_atomic_read_write(v, sizeof(*v));
2591 raw_atomic64_xor(i, v);
2595 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
2597 * @v: pointer to atomic64_t
2599 * Atomically updates @v to (@v ^ @i) with full ordering.
2601 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
2603 * Return: The original value of @v.
2605 static __always_inline s64
2606 atomic64_fetch_xor(s64 i, atomic64_t *v)
2609 instrument_atomic_read_write(v, sizeof(*v));
2610 return raw_atomic64_fetch_xor(i, v);
2614 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
2616 * @v: pointer to atomic64_t
2618 * Atomically updates @v to (@v ^ @i) with acquire ordering.
2620 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
2622 * Return: The original value of @v.
2624 static __always_inline s64
2625 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2627 instrument_atomic_read_write(v, sizeof(*v));
2628 return raw_atomic64_fetch_xor_acquire(i, v);
2632 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
2634 * @v: pointer to atomic64_t
2636 * Atomically updates @v to (@v ^ @i) with release ordering.
2638 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
2640 * Return: The original value of @v.
2642 static __always_inline s64
2643 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2646 instrument_atomic_read_write(v, sizeof(*v));
2647 return raw_atomic64_fetch_xor_release(i, v);
2651 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
2653 * @v: pointer to atomic64_t
2655 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2657 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
2659 * Return: The original value of @v.
2661 static __always_inline s64
2662 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
2664 instrument_atomic_read_write(v, sizeof(*v));
2665 return raw_atomic64_fetch_xor_relaxed(i, v);
2669 * atomic64_xchg() - atomic exchange with full ordering
2670 * @v: pointer to atomic64_t
2671 * @new: s64 value to assign
2673 * Atomically updates @v to @new with full ordering.
2675 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
2677 * Return: The original value of @v.
2679 static __always_inline s64
2680 atomic64_xchg(atomic64_t *v, s64 new)
2683 instrument_atomic_read_write(v, sizeof(*v));
2684 return raw_atomic64_xchg(v, new);
2688 * atomic64_xchg_acquire() - atomic exchange with acquire ordering
2689 * @v: pointer to atomic64_t
2690 * @new: s64 value to assign
2692 * Atomically updates @v to @new with acquire ordering.
2694 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
2696 * Return: The original value of @v.
2698 static __always_inline s64
2699 atomic64_xchg_acquire(atomic64_t *v, s64 new)
2701 instrument_atomic_read_write(v, sizeof(*v));
2702 return raw_atomic64_xchg_acquire(v, new);
2706 * atomic64_xchg_release() - atomic exchange with release ordering
2707 * @v: pointer to atomic64_t
2708 * @new: s64 value to assign
2710 * Atomically updates @v to @new with release ordering.
2712 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
2714 * Return: The original value of @v.
2716 static __always_inline s64
2717 atomic64_xchg_release(atomic64_t *v, s64 new)
2720 instrument_atomic_read_write(v, sizeof(*v));
2721 return raw_atomic64_xchg_release(v, new);
2725 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
2726 * @v: pointer to atomic64_t
2727 * @new: s64 value to assign
2729 * Atomically updates @v to @new with relaxed ordering.
2731 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
2733 * Return: The original value of @v.
2735 static __always_inline s64
2736 atomic64_xchg_relaxed(atomic64_t *v, s64 new)
2738 instrument_atomic_read_write(v, sizeof(*v));
2739 return raw_atomic64_xchg_relaxed(v, new);
2743 * atomic64_cmpxchg() - atomic compare and exchange with full ordering
2744 * @v: pointer to atomic64_t
2745 * @old: s64 value to compare with
2746 * @new: s64 value to assign
2748 * If (@v == @old), atomically updates @v to @new with full ordering.
2750 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
2752 * Return: The original value of @v.
2754 static __always_inline s64
2755 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2758 instrument_atomic_read_write(v, sizeof(*v));
2759 return raw_atomic64_cmpxchg(v, old, new);
2763 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2764 * @v: pointer to atomic64_t
2765 * @old: s64 value to compare with
2766 * @new: s64 value to assign
2768 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2770 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
2772 * Return: The original value of @v.
2774 static __always_inline s64
2775 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2777 instrument_atomic_read_write(v, sizeof(*v));
2778 return raw_atomic64_cmpxchg_acquire(v, old, new);
2782 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
2783 * @v: pointer to atomic64_t
2784 * @old: s64 value to compare with
2785 * @new: s64 value to assign
2787 * If (@v == @old), atomically updates @v to @new with release ordering.
2789 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
2791 * Return: The original value of @v.
2793 static __always_inline s64
2794 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2797 instrument_atomic_read_write(v, sizeof(*v));
2798 return raw_atomic64_cmpxchg_release(v, old, new);
2802 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2803 * @v: pointer to atomic64_t
2804 * @old: s64 value to compare with
2805 * @new: s64 value to assign
2807 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2809 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
2811 * Return: The original value of @v.
2813 static __always_inline s64
2814 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
2816 instrument_atomic_read_write(v, sizeof(*v));
2817 return raw_atomic64_cmpxchg_relaxed(v, old, new);
2821 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
2822 * @v: pointer to atomic64_t
2823 * @old: pointer to s64 value to compare with
2824 * @new: s64 value to assign
2826 * If (@v == @old), atomically updates @v to @new with full ordering.
2827 * Otherwise, updates @old to the current value of @v.
2829 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
2831 * Return: @true if the exchange occured, @false otherwise.
2833 static __always_inline bool
2834 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2837 instrument_atomic_read_write(v, sizeof(*v));
2838 instrument_atomic_read_write(old, sizeof(*old));
2839 return raw_atomic64_try_cmpxchg(v, old, new);
2843 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2844 * @v: pointer to atomic64_t
2845 * @old: pointer to s64 value to compare with
2846 * @new: s64 value to assign
2848 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2849 * Otherwise, updates @old to the current value of @v.
2851 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
2853 * Return: @true if the exchange occured, @false otherwise.
2855 static __always_inline bool
2856 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2858 instrument_atomic_read_write(v, sizeof(*v));
2859 instrument_atomic_read_write(old, sizeof(*old));
2860 return raw_atomic64_try_cmpxchg_acquire(v, old, new);
2864 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
2865 * @v: pointer to atomic64_t
2866 * @old: pointer to s64 value to compare with
2867 * @new: s64 value to assign
2869 * If (@v == @old), atomically updates @v to @new with release ordering.
2870 * Otherwise, updates @old to the current value of @v.
2872 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
2874 * Return: @true if the exchange occured, @false otherwise.
2876 static __always_inline bool
2877 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2880 instrument_atomic_read_write(v, sizeof(*v));
2881 instrument_atomic_read_write(old, sizeof(*old));
2882 return raw_atomic64_try_cmpxchg_release(v, old, new);
2886 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2887 * @v: pointer to atomic64_t
2888 * @old: pointer to s64 value to compare with
2889 * @new: s64 value to assign
2891 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2892 * Otherwise, updates @old to the current value of @v.
2894 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
2896 * Return: @true if the exchange occured, @false otherwise.
2898 static __always_inline bool
2899 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2901 instrument_atomic_read_write(v, sizeof(*v));
2902 instrument_atomic_read_write(old, sizeof(*old));
2903 return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
2907 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
2908 * @i: s64 value to add
2909 * @v: pointer to atomic64_t
2911 * Atomically updates @v to (@v - @i) with full ordering.
2913 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
2915 * Return: @true if the resulting value of @v is zero, @false otherwise.
2917 static __always_inline bool
2918 atomic64_sub_and_test(s64 i, atomic64_t *v)
2921 instrument_atomic_read_write(v, sizeof(*v));
2922 return raw_atomic64_sub_and_test(i, v);
2926 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
2927 * @v: pointer to atomic64_t
2929 * Atomically updates @v to (@v - 1) with full ordering.
2931 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
2933 * Return: @true if the resulting value of @v is zero, @false otherwise.
2935 static __always_inline bool
2936 atomic64_dec_and_test(atomic64_t *v)
2939 instrument_atomic_read_write(v, sizeof(*v));
2940 return raw_atomic64_dec_and_test(v);
2944 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
2945 * @v: pointer to atomic64_t
2947 * Atomically updates @v to (@v + 1) with full ordering.
2949 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
2951 * Return: @true if the resulting value of @v is zero, @false otherwise.
2953 static __always_inline bool
2954 atomic64_inc_and_test(atomic64_t *v)
2957 instrument_atomic_read_write(v, sizeof(*v));
2958 return raw_atomic64_inc_and_test(v);
2962 * atomic64_add_negative() - atomic add and test if negative with full ordering
2963 * @i: s64 value to add
2964 * @v: pointer to atomic64_t
2966 * Atomically updates @v to (@v + @i) with full ordering.
2968 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
2970 * Return: @true if the resulting value of @v is negative, @false otherwise.
2972 static __always_inline bool
2973 atomic64_add_negative(s64 i, atomic64_t *v)
2976 instrument_atomic_read_write(v, sizeof(*v));
2977 return raw_atomic64_add_negative(i, v);
2981 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
2982 * @i: s64 value to add
2983 * @v: pointer to atomic64_t
2985 * Atomically updates @v to (@v + @i) with acquire ordering.
2987 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
2989 * Return: @true if the resulting value of @v is negative, @false otherwise.
2991 static __always_inline bool
2992 atomic64_add_negative_acquire(s64 i, atomic64_t *v)
2994 instrument_atomic_read_write(v, sizeof(*v));
2995 return raw_atomic64_add_negative_acquire(i, v);
2999 * atomic64_add_negative_release() - atomic add and test if negative with release ordering
3000 * @i: s64 value to add
3001 * @v: pointer to atomic64_t
3003 * Atomically updates @v to (@v + @i) with release ordering.
3005 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
3007 * Return: @true if the resulting value of @v is negative, @false otherwise.
3009 static __always_inline bool
3010 atomic64_add_negative_release(s64 i, atomic64_t *v)
3013 instrument_atomic_read_write(v, sizeof(*v));
3014 return raw_atomic64_add_negative_release(i, v);
3018 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
3019 * @i: s64 value to add
3020 * @v: pointer to atomic64_t
3022 * Atomically updates @v to (@v + @i) with relaxed ordering.
3024 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
3026 * Return: @true if the resulting value of @v is negative, @false otherwise.
3028 static __always_inline bool
3029 atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
3031 instrument_atomic_read_write(v, sizeof(*v));
3032 return raw_atomic64_add_negative_relaxed(i, v);
3036 * atomic64_fetch_add_unless() - atomic add unless value with full ordering
3037 * @v: pointer to atomic64_t
3038 * @a: s64 value to add
3039 * @u: s64 value to compare with
3041 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3043 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
3045 * Return: The original value of @v.
3047 static __always_inline s64
3048 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
3051 instrument_atomic_read_write(v, sizeof(*v));
3052 return raw_atomic64_fetch_add_unless(v, a, u);
3056 * atomic64_add_unless() - atomic add unless value with full ordering
3057 * @v: pointer to atomic64_t
3058 * @a: s64 value to add
3059 * @u: s64 value to compare with
3061 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3063 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
3065 * Return: @true if @v was updated, @false otherwise.
3067 static __always_inline bool
3068 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
3071 instrument_atomic_read_write(v, sizeof(*v));
3072 return raw_atomic64_add_unless(v, a, u);
3076 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
3077 * @v: pointer to atomic64_t
3079 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
3081 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
3083 * Return: @true if @v was updated, @false otherwise.
3085 static __always_inline bool
3086 atomic64_inc_not_zero(atomic64_t *v)
3089 instrument_atomic_read_write(v, sizeof(*v));
3090 return raw_atomic64_inc_not_zero(v);
3094 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
3095 * @v: pointer to atomic64_t
3097 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
3099 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
3101 * Return: @true if @v was updated, @false otherwise.
3103 static __always_inline bool
3104 atomic64_inc_unless_negative(atomic64_t *v)
3107 instrument_atomic_read_write(v, sizeof(*v));
3108 return raw_atomic64_inc_unless_negative(v);
3112 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
3113 * @v: pointer to atomic64_t
3115 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
3117 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
3119 * Return: @true if @v was updated, @false otherwise.
3121 static __always_inline bool
3122 atomic64_dec_unless_positive(atomic64_t *v)
3125 instrument_atomic_read_write(v, sizeof(*v));
3126 return raw_atomic64_dec_unless_positive(v);
3130 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
3131 * @v: pointer to atomic64_t
3133 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
3135 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
3137 * Return: The old value of (@v - 1), regardless of whether @v was updated.
3139 static __always_inline s64
3140 atomic64_dec_if_positive(atomic64_t *v)
3143 instrument_atomic_read_write(v, sizeof(*v));
3144 return raw_atomic64_dec_if_positive(v);
3148 * atomic_long_read() - atomic load with relaxed ordering
3149 * @v: pointer to atomic_long_t
3151 * Atomically loads the value of @v with relaxed ordering.
3153 * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
3155 * Return: The value loaded from @v.
3157 static __always_inline long
3158 atomic_long_read(const atomic_long_t *v)
3160 instrument_atomic_read(v, sizeof(*v));
3161 return raw_atomic_long_read(v);
3165 * atomic_long_read_acquire() - atomic load with acquire ordering
3166 * @v: pointer to atomic_long_t
3168 * Atomically loads the value of @v with acquire ordering.
3170 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
3172 * Return: The value loaded from @v.
3174 static __always_inline long
3175 atomic_long_read_acquire(const atomic_long_t *v)
3177 instrument_atomic_read(v, sizeof(*v));
3178 return raw_atomic_long_read_acquire(v);
3182 * atomic_long_set() - atomic set with relaxed ordering
3183 * @v: pointer to atomic_long_t
3184 * @i: long value to assign
3186 * Atomically sets @v to @i with relaxed ordering.
3188 * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
3192 static __always_inline void
3193 atomic_long_set(atomic_long_t *v, long i)
3195 instrument_atomic_write(v, sizeof(*v));
3196 raw_atomic_long_set(v, i);
3200 * atomic_long_set_release() - atomic set with release ordering
3201 * @v: pointer to atomic_long_t
3202 * @i: long value to assign
3204 * Atomically sets @v to @i with release ordering.
3206 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
3210 static __always_inline void
3211 atomic_long_set_release(atomic_long_t *v, long i)
3214 instrument_atomic_write(v, sizeof(*v));
3215 raw_atomic_long_set_release(v, i);
3219 * atomic_long_add() - atomic add with relaxed ordering
3220 * @i: long value to add
3221 * @v: pointer to atomic_long_t
3223 * Atomically updates @v to (@v + @i) with relaxed ordering.
3225 * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
3229 static __always_inline void
3230 atomic_long_add(long i, atomic_long_t *v)
3232 instrument_atomic_read_write(v, sizeof(*v));
3233 raw_atomic_long_add(i, v);
3237 * atomic_long_add_return() - atomic add with full ordering
3238 * @i: long value to add
3239 * @v: pointer to atomic_long_t
3241 * Atomically updates @v to (@v + @i) with full ordering.
3243 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
3245 * Return: The updated value of @v.
3247 static __always_inline long
3248 atomic_long_add_return(long i, atomic_long_t *v)
3251 instrument_atomic_read_write(v, sizeof(*v));
3252 return raw_atomic_long_add_return(i, v);
3256 * atomic_long_add_return_acquire() - atomic add with acquire ordering
3257 * @i: long value to add
3258 * @v: pointer to atomic_long_t
3260 * Atomically updates @v to (@v + @i) with acquire ordering.
3262 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
3264 * Return: The updated value of @v.
3266 static __always_inline long
3267 atomic_long_add_return_acquire(long i, atomic_long_t *v)
3269 instrument_atomic_read_write(v, sizeof(*v));
3270 return raw_atomic_long_add_return_acquire(i, v);
3274 * atomic_long_add_return_release() - atomic add with release ordering
3275 * @i: long value to add
3276 * @v: pointer to atomic_long_t
3278 * Atomically updates @v to (@v + @i) with release ordering.
3280 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
3282 * Return: The updated value of @v.
3284 static __always_inline long
3285 atomic_long_add_return_release(long i, atomic_long_t *v)
3288 instrument_atomic_read_write(v, sizeof(*v));
3289 return raw_atomic_long_add_return_release(i, v);
3293 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
3294 * @i: long value to add
3295 * @v: pointer to atomic_long_t
3297 * Atomically updates @v to (@v + @i) with relaxed ordering.
3299 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
3301 * Return: The updated value of @v.
3303 static __always_inline long
3304 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
3306 instrument_atomic_read_write(v, sizeof(*v));
3307 return raw_atomic_long_add_return_relaxed(i, v);
3311 * atomic_long_fetch_add() - atomic add with full ordering
3312 * @i: long value to add
3313 * @v: pointer to atomic_long_t
3315 * Atomically updates @v to (@v + @i) with full ordering.
3317 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
3319 * Return: The original value of @v.
3321 static __always_inline long
3322 atomic_long_fetch_add(long i, atomic_long_t *v)
3325 instrument_atomic_read_write(v, sizeof(*v));
3326 return raw_atomic_long_fetch_add(i, v);
3330 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
3331 * @i: long value to add
3332 * @v: pointer to atomic_long_t
3334 * Atomically updates @v to (@v + @i) with acquire ordering.
3336 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
3338 * Return: The original value of @v.
3340 static __always_inline long
3341 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
3343 instrument_atomic_read_write(v, sizeof(*v));
3344 return raw_atomic_long_fetch_add_acquire(i, v);
3348 * atomic_long_fetch_add_release() - atomic add with release ordering
3349 * @i: long value to add
3350 * @v: pointer to atomic_long_t
3352 * Atomically updates @v to (@v + @i) with release ordering.
3354 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
3356 * Return: The original value of @v.
3358 static __always_inline long
3359 atomic_long_fetch_add_release(long i, atomic_long_t *v)
3362 instrument_atomic_read_write(v, sizeof(*v));
3363 return raw_atomic_long_fetch_add_release(i, v);
3367 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
3368 * @i: long value to add
3369 * @v: pointer to atomic_long_t
3371 * Atomically updates @v to (@v + @i) with relaxed ordering.
3373 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
3375 * Return: The original value of @v.
3377 static __always_inline long
3378 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
3380 instrument_atomic_read_write(v, sizeof(*v));
3381 return raw_atomic_long_fetch_add_relaxed(i, v);
3385 * atomic_long_sub() - atomic subtract with relaxed ordering
3386 * @i: long value to subtract
3387 * @v: pointer to atomic_long_t
3389 * Atomically updates @v to (@v - @i) with relaxed ordering.
3391 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
3395 static __always_inline void
3396 atomic_long_sub(long i, atomic_long_t *v)
3398 instrument_atomic_read_write(v, sizeof(*v));
3399 raw_atomic_long_sub(i, v);
3403 * atomic_long_sub_return() - atomic subtract with full ordering
3404 * @i: long value to subtract
3405 * @v: pointer to atomic_long_t
3407 * Atomically updates @v to (@v - @i) with full ordering.
3409 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
3411 * Return: The updated value of @v.
3413 static __always_inline long
3414 atomic_long_sub_return(long i, atomic_long_t *v)
3417 instrument_atomic_read_write(v, sizeof(*v));
3418 return raw_atomic_long_sub_return(i, v);
3422 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
3423 * @i: long value to subtract
3424 * @v: pointer to atomic_long_t
3426 * Atomically updates @v to (@v - @i) with acquire ordering.
3428 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
3430 * Return: The updated value of @v.
3432 static __always_inline long
3433 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
3435 instrument_atomic_read_write(v, sizeof(*v));
3436 return raw_atomic_long_sub_return_acquire(i, v);
3440 * atomic_long_sub_return_release() - atomic subtract with release ordering
3441 * @i: long value to subtract
3442 * @v: pointer to atomic_long_t
3444 * Atomically updates @v to (@v - @i) with release ordering.
3446 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
3448 * Return: The updated value of @v.
3450 static __always_inline long
3451 atomic_long_sub_return_release(long i, atomic_long_t *v)
3454 instrument_atomic_read_write(v, sizeof(*v));
3455 return raw_atomic_long_sub_return_release(i, v);
3459 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
3460 * @i: long value to subtract
3461 * @v: pointer to atomic_long_t
3463 * Atomically updates @v to (@v - @i) with relaxed ordering.
3465 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
3467 * Return: The updated value of @v.
3469 static __always_inline long
3470 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
3472 instrument_atomic_read_write(v, sizeof(*v));
3473 return raw_atomic_long_sub_return_relaxed(i, v);
3477 * atomic_long_fetch_sub() - atomic subtract with full ordering
3478 * @i: long value to subtract
3479 * @v: pointer to atomic_long_t
3481 * Atomically updates @v to (@v - @i) with full ordering.
3483 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
3485 * Return: The original value of @v.
3487 static __always_inline long
3488 atomic_long_fetch_sub(long i, atomic_long_t *v)
3491 instrument_atomic_read_write(v, sizeof(*v));
3492 return raw_atomic_long_fetch_sub(i, v);
3496 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
3497 * @i: long value to subtract
3498 * @v: pointer to atomic_long_t
3500 * Atomically updates @v to (@v - @i) with acquire ordering.
3502 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
3504 * Return: The original value of @v.
3506 static __always_inline long
3507 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
3509 instrument_atomic_read_write(v, sizeof(*v));
3510 return raw_atomic_long_fetch_sub_acquire(i, v);
3514 * atomic_long_fetch_sub_release() - atomic subtract with release ordering
3515 * @i: long value to subtract
3516 * @v: pointer to atomic_long_t
3518 * Atomically updates @v to (@v - @i) with release ordering.
3520 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
3522 * Return: The original value of @v.
3524 static __always_inline long
3525 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
3528 instrument_atomic_read_write(v, sizeof(*v));
3529 return raw_atomic_long_fetch_sub_release(i, v);
3533 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3534 * @i: long value to subtract
3535 * @v: pointer to atomic_long_t
3537 * Atomically updates @v to (@v - @i) with relaxed ordering.
3539 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
3541 * Return: The original value of @v.
3543 static __always_inline long
3544 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
3546 instrument_atomic_read_write(v, sizeof(*v));
3547 return raw_atomic_long_fetch_sub_relaxed(i, v);
3551 * atomic_long_inc() - atomic increment with relaxed ordering
3552 * @v: pointer to atomic_long_t
3554 * Atomically updates @v to (@v + 1) with relaxed ordering.
3556 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
3560 static __always_inline void
3561 atomic_long_inc(atomic_long_t *v)
3563 instrument_atomic_read_write(v, sizeof(*v));
3564 raw_atomic_long_inc(v);
3568 * atomic_long_inc_return() - atomic increment with full ordering
3569 * @v: pointer to atomic_long_t
3571 * Atomically updates @v to (@v + 1) with full ordering.
3573 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
3575 * Return: The updated value of @v.
3577 static __always_inline long
3578 atomic_long_inc_return(atomic_long_t *v)
3581 instrument_atomic_read_write(v, sizeof(*v));
3582 return raw_atomic_long_inc_return(v);
3586 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
3587 * @v: pointer to atomic_long_t
3589 * Atomically updates @v to (@v + 1) with acquire ordering.
3591 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
3593 * Return: The updated value of @v.
3595 static __always_inline long
3596 atomic_long_inc_return_acquire(atomic_long_t *v)
3598 instrument_atomic_read_write(v, sizeof(*v));
3599 return raw_atomic_long_inc_return_acquire(v);
3603 * atomic_long_inc_return_release() - atomic increment with release ordering
3604 * @v: pointer to atomic_long_t
3606 * Atomically updates @v to (@v + 1) with release ordering.
3608 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
3610 * Return: The updated value of @v.
3612 static __always_inline long
3613 atomic_long_inc_return_release(atomic_long_t *v)
3616 instrument_atomic_read_write(v, sizeof(*v));
3617 return raw_atomic_long_inc_return_release(v);
3621 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
3622 * @v: pointer to atomic_long_t
3624 * Atomically updates @v to (@v + 1) with relaxed ordering.
3626 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
3628 * Return: The updated value of @v.
3630 static __always_inline long
3631 atomic_long_inc_return_relaxed(atomic_long_t *v)
3633 instrument_atomic_read_write(v, sizeof(*v));
3634 return raw_atomic_long_inc_return_relaxed(v);
3638 * atomic_long_fetch_inc() - atomic increment with full ordering
3639 * @v: pointer to atomic_long_t
3641 * Atomically updates @v to (@v + 1) with full ordering.
3643 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
3645 * Return: The original value of @v.
3647 static __always_inline long
3648 atomic_long_fetch_inc(atomic_long_t *v)
3651 instrument_atomic_read_write(v, sizeof(*v));
3652 return raw_atomic_long_fetch_inc(v);
3656 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
3657 * @v: pointer to atomic_long_t
3659 * Atomically updates @v to (@v + 1) with acquire ordering.
3661 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
3663 * Return: The original value of @v.
3665 static __always_inline long
3666 atomic_long_fetch_inc_acquire(atomic_long_t *v)
3668 instrument_atomic_read_write(v, sizeof(*v));
3669 return raw_atomic_long_fetch_inc_acquire(v);
3673 * atomic_long_fetch_inc_release() - atomic increment with release ordering
3674 * @v: pointer to atomic_long_t
3676 * Atomically updates @v to (@v + 1) with release ordering.
3678 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
3680 * Return: The original value of @v.
3682 static __always_inline long
3683 atomic_long_fetch_inc_release(atomic_long_t *v)
3686 instrument_atomic_read_write(v, sizeof(*v));
3687 return raw_atomic_long_fetch_inc_release(v);
3691 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
3692 * @v: pointer to atomic_long_t
3694 * Atomically updates @v to (@v + 1) with relaxed ordering.
3696 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
3698 * Return: The original value of @v.
3700 static __always_inline long
3701 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
3703 instrument_atomic_read_write(v, sizeof(*v));
3704 return raw_atomic_long_fetch_inc_relaxed(v);
3708 * atomic_long_dec() - atomic decrement with relaxed ordering
3709 * @v: pointer to atomic_long_t
3711 * Atomically updates @v to (@v - 1) with relaxed ordering.
3713 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
3717 static __always_inline void
3718 atomic_long_dec(atomic_long_t *v)
3720 instrument_atomic_read_write(v, sizeof(*v));
3721 raw_atomic_long_dec(v);
3725 * atomic_long_dec_return() - atomic decrement with full ordering
3726 * @v: pointer to atomic_long_t
3728 * Atomically updates @v to (@v - 1) with full ordering.
3730 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
3732 * Return: The updated value of @v.
3734 static __always_inline long
3735 atomic_long_dec_return(atomic_long_t *v)
3738 instrument_atomic_read_write(v, sizeof(*v));
3739 return raw_atomic_long_dec_return(v);
3743 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
3744 * @v: pointer to atomic_long_t
3746 * Atomically updates @v to (@v - 1) with acquire ordering.
3748 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
3750 * Return: The updated value of @v.
3752 static __always_inline long
3753 atomic_long_dec_return_acquire(atomic_long_t *v)
3755 instrument_atomic_read_write(v, sizeof(*v));
3756 return raw_atomic_long_dec_return_acquire(v);
3760 * atomic_long_dec_return_release() - atomic decrement with release ordering
3761 * @v: pointer to atomic_long_t
3763 * Atomically updates @v to (@v - 1) with release ordering.
3765 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
3767 * Return: The updated value of @v.
3769 static __always_inline long
3770 atomic_long_dec_return_release(atomic_long_t *v)
3773 instrument_atomic_read_write(v, sizeof(*v));
3774 return raw_atomic_long_dec_return_release(v);
3778 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
3779 * @v: pointer to atomic_long_t
3781 * Atomically updates @v to (@v - 1) with relaxed ordering.
3783 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
3785 * Return: The updated value of @v.
3787 static __always_inline long
3788 atomic_long_dec_return_relaxed(atomic_long_t *v)
3790 instrument_atomic_read_write(v, sizeof(*v));
3791 return raw_atomic_long_dec_return_relaxed(v);
3795 * atomic_long_fetch_dec() - atomic decrement with full ordering
3796 * @v: pointer to atomic_long_t
3798 * Atomically updates @v to (@v - 1) with full ordering.
3800 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
3802 * Return: The original value of @v.
3804 static __always_inline long
3805 atomic_long_fetch_dec(atomic_long_t *v)
3808 instrument_atomic_read_write(v, sizeof(*v));
3809 return raw_atomic_long_fetch_dec(v);
3813 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
3814 * @v: pointer to atomic_long_t
3816 * Atomically updates @v to (@v - 1) with acquire ordering.
3818 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
3820 * Return: The original value of @v.
3822 static __always_inline long
3823 atomic_long_fetch_dec_acquire(atomic_long_t *v)
3825 instrument_atomic_read_write(v, sizeof(*v));
3826 return raw_atomic_long_fetch_dec_acquire(v);
3830 * atomic_long_fetch_dec_release() - atomic decrement with release ordering
3831 * @v: pointer to atomic_long_t
3833 * Atomically updates @v to (@v - 1) with release ordering.
3835 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
3837 * Return: The original value of @v.
3839 static __always_inline long
3840 atomic_long_fetch_dec_release(atomic_long_t *v)
3843 instrument_atomic_read_write(v, sizeof(*v));
3844 return raw_atomic_long_fetch_dec_release(v);
3848 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3849 * @v: pointer to atomic_long_t
3851 * Atomically updates @v to (@v - 1) with relaxed ordering.
3853 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
3855 * Return: The original value of @v.
3857 static __always_inline long
3858 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
3860 instrument_atomic_read_write(v, sizeof(*v));
3861 return raw_atomic_long_fetch_dec_relaxed(v);
3865 * atomic_long_and() - atomic bitwise AND with relaxed ordering
3867 * @v: pointer to atomic_long_t
3869 * Atomically updates @v to (@v & @i) with relaxed ordering.
3871 * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
3875 static __always_inline void
3876 atomic_long_and(long i, atomic_long_t *v)
3878 instrument_atomic_read_write(v, sizeof(*v));
3879 raw_atomic_long_and(i, v);
3883 * atomic_long_fetch_and() - atomic bitwise AND with full ordering
3885 * @v: pointer to atomic_long_t
3887 * Atomically updates @v to (@v & @i) with full ordering.
3889 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
3891 * Return: The original value of @v.
3893 static __always_inline long
3894 atomic_long_fetch_and(long i, atomic_long_t *v)
3897 instrument_atomic_read_write(v, sizeof(*v));
3898 return raw_atomic_long_fetch_and(i, v);
3902 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3904 * @v: pointer to atomic_long_t
3906 * Atomically updates @v to (@v & @i) with acquire ordering.
3908 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
3910 * Return: The original value of @v.
3912 static __always_inline long
3913 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
3915 instrument_atomic_read_write(v, sizeof(*v));
3916 return raw_atomic_long_fetch_and_acquire(i, v);
3920 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
3922 * @v: pointer to atomic_long_t
3924 * Atomically updates @v to (@v & @i) with release ordering.
3926 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
3928 * Return: The original value of @v.
3930 static __always_inline long
3931 atomic_long_fetch_and_release(long i, atomic_long_t *v)
3934 instrument_atomic_read_write(v, sizeof(*v));
3935 return raw_atomic_long_fetch_and_release(i, v);
3939 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3941 * @v: pointer to atomic_long_t
3943 * Atomically updates @v to (@v & @i) with relaxed ordering.
3945 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
3947 * Return: The original value of @v.
3949 static __always_inline long
3950 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
3952 instrument_atomic_read_write(v, sizeof(*v));
3953 return raw_atomic_long_fetch_and_relaxed(i, v);
3957 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
3959 * @v: pointer to atomic_long_t
3961 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3963 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
3967 static __always_inline void
3968 atomic_long_andnot(long i, atomic_long_t *v)
3970 instrument_atomic_read_write(v, sizeof(*v));
3971 raw_atomic_long_andnot(i, v);
3975 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
3977 * @v: pointer to atomic_long_t
3979 * Atomically updates @v to (@v & ~@i) with full ordering.
3981 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
3983 * Return: The original value of @v.
3985 static __always_inline long
3986 atomic_long_fetch_andnot(long i, atomic_long_t *v)
3989 instrument_atomic_read_write(v, sizeof(*v));
3990 return raw_atomic_long_fetch_andnot(i, v);
3994 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3996 * @v: pointer to atomic_long_t
3998 * Atomically updates @v to (@v & ~@i) with acquire ordering.
4000 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
4002 * Return: The original value of @v.
4004 static __always_inline long
4005 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
4007 instrument_atomic_read_write(v, sizeof(*v));
4008 return raw_atomic_long_fetch_andnot_acquire(i, v);
4012 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
4014 * @v: pointer to atomic_long_t
4016 * Atomically updates @v to (@v & ~@i) with release ordering.
4018 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
4020 * Return: The original value of @v.
4022 static __always_inline long
4023 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
4026 instrument_atomic_read_write(v, sizeof(*v));
4027 return raw_atomic_long_fetch_andnot_release(i, v);
4031 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
4033 * @v: pointer to atomic_long_t
4035 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
4037 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
4039 * Return: The original value of @v.
4041 static __always_inline long
4042 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
4044 instrument_atomic_read_write(v, sizeof(*v));
4045 return raw_atomic_long_fetch_andnot_relaxed(i, v);
4049 * atomic_long_or() - atomic bitwise OR with relaxed ordering
4051 * @v: pointer to atomic_long_t
4053 * Atomically updates @v to (@v | @i) with relaxed ordering.
4055 * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
4059 static __always_inline void
4060 atomic_long_or(long i, atomic_long_t *v)
4062 instrument_atomic_read_write(v, sizeof(*v));
4063 raw_atomic_long_or(i, v);
4067 * atomic_long_fetch_or() - atomic bitwise OR with full ordering
4069 * @v: pointer to atomic_long_t
4071 * Atomically updates @v to (@v | @i) with full ordering.
4073 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
4075 * Return: The original value of @v.
4077 static __always_inline long
4078 atomic_long_fetch_or(long i, atomic_long_t *v)
4081 instrument_atomic_read_write(v, sizeof(*v));
4082 return raw_atomic_long_fetch_or(i, v);
4086 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
4088 * @v: pointer to atomic_long_t
4090 * Atomically updates @v to (@v | @i) with acquire ordering.
4092 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
4094 * Return: The original value of @v.
4096 static __always_inline long
4097 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
4099 instrument_atomic_read_write(v, sizeof(*v));
4100 return raw_atomic_long_fetch_or_acquire(i, v);
4104 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
4106 * @v: pointer to atomic_long_t
4108 * Atomically updates @v to (@v | @i) with release ordering.
4110 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
4112 * Return: The original value of @v.
4114 static __always_inline long
4115 atomic_long_fetch_or_release(long i, atomic_long_t *v)
4118 instrument_atomic_read_write(v, sizeof(*v));
4119 return raw_atomic_long_fetch_or_release(i, v);
4123 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
4125 * @v: pointer to atomic_long_t
4127 * Atomically updates @v to (@v | @i) with relaxed ordering.
4129 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
4131 * Return: The original value of @v.
4133 static __always_inline long
4134 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
4136 instrument_atomic_read_write(v, sizeof(*v));
4137 return raw_atomic_long_fetch_or_relaxed(i, v);
4141 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
4143 * @v: pointer to atomic_long_t
4145 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4147 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
4151 static __always_inline void
4152 atomic_long_xor(long i, atomic_long_t *v)
4154 instrument_atomic_read_write(v, sizeof(*v));
4155 raw_atomic_long_xor(i, v);
4159 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
4161 * @v: pointer to atomic_long_t
4163 * Atomically updates @v to (@v ^ @i) with full ordering.
4165 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
4167 * Return: The original value of @v.
4169 static __always_inline long
4170 atomic_long_fetch_xor(long i, atomic_long_t *v)
4173 instrument_atomic_read_write(v, sizeof(*v));
4174 return raw_atomic_long_fetch_xor(i, v);
4178 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
4180 * @v: pointer to atomic_long_t
4182 * Atomically updates @v to (@v ^ @i) with acquire ordering.
4184 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
4186 * Return: The original value of @v.
4188 static __always_inline long
4189 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
4191 instrument_atomic_read_write(v, sizeof(*v));
4192 return raw_atomic_long_fetch_xor_acquire(i, v);
4196 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
4198 * @v: pointer to atomic_long_t
4200 * Atomically updates @v to (@v ^ @i) with release ordering.
4202 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
4204 * Return: The original value of @v.
4206 static __always_inline long
4207 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
4210 instrument_atomic_read_write(v, sizeof(*v));
4211 return raw_atomic_long_fetch_xor_release(i, v);
4215 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4217 * @v: pointer to atomic_long_t
4219 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4221 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
4223 * Return: The original value of @v.
4225 static __always_inline long
4226 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
4228 instrument_atomic_read_write(v, sizeof(*v));
4229 return raw_atomic_long_fetch_xor_relaxed(i, v);
4233 * atomic_long_xchg() - atomic exchange with full ordering
4234 * @v: pointer to atomic_long_t
4235 * @new: long value to assign
4237 * Atomically updates @v to @new with full ordering.
4239 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
4241 * Return: The original value of @v.
4243 static __always_inline long
4244 atomic_long_xchg(atomic_long_t *v, long new)
4247 instrument_atomic_read_write(v, sizeof(*v));
4248 return raw_atomic_long_xchg(v, new);
4252 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
4253 * @v: pointer to atomic_long_t
4254 * @new: long value to assign
4256 * Atomically updates @v to @new with acquire ordering.
4258 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
4260 * Return: The original value of @v.
4262 static __always_inline long
4263 atomic_long_xchg_acquire(atomic_long_t *v, long new)
4265 instrument_atomic_read_write(v, sizeof(*v));
4266 return raw_atomic_long_xchg_acquire(v, new);
4270 * atomic_long_xchg_release() - atomic exchange with release ordering
4271 * @v: pointer to atomic_long_t
4272 * @new: long value to assign
4274 * Atomically updates @v to @new with release ordering.
4276 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
4278 * Return: The original value of @v.
4280 static __always_inline long
4281 atomic_long_xchg_release(atomic_long_t *v, long new)
4284 instrument_atomic_read_write(v, sizeof(*v));
4285 return raw_atomic_long_xchg_release(v, new);
4289 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
4290 * @v: pointer to atomic_long_t
4291 * @new: long value to assign
4293 * Atomically updates @v to @new with relaxed ordering.
4295 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
4297 * Return: The original value of @v.
4299 static __always_inline long
4300 atomic_long_xchg_relaxed(atomic_long_t *v, long new)
4302 instrument_atomic_read_write(v, sizeof(*v));
4303 return raw_atomic_long_xchg_relaxed(v, new);
4307 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
4308 * @v: pointer to atomic_long_t
4309 * @old: long value to compare with
4310 * @new: long value to assign
4312 * If (@v == @old), atomically updates @v to @new with full ordering.
4314 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
4316 * Return: The original value of @v.
4318 static __always_inline long
4319 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
4322 instrument_atomic_read_write(v, sizeof(*v));
4323 return raw_atomic_long_cmpxchg(v, old, new);
4327 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4328 * @v: pointer to atomic_long_t
4329 * @old: long value to compare with
4330 * @new: long value to assign
4332 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4334 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
4336 * Return: The original value of @v.
4338 static __always_inline long
4339 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
4341 instrument_atomic_read_write(v, sizeof(*v));
4342 return raw_atomic_long_cmpxchg_acquire(v, old, new);
4346 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
4347 * @v: pointer to atomic_long_t
4348 * @old: long value to compare with
4349 * @new: long value to assign
4351 * If (@v == @old), atomically updates @v to @new with release ordering.
4353 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
4355 * Return: The original value of @v.
4357 static __always_inline long
4358 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
4361 instrument_atomic_read_write(v, sizeof(*v));
4362 return raw_atomic_long_cmpxchg_release(v, old, new);
4366 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4367 * @v: pointer to atomic_long_t
4368 * @old: long value to compare with
4369 * @new: long value to assign
4371 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4373 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
4375 * Return: The original value of @v.
4377 static __always_inline long
4378 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
4380 instrument_atomic_read_write(v, sizeof(*v));
4381 return raw_atomic_long_cmpxchg_relaxed(v, old, new);
4385 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
4386 * @v: pointer to atomic_long_t
4387 * @old: pointer to long value to compare with
4388 * @new: long value to assign
4390 * If (@v == @old), atomically updates @v to @new with full ordering.
4391 * Otherwise, updates @old to the current value of @v.
4393 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
4395 * Return: @true if the exchange occured, @false otherwise.
4397 static __always_inline bool
4398 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
4401 instrument_atomic_read_write(v, sizeof(*v));
4402 instrument_atomic_read_write(old, sizeof(*old));
4403 return raw_atomic_long_try_cmpxchg(v, old, new);
4407 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4408 * @v: pointer to atomic_long_t
4409 * @old: pointer to long value to compare with
4410 * @new: long value to assign
4412 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4413 * Otherwise, updates @old to the current value of @v.
4415 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
4417 * Return: @true if the exchange occured, @false otherwise.
4419 static __always_inline bool
4420 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
4422 instrument_atomic_read_write(v, sizeof(*v));
4423 instrument_atomic_read_write(old, sizeof(*old));
4424 return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
4428 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
4429 * @v: pointer to atomic_long_t
4430 * @old: pointer to long value to compare with
4431 * @new: long value to assign
4433 * If (@v == @old), atomically updates @v to @new with release ordering.
4434 * Otherwise, updates @old to the current value of @v.
4436 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
4438 * Return: @true if the exchange occured, @false otherwise.
4440 static __always_inline bool
4441 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
4444 instrument_atomic_read_write(v, sizeof(*v));
4445 instrument_atomic_read_write(old, sizeof(*old));
4446 return raw_atomic_long_try_cmpxchg_release(v, old, new);
4450 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4451 * @v: pointer to atomic_long_t
4452 * @old: pointer to long value to compare with
4453 * @new: long value to assign
4455 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4456 * Otherwise, updates @old to the current value of @v.
4458 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
4460 * Return: @true if the exchange occured, @false otherwise.
4462 static __always_inline bool
4463 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
4465 instrument_atomic_read_write(v, sizeof(*v));
4466 instrument_atomic_read_write(old, sizeof(*old));
4467 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
4471 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
4472 * @i: long value to add
4473 * @v: pointer to atomic_long_t
4475 * Atomically updates @v to (@v - @i) with full ordering.
4477 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
4479 * Return: @true if the resulting value of @v is zero, @false otherwise.
4481 static __always_inline bool
4482 atomic_long_sub_and_test(long i, atomic_long_t *v)
4485 instrument_atomic_read_write(v, sizeof(*v));
4486 return raw_atomic_long_sub_and_test(i, v);
4490 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
4491 * @v: pointer to atomic_long_t
4493 * Atomically updates @v to (@v - 1) with full ordering.
4495 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
4497 * Return: @true if the resulting value of @v is zero, @false otherwise.
4499 static __always_inline bool
4500 atomic_long_dec_and_test(atomic_long_t *v)
4503 instrument_atomic_read_write(v, sizeof(*v));
4504 return raw_atomic_long_dec_and_test(v);
4508 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
4509 * @v: pointer to atomic_long_t
4511 * Atomically updates @v to (@v + 1) with full ordering.
4513 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
4515 * Return: @true if the resulting value of @v is zero, @false otherwise.
4517 static __always_inline bool
4518 atomic_long_inc_and_test(atomic_long_t *v)
4521 instrument_atomic_read_write(v, sizeof(*v));
4522 return raw_atomic_long_inc_and_test(v);
4526 * atomic_long_add_negative() - atomic add and test if negative with full ordering
4527 * @i: long value to add
4528 * @v: pointer to atomic_long_t
4530 * Atomically updates @v to (@v + @i) with full ordering.
4532 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
4534 * Return: @true if the resulting value of @v is negative, @false otherwise.
4536 static __always_inline bool
4537 atomic_long_add_negative(long i, atomic_long_t *v)
4540 instrument_atomic_read_write(v, sizeof(*v));
4541 return raw_atomic_long_add_negative(i, v);
4545 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
4546 * @i: long value to add
4547 * @v: pointer to atomic_long_t
4549 * Atomically updates @v to (@v + @i) with acquire ordering.
4551 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
4553 * Return: @true if the resulting value of @v is negative, @false otherwise.
4555 static __always_inline bool
4556 atomic_long_add_negative_acquire(long i, atomic_long_t *v)
4558 instrument_atomic_read_write(v, sizeof(*v));
4559 return raw_atomic_long_add_negative_acquire(i, v);
4563 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
4564 * @i: long value to add
4565 * @v: pointer to atomic_long_t
4567 * Atomically updates @v to (@v + @i) with release ordering.
4569 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
4571 * Return: @true if the resulting value of @v is negative, @false otherwise.
4573 static __always_inline bool
4574 atomic_long_add_negative_release(long i, atomic_long_t *v)
4577 instrument_atomic_read_write(v, sizeof(*v));
4578 return raw_atomic_long_add_negative_release(i, v);
4582 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4583 * @i: long value to add
4584 * @v: pointer to atomic_long_t
4586 * Atomically updates @v to (@v + @i) with relaxed ordering.
4588 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
4590 * Return: @true if the resulting value of @v is negative, @false otherwise.
4592 static __always_inline bool
4593 atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
4595 instrument_atomic_read_write(v, sizeof(*v));
4596 return raw_atomic_long_add_negative_relaxed(i, v);
4600 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
4601 * @v: pointer to atomic_long_t
4602 * @a: long value to add
4603 * @u: long value to compare with
4605 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4607 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
4609 * Return: The original value of @v.
4611 static __always_inline long
4612 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
4615 instrument_atomic_read_write(v, sizeof(*v));
4616 return raw_atomic_long_fetch_add_unless(v, a, u);
4620 * atomic_long_add_unless() - atomic add unless value with full ordering
4621 * @v: pointer to atomic_long_t
4622 * @a: long value to add
4623 * @u: long value to compare with
4625 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4627 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
4629 * Return: @true if @v was updated, @false otherwise.
4631 static __always_inline bool
4632 atomic_long_add_unless(atomic_long_t *v, long a, long u)
4635 instrument_atomic_read_write(v, sizeof(*v));
4636 return raw_atomic_long_add_unless(v, a, u);
4640 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
4641 * @v: pointer to atomic_long_t
4643 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4645 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
4647 * Return: @true if @v was updated, @false otherwise.
4649 static __always_inline bool
4650 atomic_long_inc_not_zero(atomic_long_t *v)
4653 instrument_atomic_read_write(v, sizeof(*v));
4654 return raw_atomic_long_inc_not_zero(v);
4658 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
4659 * @v: pointer to atomic_long_t
4661 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4663 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
4665 * Return: @true if @v was updated, @false otherwise.
4667 static __always_inline bool
4668 atomic_long_inc_unless_negative(atomic_long_t *v)
4671 instrument_atomic_read_write(v, sizeof(*v));
4672 return raw_atomic_long_inc_unless_negative(v);
4676 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
4677 * @v: pointer to atomic_long_t
4679 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4681 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
4683 * Return: @true if @v was updated, @false otherwise.
4685 static __always_inline bool
4686 atomic_long_dec_unless_positive(atomic_long_t *v)
4689 instrument_atomic_read_write(v, sizeof(*v));
4690 return raw_atomic_long_dec_unless_positive(v);
4694 * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
4695 * @v: pointer to atomic_long_t
4697 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4699 * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
4701 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4703 static __always_inline long
4704 atomic_long_dec_if_positive(atomic_long_t *v)
4707 instrument_atomic_read_write(v, sizeof(*v));
4708 return raw_atomic_long_dec_if_positive(v);
4711 #define xchg(ptr, ...) \
4713 typeof(ptr) __ai_ptr = (ptr); \
4715 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4716 raw_xchg(__ai_ptr, __VA_ARGS__); \
4719 #define xchg_acquire(ptr, ...) \
4721 typeof(ptr) __ai_ptr = (ptr); \
4722 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4723 raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
4726 #define xchg_release(ptr, ...) \
4728 typeof(ptr) __ai_ptr = (ptr); \
4730 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4731 raw_xchg_release(__ai_ptr, __VA_ARGS__); \
4734 #define xchg_relaxed(ptr, ...) \
4736 typeof(ptr) __ai_ptr = (ptr); \
4737 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4738 raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
4741 #define cmpxchg(ptr, ...) \
4743 typeof(ptr) __ai_ptr = (ptr); \
4745 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4746 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
4749 #define cmpxchg_acquire(ptr, ...) \
4751 typeof(ptr) __ai_ptr = (ptr); \
4752 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4753 raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
4756 #define cmpxchg_release(ptr, ...) \
4758 typeof(ptr) __ai_ptr = (ptr); \
4760 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4761 raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
4764 #define cmpxchg_relaxed(ptr, ...) \
4766 typeof(ptr) __ai_ptr = (ptr); \
4767 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4768 raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
4771 #define cmpxchg64(ptr, ...) \
4773 typeof(ptr) __ai_ptr = (ptr); \
4775 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4776 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
4779 #define cmpxchg64_acquire(ptr, ...) \
4781 typeof(ptr) __ai_ptr = (ptr); \
4782 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4783 raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
4786 #define cmpxchg64_release(ptr, ...) \
4788 typeof(ptr) __ai_ptr = (ptr); \
4790 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4791 raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
4794 #define cmpxchg64_relaxed(ptr, ...) \
4796 typeof(ptr) __ai_ptr = (ptr); \
4797 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4798 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
4801 #define cmpxchg128(ptr, ...) \
4803 typeof(ptr) __ai_ptr = (ptr); \
4805 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4806 raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
4809 #define cmpxchg128_acquire(ptr, ...) \
4811 typeof(ptr) __ai_ptr = (ptr); \
4812 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4813 raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
4816 #define cmpxchg128_release(ptr, ...) \
4818 typeof(ptr) __ai_ptr = (ptr); \
4820 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4821 raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
4824 #define cmpxchg128_relaxed(ptr, ...) \
4826 typeof(ptr) __ai_ptr = (ptr); \
4827 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4828 raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
4831 #define try_cmpxchg(ptr, oldp, ...) \
4833 typeof(ptr) __ai_ptr = (ptr); \
4834 typeof(oldp) __ai_oldp = (oldp); \
4836 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4837 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4838 raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4841 #define try_cmpxchg_acquire(ptr, oldp, ...) \
4843 typeof(ptr) __ai_ptr = (ptr); \
4844 typeof(oldp) __ai_oldp = (oldp); \
4845 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4846 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4847 raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4850 #define try_cmpxchg_release(ptr, oldp, ...) \
4852 typeof(ptr) __ai_ptr = (ptr); \
4853 typeof(oldp) __ai_oldp = (oldp); \
4855 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4856 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4857 raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4860 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
4862 typeof(ptr) __ai_ptr = (ptr); \
4863 typeof(oldp) __ai_oldp = (oldp); \
4864 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4865 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4866 raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4869 #define try_cmpxchg64(ptr, oldp, ...) \
4871 typeof(ptr) __ai_ptr = (ptr); \
4872 typeof(oldp) __ai_oldp = (oldp); \
4874 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4875 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4876 raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4879 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
4881 typeof(ptr) __ai_ptr = (ptr); \
4882 typeof(oldp) __ai_oldp = (oldp); \
4883 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4884 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4885 raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4888 #define try_cmpxchg64_release(ptr, oldp, ...) \
4890 typeof(ptr) __ai_ptr = (ptr); \
4891 typeof(oldp) __ai_oldp = (oldp); \
4893 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4894 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4895 raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4898 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
4900 typeof(ptr) __ai_ptr = (ptr); \
4901 typeof(oldp) __ai_oldp = (oldp); \
4902 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4903 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4904 raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4907 #define try_cmpxchg128(ptr, oldp, ...) \
4909 typeof(ptr) __ai_ptr = (ptr); \
4910 typeof(oldp) __ai_oldp = (oldp); \
4912 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4913 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4914 raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4917 #define try_cmpxchg128_acquire(ptr, oldp, ...) \
4919 typeof(ptr) __ai_ptr = (ptr); \
4920 typeof(oldp) __ai_oldp = (oldp); \
4921 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4922 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4923 raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4926 #define try_cmpxchg128_release(ptr, oldp, ...) \
4928 typeof(ptr) __ai_ptr = (ptr); \
4929 typeof(oldp) __ai_oldp = (oldp); \
4931 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4932 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4933 raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4936 #define try_cmpxchg128_relaxed(ptr, oldp, ...) \
4938 typeof(ptr) __ai_ptr = (ptr); \
4939 typeof(oldp) __ai_oldp = (oldp); \
4940 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4941 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4942 raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4945 #define cmpxchg_local(ptr, ...) \
4947 typeof(ptr) __ai_ptr = (ptr); \
4948 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4949 raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
4952 #define cmpxchg64_local(ptr, ...) \
4954 typeof(ptr) __ai_ptr = (ptr); \
4955 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4956 raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
4959 #define cmpxchg128_local(ptr, ...) \
4961 typeof(ptr) __ai_ptr = (ptr); \
4962 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4963 raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
4966 #define sync_cmpxchg(ptr, ...) \
4968 typeof(ptr) __ai_ptr = (ptr); \
4970 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4971 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
4974 #define try_cmpxchg_local(ptr, oldp, ...) \
4976 typeof(ptr) __ai_ptr = (ptr); \
4977 typeof(oldp) __ai_oldp = (oldp); \
4978 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4979 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4980 raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4983 #define try_cmpxchg64_local(ptr, oldp, ...) \
4985 typeof(ptr) __ai_ptr = (ptr); \
4986 typeof(oldp) __ai_oldp = (oldp); \
4987 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4988 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4989 raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4992 #define try_cmpxchg128_local(ptr, oldp, ...) \
4994 typeof(ptr) __ai_ptr = (ptr); \
4995 typeof(oldp) __ai_oldp = (oldp); \
4996 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4997 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4998 raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5001 #define sync_try_cmpxchg(ptr, ...) \
5003 typeof(ptr) __ai_ptr = (ptr); \
5005 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5006 raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
5010 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5011 // 2cc4bc990fef44d3836ec108f11b610f3f438184