include/linux/atomic/atomic-instrumented.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 /*
   7  * This file provides wrappers with KASAN instrumentation for atomic operations.
   8  * To use this functionality an arch's atomic.h file needs to define all
   9  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
  10  * this file at the end. This file provides atomic_read() that forwards to
  11  * arch_atomic_read() for actual atomic operation.
  12  * Note: if an arch atomic operation is implemented by means of other atomic
  13  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
  14  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
  15  * double instrumentation.
  16  */
  17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
  18 #define _LINUX_ATOMIC_INSTRUMENTED_H
  19
  20 #include <linux/build_bug.h>
  21 #include <linux/compiler.h>
  22 #include <linux/instrumented.h>
  23
  24 static __always_inline int
  25 atomic_read(const atomic_t *v)
  26 {
  27         instrument_atomic_read(v, sizeof(*v));
  28         return arch_atomic_read(v);
  29 }
  30
  31 static __always_inline int
  32 atomic_read_acquire(const atomic_t *v)
  33 {
  34         instrument_atomic_read(v, sizeof(*v));
  35         return arch_atomic_read_acquire(v);
  36 }
  37
  38 static __always_inline void
  39 atomic_set(atomic_t *v, int i)
  40 {
  41         instrument_atomic_write(v, sizeof(*v));
  42         arch_atomic_set(v, i);
  43 }
  44
  45 static __always_inline void
  46 atomic_set_release(atomic_t *v, int i)
  47 {
  48         instrument_atomic_write(v, sizeof(*v));
  49         arch_atomic_set_release(v, i);
  50 }
  51
  52 static __always_inline void
  53 atomic_add(int i, atomic_t *v)
  54 {
  55         instrument_atomic_read_write(v, sizeof(*v));
  56         arch_atomic_add(i, v);
  57 }
  58
  59 static __always_inline int
  60 atomic_add_return(int i, atomic_t *v)
  61 {
  62         instrument_atomic_read_write(v, sizeof(*v));
  63         return arch_atomic_add_return(i, v);
  64 }
  65
  66 static __always_inline int
  67 atomic_add_return_acquire(int i, atomic_t *v)
  68 {
  69         instrument_atomic_read_write(v, sizeof(*v));
  70         return arch_atomic_add_return_acquire(i, v);
  71 }
  72
  73 static __always_inline int
  74 atomic_add_return_release(int i, atomic_t *v)
  75 {
  76         instrument_atomic_read_write(v, sizeof(*v));
  77         return arch_atomic_add_return_release(i, v);
  78 }
  79
  80 static __always_inline int
  81 atomic_add_return_relaxed(int i, atomic_t *v)
  82 {
  83         instrument_atomic_read_write(v, sizeof(*v));
  84         return arch_atomic_add_return_relaxed(i, v);
  85 }
  86
  87 static __always_inline int
  88 atomic_fetch_add(int i, atomic_t *v)
  89 {
  90         instrument_atomic_read_write(v, sizeof(*v));
  91         return arch_atomic_fetch_add(i, v);
  92 }
  93
  94 static __always_inline int
  95 atomic_fetch_add_acquire(int i, atomic_t *v)
  96 {
  97         instrument_atomic_read_write(v, sizeof(*v));
  98         return arch_atomic_fetch_add_acquire(i, v);
  99 }
 100
 101 static __always_inline int
 102 atomic_fetch_add_release(int i, atomic_t *v)
 103 {
 104         instrument_atomic_read_write(v, sizeof(*v));
 105         return arch_atomic_fetch_add_release(i, v);
 106 }
 107
 108 static __always_inline int
 109 atomic_fetch_add_relaxed(int i, atomic_t *v)
 110 {
 111         instrument_atomic_read_write(v, sizeof(*v));
 112         return arch_atomic_fetch_add_relaxed(i, v);
 113 }
 114
 115 static __always_inline void
 116 atomic_sub(int i, atomic_t *v)
 117 {
 118         instrument_atomic_read_write(v, sizeof(*v));
 119         arch_atomic_sub(i, v);
 120 }
 121
 122 static __always_inline int
 123 atomic_sub_return(int i, atomic_t *v)
 124 {
 125         instrument_atomic_read_write(v, sizeof(*v));
 126         return arch_atomic_sub_return(i, v);
 127 }
 128
 129 static __always_inline int
 130 atomic_sub_return_acquire(int i, atomic_t *v)
 131 {
 132         instrument_atomic_read_write(v, sizeof(*v));
 133         return arch_atomic_sub_return_acquire(i, v);
 134 }
 135
 136 static __always_inline int
 137 atomic_sub_return_release(int i, atomic_t *v)
 138 {
 139         instrument_atomic_read_write(v, sizeof(*v));
 140         return arch_atomic_sub_return_release(i, v);
 141 }
 142
 143 static __always_inline int
 144 atomic_sub_return_relaxed(int i, atomic_t *v)
 145 {
 146         instrument_atomic_read_write(v, sizeof(*v));
 147         return arch_atomic_sub_return_relaxed(i, v);
 148 }
 149
 150 static __always_inline int
 151 atomic_fetch_sub(int i, atomic_t *v)
 152 {
 153         instrument_atomic_read_write(v, sizeof(*v));
 154         return arch_atomic_fetch_sub(i, v);
 155 }
 156
 157 static __always_inline int
 158 atomic_fetch_sub_acquire(int i, atomic_t *v)
 159 {
 160         instrument_atomic_read_write(v, sizeof(*v));
 161         return arch_atomic_fetch_sub_acquire(i, v);
 162 }
 163
 164 static __always_inline int
 165 atomic_fetch_sub_release(int i, atomic_t *v)
 166 {
 167         instrument_atomic_read_write(v, sizeof(*v));
 168         return arch_atomic_fetch_sub_release(i, v);
 169 }
 170
 171 static __always_inline int
 172 atomic_fetch_sub_relaxed(int i, atomic_t *v)
 173 {
 174         instrument_atomic_read_write(v, sizeof(*v));
 175         return arch_atomic_fetch_sub_relaxed(i, v);
 176 }
 177
 178 static __always_inline void
 179 atomic_inc(atomic_t *v)
 180 {
 181         instrument_atomic_read_write(v, sizeof(*v));
 182         arch_atomic_inc(v);
 183 }
 184
 185 static __always_inline int
 186 atomic_inc_return(atomic_t *v)
 187 {
 188         instrument_atomic_read_write(v, sizeof(*v));
 189         return arch_atomic_inc_return(v);
 190 }
 191
 192 static __always_inline int
 193 atomic_inc_return_acquire(atomic_t *v)
 194 {
 195         instrument_atomic_read_write(v, sizeof(*v));
 196         return arch_atomic_inc_return_acquire(v);
 197 }
 198
 199 static __always_inline int
 200 atomic_inc_return_release(atomic_t *v)
 201 {
 202         instrument_atomic_read_write(v, sizeof(*v));
 203         return arch_atomic_inc_return_release(v);
 204 }
 205
 206 static __always_inline int
 207 atomic_inc_return_relaxed(atomic_t *v)
 208 {
 209         instrument_atomic_read_write(v, sizeof(*v));
 210         return arch_atomic_inc_return_relaxed(v);
 211 }
 212
 213 static __always_inline int
 214 atomic_fetch_inc(atomic_t *v)
 215 {
 216         instrument_atomic_read_write(v, sizeof(*v));
 217         return arch_atomic_fetch_inc(v);
 218 }
 219
 220 static __always_inline int
 221 atomic_fetch_inc_acquire(atomic_t *v)
 222 {
 223         instrument_atomic_read_write(v, sizeof(*v));
 224         return arch_atomic_fetch_inc_acquire(v);
 225 }
 226
 227 static __always_inline int
 228 atomic_fetch_inc_release(atomic_t *v)
 229 {
 230         instrument_atomic_read_write(v, sizeof(*v));
 231         return arch_atomic_fetch_inc_release(v);
 232 }
 233
 234 static __always_inline int
 235 atomic_fetch_inc_relaxed(atomic_t *v)
 236 {
 237         instrument_atomic_read_write(v, sizeof(*v));
 238         return arch_atomic_fetch_inc_relaxed(v);
 239 }
 240
 241 static __always_inline void
 242 atomic_dec(atomic_t *v)
 243 {
 244         instrument_atomic_read_write(v, sizeof(*v));
 245         arch_atomic_dec(v);
 246 }
 247
 248 static __always_inline int
 249 atomic_dec_return(atomic_t *v)
 250 {
 251         instrument_atomic_read_write(v, sizeof(*v));
 252         return arch_atomic_dec_return(v);
 253 }
 254
 255 static __always_inline int
 256 atomic_dec_return_acquire(atomic_t *v)
 257 {
 258         instrument_atomic_read_write(v, sizeof(*v));
 259         return arch_atomic_dec_return_acquire(v);
 260 }
 261
 262 static __always_inline int
 263 atomic_dec_return_release(atomic_t *v)
 264 {
 265         instrument_atomic_read_write(v, sizeof(*v));
 266         return arch_atomic_dec_return_release(v);
 267 }
 268
 269 static __always_inline int
 270 atomic_dec_return_relaxed(atomic_t *v)
 271 {
 272         instrument_atomic_read_write(v, sizeof(*v));
 273         return arch_atomic_dec_return_relaxed(v);
 274 }
 275
 276 static __always_inline int
 277 atomic_fetch_dec(atomic_t *v)
 278 {
 279         instrument_atomic_read_write(v, sizeof(*v));
 280         return arch_atomic_fetch_dec(v);
 281 }
 282
 283 static __always_inline int
 284 atomic_fetch_dec_acquire(atomic_t *v)
 285 {
 286         instrument_atomic_read_write(v, sizeof(*v));
 287         return arch_atomic_fetch_dec_acquire(v);
 288 }
 289
 290 static __always_inline int
 291 atomic_fetch_dec_release(atomic_t *v)
 292 {
 293         instrument_atomic_read_write(v, sizeof(*v));
 294         return arch_atomic_fetch_dec_release(v);
 295 }
 296
 297 static __always_inline int
 298 atomic_fetch_dec_relaxed(atomic_t *v)
 299 {
 300         instrument_atomic_read_write(v, sizeof(*v));
 301         return arch_atomic_fetch_dec_relaxed(v);
 302 }
 303
 304 static __always_inline void
 305 atomic_and(int i, atomic_t *v)
 306 {
 307         instrument_atomic_read_write(v, sizeof(*v));
 308         arch_atomic_and(i, v);
 309 }
 310
 311 static __always_inline int
 312 atomic_fetch_and(int i, atomic_t *v)
 313 {
 314         instrument_atomic_read_write(v, sizeof(*v));
 315         return arch_atomic_fetch_and(i, v);
 316 }
 317
 318 static __always_inline int
 319 atomic_fetch_and_acquire(int i, atomic_t *v)
 320 {
 321         instrument_atomic_read_write(v, sizeof(*v));
 322         return arch_atomic_fetch_and_acquire(i, v);
 323 }
 324
 325 static __always_inline int
 326 atomic_fetch_and_release(int i, atomic_t *v)
 327 {
 328         instrument_atomic_read_write(v, sizeof(*v));
 329         return arch_atomic_fetch_and_release(i, v);
 330 }
 331
 332 static __always_inline int
 333 atomic_fetch_and_relaxed(int i, atomic_t *v)
 334 {
 335         instrument_atomic_read_write(v, sizeof(*v));
 336         return arch_atomic_fetch_and_relaxed(i, v);
 337 }
 338
 339 static __always_inline void
 340 atomic_andnot(int i, atomic_t *v)
 341 {
 342         instrument_atomic_read_write(v, sizeof(*v));
 343         arch_atomic_andnot(i, v);
 344 }
 345
 346 static __always_inline int
 347 atomic_fetch_andnot(int i, atomic_t *v)
 348 {
 349         instrument_atomic_read_write(v, sizeof(*v));
 350         return arch_atomic_fetch_andnot(i, v);
 351 }
 352
 353 static __always_inline int
 354 atomic_fetch_andnot_acquire(int i, atomic_t *v)
 355 {
 356         instrument_atomic_read_write(v, sizeof(*v));
 357         return arch_atomic_fetch_andnot_acquire(i, v);
 358 }
 359
 360 static __always_inline int
 361 atomic_fetch_andnot_release(int i, atomic_t *v)
 362 {
 363         instrument_atomic_read_write(v, sizeof(*v));
 364         return arch_atomic_fetch_andnot_release(i, v);
 365 }
 366
 367 static __always_inline int
 368 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 369 {
 370         instrument_atomic_read_write(v, sizeof(*v));
 371         return arch_atomic_fetch_andnot_relaxed(i, v);
 372 }
 373
 374 static __always_inline void
 375 atomic_or(int i, atomic_t *v)
 376 {
 377         instrument_atomic_read_write(v, sizeof(*v));
 378         arch_atomic_or(i, v);
 379 }
 380
 381 static __always_inline int
 382 atomic_fetch_or(int i, atomic_t *v)
 383 {
 384         instrument_atomic_read_write(v, sizeof(*v));
 385         return arch_atomic_fetch_or(i, v);
 386 }
 387
 388 static __always_inline int
 389 atomic_fetch_or_acquire(int i, atomic_t *v)
 390 {
 391         instrument_atomic_read_write(v, sizeof(*v));
 392         return arch_atomic_fetch_or_acquire(i, v);
 393 }
 394
 395 static __always_inline int
 396 atomic_fetch_or_release(int i, atomic_t *v)
 397 {
 398         instrument_atomic_read_write(v, sizeof(*v));
 399         return arch_atomic_fetch_or_release(i, v);
 400 }
 401
 402 static __always_inline int
 403 atomic_fetch_or_relaxed(int i, atomic_t *v)
 404 {
 405         instrument_atomic_read_write(v, sizeof(*v));
 406         return arch_atomic_fetch_or_relaxed(i, v);
 407 }
 408
 409 static __always_inline void
 410 atomic_xor(int i, atomic_t *v)
 411 {
 412         instrument_atomic_read_write(v, sizeof(*v));
 413         arch_atomic_xor(i, v);
 414 }
 415
 416 static __always_inline int
 417 atomic_fetch_xor(int i, atomic_t *v)
 418 {
 419         instrument_atomic_read_write(v, sizeof(*v));
 420         return arch_atomic_fetch_xor(i, v);
 421 }
 422
 423 static __always_inline int
 424 atomic_fetch_xor_acquire(int i, atomic_t *v)
 425 {
 426         instrument_atomic_read_write(v, sizeof(*v));
 427         return arch_atomic_fetch_xor_acquire(i, v);
 428 }
 429
 430 static __always_inline int
 431 atomic_fetch_xor_release(int i, atomic_t *v)
 432 {
 433         instrument_atomic_read_write(v, sizeof(*v));
 434         return arch_atomic_fetch_xor_release(i, v);
 435 }
 436
 437 static __always_inline int
 438 atomic_fetch_xor_relaxed(int i, atomic_t *v)
 439 {
 440         instrument_atomic_read_write(v, sizeof(*v));
 441         return arch_atomic_fetch_xor_relaxed(i, v);
 442 }
 443
 444 static __always_inline int
 445 atomic_xchg(atomic_t *v, int i)
 446 {
 447         instrument_atomic_read_write(v, sizeof(*v));
 448         return arch_atomic_xchg(v, i);
 449 }
 450
 451 static __always_inline int
 452 atomic_xchg_acquire(atomic_t *v, int i)
 453 {
 454         instrument_atomic_read_write(v, sizeof(*v));
 455         return arch_atomic_xchg_acquire(v, i);
 456 }
 457
 458 static __always_inline int
 459 atomic_xchg_release(atomic_t *v, int i)
 460 {
 461         instrument_atomic_read_write(v, sizeof(*v));
 462         return arch_atomic_xchg_release(v, i);
 463 }
 464
 465 static __always_inline int
 466 atomic_xchg_relaxed(atomic_t *v, int i)
 467 {
 468         instrument_atomic_read_write(v, sizeof(*v));
 469         return arch_atomic_xchg_relaxed(v, i);
 470 }
 471
 472 static __always_inline int
 473 atomic_cmpxchg(atomic_t *v, int old, int new)
 474 {
 475         instrument_atomic_read_write(v, sizeof(*v));
 476         return arch_atomic_cmpxchg(v, old, new);
 477 }
 478
 479 static __always_inline int
 480 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
 481 {
 482         instrument_atomic_read_write(v, sizeof(*v));
 483         return arch_atomic_cmpxchg_acquire(v, old, new);
 484 }
 485
 486 static __always_inline int
 487 atomic_cmpxchg_release(atomic_t *v, int old, int new)
 488 {
 489         instrument_atomic_read_write(v, sizeof(*v));
 490         return arch_atomic_cmpxchg_release(v, old, new);
 491 }
 492
 493 static __always_inline int
 494 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
 495 {
 496         instrument_atomic_read_write(v, sizeof(*v));
 497         return arch_atomic_cmpxchg_relaxed(v, old, new);
 498 }
 499
 500 static __always_inline bool
 501 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 502 {
 503         instrument_atomic_read_write(v, sizeof(*v));
 504         instrument_atomic_read_write(old, sizeof(*old));
 505         return arch_atomic_try_cmpxchg(v, old, new);
 506 }
 507
 508 static __always_inline bool
 509 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 510 {
 511         instrument_atomic_read_write(v, sizeof(*v));
 512         instrument_atomic_read_write(old, sizeof(*old));
 513         return arch_atomic_try_cmpxchg_acquire(v, old, new);
 514 }
 515
 516 static __always_inline bool
 517 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 518 {
 519         instrument_atomic_read_write(v, sizeof(*v));
 520         instrument_atomic_read_write(old, sizeof(*old));
 521         return arch_atomic_try_cmpxchg_release(v, old, new);
 522 }
 523
 524 static __always_inline bool
 525 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
 526 {
 527         instrument_atomic_read_write(v, sizeof(*v));
 528         instrument_atomic_read_write(old, sizeof(*old));
 529         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
 530 }
 531
 532 static __always_inline bool
 533 atomic_sub_and_test(int i, atomic_t *v)
 534 {
 535         instrument_atomic_read_write(v, sizeof(*v));
 536         return arch_atomic_sub_and_test(i, v);
 537 }
 538
 539 static __always_inline bool
 540 atomic_dec_and_test(atomic_t *v)
 541 {
 542         instrument_atomic_read_write(v, sizeof(*v));
 543         return arch_atomic_dec_and_test(v);
 544 }
 545
 546 static __always_inline bool
 547 atomic_inc_and_test(atomic_t *v)
 548 {
 549         instrument_atomic_read_write(v, sizeof(*v));
 550         return arch_atomic_inc_and_test(v);
 551 }
 552
 553 static __always_inline bool
 554 atomic_add_negative(int i, atomic_t *v)
 555 {
 556         instrument_atomic_read_write(v, sizeof(*v));
 557         return arch_atomic_add_negative(i, v);
 558 }
 559
 560 static __always_inline int
 561 atomic_fetch_add_unless(atomic_t *v, int a, int u)
 562 {
 563         instrument_atomic_read_write(v, sizeof(*v));
 564         return arch_atomic_fetch_add_unless(v, a, u);
 565 }
 566
 567 static __always_inline bool
 568 atomic_add_unless(atomic_t *v, int a, int u)
 569 {
 570         instrument_atomic_read_write(v, sizeof(*v));
 571         return arch_atomic_add_unless(v, a, u);
 572 }
 573
 574 static __always_inline bool
 575 atomic_inc_not_zero(atomic_t *v)
 576 {
 577         instrument_atomic_read_write(v, sizeof(*v));
 578         return arch_atomic_inc_not_zero(v);
 579 }
 580
 581 static __always_inline bool
 582 atomic_inc_unless_negative(atomic_t *v)
 583 {
 584         instrument_atomic_read_write(v, sizeof(*v));
 585         return arch_atomic_inc_unless_negative(v);
 586 }
 587
 588 static __always_inline bool
 589 atomic_dec_unless_positive(atomic_t *v)
 590 {
 591         instrument_atomic_read_write(v, sizeof(*v));
 592         return arch_atomic_dec_unless_positive(v);
 593 }
 594
 595 static __always_inline int
 596 atomic_dec_if_positive(atomic_t *v)
 597 {
 598         instrument_atomic_read_write(v, sizeof(*v));
 599         return arch_atomic_dec_if_positive(v);
 600 }
 601
 602 static __always_inline s64
 603 atomic64_read(const atomic64_t *v)
 604 {
 605         instrument_atomic_read(v, sizeof(*v));
 606         return arch_atomic64_read(v);
 607 }
 608
 609 static __always_inline s64
 610 atomic64_read_acquire(const atomic64_t *v)
 611 {
 612         instrument_atomic_read(v, sizeof(*v));
 613         return arch_atomic64_read_acquire(v);
 614 }
 615
 616 static __always_inline void
 617 atomic64_set(atomic64_t *v, s64 i)
 618 {
 619         instrument_atomic_write(v, sizeof(*v));
 620         arch_atomic64_set(v, i);
 621 }
 622
 623 static __always_inline void
 624 atomic64_set_release(atomic64_t *v, s64 i)
 625 {
 626         instrument_atomic_write(v, sizeof(*v));
 627         arch_atomic64_set_release(v, i);
 628 }
 629
 630 static __always_inline void
 631 atomic64_add(s64 i, atomic64_t *v)
 632 {
 633         instrument_atomic_read_write(v, sizeof(*v));
 634         arch_atomic64_add(i, v);
 635 }
 636
 637 static __always_inline s64
 638 atomic64_add_return(s64 i, atomic64_t *v)
 639 {
 640         instrument_atomic_read_write(v, sizeof(*v));
 641         return arch_atomic64_add_return(i, v);
 642 }
 643
 644 static __always_inline s64
 645 atomic64_add_return_acquire(s64 i, atomic64_t *v)
 646 {
 647         instrument_atomic_read_write(v, sizeof(*v));
 648         return arch_atomic64_add_return_acquire(i, v);
 649 }
 650
 651 static __always_inline s64
 652 atomic64_add_return_release(s64 i, atomic64_t *v)
 653 {
 654         instrument_atomic_read_write(v, sizeof(*v));
 655         return arch_atomic64_add_return_release(i, v);
 656 }
 657
 658 static __always_inline s64
 659 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
 660 {
 661         instrument_atomic_read_write(v, sizeof(*v));
 662         return arch_atomic64_add_return_relaxed(i, v);
 663 }
 664
 665 static __always_inline s64
 666 atomic64_fetch_add(s64 i, atomic64_t *v)
 667 {
 668         instrument_atomic_read_write(v, sizeof(*v));
 669         return arch_atomic64_fetch_add(i, v);
 670 }
 671
 672 static __always_inline s64
 673 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
 674 {
 675         instrument_atomic_read_write(v, sizeof(*v));
 676         return arch_atomic64_fetch_add_acquire(i, v);
 677 }
 678
 679 static __always_inline s64
 680 atomic64_fetch_add_release(s64 i, atomic64_t *v)
 681 {
 682         instrument_atomic_read_write(v, sizeof(*v));
 683         return arch_atomic64_fetch_add_release(i, v);
 684 }
 685
 686 static __always_inline s64
 687 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
 688 {
 689         instrument_atomic_read_write(v, sizeof(*v));
 690         return arch_atomic64_fetch_add_relaxed(i, v);
 691 }
 692
 693 static __always_inline void
 694 atomic64_sub(s64 i, atomic64_t *v)
 695 {
 696         instrument_atomic_read_write(v, sizeof(*v));
 697         arch_atomic64_sub(i, v);
 698 }
 699
 700 static __always_inline s64
 701 atomic64_sub_return(s64 i, atomic64_t *v)
 702 {
 703         instrument_atomic_read_write(v, sizeof(*v));
 704         return arch_atomic64_sub_return(i, v);
 705 }
 706
 707 static __always_inline s64
 708 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
 709 {
 710         instrument_atomic_read_write(v, sizeof(*v));
 711         return arch_atomic64_sub_return_acquire(i, v);
 712 }
 713
 714 static __always_inline s64
 715 atomic64_sub_return_release(s64 i, atomic64_t *v)
 716 {
 717         instrument_atomic_read_write(v, sizeof(*v));
 718         return arch_atomic64_sub_return_release(i, v);
 719 }
 720
 721 static __always_inline s64
 722 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
 723 {
 724         instrument_atomic_read_write(v, sizeof(*v));
 725         return arch_atomic64_sub_return_relaxed(i, v);
 726 }
 727
 728 static __always_inline s64
 729 atomic64_fetch_sub(s64 i, atomic64_t *v)
 730 {
 731         instrument_atomic_read_write(v, sizeof(*v));
 732         return arch_atomic64_fetch_sub(i, v);
 733 }
 734
 735 static __always_inline s64
 736 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
 737 {
 738         instrument_atomic_read_write(v, sizeof(*v));
 739         return arch_atomic64_fetch_sub_acquire(i, v);
 740 }
 741
 742 static __always_inline s64
 743 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
 744 {
 745         instrument_atomic_read_write(v, sizeof(*v));
 746         return arch_atomic64_fetch_sub_release(i, v);
 747 }
 748
 749 static __always_inline s64
 750 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
 751 {
 752         instrument_atomic_read_write(v, sizeof(*v));
 753         return arch_atomic64_fetch_sub_relaxed(i, v);
 754 }
 755
 756 static __always_inline void
 757 atomic64_inc(atomic64_t *v)
 758 {
 759         instrument_atomic_read_write(v, sizeof(*v));
 760         arch_atomic64_inc(v);
 761 }
 762
 763 static __always_inline s64
 764 atomic64_inc_return(atomic64_t *v)
 765 {
 766         instrument_atomic_read_write(v, sizeof(*v));
 767         return arch_atomic64_inc_return(v);
 768 }
 769
 770 static __always_inline s64
 771 atomic64_inc_return_acquire(atomic64_t *v)
 772 {
 773         instrument_atomic_read_write(v, sizeof(*v));
 774         return arch_atomic64_inc_return_acquire(v);
 775 }
 776
 777 static __always_inline s64
 778 atomic64_inc_return_release(atomic64_t *v)
 779 {
 780         instrument_atomic_read_write(v, sizeof(*v));
 781         return arch_atomic64_inc_return_release(v);
 782 }
 783
 784 static __always_inline s64
 785 atomic64_inc_return_relaxed(atomic64_t *v)
 786 {
 787         instrument_atomic_read_write(v, sizeof(*v));
 788         return arch_atomic64_inc_return_relaxed(v);
 789 }
 790
 791 static __always_inline s64
 792 atomic64_fetch_inc(atomic64_t *v)
 793 {
 794         instrument_atomic_read_write(v, sizeof(*v));
 795         return arch_atomic64_fetch_inc(v);
 796 }
 797
 798 static __always_inline s64
 799 atomic64_fetch_inc_acquire(atomic64_t *v)
 800 {
 801         instrument_atomic_read_write(v, sizeof(*v));
 802         return arch_atomic64_fetch_inc_acquire(v);
 803 }
 804
 805 static __always_inline s64
 806 atomic64_fetch_inc_release(atomic64_t *v)
 807 {
 808         instrument_atomic_read_write(v, sizeof(*v));
 809         return arch_atomic64_fetch_inc_release(v);
 810 }
 811
 812 static __always_inline s64
 813 atomic64_fetch_inc_relaxed(atomic64_t *v)
 814 {
 815         instrument_atomic_read_write(v, sizeof(*v));
 816         return arch_atomic64_fetch_inc_relaxed(v);
 817 }
 818
 819 static __always_inline void
 820 atomic64_dec(atomic64_t *v)
 821 {
 822         instrument_atomic_read_write(v, sizeof(*v));
 823         arch_atomic64_dec(v);
 824 }
 825
 826 static __always_inline s64
 827 atomic64_dec_return(atomic64_t *v)
 828 {
 829         instrument_atomic_read_write(v, sizeof(*v));
 830         return arch_atomic64_dec_return(v);
 831 }
 832
 833 static __always_inline s64
 834 atomic64_dec_return_acquire(atomic64_t *v)
 835 {
 836         instrument_atomic_read_write(v, sizeof(*v));
 837         return arch_atomic64_dec_return_acquire(v);
 838 }
 839
 840 static __always_inline s64
 841 atomic64_dec_return_release(atomic64_t *v)
 842 {
 843         instrument_atomic_read_write(v, sizeof(*v));
 844         return arch_atomic64_dec_return_release(v);
 845 }
 846
 847 static __always_inline s64
 848 atomic64_dec_return_relaxed(atomic64_t *v)
 849 {
 850         instrument_atomic_read_write(v, sizeof(*v));
 851         return arch_atomic64_dec_return_relaxed(v);
 852 }
 853
 854 static __always_inline s64
 855 atomic64_fetch_dec(atomic64_t *v)
 856 {
 857         instrument_atomic_read_write(v, sizeof(*v));
 858         return arch_atomic64_fetch_dec(v);
 859 }
 860
 861 static __always_inline s64
 862 atomic64_fetch_dec_acquire(atomic64_t *v)
 863 {
 864         instrument_atomic_read_write(v, sizeof(*v));
 865         return arch_atomic64_fetch_dec_acquire(v);
 866 }
 867
 868 static __always_inline s64
 869 atomic64_fetch_dec_release(atomic64_t *v)
 870 {
 871         instrument_atomic_read_write(v, sizeof(*v));
 872         return arch_atomic64_fetch_dec_release(v);
 873 }
 874
 875 static __always_inline s64
 876 atomic64_fetch_dec_relaxed(atomic64_t *v)
 877 {
 878         instrument_atomic_read_write(v, sizeof(*v));
 879         return arch_atomic64_fetch_dec_relaxed(v);
 880 }
 881
 882 static __always_inline void
 883 atomic64_and(s64 i, atomic64_t *v)
 884 {
 885         instrument_atomic_read_write(v, sizeof(*v));
 886         arch_atomic64_and(i, v);
 887 }
 888
 889 static __always_inline s64
 890 atomic64_fetch_and(s64 i, atomic64_t *v)
 891 {
 892         instrument_atomic_read_write(v, sizeof(*v));
 893         return arch_atomic64_fetch_and(i, v);
 894 }
 895
 896 static __always_inline s64
 897 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
 898 {
 899         instrument_atomic_read_write(v, sizeof(*v));
 900         return arch_atomic64_fetch_and_acquire(i, v);
 901 }
 902
 903 static __always_inline s64
 904 atomic64_fetch_and_release(s64 i, atomic64_t *v)
 905 {
 906         instrument_atomic_read_write(v, sizeof(*v));
 907         return arch_atomic64_fetch_and_release(i, v);
 908 }
 909
 910 static __always_inline s64
 911 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
 912 {
 913         instrument_atomic_read_write(v, sizeof(*v));
 914         return arch_atomic64_fetch_and_relaxed(i, v);
 915 }
 916
 917 static __always_inline void
 918 atomic64_andnot(s64 i, atomic64_t *v)
 919 {
 920         instrument_atomic_read_write(v, sizeof(*v));
 921         arch_atomic64_andnot(i, v);
 922 }
 923
 924 static __always_inline s64
 925 atomic64_fetch_andnot(s64 i, atomic64_t *v)
 926 {
 927         instrument_atomic_read_write(v, sizeof(*v));
 928         return arch_atomic64_fetch_andnot(i, v);
 929 }
 930
 931 static __always_inline s64
 932 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
 933 {
 934         instrument_atomic_read_write(v, sizeof(*v));
 935         return arch_atomic64_fetch_andnot_acquire(i, v);
 936 }
 937
 938 static __always_inline s64
 939 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
 940 {
 941         instrument_atomic_read_write(v, sizeof(*v));
 942         return arch_atomic64_fetch_andnot_release(i, v);
 943 }
 944
 945 static __always_inline s64
 946 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
 947 {
 948         instrument_atomic_read_write(v, sizeof(*v));
 949         return arch_atomic64_fetch_andnot_relaxed(i, v);
 950 }
 951
 952 static __always_inline void
 953 atomic64_or(s64 i, atomic64_t *v)
 954 {
 955         instrument_atomic_read_write(v, sizeof(*v));
 956         arch_atomic64_or(i, v);
 957 }
 958
 959 static __always_inline s64
 960 atomic64_fetch_or(s64 i, atomic64_t *v)
 961 {
 962         instrument_atomic_read_write(v, sizeof(*v));
 963         return arch_atomic64_fetch_or(i, v);
 964 }
 965
 966 static __always_inline s64
 967 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
 968 {
 969         instrument_atomic_read_write(v, sizeof(*v));
 970         return arch_atomic64_fetch_or_acquire(i, v);
 971 }
 972
 973 static __always_inline s64
 974 atomic64_fetch_or_release(s64 i, atomic64_t *v)
 975 {
 976         instrument_atomic_read_write(v, sizeof(*v));
 977         return arch_atomic64_fetch_or_release(i, v);
 978 }
 979
 980 static __always_inline s64
 981 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
 982 {
 983         instrument_atomic_read_write(v, sizeof(*v));
 984         return arch_atomic64_fetch_or_relaxed(i, v);
 985 }
 986
 987 static __always_inline void
 988 atomic64_xor(s64 i, atomic64_t *v)
 989 {
 990         instrument_atomic_read_write(v, sizeof(*v));
 991         arch_atomic64_xor(i, v);
 992 }
 993
 994 static __always_inline s64
 995 atomic64_fetch_xor(s64 i, atomic64_t *v)
 996 {
 997         instrument_atomic_read_write(v, sizeof(*v));
 998         return arch_atomic64_fetch_xor(i, v);
 999 }
1000
1001 static __always_inline s64
1002 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1003 {
1004         instrument_atomic_read_write(v, sizeof(*v));
1005         return arch_atomic64_fetch_xor_acquire(i, v);
1006 }
1007
1008 static __always_inline s64
1009 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1010 {
1011         instrument_atomic_read_write(v, sizeof(*v));
1012         return arch_atomic64_fetch_xor_release(i, v);
1013 }
1014
1015 static __always_inline s64
1016 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1017 {
1018         instrument_atomic_read_write(v, sizeof(*v));
1019         return arch_atomic64_fetch_xor_relaxed(i, v);
1020 }
1021
1022 static __always_inline s64
1023 atomic64_xchg(atomic64_t *v, s64 i)
1024 {
1025         instrument_atomic_read_write(v, sizeof(*v));
1026         return arch_atomic64_xchg(v, i);
1027 }
1028
1029 static __always_inline s64
1030 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1031 {
1032         instrument_atomic_read_write(v, sizeof(*v));
1033         return arch_atomic64_xchg_acquire(v, i);
1034 }
1035
1036 static __always_inline s64
1037 atomic64_xchg_release(atomic64_t *v, s64 i)
1038 {
1039         instrument_atomic_read_write(v, sizeof(*v));
1040         return arch_atomic64_xchg_release(v, i);
1041 }
1042
1043 static __always_inline s64
1044 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1045 {
1046         instrument_atomic_read_write(v, sizeof(*v));
1047         return arch_atomic64_xchg_relaxed(v, i);
1048 }
1049
1050 static __always_inline s64
1051 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1052 {
1053         instrument_atomic_read_write(v, sizeof(*v));
1054         return arch_atomic64_cmpxchg(v, old, new);
1055 }
1056
1057 static __always_inline s64
1058 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1059 {
1060         instrument_atomic_read_write(v, sizeof(*v));
1061         return arch_atomic64_cmpxchg_acquire(v, old, new);
1062 }
1063
1064 static __always_inline s64
1065 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1066 {
1067         instrument_atomic_read_write(v, sizeof(*v));
1068         return arch_atomic64_cmpxchg_release(v, old, new);
1069 }
1070
1071 static __always_inline s64
1072 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1073 {
1074         instrument_atomic_read_write(v, sizeof(*v));
1075         return arch_atomic64_cmpxchg_relaxed(v, old, new);
1076 }
1077
1078 static __always_inline bool
1079 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1080 {
1081         instrument_atomic_read_write(v, sizeof(*v));
1082         instrument_atomic_read_write(old, sizeof(*old));
1083         return arch_atomic64_try_cmpxchg(v, old, new);
1084 }
1085
1086 static __always_inline bool
1087 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1088 {
1089         instrument_atomic_read_write(v, sizeof(*v));
1090         instrument_atomic_read_write(old, sizeof(*old));
1091         return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1092 }
1093
1094 static __always_inline bool
1095 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1096 {
1097         instrument_atomic_read_write(v, sizeof(*v));
1098         instrument_atomic_read_write(old, sizeof(*old));
1099         return arch_atomic64_try_cmpxchg_release(v, old, new);
1100 }
1101
1102 static __always_inline bool
1103 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1104 {
1105         instrument_atomic_read_write(v, sizeof(*v));
1106         instrument_atomic_read_write(old, sizeof(*old));
1107         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1108 }
1109
1110 static __always_inline bool
1111 atomic64_sub_and_test(s64 i, atomic64_t *v)
1112 {
1113         instrument_atomic_read_write(v, sizeof(*v));
1114         return arch_atomic64_sub_and_test(i, v);
1115 }
1116
1117 static __always_inline bool
1118 atomic64_dec_and_test(atomic64_t *v)
1119 {
1120         instrument_atomic_read_write(v, sizeof(*v));
1121         return arch_atomic64_dec_and_test(v);
1122 }
1123
1124 static __always_inline bool
1125 atomic64_inc_and_test(atomic64_t *v)
1126 {
1127         instrument_atomic_read_write(v, sizeof(*v));
1128         return arch_atomic64_inc_and_test(v);
1129 }
1130
1131 static __always_inline bool
1132 atomic64_add_negative(s64 i, atomic64_t *v)
1133 {
1134         instrument_atomic_read_write(v, sizeof(*v));
1135         return arch_atomic64_add_negative(i, v);
1136 }
1137
1138 static __always_inline s64
1139 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1140 {
1141         instrument_atomic_read_write(v, sizeof(*v));
1142         return arch_atomic64_fetch_add_unless(v, a, u);
1143 }
1144
1145 static __always_inline bool
1146 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1147 {
1148         instrument_atomic_read_write(v, sizeof(*v));
1149         return arch_atomic64_add_unless(v, a, u);
1150 }
1151
1152 static __always_inline bool
1153 atomic64_inc_not_zero(atomic64_t *v)
1154 {
1155         instrument_atomic_read_write(v, sizeof(*v));
1156         return arch_atomic64_inc_not_zero(v);
1157 }
1158
1159 static __always_inline bool
1160 atomic64_inc_unless_negative(atomic64_t *v)
1161 {
1162         instrument_atomic_read_write(v, sizeof(*v));
1163         return arch_atomic64_inc_unless_negative(v);
1164 }
1165
1166 static __always_inline bool
1167 atomic64_dec_unless_positive(atomic64_t *v)
1168 {
1169         instrument_atomic_read_write(v, sizeof(*v));
1170         return arch_atomic64_dec_unless_positive(v);
1171 }
1172
1173 static __always_inline s64
1174 atomic64_dec_if_positive(atomic64_t *v)
1175 {
1176         instrument_atomic_read_write(v, sizeof(*v));
1177         return arch_atomic64_dec_if_positive(v);
1178 }
1179
1180 static __always_inline long
1181 atomic_long_read(const atomic_long_t *v)
1182 {
1183         instrument_atomic_read(v, sizeof(*v));
1184         return arch_atomic_long_read(v);
1185 }
1186
1187 static __always_inline long
1188 atomic_long_read_acquire(const atomic_long_t *v)
1189 {
1190         instrument_atomic_read(v, sizeof(*v));
1191         return arch_atomic_long_read_acquire(v);
1192 }
1193
1194 static __always_inline void
1195 atomic_long_set(atomic_long_t *v, long i)
1196 {
1197         instrument_atomic_write(v, sizeof(*v));
1198         arch_atomic_long_set(v, i);
1199 }
1200
1201 static __always_inline void
1202 atomic_long_set_release(atomic_long_t *v, long i)
1203 {
1204         instrument_atomic_write(v, sizeof(*v));
1205         arch_atomic_long_set_release(v, i);
1206 }
1207
1208 static __always_inline void
1209 atomic_long_add(long i, atomic_long_t *v)
1210 {
1211         instrument_atomic_read_write(v, sizeof(*v));
1212         arch_atomic_long_add(i, v);
1213 }
1214
1215 static __always_inline long
1216 atomic_long_add_return(long i, atomic_long_t *v)
1217 {
1218         instrument_atomic_read_write(v, sizeof(*v));
1219         return arch_atomic_long_add_return(i, v);
1220 }
1221
1222 static __always_inline long
1223 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1224 {
1225         instrument_atomic_read_write(v, sizeof(*v));
1226         return arch_atomic_long_add_return_acquire(i, v);
1227 }
1228
1229 static __always_inline long
1230 atomic_long_add_return_release(long i, atomic_long_t *v)
1231 {
1232         instrument_atomic_read_write(v, sizeof(*v));
1233         return arch_atomic_long_add_return_release(i, v);
1234 }
1235
1236 static __always_inline long
1237 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1238 {
1239         instrument_atomic_read_write(v, sizeof(*v));
1240         return arch_atomic_long_add_return_relaxed(i, v);
1241 }
1242
1243 static __always_inline long
1244 atomic_long_fetch_add(long i, atomic_long_t *v)
1245 {
1246         instrument_atomic_read_write(v, sizeof(*v));
1247         return arch_atomic_long_fetch_add(i, v);
1248 }
1249
1250 static __always_inline long
1251 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1252 {
1253         instrument_atomic_read_write(v, sizeof(*v));
1254         return arch_atomic_long_fetch_add_acquire(i, v);
1255 }
1256
1257 static __always_inline long
1258 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1259 {
1260         instrument_atomic_read_write(v, sizeof(*v));
1261         return arch_atomic_long_fetch_add_release(i, v);
1262 }
1263
1264 static __always_inline long
1265 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1266 {
1267         instrument_atomic_read_write(v, sizeof(*v));
1268         return arch_atomic_long_fetch_add_relaxed(i, v);
1269 }
1270
1271 static __always_inline void
1272 atomic_long_sub(long i, atomic_long_t *v)
1273 {
1274         instrument_atomic_read_write(v, sizeof(*v));
1275         arch_atomic_long_sub(i, v);
1276 }
1277
1278 static __always_inline long
1279 atomic_long_sub_return(long i, atomic_long_t *v)
1280 {
1281         instrument_atomic_read_write(v, sizeof(*v));
1282         return arch_atomic_long_sub_return(i, v);
1283 }
1284
1285 static __always_inline long
1286 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1287 {
1288         instrument_atomic_read_write(v, sizeof(*v));
1289         return arch_atomic_long_sub_return_acquire(i, v);
1290 }
1291
1292 static __always_inline long
1293 atomic_long_sub_return_release(long i, atomic_long_t *v)
1294 {
1295         instrument_atomic_read_write(v, sizeof(*v));
1296         return arch_atomic_long_sub_return_release(i, v);
1297 }
1298
1299 static __always_inline long
1300 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1301 {
1302         instrument_atomic_read_write(v, sizeof(*v));
1303         return arch_atomic_long_sub_return_relaxed(i, v);
1304 }
1305
1306 static __always_inline long
1307 atomic_long_fetch_sub(long i, atomic_long_t *v)
1308 {
1309         instrument_atomic_read_write(v, sizeof(*v));
1310         return arch_atomic_long_fetch_sub(i, v);
1311 }
1312
1313 static __always_inline long
1314 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1315 {
1316         instrument_atomic_read_write(v, sizeof(*v));
1317         return arch_atomic_long_fetch_sub_acquire(i, v);
1318 }
1319
1320 static __always_inline long
1321 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1322 {
1323         instrument_atomic_read_write(v, sizeof(*v));
1324         return arch_atomic_long_fetch_sub_release(i, v);
1325 }
1326
1327 static __always_inline long
1328 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1329 {
1330         instrument_atomic_read_write(v, sizeof(*v));
1331         return arch_atomic_long_fetch_sub_relaxed(i, v);
1332 }
1333
1334 static __always_inline void
1335 atomic_long_inc(atomic_long_t *v)
1336 {
1337         instrument_atomic_read_write(v, sizeof(*v));
1338         arch_atomic_long_inc(v);
1339 }
1340
1341 static __always_inline long
1342 atomic_long_inc_return(atomic_long_t *v)
1343 {
1344         instrument_atomic_read_write(v, sizeof(*v));
1345         return arch_atomic_long_inc_return(v);
1346 }
1347
1348 static __always_inline long
1349 atomic_long_inc_return_acquire(atomic_long_t *v)
1350 {
1351         instrument_atomic_read_write(v, sizeof(*v));
1352         return arch_atomic_long_inc_return_acquire(v);
1353 }
1354
1355 static __always_inline long
1356 atomic_long_inc_return_release(atomic_long_t *v)
1357 {
1358         instrument_atomic_read_write(v, sizeof(*v));
1359         return arch_atomic_long_inc_return_release(v);
1360 }
1361
1362 static __always_inline long
1363 atomic_long_inc_return_relaxed(atomic_long_t *v)
1364 {
1365         instrument_atomic_read_write(v, sizeof(*v));
1366         return arch_atomic_long_inc_return_relaxed(v);
1367 }
1368
1369 static __always_inline long
1370 atomic_long_fetch_inc(atomic_long_t *v)
1371 {
1372         instrument_atomic_read_write(v, sizeof(*v));
1373         return arch_atomic_long_fetch_inc(v);
1374 }
1375
1376 static __always_inline long
1377 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1378 {
1379         instrument_atomic_read_write(v, sizeof(*v));
1380         return arch_atomic_long_fetch_inc_acquire(v);
1381 }
1382
1383 static __always_inline long
1384 atomic_long_fetch_inc_release(atomic_long_t *v)
1385 {
1386         instrument_atomic_read_write(v, sizeof(*v));
1387         return arch_atomic_long_fetch_inc_release(v);
1388 }
1389
1390 static __always_inline long
1391 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1392 {
1393         instrument_atomic_read_write(v, sizeof(*v));
1394         return arch_atomic_long_fetch_inc_relaxed(v);
1395 }
1396
1397 static __always_inline void
1398 atomic_long_dec(atomic_long_t *v)
1399 {
1400         instrument_atomic_read_write(v, sizeof(*v));
1401         arch_atomic_long_dec(v);
1402 }
1403
1404 static __always_inline long
1405 atomic_long_dec_return(atomic_long_t *v)
1406 {
1407         instrument_atomic_read_write(v, sizeof(*v));
1408         return arch_atomic_long_dec_return(v);
1409 }
1410
1411 static __always_inline long
1412 atomic_long_dec_return_acquire(atomic_long_t *v)
1413 {
1414         instrument_atomic_read_write(v, sizeof(*v));
1415         return arch_atomic_long_dec_return_acquire(v);
1416 }
1417
1418 static __always_inline long
1419 atomic_long_dec_return_release(atomic_long_t *v)
1420 {
1421         instrument_atomic_read_write(v, sizeof(*v));
1422         return arch_atomic_long_dec_return_release(v);
1423 }
1424
1425 static __always_inline long
1426 atomic_long_dec_return_relaxed(atomic_long_t *v)
1427 {
1428         instrument_atomic_read_write(v, sizeof(*v));
1429         return arch_atomic_long_dec_return_relaxed(v);
1430 }
1431
1432 static __always_inline long
1433 atomic_long_fetch_dec(atomic_long_t *v)
1434 {
1435         instrument_atomic_read_write(v, sizeof(*v));
1436         return arch_atomic_long_fetch_dec(v);
1437 }
1438
1439 static __always_inline long
1440 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1441 {
1442         instrument_atomic_read_write(v, sizeof(*v));
1443         return arch_atomic_long_fetch_dec_acquire(v);
1444 }
1445
1446 static __always_inline long
1447 atomic_long_fetch_dec_release(atomic_long_t *v)
1448 {
1449         instrument_atomic_read_write(v, sizeof(*v));
1450         return arch_atomic_long_fetch_dec_release(v);
1451 }
1452
1453 static __always_inline long
1454 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1455 {
1456         instrument_atomic_read_write(v, sizeof(*v));
1457         return arch_atomic_long_fetch_dec_relaxed(v);
1458 }
1459
1460 static __always_inline void
1461 atomic_long_and(long i, atomic_long_t *v)
1462 {
1463         instrument_atomic_read_write(v, sizeof(*v));
1464         arch_atomic_long_and(i, v);
1465 }
1466
1467 static __always_inline long
1468 atomic_long_fetch_and(long i, atomic_long_t *v)
1469 {
1470         instrument_atomic_read_write(v, sizeof(*v));
1471         return arch_atomic_long_fetch_and(i, v);
1472 }
1473
1474 static __always_inline long
1475 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1476 {
1477         instrument_atomic_read_write(v, sizeof(*v));
1478         return arch_atomic_long_fetch_and_acquire(i, v);
1479 }
1480
1481 static __always_inline long
1482 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1483 {
1484         instrument_atomic_read_write(v, sizeof(*v));
1485         return arch_atomic_long_fetch_and_release(i, v);
1486 }
1487
1488 static __always_inline long
1489 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1490 {
1491         instrument_atomic_read_write(v, sizeof(*v));
1492         return arch_atomic_long_fetch_and_relaxed(i, v);
1493 }
1494
1495 static __always_inline void
1496 atomic_long_andnot(long i, atomic_long_t *v)
1497 {
1498         instrument_atomic_read_write(v, sizeof(*v));
1499         arch_atomic_long_andnot(i, v);
1500 }
1501
1502 static __always_inline long
1503 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1504 {
1505         instrument_atomic_read_write(v, sizeof(*v));
1506         return arch_atomic_long_fetch_andnot(i, v);
1507 }
1508
1509 static __always_inline long
1510 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1511 {
1512         instrument_atomic_read_write(v, sizeof(*v));
1513         return arch_atomic_long_fetch_andnot_acquire(i, v);
1514 }
1515
1516 static __always_inline long
1517 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1518 {
1519         instrument_atomic_read_write(v, sizeof(*v));
1520         return arch_atomic_long_fetch_andnot_release(i, v);
1521 }
1522
1523 static __always_inline long
1524 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1525 {
1526         instrument_atomic_read_write(v, sizeof(*v));
1527         return arch_atomic_long_fetch_andnot_relaxed(i, v);
1528 }
1529
1530 static __always_inline void
1531 atomic_long_or(long i, atomic_long_t *v)
1532 {
1533         instrument_atomic_read_write(v, sizeof(*v));
1534         arch_atomic_long_or(i, v);
1535 }
1536
1537 static __always_inline long
1538 atomic_long_fetch_or(long i, atomic_long_t *v)
1539 {
1540         instrument_atomic_read_write(v, sizeof(*v));
1541         return arch_atomic_long_fetch_or(i, v);
1542 }
1543
1544 static __always_inline long
1545 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1546 {
1547         instrument_atomic_read_write(v, sizeof(*v));
1548         return arch_atomic_long_fetch_or_acquire(i, v);
1549 }
1550
1551 static __always_inline long
1552 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1553 {
1554         instrument_atomic_read_write(v, sizeof(*v));
1555         return arch_atomic_long_fetch_or_release(i, v);
1556 }
1557
1558 static __always_inline long
1559 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1560 {
1561         instrument_atomic_read_write(v, sizeof(*v));
1562         return arch_atomic_long_fetch_or_relaxed(i, v);
1563 }
1564
1565 static __always_inline void
1566 atomic_long_xor(long i, atomic_long_t *v)
1567 {
1568         instrument_atomic_read_write(v, sizeof(*v));
1569         arch_atomic_long_xor(i, v);
1570 }
1571
1572 static __always_inline long
1573 atomic_long_fetch_xor(long i, atomic_long_t *v)
1574 {
1575         instrument_atomic_read_write(v, sizeof(*v));
1576         return arch_atomic_long_fetch_xor(i, v);
1577 }
1578
1579 static __always_inline long
1580 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1581 {
1582         instrument_atomic_read_write(v, sizeof(*v));
1583         return arch_atomic_long_fetch_xor_acquire(i, v);
1584 }
1585
1586 static __always_inline long
1587 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1588 {
1589         instrument_atomic_read_write(v, sizeof(*v));
1590         return arch_atomic_long_fetch_xor_release(i, v);
1591 }
1592
1593 static __always_inline long
1594 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1595 {
1596         instrument_atomic_read_write(v, sizeof(*v));
1597         return arch_atomic_long_fetch_xor_relaxed(i, v);
1598 }
1599
1600 static __always_inline long
1601 atomic_long_xchg(atomic_long_t *v, long i)
1602 {
1603         instrument_atomic_read_write(v, sizeof(*v));
1604         return arch_atomic_long_xchg(v, i);
1605 }
1606
1607 static __always_inline long
1608 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1609 {
1610         instrument_atomic_read_write(v, sizeof(*v));
1611         return arch_atomic_long_xchg_acquire(v, i);
1612 }
1613
1614 static __always_inline long
1615 atomic_long_xchg_release(atomic_long_t *v, long i)
1616 {
1617         instrument_atomic_read_write(v, sizeof(*v));
1618         return arch_atomic_long_xchg_release(v, i);
1619 }
1620
1621 static __always_inline long
1622 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1623 {
1624         instrument_atomic_read_write(v, sizeof(*v));
1625         return arch_atomic_long_xchg_relaxed(v, i);
1626 }
1627
1628 static __always_inline long
1629 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1630 {
1631         instrument_atomic_read_write(v, sizeof(*v));
1632         return arch_atomic_long_cmpxchg(v, old, new);
1633 }
1634
1635 static __always_inline long
1636 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1637 {
1638         instrument_atomic_read_write(v, sizeof(*v));
1639         return arch_atomic_long_cmpxchg_acquire(v, old, new);
1640 }
1641
1642 static __always_inline long
1643 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1644 {
1645         instrument_atomic_read_write(v, sizeof(*v));
1646         return arch_atomic_long_cmpxchg_release(v, old, new);
1647 }
1648
1649 static __always_inline long
1650 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1651 {
1652         instrument_atomic_read_write(v, sizeof(*v));
1653         return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1654 }
1655
1656 static __always_inline bool
1657 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1658 {
1659         instrument_atomic_read_write(v, sizeof(*v));
1660         instrument_atomic_read_write(old, sizeof(*old));
1661         return arch_atomic_long_try_cmpxchg(v, old, new);
1662 }
1663
1664 static __always_inline bool
1665 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1666 {
1667         instrument_atomic_read_write(v, sizeof(*v));
1668         instrument_atomic_read_write(old, sizeof(*old));
1669         return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1670 }
1671
1672 static __always_inline bool
1673 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1674 {
1675         instrument_atomic_read_write(v, sizeof(*v));
1676         instrument_atomic_read_write(old, sizeof(*old));
1677         return arch_atomic_long_try_cmpxchg_release(v, old, new);
1678 }
1679
1680 static __always_inline bool
1681 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1682 {
1683         instrument_atomic_read_write(v, sizeof(*v));
1684         instrument_atomic_read_write(old, sizeof(*old));
1685         return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1686 }
1687
1688 static __always_inline bool
1689 atomic_long_sub_and_test(long i, atomic_long_t *v)
1690 {
1691         instrument_atomic_read_write(v, sizeof(*v));
1692         return arch_atomic_long_sub_and_test(i, v);
1693 }
1694
1695 static __always_inline bool
1696 atomic_long_dec_and_test(atomic_long_t *v)
1697 {
1698         instrument_atomic_read_write(v, sizeof(*v));
1699         return arch_atomic_long_dec_and_test(v);
1700 }
1701
1702 static __always_inline bool
1703 atomic_long_inc_and_test(atomic_long_t *v)
1704 {
1705         instrument_atomic_read_write(v, sizeof(*v));
1706         return arch_atomic_long_inc_and_test(v);
1707 }
1708
1709 static __always_inline bool
1710 atomic_long_add_negative(long i, atomic_long_t *v)
1711 {
1712         instrument_atomic_read_write(v, sizeof(*v));
1713         return arch_atomic_long_add_negative(i, v);
1714 }
1715
1716 static __always_inline long
1717 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1718 {
1719         instrument_atomic_read_write(v, sizeof(*v));
1720         return arch_atomic_long_fetch_add_unless(v, a, u);
1721 }
1722
1723 static __always_inline bool
1724 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1725 {
1726         instrument_atomic_read_write(v, sizeof(*v));
1727         return arch_atomic_long_add_unless(v, a, u);
1728 }
1729
1730 static __always_inline bool
1731 atomic_long_inc_not_zero(atomic_long_t *v)
1732 {
1733         instrument_atomic_read_write(v, sizeof(*v));
1734         return arch_atomic_long_inc_not_zero(v);
1735 }
1736
1737 static __always_inline bool
1738 atomic_long_inc_unless_negative(atomic_long_t *v)
1739 {
1740         instrument_atomic_read_write(v, sizeof(*v));
1741         return arch_atomic_long_inc_unless_negative(v);
1742 }
1743
1744 static __always_inline bool
1745 atomic_long_dec_unless_positive(atomic_long_t *v)
1746 {
1747         instrument_atomic_read_write(v, sizeof(*v));
1748         return arch_atomic_long_dec_unless_positive(v);
1749 }
1750
1751 static __always_inline long
1752 atomic_long_dec_if_positive(atomic_long_t *v)
1753 {
1754         instrument_atomic_read_write(v, sizeof(*v));
1755         return arch_atomic_long_dec_if_positive(v);
1756 }
1757
1758 #define xchg(ptr, ...) \
1759 ({ \
1760         typeof(ptr) __ai_ptr = (ptr); \
1761         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1762         arch_xchg(__ai_ptr, __VA_ARGS__); \
1763 })
1764
1765 #define xchg_acquire(ptr, ...) \
1766 ({ \
1767         typeof(ptr) __ai_ptr = (ptr); \
1768         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1769         arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1770 })
1771
1772 #define xchg_release(ptr, ...) \
1773 ({ \
1774         typeof(ptr) __ai_ptr = (ptr); \
1775         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1776         arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1777 })
1778
1779 #define xchg_relaxed(ptr, ...) \
1780 ({ \
1781         typeof(ptr) __ai_ptr = (ptr); \
1782         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1783         arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1784 })
1785
1786 #define cmpxchg(ptr, ...) \
1787 ({ \
1788         typeof(ptr) __ai_ptr = (ptr); \
1789         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1790         arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1791 })
1792
1793 #define cmpxchg_acquire(ptr, ...) \
1794 ({ \
1795         typeof(ptr) __ai_ptr = (ptr); \
1796         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1797         arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1798 })
1799
1800 #define cmpxchg_release(ptr, ...) \
1801 ({ \
1802         typeof(ptr) __ai_ptr = (ptr); \
1803         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1804         arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1805 })
1806
1807 #define cmpxchg_relaxed(ptr, ...) \
1808 ({ \
1809         typeof(ptr) __ai_ptr = (ptr); \
1810         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1811         arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1812 })
1813
1814 #define cmpxchg64(ptr, ...) \
1815 ({ \
1816         typeof(ptr) __ai_ptr = (ptr); \
1817         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1818         arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1819 })
1820
1821 #define cmpxchg64_acquire(ptr, ...) \
1822 ({ \
1823         typeof(ptr) __ai_ptr = (ptr); \
1824         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1825         arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1826 })
1827
1828 #define cmpxchg64_release(ptr, ...) \
1829 ({ \
1830         typeof(ptr) __ai_ptr = (ptr); \
1831         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1832         arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1833 })
1834
1835 #define cmpxchg64_relaxed(ptr, ...) \
1836 ({ \
1837         typeof(ptr) __ai_ptr = (ptr); \
1838         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1839         arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1840 })
1841
1842 #define try_cmpxchg(ptr, oldp, ...) \
1843 ({ \
1844         typeof(ptr) __ai_ptr = (ptr); \
1845         typeof(oldp) __ai_oldp = (oldp); \
1846         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1847         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1848         arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1849 })
1850
1851 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1852 ({ \
1853         typeof(ptr) __ai_ptr = (ptr); \
1854         typeof(oldp) __ai_oldp = (oldp); \
1855         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1856         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1857         arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1858 })
1859
1860 #define try_cmpxchg_release(ptr, oldp, ...) \
1861 ({ \
1862         typeof(ptr) __ai_ptr = (ptr); \
1863         typeof(oldp) __ai_oldp = (oldp); \
1864         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1865         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1866         arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1867 })
1868
1869 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
1870 ({ \
1871         typeof(ptr) __ai_ptr = (ptr); \
1872         typeof(oldp) __ai_oldp = (oldp); \
1873         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1874         instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1875         arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1876 })
1877
1878 #define cmpxchg_local(ptr, ...) \
1879 ({ \
1880         typeof(ptr) __ai_ptr = (ptr); \
1881         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1882         arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
1883 })
1884
1885 #define cmpxchg64_local(ptr, ...) \
1886 ({ \
1887         typeof(ptr) __ai_ptr = (ptr); \
1888         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1889         arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
1890 })
1891
1892 #define sync_cmpxchg(ptr, ...) \
1893 ({ \
1894         typeof(ptr) __ai_ptr = (ptr); \
1895         instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1896         arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1897 })
1898
1899 #define cmpxchg_double(ptr, ...) \
1900 ({ \
1901         typeof(ptr) __ai_ptr = (ptr); \
1902         instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1903         arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1904 })
1905
1906
1907 #define cmpxchg_double_local(ptr, ...) \
1908 ({ \
1909         typeof(ptr) __ai_ptr = (ptr); \
1910         instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1911         arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1912 })
1913
1914 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
1915 // 2a9553f0a9d5619f19151092df5cabbbf16ce835