tools/testing/selftests/rseq/rseq-arm.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-arm.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #define RSEQ_SIG        0x53053053
   9
  10 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb" ::: "memory", "cc")
  11 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
  12 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
  13
  14 #define rseq_smp_load_acquire(p)                                        \
  15 __extension__ ({                                                        \
  16         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  17         rseq_smp_mb();                                                  \
  18         ____p1;                                                         \
  19 })
  20
  21 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  22
  23 #define rseq_smp_store_release(p, v)                                    \
  24 do {                                                                    \
  25         rseq_smp_mb();                                                  \
  26         RSEQ_WRITE_ONCE(*p, v);                                         \
  27 } while (0)
  28
  29 #ifdef RSEQ_SKIP_FASTPATH
  30 #include "rseq-skip.h"
  31 #else /* !RSEQ_SKIP_FASTPATH */
  32
  33 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip,               \
  34                                 post_commit_offset, abort_ip)           \
  35                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
  36                 ".balign 32\n\t"                                        \
  37                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  38                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
  39                 ".popsection\n\t"
  40
  41 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip)       \
  42         __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip,                     \
  43                                 (post_commit_ip - start_ip), abort_ip)
  44
  45 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  46                 RSEQ_INJECT_ASM(1)                                      \
  47                 "adr r0, " __rseq_str(cs_label) "\n\t"                  \
  48                 "str r0, %[" __rseq_str(rseq_cs) "]\n\t"                \
  49                 __rseq_str(label) ":\n\t"
  50
  51 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  52                 RSEQ_INJECT_ASM(2)                                      \
  53                 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
  54                 "cmp %[" __rseq_str(cpu_id) "], r0\n\t"         \
  55                 "bne " __rseq_str(label) "\n\t"
  56
  57 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
  58                                 abort_label, version, flags,            \
  59                                 start_ip, post_commit_offset, abort_ip) \
  60                 ".balign 32\n\t"                                        \
  61                 __rseq_str(table_label) ":\n\t"                         \
  62                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  63                 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
  64                 ".word " __rseq_str(RSEQ_SIG) "\n\t"                    \
  65                 __rseq_str(label) ":\n\t"                               \
  66                 teardown                                                \
  67                 "b %l[" __rseq_str(abort_label) "]\n\t"
  68
  69 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
  70                               start_ip, post_commit_ip, abort_ip)       \
  71         __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
  72                                 abort_label, 0x0, 0x0, start_ip,        \
  73                                 (post_commit_ip - start_ip), abort_ip)
  74
  75 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
  76                 __rseq_str(label) ":\n\t"                               \
  77                 teardown                                                \
  78                 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
  79
  80 #define rseq_workaround_gcc_asm_size_guess()    __asm__ __volatile__("")
  81
  82 static inline __attribute__((always_inline))
  83 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  84 {
  85         RSEQ_INJECT_C(9)
  86
  87         rseq_workaround_gcc_asm_size_guess();
  88         __asm__ __volatile__ goto (
  89                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
  90                 /* Start rseq by storing table entry pointer into rseq_cs. */
  91                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
  92                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  93                 RSEQ_INJECT_ASM(3)
  94                 "ldr r0, %[v]\n\t"
  95                 "cmp %[expect], r0\n\t"
  96                 "bne %l[cmpfail]\n\t"
  97                 RSEQ_INJECT_ASM(4)
  98 #ifdef RSEQ_COMPARE_TWICE
  99                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 100                 "ldr r0, %[v]\n\t"
 101                 "cmp %[expect], r0\n\t"
 102                 "bne %l[error2]\n\t"
 103 #endif
 104                 /* final store */
 105                 "str %[newv], %[v]\n\t"
 106                 "2:\n\t"
 107                 RSEQ_INJECT_ASM(5)
 108                 "b 5f\n\t"
 109                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 110                 "5:\n\t"
 111                 : /* gcc asm goto does not allow outputs */
 112                 : [cpu_id]              "r" (cpu),
 113                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 114                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 115                   [v]                   "m" (*v),
 116                   [expect]              "r" (expect),
 117                   [newv]                "r" (newv)
 118                   RSEQ_INJECT_INPUT
 119                 : "r0", "memory", "cc"
 120                   RSEQ_INJECT_CLOBBER
 121                 : abort, cmpfail
 122 #ifdef RSEQ_COMPARE_TWICE
 123                   , error1, error2
 124 #endif
 125         );
 126         rseq_workaround_gcc_asm_size_guess();
 127         return 0;
 128 abort:
 129         rseq_workaround_gcc_asm_size_guess();
 130         RSEQ_INJECT_FAILED
 131         return -1;
 132 cmpfail:
 133         rseq_workaround_gcc_asm_size_guess();
 134         return 1;
 135 #ifdef RSEQ_COMPARE_TWICE
 136 error1:
 137         rseq_bug("cpu_id comparison failed");
 138 error2:
 139         rseq_bug("expected value comparison failed");
 140 #endif
 141 }
 142
 143 static inline __attribute__((always_inline))
 144 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 145                                off_t voffp, intptr_t *load, int cpu)
 146 {
 147         RSEQ_INJECT_C(9)
 148
 149         rseq_workaround_gcc_asm_size_guess();
 150         __asm__ __volatile__ goto (
 151                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 152                 /* Start rseq by storing table entry pointer into rseq_cs. */
 153                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 154                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 155                 RSEQ_INJECT_ASM(3)
 156                 "ldr r0, %[v]\n\t"
 157                 "cmp %[expectnot], r0\n\t"
 158                 "beq %l[cmpfail]\n\t"
 159                 RSEQ_INJECT_ASM(4)
 160 #ifdef RSEQ_COMPARE_TWICE
 161                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 162                 "ldr r0, %[v]\n\t"
 163                 "cmp %[expectnot], r0\n\t"
 164                 "beq %l[error2]\n\t"
 165 #endif
 166                 "str r0, %[load]\n\t"
 167                 "add r0, %[voffp]\n\t"
 168                 "ldr r0, [r0]\n\t"
 169                 /* final store */
 170                 "str r0, %[v]\n\t"
 171                 "2:\n\t"
 172                 RSEQ_INJECT_ASM(5)
 173                 "b 5f\n\t"
 174                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 175                 "5:\n\t"
 176                 : /* gcc asm goto does not allow outputs */
 177                 : [cpu_id]              "r" (cpu),
 178                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 179                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 180                   /* final store input */
 181                   [v]                   "m" (*v),
 182                   [expectnot]           "r" (expectnot),
 183                   [voffp]               "Ir" (voffp),
 184                   [load]                "m" (*load)
 185                   RSEQ_INJECT_INPUT
 186                 : "r0", "memory", "cc"
 187                   RSEQ_INJECT_CLOBBER
 188                 : abort, cmpfail
 189 #ifdef RSEQ_COMPARE_TWICE
 190                   , error1, error2
 191 #endif
 192         );
 193         rseq_workaround_gcc_asm_size_guess();
 194         return 0;
 195 abort:
 196         rseq_workaround_gcc_asm_size_guess();
 197         RSEQ_INJECT_FAILED
 198         return -1;
 199 cmpfail:
 200         rseq_workaround_gcc_asm_size_guess();
 201         return 1;
 202 #ifdef RSEQ_COMPARE_TWICE
 203 error1:
 204         rseq_bug("cpu_id comparison failed");
 205 error2:
 206         rseq_bug("expected value comparison failed");
 207 #endif
 208 }
 209
 210 static inline __attribute__((always_inline))
 211 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 212 {
 213         RSEQ_INJECT_C(9)
 214
 215         rseq_workaround_gcc_asm_size_guess();
 216         __asm__ __volatile__ goto (
 217                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 218                 /* Start rseq by storing table entry pointer into rseq_cs. */
 219                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 220                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 221                 RSEQ_INJECT_ASM(3)
 222 #ifdef RSEQ_COMPARE_TWICE
 223                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 224 #endif
 225                 "ldr r0, %[v]\n\t"
 226                 "add r0, %[count]\n\t"
 227                 /* final store */
 228                 "str r0, %[v]\n\t"
 229                 "2:\n\t"
 230                 RSEQ_INJECT_ASM(4)
 231                 "b 5f\n\t"
 232                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 233                 "5:\n\t"
 234                 : /* gcc asm goto does not allow outputs */
 235                 : [cpu_id]              "r" (cpu),
 236                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 237                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 238                   [v]                   "m" (*v),
 239                   [count]               "Ir" (count)
 240                   RSEQ_INJECT_INPUT
 241                 : "r0", "memory", "cc"
 242                   RSEQ_INJECT_CLOBBER
 243                 : abort
 244 #ifdef RSEQ_COMPARE_TWICE
 245                   , error1
 246 #endif
 247         );
 248         rseq_workaround_gcc_asm_size_guess();
 249         return 0;
 250 abort:
 251         rseq_workaround_gcc_asm_size_guess();
 252         RSEQ_INJECT_FAILED
 253         return -1;
 254 #ifdef RSEQ_COMPARE_TWICE
 255 error1:
 256         rseq_bug("cpu_id comparison failed");
 257 #endif
 258 }
 259
 260 static inline __attribute__((always_inline))
 261 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 262                                  intptr_t *v2, intptr_t newv2,
 263                                  intptr_t newv, int cpu)
 264 {
 265         RSEQ_INJECT_C(9)
 266
 267         rseq_workaround_gcc_asm_size_guess();
 268         __asm__ __volatile__ goto (
 269                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 270                 /* Start rseq by storing table entry pointer into rseq_cs. */
 271                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 272                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 273                 RSEQ_INJECT_ASM(3)
 274                 "ldr r0, %[v]\n\t"
 275                 "cmp %[expect], r0\n\t"
 276                 "bne %l[cmpfail]\n\t"
 277                 RSEQ_INJECT_ASM(4)
 278 #ifdef RSEQ_COMPARE_TWICE
 279                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 280                 "ldr r0, %[v]\n\t"
 281                 "cmp %[expect], r0\n\t"
 282                 "bne %l[error2]\n\t"
 283 #endif
 284                 /* try store */
 285                 "str %[newv2], %[v2]\n\t"
 286                 RSEQ_INJECT_ASM(5)
 287                 /* final store */
 288                 "str %[newv], %[v]\n\t"
 289                 "2:\n\t"
 290                 RSEQ_INJECT_ASM(6)
 291                 "b 5f\n\t"
 292                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 293                 "5:\n\t"
 294                 : /* gcc asm goto does not allow outputs */
 295                 : [cpu_id]              "r" (cpu),
 296                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 297                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 298                   /* try store input */
 299                   [v2]                  "m" (*v2),
 300                   [newv2]               "r" (newv2),
 301                   /* final store input */
 302                   [v]                   "m" (*v),
 303                   [expect]              "r" (expect),
 304                   [newv]                "r" (newv)
 305                   RSEQ_INJECT_INPUT
 306                 : "r0", "memory", "cc"
 307                   RSEQ_INJECT_CLOBBER
 308                 : abort, cmpfail
 309 #ifdef RSEQ_COMPARE_TWICE
 310                   , error1, error2
 311 #endif
 312         );
 313         rseq_workaround_gcc_asm_size_guess();
 314         return 0;
 315 abort:
 316         rseq_workaround_gcc_asm_size_guess();
 317         RSEQ_INJECT_FAILED
 318         return -1;
 319 cmpfail:
 320         rseq_workaround_gcc_asm_size_guess();
 321         return 1;
 322 #ifdef RSEQ_COMPARE_TWICE
 323 error1:
 324         rseq_bug("cpu_id comparison failed");
 325 error2:
 326         rseq_bug("expected value comparison failed");
 327 #endif
 328 }
 329
 330 static inline __attribute__((always_inline))
 331 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 332                                          intptr_t *v2, intptr_t newv2,
 333                                          intptr_t newv, int cpu)
 334 {
 335         RSEQ_INJECT_C(9)
 336
 337         rseq_workaround_gcc_asm_size_guess();
 338         __asm__ __volatile__ goto (
 339                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 340                 /* Start rseq by storing table entry pointer into rseq_cs. */
 341                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 342                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 343                 RSEQ_INJECT_ASM(3)
 344                 "ldr r0, %[v]\n\t"
 345                 "cmp %[expect], r0\n\t"
 346                 "bne %l[cmpfail]\n\t"
 347                 RSEQ_INJECT_ASM(4)
 348 #ifdef RSEQ_COMPARE_TWICE
 349                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 350                 "ldr r0, %[v]\n\t"
 351                 "cmp %[expect], r0\n\t"
 352                 "bne %l[error2]\n\t"
 353 #endif
 354                 /* try store */
 355                 "str %[newv2], %[v2]\n\t"
 356                 RSEQ_INJECT_ASM(5)
 357                 "dmb\n\t"       /* full mb provides store-release */
 358                 /* final store */
 359                 "str %[newv], %[v]\n\t"
 360                 "2:\n\t"
 361                 RSEQ_INJECT_ASM(6)
 362                 "b 5f\n\t"
 363                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 364                 "5:\n\t"
 365                 : /* gcc asm goto does not allow outputs */
 366                 : [cpu_id]              "r" (cpu),
 367                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 368                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 369                   /* try store input */
 370                   [v2]                  "m" (*v2),
 371                   [newv2]               "r" (newv2),
 372                   /* final store input */
 373                   [v]                   "m" (*v),
 374                   [expect]              "r" (expect),
 375                   [newv]                "r" (newv)
 376                   RSEQ_INJECT_INPUT
 377                 : "r0", "memory", "cc"
 378                   RSEQ_INJECT_CLOBBER
 379                 : abort, cmpfail
 380 #ifdef RSEQ_COMPARE_TWICE
 381                   , error1, error2
 382 #endif
 383         );
 384         rseq_workaround_gcc_asm_size_guess();
 385         return 0;
 386 abort:
 387         rseq_workaround_gcc_asm_size_guess();
 388         RSEQ_INJECT_FAILED
 389         return -1;
 390 cmpfail:
 391         rseq_workaround_gcc_asm_size_guess();
 392         return 1;
 393 #ifdef RSEQ_COMPARE_TWICE
 394 error1:
 395         rseq_bug("cpu_id comparison failed");
 396 error2:
 397         rseq_bug("expected value comparison failed");
 398 #endif
 399 }
 400
 401 static inline __attribute__((always_inline))
 402 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 403                               intptr_t *v2, intptr_t expect2,
 404                               intptr_t newv, int cpu)
 405 {
 406         RSEQ_INJECT_C(9)
 407
 408         rseq_workaround_gcc_asm_size_guess();
 409         __asm__ __volatile__ goto (
 410                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 411                 /* Start rseq by storing table entry pointer into rseq_cs. */
 412                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 413                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 414                 RSEQ_INJECT_ASM(3)
 415                 "ldr r0, %[v]\n\t"
 416                 "cmp %[expect], r0\n\t"
 417                 "bne %l[cmpfail]\n\t"
 418                 RSEQ_INJECT_ASM(4)
 419                 "ldr r0, %[v2]\n\t"
 420                 "cmp %[expect2], r0\n\t"
 421                 "bne %l[cmpfail]\n\t"
 422                 RSEQ_INJECT_ASM(5)
 423 #ifdef RSEQ_COMPARE_TWICE
 424                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 425                 "ldr r0, %[v]\n\t"
 426                 "cmp %[expect], r0\n\t"
 427                 "bne %l[error2]\n\t"
 428                 "ldr r0, %[v2]\n\t"
 429                 "cmp %[expect2], r0\n\t"
 430                 "bne %l[error3]\n\t"
 431 #endif
 432                 /* final store */
 433                 "str %[newv], %[v]\n\t"
 434                 "2:\n\t"
 435                 RSEQ_INJECT_ASM(6)
 436                 "b 5f\n\t"
 437                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 438                 "5:\n\t"
 439                 : /* gcc asm goto does not allow outputs */
 440                 : [cpu_id]              "r" (cpu),
 441                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 442                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 443                   /* cmp2 input */
 444                   [v2]                  "m" (*v2),
 445                   [expect2]             "r" (expect2),
 446                   /* final store input */
 447                   [v]                   "m" (*v),
 448                   [expect]              "r" (expect),
 449                   [newv]                "r" (newv)
 450                   RSEQ_INJECT_INPUT
 451                 : "r0", "memory", "cc"
 452                   RSEQ_INJECT_CLOBBER
 453                 : abort, cmpfail
 454 #ifdef RSEQ_COMPARE_TWICE
 455                   , error1, error2, error3
 456 #endif
 457         );
 458         rseq_workaround_gcc_asm_size_guess();
 459         return 0;
 460 abort:
 461         rseq_workaround_gcc_asm_size_guess();
 462         RSEQ_INJECT_FAILED
 463         return -1;
 464 cmpfail:
 465         rseq_workaround_gcc_asm_size_guess();
 466         return 1;
 467 #ifdef RSEQ_COMPARE_TWICE
 468 error1:
 469         rseq_bug("cpu_id comparison failed");
 470 error2:
 471         rseq_bug("1st expected value comparison failed");
 472 error3:
 473         rseq_bug("2nd expected value comparison failed");
 474 #endif
 475 }
 476
 477 static inline __attribute__((always_inline))
 478 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 479                                  void *dst, void *src, size_t len,
 480                                  intptr_t newv, int cpu)
 481 {
 482         uint32_t rseq_scratch[3];
 483
 484         RSEQ_INJECT_C(9)
 485
 486         rseq_workaround_gcc_asm_size_guess();
 487         __asm__ __volatile__ goto (
 488                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 489                 "str %[src], %[rseq_scratch0]\n\t"
 490                 "str %[dst], %[rseq_scratch1]\n\t"
 491                 "str %[len], %[rseq_scratch2]\n\t"
 492                 /* Start rseq by storing table entry pointer into rseq_cs. */
 493                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 494                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 495                 RSEQ_INJECT_ASM(3)
 496                 "ldr r0, %[v]\n\t"
 497                 "cmp %[expect], r0\n\t"
 498                 "bne 5f\n\t"
 499                 RSEQ_INJECT_ASM(4)
 500 #ifdef RSEQ_COMPARE_TWICE
 501                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 502                 "ldr r0, %[v]\n\t"
 503                 "cmp %[expect], r0\n\t"
 504                 "bne 7f\n\t"
 505 #endif
 506                 /* try memcpy */
 507                 "cmp %[len], #0\n\t" \
 508                 "beq 333f\n\t" \
 509                 "222:\n\t" \
 510                 "ldrb %%r0, [%[src]]\n\t" \
 511                 "strb %%r0, [%[dst]]\n\t" \
 512                 "adds %[src], #1\n\t" \
 513                 "adds %[dst], #1\n\t" \
 514                 "subs %[len], #1\n\t" \
 515                 "bne 222b\n\t" \
 516                 "333:\n\t" \
 517                 RSEQ_INJECT_ASM(5)
 518                 /* final store */
 519                 "str %[newv], %[v]\n\t"
 520                 "2:\n\t"
 521                 RSEQ_INJECT_ASM(6)
 522                 /* teardown */
 523                 "ldr %[len], %[rseq_scratch2]\n\t"
 524                 "ldr %[dst], %[rseq_scratch1]\n\t"
 525                 "ldr %[src], %[rseq_scratch0]\n\t"
 526                 "b 8f\n\t"
 527                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 528                                       /* teardown */
 529                                       "ldr %[len], %[rseq_scratch2]\n\t"
 530                                       "ldr %[dst], %[rseq_scratch1]\n\t"
 531                                       "ldr %[src], %[rseq_scratch0]\n\t",
 532                                       abort, 1b, 2b, 4f)
 533                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 534                                         /* teardown */
 535                                         "ldr %[len], %[rseq_scratch2]\n\t"
 536                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 537                                         "ldr %[src], %[rseq_scratch0]\n\t",
 538                                         cmpfail)
 539 #ifdef RSEQ_COMPARE_TWICE
 540                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 541                                         /* teardown */
 542                                         "ldr %[len], %[rseq_scratch2]\n\t"
 543                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 544                                         "ldr %[src], %[rseq_scratch0]\n\t",
 545                                         error1)
 546                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 547                                         /* teardown */
 548                                         "ldr %[len], %[rseq_scratch2]\n\t"
 549                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 550                                         "ldr %[src], %[rseq_scratch0]\n\t",
 551                                         error2)
 552 #endif
 553                 "8:\n\t"
 554                 : /* gcc asm goto does not allow outputs */
 555                 : [cpu_id]              "r" (cpu),
 556                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 557                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 558                   /* final store input */
 559                   [v]                   "m" (*v),
 560                   [expect]              "r" (expect),
 561                   [newv]                "r" (newv),
 562                   /* try memcpy input */
 563                   [dst]                 "r" (dst),
 564                   [src]                 "r" (src),
 565                   [len]                 "r" (len),
 566                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 567                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 568                   [rseq_scratch2]       "m" (rseq_scratch[2])
 569                   RSEQ_INJECT_INPUT
 570                 : "r0", "memory", "cc"
 571                   RSEQ_INJECT_CLOBBER
 572                 : abort, cmpfail
 573 #ifdef RSEQ_COMPARE_TWICE
 574                   , error1, error2
 575 #endif
 576         );
 577         rseq_workaround_gcc_asm_size_guess();
 578         return 0;
 579 abort:
 580         rseq_workaround_gcc_asm_size_guess();
 581         RSEQ_INJECT_FAILED
 582         return -1;
 583 cmpfail:
 584         rseq_workaround_gcc_asm_size_guess();
 585         return 1;
 586 #ifdef RSEQ_COMPARE_TWICE
 587 error1:
 588         rseq_workaround_gcc_asm_size_guess();
 589         rseq_bug("cpu_id comparison failed");
 590 error2:
 591         rseq_workaround_gcc_asm_size_guess();
 592         rseq_bug("expected value comparison failed");
 593 #endif
 594 }
 595
 596 static inline __attribute__((always_inline))
 597 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 598                                          void *dst, void *src, size_t len,
 599                                          intptr_t newv, int cpu)
 600 {
 601         uint32_t rseq_scratch[3];
 602
 603         RSEQ_INJECT_C(9)
 604
 605         rseq_workaround_gcc_asm_size_guess();
 606         __asm__ __volatile__ goto (
 607                 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
 608                 "str %[src], %[rseq_scratch0]\n\t"
 609                 "str %[dst], %[rseq_scratch1]\n\t"
 610                 "str %[len], %[rseq_scratch2]\n\t"
 611                 /* Start rseq by storing table entry pointer into rseq_cs. */
 612                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 613                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 614                 RSEQ_INJECT_ASM(3)
 615                 "ldr r0, %[v]\n\t"
 616                 "cmp %[expect], r0\n\t"
 617                 "bne 5f\n\t"
 618                 RSEQ_INJECT_ASM(4)
 619 #ifdef RSEQ_COMPARE_TWICE
 620                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 621                 "ldr r0, %[v]\n\t"
 622                 "cmp %[expect], r0\n\t"
 623                 "bne 7f\n\t"
 624 #endif
 625                 /* try memcpy */
 626                 "cmp %[len], #0\n\t" \
 627                 "beq 333f\n\t" \
 628                 "222:\n\t" \
 629                 "ldrb %%r0, [%[src]]\n\t" \
 630                 "strb %%r0, [%[dst]]\n\t" \
 631                 "adds %[src], #1\n\t" \
 632                 "adds %[dst], #1\n\t" \
 633                 "subs %[len], #1\n\t" \
 634                 "bne 222b\n\t" \
 635                 "333:\n\t" \
 636                 RSEQ_INJECT_ASM(5)
 637                 "dmb\n\t"       /* full mb provides store-release */
 638                 /* final store */
 639                 "str %[newv], %[v]\n\t"
 640                 "2:\n\t"
 641                 RSEQ_INJECT_ASM(6)
 642                 /* teardown */
 643                 "ldr %[len], %[rseq_scratch2]\n\t"
 644                 "ldr %[dst], %[rseq_scratch1]\n\t"
 645                 "ldr %[src], %[rseq_scratch0]\n\t"
 646                 "b 8f\n\t"
 647                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 648                                       /* teardown */
 649                                       "ldr %[len], %[rseq_scratch2]\n\t"
 650                                       "ldr %[dst], %[rseq_scratch1]\n\t"
 651                                       "ldr %[src], %[rseq_scratch0]\n\t",
 652                                       abort, 1b, 2b, 4f)
 653                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 654                                         /* teardown */
 655                                         "ldr %[len], %[rseq_scratch2]\n\t"
 656                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 657                                         "ldr %[src], %[rseq_scratch0]\n\t",
 658                                         cmpfail)
 659 #ifdef RSEQ_COMPARE_TWICE
 660                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 661                                         /* teardown */
 662                                         "ldr %[len], %[rseq_scratch2]\n\t"
 663                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 664                                         "ldr %[src], %[rseq_scratch0]\n\t",
 665                                         error1)
 666                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 667                                         /* teardown */
 668                                         "ldr %[len], %[rseq_scratch2]\n\t"
 669                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 670                                         "ldr %[src], %[rseq_scratch0]\n\t",
 671                                         error2)
 672 #endif
 673                 "8:\n\t"
 674                 : /* gcc asm goto does not allow outputs */
 675                 : [cpu_id]              "r" (cpu),
 676                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 677                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 678                   /* final store input */
 679                   [v]                   "m" (*v),
 680                   [expect]              "r" (expect),
 681                   [newv]                "r" (newv),
 682                   /* try memcpy input */
 683                   [dst]                 "r" (dst),
 684                   [src]                 "r" (src),
 685                   [len]                 "r" (len),
 686                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 687                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 688                   [rseq_scratch2]       "m" (rseq_scratch[2])
 689                   RSEQ_INJECT_INPUT
 690                 : "r0", "memory", "cc"
 691                   RSEQ_INJECT_CLOBBER
 692                 : abort, cmpfail
 693 #ifdef RSEQ_COMPARE_TWICE
 694                   , error1, error2
 695 #endif
 696         );
 697         rseq_workaround_gcc_asm_size_guess();
 698         return 0;
 699 abort:
 700         rseq_workaround_gcc_asm_size_guess();
 701         RSEQ_INJECT_FAILED
 702         return -1;
 703 cmpfail:
 704         rseq_workaround_gcc_asm_size_guess();
 705         return 1;
 706 #ifdef RSEQ_COMPARE_TWICE
 707 error1:
 708         rseq_workaround_gcc_asm_size_guess();
 709         rseq_bug("cpu_id comparison failed");
 710 error2:
 711         rseq_workaround_gcc_asm_size_guess();
 712         rseq_bug("expected value comparison failed");
 713 #endif
 714 }
 715
 716 #endif /* !RSEQ_SKIP_FASTPATH */