1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
3 * Author: Paul Burton <paul.burton@mips.com>
4 * (C) Copyright 2018 MIPS Tech LLC
7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #define RSEQ_SIG 0x53053053
12 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory")
13 #define rseq_smp_rmb() rseq_smp_mb()
14 #define rseq_smp_wmb() rseq_smp_mb()
16 #define rseq_smp_load_acquire(p) \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
25 #define rseq_smp_store_release(p, v) \
28 RSEQ_WRITE_ONCE(*p, v); \
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
35 #if _MIPS_SZLONG == 64
36 # define LONG ".dword"
37 # define LONG_LA "dla"
40 # define LONG_ADDI "daddiu"
41 # define U32_U64_PAD(x) x
42 #elif _MIPS_SZLONG == 32
47 # define LONG_ADDI "addiu"
49 # define U32_U64_PAD(x) "0x0, " x
51 # define U32_U64_PAD(x) x ", 0x0"
54 # error unsupported _MIPS_SZLONG
57 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
58 post_commit_offset, abort_ip) \
59 ".pushsection __rseq_table, \"aw\"\n\t" \
61 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
63 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
64 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
67 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
68 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
69 (post_commit_ip - start_ip), abort_ip)
71 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
73 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
74 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
75 __rseq_str(label) ":\n\t"
77 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
79 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
80 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
82 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
83 abort_label, version, flags, \
84 start_ip, post_commit_offset, abort_ip) \
86 __rseq_str(table_label) ":\n\t" \
87 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
88 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
89 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
90 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
91 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
92 __rseq_str(label) ":\n\t" \
94 "b %l[" __rseq_str(abort_label) "]\n\t"
96 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
97 start_ip, post_commit_ip, abort_ip) \
98 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
99 abort_label, 0x0, 0x0, start_ip, \
100 (post_commit_ip - start_ip), abort_ip)
102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 __rseq_str(label) ":\n\t" \
105 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
107 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
109 static inline __attribute__((always_inline))
110 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
114 rseq_workaround_gcc_asm_size_guess();
115 __asm__ __volatile__ goto (
116 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
117 /* Start rseq by storing table entry pointer into rseq_cs. */
118 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
119 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
121 LONG_L " $4, %[v]\n\t"
122 "bne $4, %[expect], %l[cmpfail]\n\t"
124 #ifdef RSEQ_COMPARE_TWICE
125 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
126 LONG_L " $4, %[v]\n\t"
127 "bne $4, %[expect], %l[error2]\n\t"
130 LONG_S " %[newv], %[v]\n\t"
134 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
136 : /* gcc asm goto does not allow outputs */
137 : [cpu_id] "r" (cpu),
138 [current_cpu_id] "m" (__rseq_abi.cpu_id),
139 [rseq_cs] "m" (__rseq_abi.rseq_cs),
141 [expect] "r" (expect),
147 #ifdef RSEQ_COMPARE_TWICE
151 rseq_workaround_gcc_asm_size_guess();
154 rseq_workaround_gcc_asm_size_guess();
158 rseq_workaround_gcc_asm_size_guess();
160 #ifdef RSEQ_COMPARE_TWICE
162 rseq_bug("cpu_id comparison failed");
164 rseq_bug("expected value comparison failed");
168 static inline __attribute__((always_inline))
169 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
170 off_t voffp, intptr_t *load, int cpu)
174 rseq_workaround_gcc_asm_size_guess();
175 __asm__ __volatile__ goto (
176 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
177 /* Start rseq by storing table entry pointer into rseq_cs. */
178 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
179 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
181 LONG_L " $4, %[v]\n\t"
182 "beq $4, %[expectnot], %l[cmpfail]\n\t"
184 #ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
186 LONG_L " $4, %[v]\n\t"
187 "beq $4, %[expectnot], %l[error2]\n\t"
189 LONG_S " $4, %[load]\n\t"
190 LONG_ADDI " $4, %[voffp]\n\t"
191 LONG_L " $4, 0($4)\n\t"
193 LONG_S " $4, %[v]\n\t"
197 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
199 : /* gcc asm goto does not allow outputs */
200 : [cpu_id] "r" (cpu),
201 [current_cpu_id] "m" (__rseq_abi.cpu_id),
202 [rseq_cs] "m" (__rseq_abi.rseq_cs),
203 /* final store input */
205 [expectnot] "r" (expectnot),
206 [voffp] "Ir" (voffp),
212 #ifdef RSEQ_COMPARE_TWICE
216 rseq_workaround_gcc_asm_size_guess();
219 rseq_workaround_gcc_asm_size_guess();
223 rseq_workaround_gcc_asm_size_guess();
225 #ifdef RSEQ_COMPARE_TWICE
227 rseq_bug("cpu_id comparison failed");
229 rseq_bug("expected value comparison failed");
233 static inline __attribute__((always_inline))
234 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
238 rseq_workaround_gcc_asm_size_guess();
239 __asm__ __volatile__ goto (
240 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
241 /* Start rseq by storing table entry pointer into rseq_cs. */
242 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
245 #ifdef RSEQ_COMPARE_TWICE
246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
248 LONG_L " $4, %[v]\n\t"
249 LONG_ADDI " $4, %[count]\n\t"
251 LONG_S " $4, %[v]\n\t"
255 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
257 : /* gcc asm goto does not allow outputs */
258 : [cpu_id] "r" (cpu),
259 [current_cpu_id] "m" (__rseq_abi.cpu_id),
260 [rseq_cs] "m" (__rseq_abi.rseq_cs),
267 #ifdef RSEQ_COMPARE_TWICE
271 rseq_workaround_gcc_asm_size_guess();
274 rseq_workaround_gcc_asm_size_guess();
277 #ifdef RSEQ_COMPARE_TWICE
279 rseq_bug("cpu_id comparison failed");
283 static inline __attribute__((always_inline))
284 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
285 intptr_t *v2, intptr_t newv2,
286 intptr_t newv, int cpu)
290 rseq_workaround_gcc_asm_size_guess();
291 __asm__ __volatile__ goto (
292 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
293 /* Start rseq by storing table entry pointer into rseq_cs. */
294 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
297 LONG_L " $4, %[v]\n\t"
298 "bne $4, %[expect], %l[cmpfail]\n\t"
300 #ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
302 LONG_L " $4, %[v]\n\t"
303 "bne $4, %[expect], %l[error2]\n\t"
306 LONG_S " %[newv2], %[v2]\n\t"
309 LONG_S " %[newv], %[v]\n\t"
313 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
315 : /* gcc asm goto does not allow outputs */
316 : [cpu_id] "r" (cpu),
317 [current_cpu_id] "m" (__rseq_abi.cpu_id),
318 [rseq_cs] "m" (__rseq_abi.rseq_cs),
319 /* try store input */
322 /* final store input */
324 [expect] "r" (expect),
330 #ifdef RSEQ_COMPARE_TWICE
334 rseq_workaround_gcc_asm_size_guess();
337 rseq_workaround_gcc_asm_size_guess();
341 rseq_workaround_gcc_asm_size_guess();
343 #ifdef RSEQ_COMPARE_TWICE
345 rseq_bug("cpu_id comparison failed");
347 rseq_bug("expected value comparison failed");
351 static inline __attribute__((always_inline))
352 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
353 intptr_t *v2, intptr_t newv2,
354 intptr_t newv, int cpu)
358 rseq_workaround_gcc_asm_size_guess();
359 __asm__ __volatile__ goto (
360 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
361 /* Start rseq by storing table entry pointer into rseq_cs. */
362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
365 LONG_L " $4, %[v]\n\t"
366 "bne $4, %[expect], %l[cmpfail]\n\t"
368 #ifdef RSEQ_COMPARE_TWICE
369 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
370 LONG_L " $4, %[v]\n\t"
371 "bne $4, %[expect], %l[error2]\n\t"
374 LONG_S " %[newv2], %[v2]\n\t"
376 "sync\n\t" /* full sync provides store-release */
378 LONG_S " %[newv], %[v]\n\t"
382 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
384 : /* gcc asm goto does not allow outputs */
385 : [cpu_id] "r" (cpu),
386 [current_cpu_id] "m" (__rseq_abi.cpu_id),
387 [rseq_cs] "m" (__rseq_abi.rseq_cs),
388 /* try store input */
391 /* final store input */
393 [expect] "r" (expect),
399 #ifdef RSEQ_COMPARE_TWICE
403 rseq_workaround_gcc_asm_size_guess();
406 rseq_workaround_gcc_asm_size_guess();
410 rseq_workaround_gcc_asm_size_guess();
412 #ifdef RSEQ_COMPARE_TWICE
414 rseq_bug("cpu_id comparison failed");
416 rseq_bug("expected value comparison failed");
420 static inline __attribute__((always_inline))
421 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
422 intptr_t *v2, intptr_t expect2,
423 intptr_t newv, int cpu)
427 rseq_workaround_gcc_asm_size_guess();
428 __asm__ __volatile__ goto (
429 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
430 /* Start rseq by storing table entry pointer into rseq_cs. */
431 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
432 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
434 LONG_L " $4, %[v]\n\t"
435 "bne $4, %[expect], %l[cmpfail]\n\t"
437 LONG_L " $4, %[v2]\n\t"
438 "bne $4, %[expect2], %l[cmpfail]\n\t"
440 #ifdef RSEQ_COMPARE_TWICE
441 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
442 LONG_L " $4, %[v]\n\t"
443 "bne $4, %[expect], %l[error2]\n\t"
444 LONG_L " $4, %[v2]\n\t"
445 "bne $4, %[expect2], %l[error3]\n\t"
448 LONG_S " %[newv], %[v]\n\t"
452 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
454 : /* gcc asm goto does not allow outputs */
455 : [cpu_id] "r" (cpu),
456 [current_cpu_id] "m" (__rseq_abi.cpu_id),
457 [rseq_cs] "m" (__rseq_abi.rseq_cs),
460 [expect2] "r" (expect2),
461 /* final store input */
463 [expect] "r" (expect),
469 #ifdef RSEQ_COMPARE_TWICE
470 , error1, error2, error3
473 rseq_workaround_gcc_asm_size_guess();
476 rseq_workaround_gcc_asm_size_guess();
480 rseq_workaround_gcc_asm_size_guess();
482 #ifdef RSEQ_COMPARE_TWICE
484 rseq_bug("cpu_id comparison failed");
486 rseq_bug("1st expected value comparison failed");
488 rseq_bug("2nd expected value comparison failed");
492 static inline __attribute__((always_inline))
493 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
494 void *dst, void *src, size_t len,
495 intptr_t newv, int cpu)
497 uintptr_t rseq_scratch[3];
501 rseq_workaround_gcc_asm_size_guess();
502 __asm__ __volatile__ goto (
503 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
504 LONG_S " %[src], %[rseq_scratch0]\n\t"
505 LONG_S " %[dst], %[rseq_scratch1]\n\t"
506 LONG_S " %[len], %[rseq_scratch2]\n\t"
507 /* Start rseq by storing table entry pointer into rseq_cs. */
508 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
509 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
511 LONG_L " $4, %[v]\n\t"
512 "bne $4, %[expect], 5f\n\t"
514 #ifdef RSEQ_COMPARE_TWICE
515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
516 LONG_L " $4, %[v]\n\t"
517 "bne $4, %[expect], 7f\n\t"
520 "beqz %[len], 333f\n\t" \
522 "lb $4, 0(%[src])\n\t" \
523 "sb $4, 0(%[dst])\n\t" \
524 LONG_ADDI " %[src], 1\n\t" \
525 LONG_ADDI " %[dst], 1\n\t" \
526 LONG_ADDI " %[len], -1\n\t" \
527 "bnez %[len], 222b\n\t" \
531 LONG_S " %[newv], %[v]\n\t"
535 LONG_L " %[len], %[rseq_scratch2]\n\t"
536 LONG_L " %[dst], %[rseq_scratch1]\n\t"
537 LONG_L " %[src], %[rseq_scratch0]\n\t"
539 RSEQ_ASM_DEFINE_ABORT(3, 4,
541 LONG_L " %[len], %[rseq_scratch2]\n\t"
542 LONG_L " %[dst], %[rseq_scratch1]\n\t"
543 LONG_L " %[src], %[rseq_scratch0]\n\t",
545 RSEQ_ASM_DEFINE_CMPFAIL(5,
547 LONG_L " %[len], %[rseq_scratch2]\n\t"
548 LONG_L " %[dst], %[rseq_scratch1]\n\t"
549 LONG_L " %[src], %[rseq_scratch0]\n\t",
551 #ifdef RSEQ_COMPARE_TWICE
552 RSEQ_ASM_DEFINE_CMPFAIL(6,
554 LONG_L " %[len], %[rseq_scratch2]\n\t"
555 LONG_L " %[dst], %[rseq_scratch1]\n\t"
556 LONG_L " %[src], %[rseq_scratch0]\n\t",
558 RSEQ_ASM_DEFINE_CMPFAIL(7,
560 LONG_L " %[len], %[rseq_scratch2]\n\t"
561 LONG_L " %[dst], %[rseq_scratch1]\n\t"
562 LONG_L " %[src], %[rseq_scratch0]\n\t",
566 : /* gcc asm goto does not allow outputs */
567 : [cpu_id] "r" (cpu),
568 [current_cpu_id] "m" (__rseq_abi.cpu_id),
569 [rseq_cs] "m" (__rseq_abi.rseq_cs),
570 /* final store input */
572 [expect] "r" (expect),
574 /* try memcpy input */
578 [rseq_scratch0] "m" (rseq_scratch[0]),
579 [rseq_scratch1] "m" (rseq_scratch[1]),
580 [rseq_scratch2] "m" (rseq_scratch[2])
585 #ifdef RSEQ_COMPARE_TWICE
589 rseq_workaround_gcc_asm_size_guess();
592 rseq_workaround_gcc_asm_size_guess();
596 rseq_workaround_gcc_asm_size_guess();
598 #ifdef RSEQ_COMPARE_TWICE
600 rseq_workaround_gcc_asm_size_guess();
601 rseq_bug("cpu_id comparison failed");
603 rseq_workaround_gcc_asm_size_guess();
604 rseq_bug("expected value comparison failed");
608 static inline __attribute__((always_inline))
609 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
610 void *dst, void *src, size_t len,
611 intptr_t newv, int cpu)
613 uintptr_t rseq_scratch[3];
617 rseq_workaround_gcc_asm_size_guess();
618 __asm__ __volatile__ goto (
619 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
620 LONG_S " %[src], %[rseq_scratch0]\n\t"
621 LONG_S " %[dst], %[rseq_scratch1]\n\t"
622 LONG_S " %[len], %[rseq_scratch2]\n\t"
623 /* Start rseq by storing table entry pointer into rseq_cs. */
624 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
625 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
627 LONG_L " $4, %[v]\n\t"
628 "bne $4, %[expect], 5f\n\t"
630 #ifdef RSEQ_COMPARE_TWICE
631 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
632 LONG_L " $4, %[v]\n\t"
633 "bne $4, %[expect], 7f\n\t"
636 "beqz %[len], 333f\n\t" \
638 "lb $4, 0(%[src])\n\t" \
639 "sb $4, 0(%[dst])\n\t" \
640 LONG_ADDI " %[src], 1\n\t" \
641 LONG_ADDI " %[dst], 1\n\t" \
642 LONG_ADDI " %[len], -1\n\t" \
643 "bnez %[len], 222b\n\t" \
646 "sync\n\t" /* full sync provides store-release */
648 LONG_S " %[newv], %[v]\n\t"
652 LONG_L " %[len], %[rseq_scratch2]\n\t"
653 LONG_L " %[dst], %[rseq_scratch1]\n\t"
654 LONG_L " %[src], %[rseq_scratch0]\n\t"
656 RSEQ_ASM_DEFINE_ABORT(3, 4,
658 LONG_L " %[len], %[rseq_scratch2]\n\t"
659 LONG_L " %[dst], %[rseq_scratch1]\n\t"
660 LONG_L " %[src], %[rseq_scratch0]\n\t",
662 RSEQ_ASM_DEFINE_CMPFAIL(5,
664 LONG_L " %[len], %[rseq_scratch2]\n\t"
665 LONG_L " %[dst], %[rseq_scratch1]\n\t"
666 LONG_L " %[src], %[rseq_scratch0]\n\t",
668 #ifdef RSEQ_COMPARE_TWICE
669 RSEQ_ASM_DEFINE_CMPFAIL(6,
671 LONG_L " %[len], %[rseq_scratch2]\n\t"
672 LONG_L " %[dst], %[rseq_scratch1]\n\t"
673 LONG_L " %[src], %[rseq_scratch0]\n\t",
675 RSEQ_ASM_DEFINE_CMPFAIL(7,
677 LONG_L " %[len], %[rseq_scratch2]\n\t"
678 LONG_L " %[dst], %[rseq_scratch1]\n\t"
679 LONG_L " %[src], %[rseq_scratch0]\n\t",
683 : /* gcc asm goto does not allow outputs */
684 : [cpu_id] "r" (cpu),
685 [current_cpu_id] "m" (__rseq_abi.cpu_id),
686 [rseq_cs] "m" (__rseq_abi.rseq_cs),
687 /* final store input */
689 [expect] "r" (expect),
691 /* try memcpy input */
695 [rseq_scratch0] "m" (rseq_scratch[0]),
696 [rseq_scratch1] "m" (rseq_scratch[1]),
697 [rseq_scratch2] "m" (rseq_scratch[2])
702 #ifdef RSEQ_COMPARE_TWICE
706 rseq_workaround_gcc_asm_size_guess();
709 rseq_workaround_gcc_asm_size_guess();
713 rseq_workaround_gcc_asm_size_guess();
715 #ifdef RSEQ_COMPARE_TWICE
717 rseq_workaround_gcc_asm_size_guess();
718 rseq_bug("cpu_id comparison failed");
720 rseq_workaround_gcc_asm_size_guess();
721 rseq_bug("expected value comparison failed");
725 #endif /* !RSEQ_SKIP_FASTPATH */