1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright (c) 2012-2021, Arm Limited.
5 * Adapted from the original at:
6 * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/strcmp.S
9 #include <linux/linkage.h>
10 #include <asm/assembler.h>
17 #define L(label) .L ## label
19 #define REP8_01 0x0101010101010101
20 #define REP8_7f 0x7f7f7f7f7f7f7f7f
21 #define REP8_80 0x8080808080808080
23 /* Parameters and result. */
28 /* Internal variables. */
42 /* Start of performance-critical section -- one 64B cache line. */
44 SYM_FUNC_START_WEAK_PI(strcmp)
46 mov zeroones, #REP8_01
51 /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
52 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
53 can be done in parallel across the entire word. */
58 sub tmp1, data1, zeroones
59 orr tmp2, data1, #REP8_7f
60 eor diff, data1, data2 /* Non-zero if differences found. */
61 bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
62 orr syndrome, diff, has_nul
63 cbz syndrome, L(loop_aligned)
64 /* End of performance-critical section -- one 64B cache line. */
68 rev syndrome, syndrome
70 /* The MS-non-zero bit of the syndrome marks either the first bit
71 that is different, or the top bit of the first zero byte.
72 Shifting left now will bring the critical information into the
78 /* But we need to zero-extend (char is unsigned) the value and then
79 perform a signed 32-bit subtraction. */
81 sub result, data1, data2, lsr #56
84 /* For big-endian we cannot use the trick with the syndrome value
85 as carry-propagation can corrupt the upper bits if the trailing
86 bytes in the string contain 0x01. */
87 /* However, if there is no NUL byte in the dword, we can generate
88 the result directly. We can't just subtract the bytes as the
89 MSB might be significant. */
93 cneg result, result, lo
96 /* Re-compute the NUL-byte detection, using a byte-reversed value. */
98 sub tmp1, tmp3, zeroones
99 orr tmp2, tmp3, #REP8_7f
100 bic has_nul, tmp1, tmp2
102 orr syndrome, diff, has_nul
104 /* The MS-non-zero bit of the syndrome marks either the first bit
105 that is different, or the top bit of the first zero byte.
106 Shifting left now will bring the critical information into the
108 lsl data1, data1, pos
109 lsl data2, data2, pos
110 /* But we need to zero-extend (char is unsigned) the value and then
111 perform a signed 32-bit subtraction. */
112 lsr data1, data1, #56
113 sub result, data1, data2, lsr #56
118 /* Sources are mutually aligned, but are not currently at an
119 alignment boundary. Round down the addresses and then mask off
120 the bytes that preceed the start point. */
123 lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
124 ldr data1, [src1], #8
125 neg tmp1, tmp1 /* Bits to alignment -64. */
126 ldr data2, [src2], #8
129 /* Big-endian. Early bytes are at MSB. */
130 lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
132 /* Little-endian. Early bytes are at LSB. */
133 lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
135 orr data1, data1, tmp2
136 orr data2, data2, tmp2
140 /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always
141 checking to make sure that we don't access beyond page boundary in
144 b.eq L(loop_misaligned)
146 ldrb data1w, [src1], #1
147 ldrb data2w, [src2], #1
149 ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
152 b.ne L(do_misaligned)
155 /* Test if we are within the last dword of the end of a 4K page. If
156 yes then jump back to the misaligned loop to copy a byte at a time. */
157 and tmp1, src2, #0xff8
158 eor tmp1, tmp1, #0xff8
159 cbz tmp1, L(do_misaligned)
160 ldr data1, [src1], #8
161 ldr data2, [src2], #8
163 sub tmp1, data1, zeroones
164 orr tmp2, data1, #REP8_7f
165 eor diff, data1, data2 /* Non-zero if differences found. */
166 bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
167 orr syndrome, diff, has_nul
168 cbz syndrome, L(loop_misaligned)
172 sub result, data1, data2
175 SYM_FUNC_END_PI(strcmp)
176 EXPORT_SYMBOL_NOHWKASAN(strcmp)