1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright 2002 Andi Kleen */
4 #include <linux/linkage.h>
6 #include <asm/cpufeatures.h>
7 #include <asm/mcsafe_test.h>
8 #include <asm/alternative-asm.h>
9 #include <asm/export.h>
12 * We build a jump to memcpy_orig by default which gets NOPped out on
13 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
14 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
15 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
19 * memcpy - Copy a memory block.
27 * rax original destination
30 SYM_FUNC_START_WEAK(memcpy)
31 ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
32 "jmp memcpy_erms", X86_FEATURE_ERMS
45 EXPORT_SYMBOL(__memcpy)
48 * memcpy_erms() - enhanced fast string memcpy. This is faster and
49 * simpler than memcpy. Use memcpy_erms when possible.
65 * We check whether memory false dependence could occur,
66 * then jump to corresponding copy mode.
75 * Move in blocks of 4x8 bytes:
88 jae .Lcopy_forward_loop
94 * Calculate copy position to tail.
100 * At most 3 ALU operations in one cycle,
101 * so append NOPS in the same 16 bytes trunk.
104 .Lcopy_backward_loop:
108 movq -3*8(%rsi), %r10
109 movq -4*8(%rsi), %r11
110 leaq -4*8(%rsi), %rsi
113 movq %r10, -3*8(%rdi)
114 movq %r11, -4*8(%rdi)
115 leaq -4*8(%rdi), %rdi
116 jae .Lcopy_backward_loop
119 * Calculate copy position to head.
129 * Move data from 16 bytes to 31 bytes.
133 movq -2*8(%rsi, %rdx), %r10
134 movq -1*8(%rsi, %rdx), %r11
137 movq %r10, -2*8(%rdi, %rdx)
138 movq %r11, -1*8(%rdi, %rdx)
145 * Move data from 8 bytes to 15 bytes.
148 movq -1*8(%rsi, %rdx), %r9
150 movq %r9, -1*8(%rdi, %rdx)
158 * Move data from 4 bytes to 7 bytes.
161 movl -4(%rsi, %rdx), %r8d
163 movl %r8d, -4(%rdi, %rdx)
170 * Move data from 1 bytes to 3 bytes.
175 movzbq (%rsi, %rdx), %r9
177 movb %r9b, (%rdi, %rdx)
190 * __memcpy_mcsafe - memory copy with machine check exception handling
191 * Note that we only catch machine checks when reading the source addresses.
192 * Writes to target are posted and don't generate machine checks.
194 ENTRY(__memcpy_mcsafe)
196 /* Less than 8 bytes? Go to byte copy loop */
199 /* Check for bad alignment of source */
201 /* Already aligned */
204 /* Copy one byte at a time until source is 8-byte aligned */
210 .L_read_leading_bytes:
212 MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes
213 MCSAFE_TEST_DST %rdi 1 .E_leading_bytes
214 .L_write_leading_bytes:
219 jnz .L_read_leading_bytes
229 MCSAFE_TEST_SRC %rsi 8 .E_read_words
230 MCSAFE_TEST_DST %rdi 8 .E_write_words
238 /* Any trailing bytes? */
241 jz .L_done_memcpy_trap
243 /* Copy trailing bytes */
245 .L_read_trailing_bytes:
247 MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes
248 MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes
249 .L_write_trailing_bytes:
254 jnz .L_read_trailing_bytes
256 /* Copy successful. Return zero */
261 ENDPROC(__memcpy_mcsafe)
262 EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
264 .section .fixup, "ax"
266 * Return number of bytes not copied for any failure. Note that
267 * there is no "tail" handling since the source buffer is 8-byte
268 * aligned and poison is cacheline aligned.
279 * For write fault handling, given the destination is unaligned,
280 * we handle faults on multi-byte writes with a byte-by-byte
281 * copy up to the write-protected page.
287 jmp mcsafe_handle_tail
291 _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
292 _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
293 _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
294 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
295 _ASM_EXTABLE(.L_write_words, .E_write_words)
296 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)