1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/string.h>
3 #include <linux/export.h>
9 __visible void *memcpy(void *to, const void *from, size_t n)
11 return __memcpy(to, from, n);
13 EXPORT_SYMBOL(memcpy);
15 __visible void *memset(void *s, int c, size_t count)
17 return __memset(s, c, count);
19 EXPORT_SYMBOL(memset);
21 __visible void *memmove(void *dest, const void *src, size_t n)
23 int d0,d1,d2,d3,d4,d5;
27 /* Handle more 16 bytes in loop */
31 /* Decide forward/backward copy mode */
36 * movs instruction have many startup latency
37 * so we handle small size by general register.
42 * movs instruction is only good for aligned case.
52 * We gobble 16 bytes forward in each loop.
64 "lea 0x10(%1), %1\n\t"
65 "lea 0x10(%2), %2\n\t"
71 * Handle data forward by movs.
75 "mov -4(%1, %0), %3\n\t"
76 "lea -4(%2, %0), %4\n\t"
82 * Handle data backward by movs.
88 "lea -4(%1, %0), %1\n\t"
89 "lea -4(%2, %0), %2\n\t"
98 * Start to prepare for backward copy.
110 * Calculate copy position to tail.
118 * We gobble 16 bytes backward in each loop.
123 "mov -1*4(%1), %3\n\t"
124 "mov -2*4(%1), %4\n\t"
125 "mov %3, -1*4(%2)\n\t"
126 "mov %4, -2*4(%2)\n\t"
127 "mov -3*4(%1), %3\n\t"
128 "mov -4*4(%1), %4\n\t"
129 "mov %3, -3*4(%2)\n\t"
130 "mov %4, -4*4(%2)\n\t"
131 "lea -0x10(%1), %1\n\t"
132 "lea -0x10(%2), %2\n\t"
135 * Calculate copy position to head.
142 * Move data from 8 bytes to 15 bytes.
148 "mov 0*4(%1), %3\n\t"
149 "mov 1*4(%1), %4\n\t"
150 "mov -2*4(%1, %0), %5\n\t"
151 "mov -1*4(%1, %0), %1\n\t"
153 "mov %3, 0*4(%2)\n\t"
154 "mov %4, 1*4(%2)\n\t"
155 "mov %5, -2*4(%2, %0)\n\t"
156 "mov %1, -1*4(%2, %0)\n\t"
160 * Move data from 4 bytes to 7 bytes.
166 "mov 0*4(%1), %3\n\t"
167 "mov -1*4(%1, %0), %4\n\t"
168 "mov %3, 0*4(%2)\n\t"
169 "mov %4, -1*4(%2, %0)\n\t"
173 * Move data from 2 bytes to 3 bytes.
179 "movw 0*2(%1), %%dx\n\t"
180 "movw -1*2(%1, %0), %%bx\n\t"
181 "movw %%dx, 0*2(%2)\n\t"
182 "movw %%bx, -1*2(%2, %0)\n\t"
186 * Move data for 1 byte.
192 "movb (%1), %%cl\n\t"
193 "movb %%cl, (%2)\n\t"
196 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
197 "=r" (d3),"=r" (d4), "=r"(d5)
206 EXPORT_SYMBOL(memmove);