GNU Linux-libre 4.14.266-gnu1
[releases.git] / arch / x86 / lib / memcpy_32.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/string.h>
3 #include <linux/export.h>
4
5 #undef memcpy
6 #undef memset
7
8 __visible void *memcpy(void *to, const void *from, size_t n)
9 {
10 #if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE)
11         return __memcpy3d(to, from, n);
12 #else
13         return __memcpy(to, from, n);
14 #endif
15 }
16 EXPORT_SYMBOL(memcpy);
17
18 __visible void *memset(void *s, int c, size_t count)
19 {
20         return __memset(s, c, count);
21 }
22 EXPORT_SYMBOL(memset);
23
24 __visible void *memmove(void *dest, const void *src, size_t n)
25 {
26         int d0,d1,d2,d3,d4,d5;
27         char *ret = dest;
28
29         __asm__ __volatile__(
30                 /* Handle more 16 bytes in loop */
31                 "cmp $0x10, %0\n\t"
32                 "jb     1f\n\t"
33
34                 /* Decide forward/backward copy mode */
35                 "cmp %2, %1\n\t"
36                 "jb     2f\n\t"
37
38                 /*
39                  * movs instruction have many startup latency
40                  * so we handle small size by general register.
41                  */
42                 "cmp  $680, %0\n\t"
43                 "jb 3f\n\t"
44                 /*
45                  * movs instruction is only good for aligned case.
46                  */
47                 "mov %1, %3\n\t"
48                 "xor %2, %3\n\t"
49                 "and $0xff, %3\n\t"
50                 "jz 4f\n\t"
51                 "3:\n\t"
52                 "sub $0x10, %0\n\t"
53
54                 /*
55                  * We gobble 16 bytes forward in each loop.
56                  */
57                 "3:\n\t"
58                 "sub $0x10, %0\n\t"
59                 "mov 0*4(%1), %3\n\t"
60                 "mov 1*4(%1), %4\n\t"
61                 "mov  %3, 0*4(%2)\n\t"
62                 "mov  %4, 1*4(%2)\n\t"
63                 "mov 2*4(%1), %3\n\t"
64                 "mov 3*4(%1), %4\n\t"
65                 "mov  %3, 2*4(%2)\n\t"
66                 "mov  %4, 3*4(%2)\n\t"
67                 "lea  0x10(%1), %1\n\t"
68                 "lea  0x10(%2), %2\n\t"
69                 "jae 3b\n\t"
70                 "add $0x10, %0\n\t"
71                 "jmp 1f\n\t"
72
73                 /*
74                  * Handle data forward by movs.
75                  */
76                 ".p2align 4\n\t"
77                 "4:\n\t"
78                 "mov -4(%1, %0), %3\n\t"
79                 "lea -4(%2, %0), %4\n\t"
80                 "shr $2, %0\n\t"
81                 "rep movsl\n\t"
82                 "mov %3, (%4)\n\t"
83                 "jmp 11f\n\t"
84                 /*
85                  * Handle data backward by movs.
86                  */
87                 ".p2align 4\n\t"
88                 "6:\n\t"
89                 "mov (%1), %3\n\t"
90                 "mov %2, %4\n\t"
91                 "lea -4(%1, %0), %1\n\t"
92                 "lea -4(%2, %0), %2\n\t"
93                 "shr $2, %0\n\t"
94                 "std\n\t"
95                 "rep movsl\n\t"
96                 "mov %3,(%4)\n\t"
97                 "cld\n\t"
98                 "jmp 11f\n\t"
99
100                 /*
101                  * Start to prepare for backward copy.
102                  */
103                 ".p2align 4\n\t"
104                 "2:\n\t"
105                 "cmp  $680, %0\n\t"
106                 "jb 5f\n\t"
107                 "mov %1, %3\n\t"
108                 "xor %2, %3\n\t"
109                 "and $0xff, %3\n\t"
110                 "jz 6b\n\t"
111
112                 /*
113                  * Calculate copy position to tail.
114                  */
115                 "5:\n\t"
116                 "add %0, %1\n\t"
117                 "add %0, %2\n\t"
118                 "sub $0x10, %0\n\t"
119
120                 /*
121                  * We gobble 16 bytes backward in each loop.
122                  */
123                 "7:\n\t"
124                 "sub $0x10, %0\n\t"
125
126                 "mov -1*4(%1), %3\n\t"
127                 "mov -2*4(%1), %4\n\t"
128                 "mov  %3, -1*4(%2)\n\t"
129                 "mov  %4, -2*4(%2)\n\t"
130                 "mov -3*4(%1), %3\n\t"
131                 "mov -4*4(%1), %4\n\t"
132                 "mov  %3, -3*4(%2)\n\t"
133                 "mov  %4, -4*4(%2)\n\t"
134                 "lea  -0x10(%1), %1\n\t"
135                 "lea  -0x10(%2), %2\n\t"
136                 "jae 7b\n\t"
137                 /*
138                  * Calculate copy position to head.
139                  */
140                 "add $0x10, %0\n\t"
141                 "sub %0, %1\n\t"
142                 "sub %0, %2\n\t"
143
144                 /*
145                  * Move data from 8 bytes to 15 bytes.
146                  */
147                 ".p2align 4\n\t"
148                 "1:\n\t"
149                 "cmp $8, %0\n\t"
150                 "jb 8f\n\t"
151                 "mov 0*4(%1), %3\n\t"
152                 "mov 1*4(%1), %4\n\t"
153                 "mov -2*4(%1, %0), %5\n\t"
154                 "mov -1*4(%1, %0), %1\n\t"
155
156                 "mov  %3, 0*4(%2)\n\t"
157                 "mov  %4, 1*4(%2)\n\t"
158                 "mov  %5, -2*4(%2, %0)\n\t"
159                 "mov  %1, -1*4(%2, %0)\n\t"
160                 "jmp 11f\n\t"
161
162                 /*
163                  * Move data from 4 bytes to 7 bytes.
164                  */
165                 ".p2align 4\n\t"
166                 "8:\n\t"
167                 "cmp $4, %0\n\t"
168                 "jb 9f\n\t"
169                 "mov 0*4(%1), %3\n\t"
170                 "mov -1*4(%1, %0), %4\n\t"
171                 "mov  %3, 0*4(%2)\n\t"
172                 "mov  %4, -1*4(%2, %0)\n\t"
173                 "jmp 11f\n\t"
174
175                 /*
176                  * Move data from 2 bytes to 3 bytes.
177                  */
178                 ".p2align 4\n\t"
179                 "9:\n\t"
180                 "cmp $2, %0\n\t"
181                 "jb 10f\n\t"
182                 "movw 0*2(%1), %%dx\n\t"
183                 "movw -1*2(%1, %0), %%bx\n\t"
184                 "movw %%dx, 0*2(%2)\n\t"
185                 "movw %%bx, -1*2(%2, %0)\n\t"
186                 "jmp 11f\n\t"
187
188                 /*
189                  * Move data for 1 byte.
190                  */
191                 ".p2align 4\n\t"
192                 "10:\n\t"
193                 "cmp $1, %0\n\t"
194                 "jb 11f\n\t"
195                 "movb (%1), %%cl\n\t"
196                 "movb %%cl, (%2)\n\t"
197                 ".p2align 4\n\t"
198                 "11:"
199                 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
200                   "=r" (d3),"=r" (d4), "=r"(d5)
201                 :"0" (n),
202                  "1" (src),
203                  "2" (dest)
204                 :"memory");
205
206         return ret;
207
208 }
209 EXPORT_SYMBOL(memmove);