1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
6 * -__clear_user( ) called multiple times during elf load was byte loop
7 * converted to do as much word clear as possible.
10 * -Hand crafted constant propagation for "constant" copy sizes
11 * -stock kernel shrunk by 33K at -O3
14 * -Added option to (UN)inline copy_(to|from)_user to reduce code sz
15 * -kernel shrunk by 200K even at -O3 (gcc 4.2.1)
16 * -Enabled when doing -Os
18 * Amit Bhor, Sameer Dhavale: Codito Technologies 2004
21 #ifndef _ASM_ARC_UACCESS_H
22 #define _ASM_ARC_UACCESS_H
24 #include <linux/string.h> /* for generic string functions */
26 /*********** Single byte/hword/word copies ******************/
28 #define __get_user_fn(sz, u, k) \
30 long __ret = 0; /* success by default */ \
32 case 1: __arc_get_user_one(*(k), u, "ldb", __ret); break; \
33 case 2: __arc_get_user_one(*(k), u, "ldw", __ret); break; \
34 case 4: __arc_get_user_one(*(k), u, "ld", __ret); break; \
35 case 8: __arc_get_user_one_64(*(k), u, __ret); break; \
41 * Returns 0 on success, -EFAULT if not.
42 * @ret already contains 0 - given that errors will be less likely
43 * (hence +r asm constraint below).
44 * In case of error, fixup code will make it -EFAULT
46 #define __arc_get_user_one(dst, src, op, ret) \
47 __asm__ __volatile__( \
50 " .section .fixup, \"ax\"\n" \
52 "3: # return -EFAULT\n" \
54 " # zero out dst ptr\n" \
58 " .section __ex_table, \"a\"\n" \
63 : "+r" (ret), "=r" (dst) \
64 : "r" (src), "ir" (-EFAULT))
66 #define __arc_get_user_one_64(dst, src, ret) \
67 __asm__ __volatile__( \
69 "4: ld %R1,[%2, 4]\n" \
71 " .section .fixup, \"ax\"\n" \
73 "3: # return -EFAULT\n" \
75 " # zero out dst ptr\n" \
80 " .section __ex_table, \"a\"\n" \
86 : "+r" (ret), "=r" (dst) \
87 : "r" (src), "ir" (-EFAULT))
89 #define __put_user_fn(sz, u, k) \
91 long __ret = 0; /* success by default */ \
93 case 1: __arc_put_user_one(*(k), u, "stb", __ret); break; \
94 case 2: __arc_put_user_one(*(k), u, "stw", __ret); break; \
95 case 4: __arc_put_user_one(*(k), u, "st", __ret); break; \
96 case 8: __arc_put_user_one_64(*(k), u, __ret); break; \
101 #define __arc_put_user_one(src, dst, op, ret) \
102 __asm__ __volatile__( \
103 "1: "op" %1,[%2]\n" \
105 " .section .fixup, \"ax\"\n" \
110 " .section __ex_table, \"a\"\n" \
116 : "r" (src), "r" (dst), "ir" (-EFAULT))
118 #define __arc_put_user_one_64(src, dst, ret) \
119 __asm__ __volatile__( \
121 "4: st %R1,[%2, 4]\n" \
123 " .section .fixup, \"ax\"\n" \
128 " .section __ex_table, \"a\"\n" \
135 : "r" (src), "r" (dst), "ir" (-EFAULT))
138 static inline unsigned long
139 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
143 unsigned long tmp1, tmp2, tmp3, tmp4;
144 unsigned long orig_n = n;
149 /* fallback for unaligned access when hardware doesn't support */
150 if (!IS_ENABLED(CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS) &&
151 (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3))) {
155 __asm__ __volatile__ (
156 " mov.f lp_count, %0 \n"
158 "1: ldb.ab %1, [%3, 1] \n"
159 " stb.ab %1, [%2, 1] \n"
162 " .section .fixup, \"ax\" \n"
166 " .section __ex_table, \"a\" \n"
173 * Note as an '&' earlyclobber operand to make sure the
174 * temporary register inside the loop is not the same as
177 "=&r" (tmp), "+r" (to), "+r" (from)
179 : "lp_count", "memory");
185 * Hand-crafted constant propagation to reduce code sz of the
186 * laddered copy 16x,8,4,2,1
188 if (__builtin_constant_p(orig_n)) {
192 orig_n = orig_n % 16;
194 __asm__ __volatile__(
195 " lsr lp_count, %7,4 \n"
197 "1: ld.ab %3, [%2, 4] \n"
198 "11: ld.ab %4, [%2, 4] \n"
199 "12: ld.ab %5, [%2, 4] \n"
200 "13: ld.ab %6, [%2, 4] \n"
201 " st.ab %3, [%1, 4] \n"
202 " st.ab %4, [%1, 4] \n"
203 " st.ab %5, [%1, 4] \n"
204 " st.ab %6, [%1, 4] \n"
207 " .section .fixup, \"ax\" \n"
211 " .section __ex_table, \"a\" \n"
218 : "+r" (res), "+r"(to), "+r"(from),
219 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
221 : "lp_count", "memory");
226 __asm__ __volatile__(
227 "14: ld.ab %3, [%2,4] \n"
228 "15: ld.ab %4, [%2,4] \n"
229 " st.ab %3, [%1,4] \n"
230 " st.ab %4, [%1,4] \n"
233 " .section .fixup, \"ax\" \n"
237 " .section __ex_table, \"a\" \n"
242 : "+r" (res), "+r"(to), "+r"(from),
243 "=r"(tmp1), "=r"(tmp2)
250 __asm__ __volatile__(
251 "16: ld.ab %3, [%2,4] \n"
252 " st.ab %3, [%1,4] \n"
255 " .section .fixup, \"ax\" \n"
259 " .section __ex_table, \"a\" \n"
263 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
270 __asm__ __volatile__(
271 "17: ldw.ab %3, [%2,2] \n"
272 " stw.ab %3, [%1,2] \n"
275 " .section .fixup, \"ax\" \n"
279 " .section __ex_table, \"a\" \n"
283 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
288 __asm__ __volatile__(
289 "18: ldb.ab %3, [%2,2] \n"
290 " stb.ab %3, [%1,2] \n"
293 " .section .fixup, \"ax\" \n"
297 " .section __ex_table, \"a\" \n"
301 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
305 } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
307 __asm__ __volatile__(
309 " lsr.f lp_count, %3,4 \n" /* 16x bytes */
311 "1: ld.ab %5, [%2, 4] \n"
312 "11: ld.ab %6, [%2, 4] \n"
313 "12: ld.ab %7, [%2, 4] \n"
314 "13: ld.ab %8, [%2, 4] \n"
315 " st.ab %5, [%1, 4] \n"
316 " st.ab %6, [%1, 4] \n"
317 " st.ab %7, [%1, 4] \n"
318 " st.ab %8, [%1, 4] \n"
320 "3: and.f %3,%3,0xf \n" /* stragglers */
322 " bbit0 %3,3,31f \n" /* 8 bytes left */
323 "14: ld.ab %5, [%2,4] \n"
324 "15: ld.ab %6, [%2,4] \n"
325 " st.ab %5, [%1,4] \n"
326 " st.ab %6, [%1,4] \n"
328 "31: bbit0 %3,2,32f \n" /* 4 bytes left */
329 "16: ld.ab %5, [%2,4] \n"
330 " st.ab %5, [%1,4] \n"
332 "32: bbit0 %3,1,33f \n" /* 2 bytes left */
333 "17: ldw.ab %5, [%2,2] \n"
334 " stw.ab %5, [%1,2] \n"
336 "33: bbit0 %3,0,34f \n"
337 "18: ldb.ab %5, [%2,1] \n" /* 1 byte left */
338 " stb.ab %5, [%1,1] \n"
341 " .section .fixup, \"ax\" \n"
345 " .section __ex_table, \"a\" \n"
357 : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
358 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
360 : "lp_count", "memory");
366 static inline unsigned long
367 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
371 unsigned long tmp1, tmp2, tmp3, tmp4;
372 unsigned long orig_n = n;
377 /* fallback for unaligned access when hardware doesn't support */
378 if (!IS_ENABLED(CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS) &&
379 (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3))) {
383 __asm__ __volatile__(
384 " mov.f lp_count, %0 \n"
386 " ldb.ab %1, [%3, 1] \n"
387 "1: stb.ab %1, [%2, 1] \n"
390 " .section .fixup, \"ax\" \n"
394 " .section __ex_table, \"a\" \n"
400 /* Note as an '&' earlyclobber operand to make sure the
401 * temporary register inside the loop is not the same as
404 "=&r" (tmp), "+r" (to), "+r" (from)
406 : "lp_count", "memory");
411 if (__builtin_constant_p(orig_n)) {
415 orig_n = orig_n % 16;
417 __asm__ __volatile__(
418 " lsr lp_count, %7,4 \n"
420 " ld.ab %3, [%2, 4] \n"
421 " ld.ab %4, [%2, 4] \n"
422 " ld.ab %5, [%2, 4] \n"
423 " ld.ab %6, [%2, 4] \n"
424 "1: st.ab %3, [%1, 4] \n"
425 "11: st.ab %4, [%1, 4] \n"
426 "12: st.ab %5, [%1, 4] \n"
427 "13: st.ab %6, [%1, 4] \n"
430 " .section .fixup, \"ax\" \n"
434 " .section __ex_table, \"a\" \n"
441 : "+r" (res), "+r"(to), "+r"(from),
442 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
444 : "lp_count", "memory");
449 __asm__ __volatile__(
450 " ld.ab %3, [%2,4] \n"
451 " ld.ab %4, [%2,4] \n"
452 "14: st.ab %3, [%1,4] \n"
453 "15: st.ab %4, [%1,4] \n"
456 " .section .fixup, \"ax\" \n"
460 " .section __ex_table, \"a\" \n"
465 : "+r" (res), "+r"(to), "+r"(from),
466 "=r"(tmp1), "=r"(tmp2)
473 __asm__ __volatile__(
474 " ld.ab %3, [%2,4] \n"
475 "16: st.ab %3, [%1,4] \n"
478 " .section .fixup, \"ax\" \n"
482 " .section __ex_table, \"a\" \n"
486 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
493 __asm__ __volatile__(
494 " ldw.ab %3, [%2,2] \n"
495 "17: stw.ab %3, [%1,2] \n"
498 " .section .fixup, \"ax\" \n"
502 " .section __ex_table, \"a\" \n"
506 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
511 __asm__ __volatile__(
512 " ldb.ab %3, [%2,1] \n"
513 "18: stb.ab %3, [%1,1] \n"
516 " .section .fixup, \"ax\" \n"
520 " .section __ex_table, \"a\" \n"
524 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
528 } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
530 __asm__ __volatile__(
532 " lsr.f lp_count, %3,4 \n" /* 16x bytes */
534 " ld.ab %5, [%2, 4] \n"
535 " ld.ab %6, [%2, 4] \n"
536 " ld.ab %7, [%2, 4] \n"
537 " ld.ab %8, [%2, 4] \n"
538 "1: st.ab %5, [%1, 4] \n"
539 "11: st.ab %6, [%1, 4] \n"
540 "12: st.ab %7, [%1, 4] \n"
541 "13: st.ab %8, [%1, 4] \n"
543 "3: and.f %3,%3,0xf \n" /* stragglers */
545 " bbit0 %3,3,31f \n" /* 8 bytes left */
546 " ld.ab %5, [%2,4] \n"
547 " ld.ab %6, [%2,4] \n"
548 "14: st.ab %5, [%1,4] \n"
549 "15: st.ab %6, [%1,4] \n"
550 " sub.f %0, %0, 8 \n"
551 "31: bbit0 %3,2,32f \n" /* 4 bytes left */
552 " ld.ab %5, [%2,4] \n"
553 "16: st.ab %5, [%1,4] \n"
554 " sub.f %0, %0, 4 \n"
555 "32: bbit0 %3,1,33f \n" /* 2 bytes left */
556 " ldw.ab %5, [%2,2] \n"
557 "17: stw.ab %5, [%1,2] \n"
558 " sub.f %0, %0, 2 \n"
559 "33: bbit0 %3,0,34f \n"
560 " ldb.ab %5, [%2,1] \n" /* 1 byte left */
561 "18: stb.ab %5, [%1,1] \n"
562 " sub.f %0, %0, 1 \n"
564 " .section .fixup, \"ax\" \n"
568 " .section __ex_table, \"a\" \n"
580 : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
581 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
583 : "lp_count", "memory");
589 static inline unsigned long __clear_user(void __user *to, unsigned long n)
592 unsigned char *d_char = to;
594 __asm__ __volatile__(
595 " bbit0 %0, 0, 1f \n"
596 "75: stb.ab %2, [%0,1] \n"
598 "1: bbit0 %0, 1, 2f \n"
599 "76: stw.ab %2, [%0,2] \n"
601 "2: asr.f lp_count, %1, 2 \n"
603 "77: st.ab %2, [%0,4] \n"
605 "3: bbit0 %1, 1, 4f \n"
606 "78: stw.ab %2, [%0,2] \n"
608 "4: bbit0 %1, 0, 5f \n"
609 "79: stb.ab %2, [%0,1] \n"
612 " .section .fixup, \"ax\" \n"
616 " .section __ex_table, \"a\" \n"
624 : "+r"(d_char), "+r"(res)
626 : "lp_count", "memory");
631 #define INLINE_COPY_TO_USER
632 #define INLINE_COPY_FROM_USER
634 #define __clear_user __clear_user
636 #include <asm-generic/uaccess.h>