1 /* SPDX-License-Identifier: GPL-2.0 */
3 * __get_user functions.
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
18 * Inputs: %[r|e]ax contains the address.
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
29 #include <linux/export.h>
30 #include <linux/linkage.h>
31 #include <asm/page_types.h>
32 #include <asm/errno.h>
33 #include <asm/asm-offsets.h>
34 #include <asm/thread_info.h>
38 #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
40 .macro check_range size:req
41 .if IS_ENABLED(CONFIG_X86_64)
46 cmp $TASK_SIZE_MAX-\size+1, %eax
48 sbb %edx, %edx /* array_index_mask_nospec() */
54 SYM_FUNC_START(__get_user_1)
57 1: movzbl (%_ASM_AX),%edx
61 SYM_FUNC_END(__get_user_1)
62 EXPORT_SYMBOL(__get_user_1)
64 SYM_FUNC_START(__get_user_2)
67 2: movzwl (%_ASM_AX),%edx
71 SYM_FUNC_END(__get_user_2)
72 EXPORT_SYMBOL(__get_user_2)
74 SYM_FUNC_START(__get_user_4)
77 3: movl (%_ASM_AX),%edx
81 SYM_FUNC_END(__get_user_4)
82 EXPORT_SYMBOL(__get_user_4)
84 SYM_FUNC_START(__get_user_8)
88 4: movq (%_ASM_AX),%rdx
90 4: movl (%_ASM_AX),%edx
91 5: movl 4(%_ASM_AX),%ecx
96 SYM_FUNC_END(__get_user_8)
97 EXPORT_SYMBOL(__get_user_8)
99 /* .. and the same for __get_user, just without the range checks */
100 SYM_FUNC_START(__get_user_nocheck_1)
103 6: movzbl (%_ASM_AX),%edx
107 SYM_FUNC_END(__get_user_nocheck_1)
108 EXPORT_SYMBOL(__get_user_nocheck_1)
110 SYM_FUNC_START(__get_user_nocheck_2)
113 7: movzwl (%_ASM_AX),%edx
117 SYM_FUNC_END(__get_user_nocheck_2)
118 EXPORT_SYMBOL(__get_user_nocheck_2)
120 SYM_FUNC_START(__get_user_nocheck_4)
123 8: movl (%_ASM_AX),%edx
127 SYM_FUNC_END(__get_user_nocheck_4)
128 EXPORT_SYMBOL(__get_user_nocheck_4)
130 SYM_FUNC_START(__get_user_nocheck_8)
134 9: movq (%_ASM_AX),%rdx
136 9: movl (%_ASM_AX),%edx
137 10: movl 4(%_ASM_AX),%ecx
142 SYM_FUNC_END(__get_user_nocheck_8)
143 EXPORT_SYMBOL(__get_user_nocheck_8)
146 SYM_CODE_START_LOCAL(__get_user_handle_exception)
150 mov $(-EFAULT),%_ASM_AX
152 SYM_CODE_END(__get_user_handle_exception)
155 SYM_CODE_START_LOCAL(__get_user_8_handle_exception)
160 mov $(-EFAULT),%_ASM_AX
162 SYM_CODE_END(__get_user_8_handle_exception)
166 _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
167 _ASM_EXTABLE_UA(2b, __get_user_handle_exception)
168 _ASM_EXTABLE_UA(3b, __get_user_handle_exception)
170 _ASM_EXTABLE_UA(4b, __get_user_handle_exception)
172 _ASM_EXTABLE_UA(4b, __get_user_8_handle_exception)
173 _ASM_EXTABLE_UA(5b, __get_user_8_handle_exception)
177 _ASM_EXTABLE_UA(6b, __get_user_handle_exception)
178 _ASM_EXTABLE_UA(7b, __get_user_handle_exception)
179 _ASM_EXTABLE_UA(8b, __get_user_handle_exception)
181 _ASM_EXTABLE_UA(9b, __get_user_handle_exception)
183 _ASM_EXTABLE_UA(9b, __get_user_8_handle_exception)
184 _ASM_EXTABLE_UA(10b, __get_user_8_handle_exception)