1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Author: Huacai Chen <chenhuacai@loongson.cn>
4 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
9 #include <linux/sched.h>
10 #include <linux/sched/task_stack.h>
11 #include <linux/ptrace.h>
12 #include <linux/thread_info.h>
13 #include <linux/bitops.h>
16 #include <asm/cpu-features.h>
17 #include <asm/current.h>
18 #include <asm/loongarch.h>
19 #include <asm/processor.h>
20 #include <asm/ptrace.h>
24 extern void kernel_fpu_begin(void);
25 extern void kernel_fpu_end(void);
27 extern void _init_fpu(unsigned int);
28 extern void _save_fp(struct loongarch_fpu *);
29 extern void _restore_fp(struct loongarch_fpu *);
31 extern void _save_lsx(struct loongarch_fpu *fpu);
32 extern void _restore_lsx(struct loongarch_fpu *fpu);
33 extern void _init_lsx_upper(void);
34 extern void _restore_lsx_upper(struct loongarch_fpu *fpu);
36 extern void _save_lasx(struct loongarch_fpu *fpu);
37 extern void _restore_lasx(struct loongarch_fpu *fpu);
38 extern void _init_lasx_upper(void);
39 extern void _restore_lasx_upper(struct loongarch_fpu *fpu);
41 static inline void enable_lsx(void);
42 static inline void disable_lsx(void);
43 static inline void save_lsx(struct task_struct *t);
44 static inline void restore_lsx(struct task_struct *t);
46 static inline void enable_lasx(void);
47 static inline void disable_lasx(void);
48 static inline void save_lasx(struct task_struct *t);
49 static inline void restore_lasx(struct task_struct *t);
52 * Mask the FCSR Cause bits according to the Enable bits, observing
53 * that Unimplemented is always enabled.
55 static inline unsigned long mask_fcsr_x(unsigned long fcsr)
57 return fcsr & ((fcsr & FPU_CSR_ALL_E) <<
58 (ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E)));
61 static inline int is_fp_enabled(void)
63 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_FPEN) ?
67 static inline int is_lsx_enabled(void)
72 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ?
76 static inline int is_lasx_enabled(void)
81 return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ?
85 static inline int is_simd_enabled(void)
87 return is_lsx_enabled() | is_lasx_enabled();
90 #define enable_fpu() set_csr_euen(CSR_EUEN_FPEN)
92 #define disable_fpu() clear_csr_euen(CSR_EUEN_FPEN)
94 #define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU)
96 static inline int is_fpu_owner(void)
98 return test_thread_flag(TIF_USEDFPU);
101 static inline void __own_fpu(void)
104 set_thread_flag(TIF_USEDFPU);
105 KSTK_EUEN(current) |= CSR_EUEN_FPEN;
108 static inline void own_fpu_inatomic(int restore)
110 if (cpu_has_fpu && !is_fpu_owner()) {
113 _restore_fp(¤t->thread.fpu);
117 static inline void own_fpu(int restore)
120 own_fpu_inatomic(restore);
124 static inline void lose_fpu_inatomic(int save, struct task_struct *tsk)
126 if (is_fpu_owner()) {
127 if (!is_simd_enabled()) {
129 _save_fp(&tsk->thread.fpu);
133 if (!is_lasx_enabled())
141 clear_tsk_thread_flag(tsk, TIF_USEDSIMD);
143 clear_tsk_thread_flag(tsk, TIF_USEDFPU);
145 KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
148 static inline void lose_fpu(int save)
151 lose_fpu_inatomic(save, current);
155 static inline void init_fpu(void)
157 unsigned int fcsr = current->thread.fpu.fcsr;
164 static inline void save_fp(struct task_struct *tsk)
167 _save_fp(&tsk->thread.fpu);
170 static inline void restore_fp(struct task_struct *tsk)
173 _restore_fp(&tsk->thread.fpu);
176 static inline void save_fpu_regs(struct task_struct *tsk)
180 if (tsk == current) {
183 euen = csr_read32(LOONGARCH_CSR_EUEN);
185 #ifdef CONFIG_CPU_HAS_LASX
186 if (euen & CSR_EUEN_LASXEN)
187 _save_lasx(¤t->thread.fpu);
190 #ifdef CONFIG_CPU_HAS_LSX
191 if (euen & CSR_EUEN_LSXEN)
192 _save_lsx(¤t->thread.fpu);
195 if (euen & CSR_EUEN_FPEN)
196 _save_fp(¤t->thread.fpu);
202 static inline int is_simd_owner(void)
204 return test_thread_flag(TIF_USEDSIMD);
207 #ifdef CONFIG_CPU_HAS_LSX
209 static inline void enable_lsx(void)
212 csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN);
215 static inline void disable_lsx(void)
218 csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN);
221 static inline void save_lsx(struct task_struct *t)
224 _save_lsx(&t->thread.fpu);
227 static inline void restore_lsx(struct task_struct *t)
230 _restore_lsx(&t->thread.fpu);
233 static inline void init_lsx_upper(void)
239 static inline void restore_lsx_upper(struct task_struct *t)
242 _restore_lsx_upper(&t->thread.fpu);
246 static inline void enable_lsx(void) {}
247 static inline void disable_lsx(void) {}
248 static inline void save_lsx(struct task_struct *t) {}
249 static inline void restore_lsx(struct task_struct *t) {}
250 static inline void init_lsx_upper(void) {}
251 static inline void restore_lsx_upper(struct task_struct *t) {}
254 #ifdef CONFIG_CPU_HAS_LASX
256 static inline void enable_lasx(void)
260 csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN);
263 static inline void disable_lasx(void)
266 csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN);
269 static inline void save_lasx(struct task_struct *t)
272 _save_lasx(&t->thread.fpu);
275 static inline void restore_lasx(struct task_struct *t)
278 _restore_lasx(&t->thread.fpu);
281 static inline void init_lasx_upper(void)
287 static inline void restore_lasx_upper(struct task_struct *t)
290 _restore_lasx_upper(&t->thread.fpu);
294 static inline void enable_lasx(void) {}
295 static inline void disable_lasx(void) {}
296 static inline void save_lasx(struct task_struct *t) {}
297 static inline void restore_lasx(struct task_struct *t) {}
298 static inline void init_lasx_upper(void) {}
299 static inline void restore_lasx_upper(struct task_struct *t) {}
302 static inline int thread_lsx_context_live(void)
307 return test_thread_flag(TIF_LSX_CTX_LIVE);
310 static inline int thread_lasx_context_live(void)
315 return test_thread_flag(TIF_LASX_CTX_LIVE);
318 #endif /* _ASM_FPU_H */