arch/loongarch/kernel/kfpu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2023 Loongson Technology Corporation Limited
   4  */
   5
   6 #include <linux/cpu.h>
   7 #include <linux/init.h>
   8 #include <asm/fpu.h>
   9 #include <asm/smp.h>
  10
  11 static unsigned int euen_mask = CSR_EUEN_FPEN;
  12
  13 /*
  14  * The critical section between kernel_fpu_begin() and kernel_fpu_end()
  15  * is non-reentrant. It is the caller's responsibility to avoid reentrance.
  16  * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example.
  17  */
  18 static DEFINE_PER_CPU(bool, in_kernel_fpu);
  19 static DEFINE_PER_CPU(unsigned int, euen_current);
  20
  21 void kernel_fpu_begin(void)
  22 {
  23         unsigned int *euen_curr;
  24
  25         preempt_disable();
  26
  27         WARN_ON(this_cpu_read(in_kernel_fpu));
  28
  29         this_cpu_write(in_kernel_fpu, true);
  30         euen_curr = this_cpu_ptr(&euen_current);
  31
  32         *euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN);
  33
  34 #ifdef CONFIG_CPU_HAS_LASX
  35         if (*euen_curr & CSR_EUEN_LASXEN)
  36                 _save_lasx(&current->thread.fpu);
  37         else
  38 #endif
  39 #ifdef CONFIG_CPU_HAS_LSX
  40         if (*euen_curr & CSR_EUEN_LSXEN)
  41                 _save_lsx(&current->thread.fpu);
  42         else
  43 #endif
  44         if (*euen_curr & CSR_EUEN_FPEN)
  45                 _save_fp(&current->thread.fpu);
  46
  47         write_fcsr(LOONGARCH_FCSR0, 0);
  48 }
  49 EXPORT_SYMBOL_GPL(kernel_fpu_begin);
  50
  51 void kernel_fpu_end(void)
  52 {
  53         unsigned int *euen_curr;
  54
  55         WARN_ON(!this_cpu_read(in_kernel_fpu));
  56
  57         euen_curr = this_cpu_ptr(&euen_current);
  58
  59 #ifdef CONFIG_CPU_HAS_LASX
  60         if (*euen_curr & CSR_EUEN_LASXEN)
  61                 _restore_lasx(&current->thread.fpu);
  62         else
  63 #endif
  64 #ifdef CONFIG_CPU_HAS_LSX
  65         if (*euen_curr & CSR_EUEN_LSXEN)
  66                 _restore_lsx(&current->thread.fpu);
  67         else
  68 #endif
  69         if (*euen_curr & CSR_EUEN_FPEN)
  70                 _restore_fp(&current->thread.fpu);
  71
  72         *euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN);
  73
  74         this_cpu_write(in_kernel_fpu, false);
  75
  76         preempt_enable();
  77 }
  78 EXPORT_SYMBOL_GPL(kernel_fpu_end);
  79
  80 static int __init init_euen_mask(void)
  81 {
  82         if (cpu_has_lsx)
  83                 euen_mask |= CSR_EUEN_LSXEN;
  84
  85         if (cpu_has_lasx)
  86                 euen_mask |= CSR_EUEN_LASXEN;
  87
  88         return 0;
  89 }
  90 arch_initcall(init_euen_mask);