arch/s390/lib/mem.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * String handling functions.
   4  *
   5  * Copyright IBM Corp. 2012
   6  */
   7
   8 #include <linux/export.h>
   9 #include <linux/linkage.h>
  10 #include <asm/nospec-insn.h>
  11
  12         GEN_BR_THUNK %r14
  13
  14 /*
  15  * void *memmove(void *dest, const void *src, size_t n)
  16  */
  17 SYM_FUNC_START(__memmove)
  18         ltgr    %r4,%r4
  19         lgr     %r1,%r2
  20         jz      .Lmemmove_exit
  21         aghi    %r4,-1
  22         clgr    %r2,%r3
  23         jnh     .Lmemmove_forward
  24         la      %r5,1(%r4,%r3)
  25         clgr    %r2,%r5
  26         jl      .Lmemmove_reverse
  27 .Lmemmove_forward:
  28         srlg    %r0,%r4,8
  29         ltgr    %r0,%r0
  30         jz      .Lmemmove_forward_remainder
  31 .Lmemmove_forward_loop:
  32         mvc     0(256,%r1),0(%r3)
  33         la      %r1,256(%r1)
  34         la      %r3,256(%r3)
  35         brctg   %r0,.Lmemmove_forward_loop
  36 .Lmemmove_forward_remainder:
  37         larl    %r5,.Lmemmove_mvc
  38         ex      %r4,0(%r5)
  39 .Lmemmove_exit:
  40         BR_EX   %r14
  41 .Lmemmove_reverse:
  42         ic      %r0,0(%r4,%r3)
  43         stc     %r0,0(%r4,%r1)
  44         brctg   %r4,.Lmemmove_reverse
  45         ic      %r0,0(%r4,%r3)
  46         stc     %r0,0(%r4,%r1)
  47         BR_EX   %r14
  48 .Lmemmove_mvc:
  49         mvc     0(1,%r1),0(%r3)
  50 SYM_FUNC_END(__memmove)
  51 EXPORT_SYMBOL(__memmove)
  52
  53 SYM_FUNC_ALIAS(memmove, __memmove)
  54 EXPORT_SYMBOL(memmove)
  55
  56 /*
  57  * memset implementation
  58  *
  59  * This code corresponds to the C construct below. We do distinguish
  60  * between clearing (c == 0) and setting a memory array (c != 0) simply
  61  * because nearly all memset invocations in the kernel clear memory and
  62  * the xc instruction is preferred in such cases.
  63  *
  64  * void *memset(void *s, int c, size_t n)
  65  * {
  66  *      if (likely(c == 0))
  67  *              return __builtin_memset(s, 0, n);
  68  *      return __builtin_memset(s, c, n);
  69  * }
  70  */
  71 SYM_FUNC_START(__memset)
  72         ltgr    %r4,%r4
  73         jz      .Lmemset_exit
  74         ltgr    %r3,%r3
  75         jnz     .Lmemset_fill
  76         aghi    %r4,-1
  77         srlg    %r3,%r4,8
  78         ltgr    %r3,%r3
  79         lgr     %r1,%r2
  80         jz      .Lmemset_clear_remainder
  81 .Lmemset_clear_loop:
  82         xc      0(256,%r1),0(%r1)
  83         la      %r1,256(%r1)
  84         brctg   %r3,.Lmemset_clear_loop
  85 .Lmemset_clear_remainder:
  86         larl    %r3,.Lmemset_xc
  87         ex      %r4,0(%r3)
  88 .Lmemset_exit:
  89         BR_EX   %r14
  90 .Lmemset_fill:
  91         cghi    %r4,1
  92         lgr     %r1,%r2
  93         je      .Lmemset_fill_exit
  94         aghi    %r4,-2
  95         srlg    %r5,%r4,8
  96         ltgr    %r5,%r5
  97         jz      .Lmemset_fill_remainder
  98 .Lmemset_fill_loop:
  99         stc     %r3,0(%r1)
 100         mvc     1(255,%r1),0(%r1)
 101         la      %r1,256(%r1)
 102         brctg   %r5,.Lmemset_fill_loop
 103 .Lmemset_fill_remainder:
 104         stc     %r3,0(%r1)
 105         larl    %r5,.Lmemset_mvc
 106         ex      %r4,0(%r5)
 107         BR_EX   %r14
 108 .Lmemset_fill_exit:
 109         stc     %r3,0(%r1)
 110         BR_EX   %r14
 111 .Lmemset_xc:
 112         xc      0(1,%r1),0(%r1)
 113 .Lmemset_mvc:
 114         mvc     1(1,%r1),0(%r1)
 115 SYM_FUNC_END(__memset)
 116 EXPORT_SYMBOL(__memset)
 117
 118 SYM_FUNC_ALIAS(memset, __memset)
 119 EXPORT_SYMBOL(memset)
 120
 121 /*
 122  * memcpy implementation
 123  *
 124  * void *memcpy(void *dest, const void *src, size_t n)
 125  */
 126 SYM_FUNC_START(__memcpy)
 127         ltgr    %r4,%r4
 128         jz      .Lmemcpy_exit
 129         aghi    %r4,-1
 130         srlg    %r5,%r4,8
 131         ltgr    %r5,%r5
 132         lgr     %r1,%r2
 133         jnz     .Lmemcpy_loop
 134 .Lmemcpy_remainder:
 135         larl    %r5,.Lmemcpy_mvc
 136         ex      %r4,0(%r5)
 137 .Lmemcpy_exit:
 138         BR_EX   %r14
 139 .Lmemcpy_loop:
 140         mvc     0(256,%r1),0(%r3)
 141         la      %r1,256(%r1)
 142         la      %r3,256(%r3)
 143         brctg   %r5,.Lmemcpy_loop
 144         j       .Lmemcpy_remainder
 145 .Lmemcpy_mvc:
 146         mvc     0(1,%r1),0(%r3)
 147 SYM_FUNC_END(__memcpy)
 148 EXPORT_SYMBOL(__memcpy)
 149
 150 SYM_FUNC_ALIAS(memcpy, __memcpy)
 151 EXPORT_SYMBOL(memcpy)
 152
 153 /*
 154  * __memset16/32/64
 155  *
 156  * void *__memset16(uint16_t *s, uint16_t v, size_t count)
 157  * void *__memset32(uint32_t *s, uint32_t v, size_t count)
 158  * void *__memset64(uint64_t *s, uint64_t v, size_t count)
 159  */
 160 .macro __MEMSET bits,bytes,insn
 161 SYM_FUNC_START(__memset\bits)
 162         ltgr    %r4,%r4
 163         jz      .L__memset_exit\bits
 164         cghi    %r4,\bytes
 165         je      .L__memset_store\bits
 166         aghi    %r4,-(\bytes+1)
 167         srlg    %r5,%r4,8
 168         ltgr    %r5,%r5
 169         lgr     %r1,%r2
 170         jz      .L__memset_remainder\bits
 171 .L__memset_loop\bits:
 172         \insn   %r3,0(%r1)
 173         mvc     \bytes(256-\bytes,%r1),0(%r1)
 174         la      %r1,256(%r1)
 175         brctg   %r5,.L__memset_loop\bits
 176 .L__memset_remainder\bits:
 177         \insn   %r3,0(%r1)
 178         larl    %r5,.L__memset_mvc\bits
 179         ex      %r4,0(%r5)
 180         BR_EX   %r14
 181 .L__memset_store\bits:
 182         \insn   %r3,0(%r2)
 183 .L__memset_exit\bits:
 184         BR_EX   %r14
 185 .L__memset_mvc\bits:
 186         mvc     \bytes(1,%r1),0(%r1)
 187 SYM_FUNC_END(__memset\bits)
 188 .endm
 189
 190 __MEMSET 16,2,sth
 191 EXPORT_SYMBOL(__memset16)
 192
 193 __MEMSET 32,4,st
 194 EXPORT_SYMBOL(__memset32)
 195
 196 __MEMSET 64,8,stg
 197 EXPORT_SYMBOL(__memset64)