2 * Memory copy functions for 32-bit PowerPC.
4 * Copyright (C) 1996-2005 Paul Mackerras.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
11 #include <asm/processor.h>
12 #include <asm/cache.h>
13 #include <asm/errno.h>
14 #include <asm/ppc_asm.h>
15 #include <asm/export.h>
17 #define COPY_16_BYTES \
27 #define COPY_16_BYTES_WITHEX(n) \
45 #define COPY_16_BYTES_EXCODE(n) \
47 addi r5,r5,-(16 * n); \
50 addi r5,r5,-(16 * n); \
52 .section __ex_table,"a"; \
54 .long 8 ## n ## 0b,9 ## n ## 0b; \
55 .long 8 ## n ## 1b,9 ## n ## 0b; \
56 .long 8 ## n ## 2b,9 ## n ## 0b; \
57 .long 8 ## n ## 3b,9 ## n ## 0b; \
58 .long 8 ## n ## 4b,9 ## n ## 1b; \
59 .long 8 ## n ## 5b,9 ## n ## 1b; \
60 .long 8 ## n ## 6b,9 ## n ## 1b; \
61 .long 8 ## n ## 7b,9 ## n ## 1b; \
65 .stabs "arch/powerpc/lib/",N_SO,0,0,0f
66 .stabs "copy_32.S",N_SO,0,0,0f
69 CACHELINE_BYTES = L1_CACHE_BYTES
70 LG_CACHELINE_BYTES = L1_CACHE_SHIFT
71 CACHELINE_MASK = (L1_CACHE_BYTES-1)
74 * Use dcbz on the complete cache lines in the destination
75 * to set them to zero. This requires that the destination
76 * area is cacheable. -- paulus
78 * During early init, cache might not be active yet, so dcbz cannot be used.
79 * We therefore skip the optimised bloc that uses dcbz. This jump is
80 * replaced by a nop once cache is active. This is done in machine_init()
95 bne 2f /* Use normal procedure if r4 is not zero */
97 _GLOBAL(memset_nocache_branch)
98 b 2f /* Skip optimised bloc until cache is enabled */
100 clrlwi r7,r6,32-LG_CACHELINE_BYTES
102 srwi r9,r8,LG_CACHELINE_BYTES
103 addic. r9,r9,-1 /* total number of complete cachelines */
105 xori r0,r7,CACHELINE_MASK & ~3
114 addi r6,r6,CACHELINE_BYTES
116 clrlwi r5,r8,32-LG_CACHELINE_BYTES
134 * This version uses dcbz on the complete cache lines in the
135 * destination area to reduce memory traffic. This requires that
136 * the destination area is cacheable.
137 * We only use this version if the source and dest don't overlap.
140 * During early init, cache might not be active yet, so dcbz cannot be used.
141 * We therefore jump to generic_memcpy which doesn't use dcbz. This jump is
142 * replaced by a nop once cache is active. This is done in machine_init()
151 add r7,r3,r5 /* test if the src & dst overlap */
155 crand 0,0,4 /* cr0.lt &= cr1.lt */
156 blt generic_memcpy /* if regions overlap */
161 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
164 cmplw 0,r5,r0 /* is this more than total to do? */
165 blt 63f /* if not much to do */
166 andi. r8,r0,3 /* get it word-aligned first */
170 70: lbz r9,4(r4) /* do some bytes */
178 72: lwzu r9,4(r4) /* do some words */
182 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
183 clrlwi r5,r5,32-LG_CACHELINE_BYTES
190 #if L1_CACHE_BYTES >= 32
192 #if L1_CACHE_BYTES >= 64
195 #if L1_CACHE_BYTES >= 128
221 EXPORT_SYMBOL(memcpy)
222 EXPORT_SYMBOL(memmove)
228 beq 2f /* if less than 8 bytes to do */
229 andi. r0,r6,3 /* get dest word aligned */
260 rlwinm. r7,r5,32-3,3,31
265 _GLOBAL(backwards_memcpy)
266 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
296 rlwinm. r7,r5,32-3,3,31
301 _GLOBAL(__copy_tofrom_user)
305 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
308 cmplw 0,r5,r0 /* is this more than total to do? */
309 blt 63f /* if not much to do */
310 andi. r8,r0,3 /* get it word-aligned first */
313 70: lbz r9,4(r4) /* do some bytes */
322 72: lwzu r9,4(r4) /* do some words */
326 .section __ex_table,"a"
334 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
335 clrlwi r5,r5,32-LG_CACHELINE_BYTES
339 /* Here we decide how far ahead to prefetch the source */
345 #if MAX_COPY_PREFETCH > 1
346 /* Heuristically, for large transfers we prefetch
347 MAX_COPY_PREFETCH cachelines ahead. For small transfers
348 we prefetch 1 cacheline ahead. */
349 cmpwi r0,MAX_COPY_PREFETCH
351 li r7,MAX_COPY_PREFETCH
354 addi r3,r3,CACHELINE_BYTES
358 addi r3,r3,CACHELINE_BYTES
359 #endif /* MAX_COPY_PREFETCH > 1 */
367 .section __ex_table,"a"
371 /* the main body of the cacheline loop */
372 COPY_16_BYTES_WITHEX(0)
373 #if L1_CACHE_BYTES >= 32
374 COPY_16_BYTES_WITHEX(1)
375 #if L1_CACHE_BYTES >= 64
376 COPY_16_BYTES_WITHEX(2)
377 COPY_16_BYTES_WITHEX(3)
378 #if L1_CACHE_BYTES >= 128
379 COPY_16_BYTES_WITHEX(4)
380 COPY_16_BYTES_WITHEX(5)
381 COPY_16_BYTES_WITHEX(6)
382 COPY_16_BYTES_WITHEX(7)
410 /* read fault, initial single-byte copy */
413 /* write fault, initial single-byte copy */
418 /* read fault, initial word copy */
421 /* write fault, initial word copy */
427 * this stuff handles faults in the cacheline loop and branches to either
428 * 104f (if in read part) or 105f (if in write part), after updating r5
430 COPY_16_BYTES_EXCODE(0)
431 #if L1_CACHE_BYTES >= 32
432 COPY_16_BYTES_EXCODE(1)
433 #if L1_CACHE_BYTES >= 64
434 COPY_16_BYTES_EXCODE(2)
435 COPY_16_BYTES_EXCODE(3)
436 #if L1_CACHE_BYTES >= 128
437 COPY_16_BYTES_EXCODE(4)
438 COPY_16_BYTES_EXCODE(5)
439 COPY_16_BYTES_EXCODE(6)
440 COPY_16_BYTES_EXCODE(7)
445 /* read fault in cacheline loop */
448 /* fault on dcbz (effectively a write fault) */
449 /* or write fault in cacheline loop */
451 92: li r3,LG_CACHELINE_BYTES
455 /* read fault in final word loop */
458 /* write fault in final word loop */
463 /* read fault in final byte loop */
466 /* write fault in final byte loop */
471 * At this stage the number of bytes not copied is
472 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
477 beq 120f /* shouldn't happen */
480 /* for a read fault, first try to continue the copy one byte at a time */
487 /* then clear out the destination: r3 bytes starting at 4(r6) */
503 .section __ex_table,"a"
514 EXPORT_SYMBOL(__copy_tofrom_user)