2 * arch/alpha/lib/ev6-memset.S
4 * This is an efficient (and relatively small) implementation of the C library
5 * "memset()" function for the 21264 implementation of Alpha.
7 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
9 * Much of the information about 21264 scheduling/coding comes from:
10 * Compiler Writer's Guide for the Alpha 21264
11 * abbreviated as 'CWG' in other comments here
12 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
13 * Scheduling notation:
15 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
16 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
17 * The algorithm for the leading and trailing quadwords remains the same,
18 * however the loop has been unrolled to enable better memory throughput,
19 * and the code has been replicated for each of the entry points: __memset
20 * and __memsetw to permit better scheduling to eliminate the stalling
21 * encountered during the mask replication.
22 * A future enhancement might be to put in a byte store loop for really
23 * small (say < 32 bytes) memset()s. Whether or not that change would be
24 * a win in the kernel would depend upon the contextual usage.
25 * WARNING: Maintaining this is going to be more work than the above version,
26 * as fixes will need to be made in multiple places. The performance gain
29 #include <asm/export.h>
37 .globl __constant_c_memset
46 * Serious stalling happens. The only way to mitigate this is to
47 * undertake a major re-write to interleave the constant materialization
48 * with other parts of the fall-through code. This is important, even
49 * though it makes maintenance tougher.
52 and $17,255,$1 # E : 00000000000000ch
53 insbl $17,1,$2 # U : 000000000000ch00
54 bis $16,$16,$0 # E : return value
55 ble $18,end_b # U : zero length requested?
57 addq $18,$16,$6 # E : max address to write to
58 bis $1,$2,$17 # E : 000000000000chch
59 insbl $1,2,$3 # U : 0000000000ch0000
60 insbl $1,3,$4 # U : 00000000ch000000
62 or $3,$4,$3 # E : 00000000chch0000
63 inswl $17,4,$5 # U : 0000chch00000000
64 xor $16,$6,$1 # E : will complete write be within one quadword?
65 inswl $17,6,$2 # U : chch000000000000
67 or $17,$3,$17 # E : 00000000chchchch
68 or $2,$5,$2 # E : chchchch00000000
69 bic $1,7,$1 # E : fit within a single quadword?
70 and $16,7,$3 # E : Target addr misalignment
72 or $17,$2,$17 # E : chchchchchchchch
73 beq $1,within_quad_b # U :
75 beq $3,aligned_b # U : target is 0mod8
78 * Target address is misaligned, and won't fit within a quadword
80 ldq_u $4,0($16) # L : Fetch first partial
81 bis $16,$16,$5 # E : Save the address
82 insql $17,$16,$2 # U : Insert new bytes
83 subq $3,8,$3 # E : Invert (for addressing uses)
85 addq $18,$3,$18 # E : $18 is new count ($3 is negative)
86 mskql $4,$16,$4 # U : clear relevant parts of the quad
87 subq $16,$3,$16 # E : $16 is new aligned destination
88 bis $2,$4,$1 # E : Final bytes
91 stq_u $1,0($5) # L : Store result
98 * We are now guaranteed to be quad aligned, with at least
99 * one partial quad to write.
102 sra $18,3,$3 # U : Number of remaining quads to write
103 and $18,7,$18 # E : Number of trailing bytes to write
104 bis $16,$16,$5 # E : Save dest address
105 beq $3,no_quad_b # U : tail stuff only
108 * it's worth the effort to unroll this and use wh64 if possible
109 * Lifted a bunch of code from clear_user.S
110 * At this point, entry values are:
111 * $16 Current destination address
113 * $6 The max quadword address to write to
114 * $18 Number trailer bytes
115 * $3 Number quads to write
118 and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop)
119 subq $3, 16, $4 # E : Only try to unroll if > 128 bytes
120 subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64)
124 * We know we've got at least 16 quads, minimum of one trip
125 * through unrolled loop. Do a quad at a time to get us 0mod64
132 beq $1, $bigalign_b # U :
136 subq $3, 1, $3 # E : For consistency later
137 addq $1, 8, $1 # E : Increment towards zero for alignment
138 addq $5, 8, $4 # E : Initial wh64 address (filler instruction)
142 addq $5, 8, $5 # E : Inc address
143 blt $1, $alignmod64_b # U :
147 * $3 - number quads left to go
148 * $5 - target address (aligned 0mod64)
149 * $17 - mask of stuff to store
150 * Scratch registers available: $7, $2, $4, $1
151 * we know that we'll be taking a minimum of one trip through
152 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
153 * Assumes the wh64 needs to be for 2 trips through the loop in the future
154 * The wh64 is issued on for the starting destination address for trip +2
155 * through the loop, and if there are less than two trips left, the target
156 * address will be for the current trip.
160 wh64 ($4) # L1 : memory subsystem write hint
161 subq $3, 24, $2 # E : For determining future wh64 addresses
165 addq $5, 128, $4 # E : speculative target of next wh64
167 stq $17, 16($5) # L :
168 addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr)
170 stq $17, 24($5) # L :
171 stq $17, 32($5) # L :
172 cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle
175 stq $17, 40($5) # L :
176 stq $17, 48($5) # L :
177 subq $3, 16, $2 # E : Repeat the loop at least once more?
180 stq $17, 56($5) # L :
181 addq $5, 64, $5 # E :
183 bge $2, $do_wh64_b # U :
188 beq $3, no_quad_b # U : Might have finished already
192 * Simple loop for trailing quadwords, or for small amounts
193 * of data (where we can't use an unrolled loop and wh64)
197 subq $3,1,$3 # E : Decrement number quads left
198 addq $5,8,$5 # E : Inc address
199 bne $3,loop_b # U : more?
203 * Write 0..7 trailing bytes.
206 beq $18,end_b # U : All done?
208 mskqh $7,$6,$2 # U : Mask final quad
210 insqh $17,$6,$4 # U : New bits
211 bis $2,$4,$1 # E : Put it all together
212 stq $1,0($5) # L : And back to memory
213 ret $31,($26),1 # L0 :
216 ldq_u $1,0($16) # L :
217 insql $17,$16,$2 # U : New bits
218 mskql $1,$16,$4 # U : Clear old
219 bis $2,$4,$2 # E : New result
224 stq_u $1,0($16) # L :
230 ret $31,($26),1 # L0 :
232 EXPORT_SYMBOL(___memset)
235 * This is the original body of code, prior to replication and
236 * rescheduling. Leave it here, as there may be calls to this
240 .ent __constant_c_memset
245 addq $18,$16,$6 # E : max address to write to
246 bis $16,$16,$0 # E : return value
247 xor $16,$6,$1 # E : will complete write be within one quadword?
248 ble $18,end # U : zero length requested?
250 bic $1,7,$1 # E : fit within a single quadword
251 beq $1,within_one_quad # U :
252 and $16,7,$3 # E : Target addr misalignment
253 beq $3,aligned # U : target is 0mod8
256 * Target address is misaligned, and won't fit within a quadword
258 ldq_u $4,0($16) # L : Fetch first partial
259 bis $16,$16,$5 # E : Save the address
260 insql $17,$16,$2 # U : Insert new bytes
261 subq $3,8,$3 # E : Invert (for addressing uses)
263 addq $18,$3,$18 # E : $18 is new count ($3 is negative)
264 mskql $4,$16,$4 # U : clear relevant parts of the quad
265 subq $16,$3,$16 # E : $16 is new aligned destination
266 bis $2,$4,$1 # E : Final bytes
269 stq_u $1,0($5) # L : Store result
276 * We are now guaranteed to be quad aligned, with at least
277 * one partial quad to write.
280 sra $18,3,$3 # U : Number of remaining quads to write
281 and $18,7,$18 # E : Number of trailing bytes to write
282 bis $16,$16,$5 # E : Save dest address
283 beq $3,no_quad # U : tail stuff only
286 * it's worth the effort to unroll this and use wh64 if possible
287 * Lifted a bunch of code from clear_user.S
288 * At this point, entry values are:
289 * $16 Current destination address
291 * $6 The max quadword address to write to
292 * $18 Number trailer bytes
293 * $3 Number quads to write
296 and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop)
297 subq $3, 16, $4 # E : Only try to unroll if > 128 bytes
298 subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64)
302 * We know we've got at least 16 quads, minimum of one trip
303 * through unrolled loop. Do a quad at a time to get us 0mod64
310 beq $1, $bigalign # U :
314 subq $3, 1, $3 # E : For consistency later
315 addq $1, 8, $1 # E : Increment towards zero for alignment
316 addq $5, 8, $4 # E : Initial wh64 address (filler instruction)
320 addq $5, 8, $5 # E : Inc address
321 blt $1, $alignmod64 # U :
325 * $3 - number quads left to go
326 * $5 - target address (aligned 0mod64)
327 * $17 - mask of stuff to store
328 * Scratch registers available: $7, $2, $4, $1
329 * we know that we'll be taking a minimum of one trip through
330 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
331 * Assumes the wh64 needs to be for 2 trips through the loop in the future
332 * The wh64 is issued on for the starting destination address for trip +2
333 * through the loop, and if there are less than two trips left, the target
334 * address will be for the current trip.
338 wh64 ($4) # L1 : memory subsystem write hint
339 subq $3, 24, $2 # E : For determining future wh64 addresses
343 addq $5, 128, $4 # E : speculative target of next wh64
345 stq $17, 16($5) # L :
346 addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr)
348 stq $17, 24($5) # L :
349 stq $17, 32($5) # L :
350 cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle
353 stq $17, 40($5) # L :
354 stq $17, 48($5) # L :
355 subq $3, 16, $2 # E : Repeat the loop at least once more?
358 stq $17, 56($5) # L :
359 addq $5, 64, $5 # E :
361 bge $2, $do_wh64 # U :
366 beq $3, no_quad # U : Might have finished already
370 * Simple loop for trailing quadwords, or for small amounts
371 * of data (where we can't use an unrolled loop and wh64)
375 subq $3,1,$3 # E : Decrement number quads left
376 addq $5,8,$5 # E : Inc address
377 bne $3,loop # U : more?
381 * Write 0..7 trailing bytes.
384 beq $18,end # U : All done?
386 mskqh $7,$6,$2 # U : Mask final quad
388 insqh $17,$6,$4 # U : New bits
389 bis $2,$4,$1 # E : Put it all together
390 stq $1,0($5) # L : And back to memory
391 ret $31,($26),1 # L0 :
394 ldq_u $1,0($16) # L :
395 insql $17,$16,$2 # U : New bits
396 mskql $1,$16,$4 # U : Clear old
397 bis $2,$4,$2 # E : New result
402 stq_u $1,0($16) # L :
408 ret $31,($26),1 # L0 :
409 .end __constant_c_memset
410 EXPORT_SYMBOL(__constant_c_memset)
413 * This is a replicant of the __constant_c_memset code, rescheduled
414 * to mask stalls. Note that entry point names also had to change
423 inswl $17,0,$5 # U : 000000000000c1c2
424 inswl $17,2,$2 # U : 00000000c1c20000
425 bis $16,$16,$0 # E : return value
426 addq $18,$16,$6 # E : max address to write to
428 ble $18, end_w # U : zero length requested?
429 inswl $17,4,$3 # U : 0000c1c200000000
430 inswl $17,6,$4 # U : c1c2000000000000
431 xor $16,$6,$1 # E : will complete write be within one quadword?
433 or $2,$5,$2 # E : 00000000c1c2c1c2
434 or $3,$4,$17 # E : c1c2c1c200000000
435 bic $1,7,$1 # E : fit within a single quadword
436 and $16,7,$3 # E : Target addr misalignment
438 or $17,$2,$17 # E : c1c2c1c2c1c2c1c2
439 beq $1,within_quad_w # U :
441 beq $3,aligned_w # U : target is 0mod8
444 * Target address is misaligned, and won't fit within a quadword
446 ldq_u $4,0($16) # L : Fetch first partial
447 bis $16,$16,$5 # E : Save the address
448 insql $17,$16,$2 # U : Insert new bytes
449 subq $3,8,$3 # E : Invert (for addressing uses)
451 addq $18,$3,$18 # E : $18 is new count ($3 is negative)
452 mskql $4,$16,$4 # U : clear relevant parts of the quad
453 subq $16,$3,$16 # E : $16 is new aligned destination
454 bis $2,$4,$1 # E : Final bytes
457 stq_u $1,0($5) # L : Store result
464 * We are now guaranteed to be quad aligned, with at least
465 * one partial quad to write.
468 sra $18,3,$3 # U : Number of remaining quads to write
469 and $18,7,$18 # E : Number of trailing bytes to write
470 bis $16,$16,$5 # E : Save dest address
471 beq $3,no_quad_w # U : tail stuff only
474 * it's worth the effort to unroll this and use wh64 if possible
475 * Lifted a bunch of code from clear_user.S
476 * At this point, entry values are:
477 * $16 Current destination address
479 * $6 The max quadword address to write to
480 * $18 Number trailer bytes
481 * $3 Number quads to write
484 and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop)
485 subq $3, 16, $4 # E : Only try to unroll if > 128 bytes
486 subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64)
490 * We know we've got at least 16 quads, minimum of one trip
491 * through unrolled loop. Do a quad at a time to get us 0mod64
498 beq $1, $bigalign_w # U :
502 subq $3, 1, $3 # E : For consistency later
503 addq $1, 8, $1 # E : Increment towards zero for alignment
504 addq $5, 8, $4 # E : Initial wh64 address (filler instruction)
508 addq $5, 8, $5 # E : Inc address
509 blt $1, $alignmod64_w # U :
513 * $3 - number quads left to go
514 * $5 - target address (aligned 0mod64)
515 * $17 - mask of stuff to store
516 * Scratch registers available: $7, $2, $4, $1
517 * we know that we'll be taking a minimum of one trip through
518 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
519 * Assumes the wh64 needs to be for 2 trips through the loop in the future
520 * The wh64 is issued on for the starting destination address for trip +2
521 * through the loop, and if there are less than two trips left, the target
522 * address will be for the current trip.
526 wh64 ($4) # L1 : memory subsystem write hint
527 subq $3, 24, $2 # E : For determining future wh64 addresses
531 addq $5, 128, $4 # E : speculative target of next wh64
533 stq $17, 16($5) # L :
534 addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr)
536 stq $17, 24($5) # L :
537 stq $17, 32($5) # L :
538 cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle
541 stq $17, 40($5) # L :
542 stq $17, 48($5) # L :
543 subq $3, 16, $2 # E : Repeat the loop at least once more?
546 stq $17, 56($5) # L :
547 addq $5, 64, $5 # E :
549 bge $2, $do_wh64_w # U :
554 beq $3, no_quad_w # U : Might have finished already
558 * Simple loop for trailing quadwords, or for small amounts
559 * of data (where we can't use an unrolled loop and wh64)
563 subq $3,1,$3 # E : Decrement number quads left
564 addq $5,8,$5 # E : Inc address
565 bne $3,loop_w # U : more?
569 * Write 0..7 trailing bytes.
572 beq $18,end_w # U : All done?
574 mskqh $7,$6,$2 # U : Mask final quad
576 insqh $17,$6,$4 # U : New bits
577 bis $2,$4,$1 # E : Put it all together
578 stq $1,0($5) # L : And back to memory
579 ret $31,($26),1 # L0 :
582 ldq_u $1,0($16) # L :
583 insql $17,$16,$2 # U : New bits
584 mskql $1,$16,$4 # U : Clear old
585 bis $2,$4,$2 # E : New result
590 stq_u $1,0($16) # L :
596 ret $31,($26),1 # L0 :
599 EXPORT_SYMBOL(__memsetw)
603 EXPORT_SYMBOL(memset)
604 EXPORT_SYMBOL(__memset)