2 * Copyright 2007-2008 Analog Devices Inc.
3 * Philippe Gerum <rpm@xenomai.org>
5 * Licensed under the GPL-2 or later.
8 #include <linux/linkage.h>
9 #include <asm/blackfin.h>
10 #include <asm/cache.h>
11 #include <asm/asm-offsets.h>
12 #include <asm/rwlock.h>
17 .macro coreslot_loadaddr reg:req
18 \reg\().l = _corelock;
19 \reg\().h = _corelock;
22 .macro safe_testset addr:req, scratch:req
33 * r0 = address of atomic data to flush and invalidate (32bit).
35 * Clear interrupts and return the old mask.
36 * We assume that no atomic data can span cachelines.
47 if cc jump .Ldone_corelock;
52 /* flush core internal write buffer before invalidate dcache */
57 ENDPROC(_get_core_lock)
60 * r0 = address of atomic data in uncacheable memory region (32bit).
62 * Clear interrupts and return the old mask.
66 ENTRY(_get_core_lock_noflush)
69 .Lretry_corelock_noflush:
71 if cc jump .Ldone_corelock_noflush;
73 jump .Lretry_corelock_noflush
74 .Ldone_corelock_noflush:
76 * SMP kgdb runs into dead loop without NOP here, when one core
77 * single steps over get_core_lock_noflush and the other executes
78 * get_core_lock as a slave node.
83 ENDPROC(_get_core_lock_noflush)
86 * r0 = interrupt mask to restore.
87 * r1 = address of atomic data to flush and invalidate (32bit).
89 * Interrupts are masked on entry (see _get_core_lock).
93 /* Write-through cache assumed, so no flush needed here. */
100 ENDPROC(_put_core_lock)
102 #ifdef __ARCH_SYNC_CORE_DCACHE
104 ENTRY(___raw_smp_mark_barrier_asm)
110 call _get_core_lock_noflush;
113 * Calculate current core mask
120 * Set bit of other cores in barrier mask. Don't change current core bit.
122 p1.l = _barrier_mask;
123 p1.h = _barrier_mask;
141 ENDPROC(___raw_smp_mark_barrier_asm)
143 ENTRY(___raw_smp_check_barrier_asm)
149 call _get_core_lock_noflush;
152 * Calculate current core mask
159 * Clear current core bit in barrier mask if it is set.
161 p1.l = _barrier_mask;
162 p1.h = _barrier_mask;
175 * Invalidate the entire D-cache of current core.
178 call _resync_core_dcache
190 ENDPROC(___raw_smp_check_barrier_asm)
194 * r1 = address of atomic data
196 * Clobbers: r2:0, p1:0
198 _start_lock_coherent:
206 * Determine whether the atomic data was previously
207 * owned by another CPU (=r6).
215 r1 >>= 28; /* CPU fingerprints are stored in the high nibble. */
223 * Release the core lock now, but keep IRQs disabled while we are
224 * performing the remaining housekeeping chores for the current CPU.
226 coreslot_loadaddr p0;
231 * If another CPU has owned the same atomic section before us,
232 * then our D-cached copy of the shared data protected by the
233 * current spin/write_lock may be obsolete.
236 if cc jump .Lcache_synced
239 * Invalidate the entire D-cache of the current core.
242 call _resync_core_dcache
254 * r1 = address of atomic data
256 * Clobbers: r2:0, p1:0
271 #endif /* __ARCH_SYNC_CORE_DCACHE */
274 * r0 = &spinlock->lock
276 * Clobbers: r3:0, p1:0
278 ENTRY(___raw_spin_is_locked_asm)
283 cc = bittst( r3, 0 );
290 ENDPROC(___raw_spin_is_locked_asm)
293 * r0 = &spinlock->lock
295 * Clobbers: r3:0, p1:0
297 ENTRY(___raw_spin_lock_asm)
304 cc = bittst( r2, 0 );
305 if cc jump .Lbusy_spinlock
306 #ifdef __ARCH_SYNC_CORE_DCACHE
308 bitset ( r2, 0 ); /* Raise the lock bit. */
310 call _start_lock_coherent
320 /* We don't touch the atomic area if busy, so that flush
321 will behave like nop in _put_core_lock. */
325 jump .Lretry_spinlock
326 ENDPROC(___raw_spin_lock_asm)
329 * r0 = &spinlock->lock
331 * Clobbers: r3:0, p1:0
333 ENTRY(___raw_spin_trylock_asm)
339 cc = bittst( r3, 0 );
340 if cc jump .Lfailed_trylock
341 #ifdef __ARCH_SYNC_CORE_DCACHE
342 bitset ( r3, 0 ); /* Raise the lock bit. */
344 call _start_lock_coherent
358 ENDPROC(___raw_spin_trylock_asm)
361 * r0 = &spinlock->lock
363 * Clobbers: r2:0, p1:0
365 ENTRY(___raw_spin_unlock_asm)
373 #ifdef __ARCH_SYNC_CORE_DCACHE
374 call _end_lock_coherent
380 ENDPROC(___raw_spin_unlock_asm)
385 * Clobbers: r2:0, p1:0
387 ENTRY(___raw_read_lock_asm)
396 if cc jump .Lrdlock_failed
398 #ifdef __ARCH_SYNC_CORE_DCACHE
399 call _start_lock_coherent
417 if cc jump .Lrdlock_wait;
419 ENDPROC(___raw_read_lock_asm)
424 * Clobbers: r3:0, p1:0
426 ENTRY(___raw_read_trylock_asm)
432 if cc jump .Lfailed_tryrdlock;
436 #ifdef __ARCH_SYNC_CORE_DCACHE
437 call _start_lock_coherent
450 ENDPROC(___raw_read_trylock_asm)
455 * Note: Processing controlled by a reader lock should not have
456 * any side-effect on cache issues with the other core, so we
457 * just release the core lock and exit (no _end_lock_coherent).
459 * Clobbers: r3:0, p1:0
461 ENTRY(___raw_read_unlock_asm)
472 ENDPROC(___raw_read_unlock_asm)
477 * Clobbers: r3:0, p1:0
479 ENTRY(___raw_write_lock_asm)
481 r3.l = lo(RW_LOCK_BIAS);
482 r3.h = hi(RW_LOCK_BIAS);
488 #ifdef __ARCH_SYNC_CORE_DCACHE
496 if !cc jump .Lwrlock_wait
499 #ifdef __ARCH_SYNC_CORE_DCACHE
500 call _start_lock_coherent
514 #ifdef __ARCH_SYNC_CORE_DCACHE
519 if !cc jump .Lwrlock_wait;
521 ENDPROC(___raw_write_lock_asm)
526 * Clobbers: r3:0, p1:0
528 ENTRY(___raw_write_trylock_asm)
533 r2.l = lo(RW_LOCK_BIAS);
534 r2.h = hi(RW_LOCK_BIAS);
536 if !cc jump .Lfailed_trywrlock;
537 #ifdef __ARCH_SYNC_CORE_DCACHE
545 #ifdef __ARCH_SYNC_CORE_DCACHE
546 call _start_lock_coherent
560 ENDPROC(___raw_write_trylock_asm)
565 * Clobbers: r3:0, p1:0
567 ENTRY(___raw_write_unlock_asm)
569 r3.l = lo(RW_LOCK_BIAS);
570 r3.h = hi(RW_LOCK_BIAS);
577 #ifdef __ARCH_SYNC_CORE_DCACHE
578 call _end_lock_coherent
584 ENDPROC(___raw_write_unlock_asm)
590 * ADD a signed value to a 32bit word and return the new value atomically.
591 * Clobbers: r3:0, p1:0
593 ENTRY(___raw_atomic_add_asm)
606 ENDPROC(___raw_atomic_add_asm)
612 * ADD a signed value to a 32bit word and return the old value atomically.
613 * Clobbers: r3:0, p1:0
615 ENTRY(___raw_atomic_xadd_asm)
628 ENDPROC(___raw_atomic_add_asm)
634 * AND the mask bits from a 32bit word and return the old 32bit value
636 * Clobbers: r3:0, p1:0
638 ENTRY(___raw_atomic_and_asm)
651 ENDPROC(___raw_atomic_and_asm)
657 * OR the mask bits into a 32bit word and return the old 32bit value
659 * Clobbers: r3:0, p1:0
661 ENTRY(___raw_atomic_or_asm)
674 ENDPROC(___raw_atomic_or_asm)
680 * XOR the mask bits with a 32bit word and return the old 32bit value
682 * Clobbers: r3:0, p1:0
684 ENTRY(___raw_atomic_xor_asm)
697 ENDPROC(___raw_atomic_xor_asm)
703 * Perform a logical AND between the mask bits and a 32bit word, and
704 * return the masked value. We need this on this architecture in
705 * order to invalidate the local cache before testing.
707 * Clobbers: r3:0, p1:0
709 ENTRY(___raw_atomic_test_asm)
712 r1 = -L1_CACHE_BYTES;
715 /* flush core internal write buffer before invalidate dcache */
722 ENDPROC(___raw_atomic_test_asm)
728 * Swap *ptr with value and return the old 32bit value atomically.
729 * Clobbers: r3:0, p1:0
731 #define __do_xchg(src, dst) \
735 call _get_core_lock; \
740 call _put_core_lock; \
745 ENTRY(___raw_xchg_1_asm)
746 __do_xchg(b[p1] (z), b[p1])
747 ENDPROC(___raw_xchg_1_asm)
749 ENTRY(___raw_xchg_2_asm)
750 __do_xchg(w[p1] (z), w[p1])
751 ENDPROC(___raw_xchg_2_asm)
753 ENTRY(___raw_xchg_4_asm)
754 __do_xchg([p1], [p1])
755 ENDPROC(___raw_xchg_4_asm)
762 * Swap *ptr with new if *ptr == old and return the previous *ptr
765 * Clobbers: r3:0, p1:0
767 #define __do_cmpxchg(src, dst) \
773 call _get_core_lock; \
780 call _put_core_lock; \
786 ENTRY(___raw_cmpxchg_1_asm)
787 __do_cmpxchg(b[p1] (z), b[p1])
788 ENDPROC(___raw_cmpxchg_1_asm)
790 ENTRY(___raw_cmpxchg_2_asm)
791 __do_cmpxchg(w[p1] (z), w[p1])
792 ENDPROC(___raw_cmpxchg_2_asm)
794 ENTRY(___raw_cmpxchg_4_asm)
795 __do_cmpxchg([p1], [p1])
796 ENDPROC(___raw_cmpxchg_4_asm)
802 * Set a bit in a 32bit word and return the old 32bit value atomically.
803 * Clobbers: r3:0, p1:0
805 ENTRY(___raw_bit_set_asm)
809 jump ___raw_atomic_or_asm
810 ENDPROC(___raw_bit_set_asm)
816 * Clear a bit in a 32bit word and return the old 32bit value atomically.
817 * Clobbers: r3:0, p1:0
819 ENTRY(___raw_bit_clear_asm)
823 jump ___raw_atomic_and_asm
824 ENDPROC(___raw_bit_clear_asm)
830 * Toggle a bit in a 32bit word and return the old 32bit value atomically.
831 * Clobbers: r3:0, p1:0
833 ENTRY(___raw_bit_toggle_asm)
837 jump ___raw_atomic_xor_asm
838 ENDPROC(___raw_bit_toggle_asm)
844 * Test-and-set a bit in a 32bit word and return the old bit value atomically.
845 * Clobbers: r3:0, p1:0
847 ENTRY(___raw_bit_test_set_asm)
850 call ___raw_bit_set_asm
861 ENDPROC(___raw_bit_test_set_asm)
867 * Test-and-clear a bit in a 32bit word and return the old bit value atomically.
868 * Clobbers: r3:0, p1:0
870 ENTRY(___raw_bit_test_clear_asm)
873 call ___raw_bit_clear_asm
884 ENDPROC(___raw_bit_test_clear_asm)
890 * Test-and-toggle a bit in a 32bit word,
891 * and return the old bit value atomically.
892 * Clobbers: r3:0, p1:0
894 ENTRY(___raw_bit_test_toggle_asm)
897 call ___raw_bit_toggle_asm
908 ENDPROC(___raw_bit_test_toggle_asm)
914 * Test a bit in a 32bit word and return its value.
915 * We need this on this architecture in order to invalidate
916 * the local cache before testing.
918 * Clobbers: r3:0, p1:0
920 ENTRY(___raw_bit_test_asm)
924 jump ___raw_atomic_test_asm
925 ENDPROC(___raw_bit_test_asm)
930 * Fetch and return an uncached 32bit value.
932 * Clobbers: r2:0, p1:0
934 ENTRY(___raw_uncached_fetch_asm)
936 r1 = -L1_CACHE_BYTES;
939 /* flush core internal write buffer before invalidate dcache */
945 ENDPROC(___raw_uncached_fetch_asm)