1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* 64-bit atomic xchg() and cmpxchg() definitions.
4 * Copyright (C) 1996, 1997, 2000 David S. Miller (davem@redhat.com)
7 #ifndef __ARCH_SPARC64_CMPXCHG__
8 #define __ARCH_SPARC64_CMPXCHG__
10 static inline unsigned long
11 __cmpxchg_u32(volatile int *m, int old, int new)
13 __asm__ __volatile__("cas [%2], %3, %0"
15 : "0" (new), "r" (m), "r" (old)
21 static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int val)
23 unsigned long tmp1, tmp2;
30 " bne,a,pn %%icc, 1b\n"
32 : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2)
38 static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long val)
40 unsigned long tmp1, tmp2;
45 " casx [%4], %2, %0\n"
47 " bne,a,pn %%xcc, 1b\n"
49 : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2)
55 #define arch_xchg(ptr,x) \
56 ({ __typeof__(*(ptr)) __ret; \
57 __ret = (__typeof__(*(ptr))) \
58 __arch_xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
62 void __xchg_called_with_bad_pointer(void);
65 * Use 4 byte cas instruction to achieve 2 byte xchg. Main logic
66 * here is to get the bit shift of the byte we are interested in.
67 * The XOR is handy for reversing the bits for big-endian byte order.
69 static inline unsigned long
70 xchg16(__volatile__ unsigned short *m, unsigned short val)
72 unsigned long maddr = (unsigned long)m;
73 int bit_shift = (((unsigned long)m & 2) ^ 2) << 3;
74 unsigned int mask = 0xffff << bit_shift;
75 unsigned int *ptr = (unsigned int *) (maddr & ~2);
76 unsigned int old32, new32, load32;
78 /* Read the old value */
83 new32 = (load32 & (~mask)) | val << bit_shift;
84 load32 = __cmpxchg_u32(ptr, old32, new32);
85 } while (load32 != old32);
87 return (load32 & mask) >> bit_shift;
90 static __always_inline unsigned long
91 __arch_xchg(unsigned long x, __volatile__ void * ptr, int size)
95 return xchg16(ptr, x);
97 return xchg32(ptr, x);
99 return xchg64(ptr, x);
101 __xchg_called_with_bad_pointer();
106 * Atomic compare and exchange. Compare OLD with MEM, if identical,
107 * store NEW in MEM. Return the initial value in MEM. Success is
108 * indicated by comparing RETURN with OLD.
111 #include <asm-generic/cmpxchg-local.h>
114 static inline unsigned long
115 __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
117 __asm__ __volatile__("casx [%2], %3, %0"
119 : "0" (new), "r" (m), "r" (old)
126 * Use 4 byte cas instruction to achieve 1 byte cmpxchg. Main logic
127 * here is to get the bit shift of the byte we are interested in.
128 * The XOR is handy for reversing the bits for big-endian byte order
130 static inline unsigned long
131 __cmpxchg_u8(volatile unsigned char *m, unsigned char old, unsigned char new)
133 unsigned long maddr = (unsigned long)m;
134 int bit_shift = (((unsigned long)m & 3) ^ 3) << 3;
135 unsigned int mask = 0xff << bit_shift;
136 unsigned int *ptr = (unsigned int *) (maddr & ~3);
137 unsigned int old32, new32, load;
138 unsigned int load32 = *ptr;
141 new32 = (load32 & ~mask) | (new << bit_shift);
142 old32 = (load32 & ~mask) | (old << bit_shift);
143 load32 = __cmpxchg_u32(ptr, old32, new32);
146 load = (load32 & mask) >> bit_shift;
147 } while (load == old);
152 /* This function doesn't exist, so you'll get a linker error
153 if something tries to do an invalid cmpxchg(). */
154 void __cmpxchg_called_with_bad_pointer(void);
156 static inline unsigned long
157 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
161 return __cmpxchg_u8(ptr, old, new);
163 return __cmpxchg_u32(ptr, old, new);
165 return __cmpxchg_u64(ptr, old, new);
167 __cmpxchg_called_with_bad_pointer();
171 #define arch_cmpxchg(ptr,o,n) \
173 __typeof__(*(ptr)) _o_ = (o); \
174 __typeof__(*(ptr)) _n_ = (n); \
175 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
176 (unsigned long)_n_, sizeof(*(ptr))); \
180 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
184 static inline unsigned long __cmpxchg_local(volatile void *ptr,
186 unsigned long new, int size)
190 case 8: return __cmpxchg(ptr, old, new, size);
192 return __generic_cmpxchg_local(ptr, old, new, size);
198 #define arch_cmpxchg_local(ptr, o, n) \
199 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \
200 (unsigned long)(n), sizeof(*(ptr))))
201 #define arch_cmpxchg64_local(ptr, o, n) \
203 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
204 arch_cmpxchg_local((ptr), (o), (n)); \
206 #define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n))
208 #endif /* __ARCH_SPARC64_CMPXCHG__ */