1 #ifndef _ASM_GENERIC_PERCPU_H_
2 #define _ASM_GENERIC_PERCPU_H_
4 #include <linux/compiler.h>
5 #include <linux/threads.h>
6 #include <linux/percpu-defs.h>
11 * per_cpu_offset() is the offset that has to be added to a
12 * percpu variable to get to the instance for a certain processor.
14 * Most arches use the __per_cpu_offset array for those offsets but
15 * some arches have their own ways of determining the offset (x86_64, s390).
17 #ifndef __per_cpu_offset
18 extern unsigned long __per_cpu_offset[NR_CPUS];
20 #define per_cpu_offset(x) (__per_cpu_offset[x])
24 * Determine the offset for the currently active processor.
25 * An arch may define __my_cpu_offset to provide a more effective
26 * means of obtaining the offset to the per cpu variables of the
29 #ifndef __my_cpu_offset
30 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
32 #ifdef CONFIG_DEBUG_PREEMPT
33 #define my_cpu_offset per_cpu_offset(smp_processor_id())
35 #define my_cpu_offset __my_cpu_offset
39 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
40 * translations for raw_cpu_ptr().
42 #ifndef arch_raw_cpu_ptr
43 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
46 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
47 extern void setup_per_cpu_areas(void);
52 #ifndef PER_CPU_BASE_SECTION
54 #define PER_CPU_BASE_SECTION ".data..percpu"
56 #define PER_CPU_BASE_SECTION ".data"
60 #ifndef PER_CPU_ATTRIBUTES
61 #define PER_CPU_ATTRIBUTES
64 #ifndef PER_CPU_DEF_ATTRIBUTES
65 #define PER_CPU_DEF_ATTRIBUTES
68 #define raw_cpu_generic_read(pcp) \
70 *raw_cpu_ptr(&(pcp)); \
73 #define raw_cpu_generic_to_op(pcp, val, op) \
75 *raw_cpu_ptr(&(pcp)) op val; \
78 #define raw_cpu_generic_add_return(pcp, val) \
80 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
86 #define raw_cpu_generic_xchg(pcp, nval) \
88 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
95 #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
97 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
100 if (__ret == (oval)) \
105 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
107 typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1)); \
108 typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2)); \
110 if (*__p1 == (oval1) && *__p2 == (oval2)) { \
118 #define __this_cpu_generic_read_nopreempt(pcp) \
121 preempt_disable_notrace(); \
122 __ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \
123 preempt_enable_notrace(); \
127 #define __this_cpu_generic_read_noirq(pcp) \
130 unsigned long __flags; \
131 raw_local_irq_save(__flags); \
132 __ret = raw_cpu_generic_read(pcp); \
133 raw_local_irq_restore(__flags); \
137 #define this_cpu_generic_read(pcp) \
140 if (__native_word(pcp)) \
141 __ret = __this_cpu_generic_read_nopreempt(pcp); \
143 __ret = __this_cpu_generic_read_noirq(pcp); \
147 #define this_cpu_generic_to_op(pcp, val, op) \
149 unsigned long __flags; \
150 raw_local_irq_save(__flags); \
151 raw_cpu_generic_to_op(pcp, val, op); \
152 raw_local_irq_restore(__flags); \
156 #define this_cpu_generic_add_return(pcp, val) \
159 unsigned long __flags; \
160 raw_local_irq_save(__flags); \
161 __ret = raw_cpu_generic_add_return(pcp, val); \
162 raw_local_irq_restore(__flags); \
166 #define this_cpu_generic_xchg(pcp, nval) \
169 unsigned long __flags; \
170 raw_local_irq_save(__flags); \
171 __ret = raw_cpu_generic_xchg(pcp, nval); \
172 raw_local_irq_restore(__flags); \
176 #define this_cpu_generic_cmpxchg(pcp, oval, nval) \
179 unsigned long __flags; \
180 raw_local_irq_save(__flags); \
181 __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
182 raw_local_irq_restore(__flags); \
186 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
189 unsigned long __flags; \
190 raw_local_irq_save(__flags); \
191 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
192 oval1, oval2, nval1, nval2); \
193 raw_local_irq_restore(__flags); \
197 #ifndef raw_cpu_read_1
198 #define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
200 #ifndef raw_cpu_read_2
201 #define raw_cpu_read_2(pcp) raw_cpu_generic_read(pcp)
203 #ifndef raw_cpu_read_4
204 #define raw_cpu_read_4(pcp) raw_cpu_generic_read(pcp)
206 #ifndef raw_cpu_read_8
207 #define raw_cpu_read_8(pcp) raw_cpu_generic_read(pcp)
210 #ifndef raw_cpu_write_1
211 #define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
213 #ifndef raw_cpu_write_2
214 #define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
216 #ifndef raw_cpu_write_4
217 #define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
219 #ifndef raw_cpu_write_8
220 #define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
223 #ifndef raw_cpu_add_1
224 #define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
226 #ifndef raw_cpu_add_2
227 #define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
229 #ifndef raw_cpu_add_4
230 #define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
232 #ifndef raw_cpu_add_8
233 #define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
236 #ifndef raw_cpu_and_1
237 #define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
239 #ifndef raw_cpu_and_2
240 #define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
242 #ifndef raw_cpu_and_4
243 #define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
245 #ifndef raw_cpu_and_8
246 #define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
250 #define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
253 #define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
256 #define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
259 #define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
262 #ifndef raw_cpu_add_return_1
263 #define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
265 #ifndef raw_cpu_add_return_2
266 #define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
268 #ifndef raw_cpu_add_return_4
269 #define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
271 #ifndef raw_cpu_add_return_8
272 #define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
275 #ifndef raw_cpu_xchg_1
276 #define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
278 #ifndef raw_cpu_xchg_2
279 #define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
281 #ifndef raw_cpu_xchg_4
282 #define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
284 #ifndef raw_cpu_xchg_8
285 #define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
288 #ifndef raw_cpu_cmpxchg_1
289 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
290 raw_cpu_generic_cmpxchg(pcp, oval, nval)
292 #ifndef raw_cpu_cmpxchg_2
293 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
294 raw_cpu_generic_cmpxchg(pcp, oval, nval)
296 #ifndef raw_cpu_cmpxchg_4
297 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
298 raw_cpu_generic_cmpxchg(pcp, oval, nval)
300 #ifndef raw_cpu_cmpxchg_8
301 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
302 raw_cpu_generic_cmpxchg(pcp, oval, nval)
305 #ifndef raw_cpu_cmpxchg_double_1
306 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
307 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
309 #ifndef raw_cpu_cmpxchg_double_2
310 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
311 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
313 #ifndef raw_cpu_cmpxchg_double_4
314 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
315 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
317 #ifndef raw_cpu_cmpxchg_double_8
318 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
319 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
322 #ifndef this_cpu_read_1
323 #define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
325 #ifndef this_cpu_read_2
326 #define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
328 #ifndef this_cpu_read_4
329 #define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
331 #ifndef this_cpu_read_8
332 #define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
335 #ifndef this_cpu_write_1
336 #define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
338 #ifndef this_cpu_write_2
339 #define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
341 #ifndef this_cpu_write_4
342 #define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
344 #ifndef this_cpu_write_8
345 #define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
348 #ifndef this_cpu_add_1
349 #define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
351 #ifndef this_cpu_add_2
352 #define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
354 #ifndef this_cpu_add_4
355 #define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
357 #ifndef this_cpu_add_8
358 #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
361 #ifndef this_cpu_and_1
362 #define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
364 #ifndef this_cpu_and_2
365 #define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
367 #ifndef this_cpu_and_4
368 #define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
370 #ifndef this_cpu_and_8
371 #define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
374 #ifndef this_cpu_or_1
375 #define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
377 #ifndef this_cpu_or_2
378 #define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
380 #ifndef this_cpu_or_4
381 #define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
383 #ifndef this_cpu_or_8
384 #define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
387 #ifndef this_cpu_add_return_1
388 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
390 #ifndef this_cpu_add_return_2
391 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
393 #ifndef this_cpu_add_return_4
394 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
396 #ifndef this_cpu_add_return_8
397 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
400 #ifndef this_cpu_xchg_1
401 #define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
403 #ifndef this_cpu_xchg_2
404 #define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
406 #ifndef this_cpu_xchg_4
407 #define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
409 #ifndef this_cpu_xchg_8
410 #define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
413 #ifndef this_cpu_cmpxchg_1
414 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
415 this_cpu_generic_cmpxchg(pcp, oval, nval)
417 #ifndef this_cpu_cmpxchg_2
418 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
419 this_cpu_generic_cmpxchg(pcp, oval, nval)
421 #ifndef this_cpu_cmpxchg_4
422 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
423 this_cpu_generic_cmpxchg(pcp, oval, nval)
425 #ifndef this_cpu_cmpxchg_8
426 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
427 this_cpu_generic_cmpxchg(pcp, oval, nval)
430 #ifndef this_cpu_cmpxchg_double_1
431 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
432 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
434 #ifndef this_cpu_cmpxchg_double_2
435 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
436 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
438 #ifndef this_cpu_cmpxchg_double_4
439 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
440 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
442 #ifndef this_cpu_cmpxchg_double_8
443 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
444 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
447 #endif /* _ASM_GENERIC_PERCPU_H_ */