1 #ifndef _ASM_GENERIC_PERCPU_H_
2 #define _ASM_GENERIC_PERCPU_H_
4 #include <linux/compiler.h>
5 #include <linux/threads.h>
6 #include <linux/percpu-defs.h>
11 * per_cpu_offset() is the offset that has to be added to a
12 * percpu variable to get to the instance for a certain processor.
14 * Most arches use the __per_cpu_offset array for those offsets but
15 * some arches have their own ways of determining the offset (x86_64, s390).
17 #ifndef __per_cpu_offset
18 extern unsigned long __per_cpu_offset[NR_CPUS];
20 #define per_cpu_offset(x) (__per_cpu_offset[x])
24 * Determine the offset for the currently active processor.
25 * An arch may define __my_cpu_offset to provide a more effective
26 * means of obtaining the offset to the per cpu variables of the
29 #ifndef __my_cpu_offset
30 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
32 #ifdef CONFIG_DEBUG_PREEMPT
33 #define my_cpu_offset per_cpu_offset(smp_processor_id())
35 #define my_cpu_offset __my_cpu_offset
39 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
40 * translations for raw_cpu_ptr().
42 #ifndef arch_raw_cpu_ptr
43 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
46 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
47 extern void setup_per_cpu_areas(void);
52 #ifndef PER_CPU_BASE_SECTION
54 #define PER_CPU_BASE_SECTION ".data..percpu"
56 #define PER_CPU_BASE_SECTION ".data"
60 #ifndef PER_CPU_ATTRIBUTES
61 #define PER_CPU_ATTRIBUTES
64 #ifndef PER_CPU_DEF_ATTRIBUTES
65 #define PER_CPU_DEF_ATTRIBUTES
68 #define raw_cpu_generic_to_op(pcp, val, op) \
70 *raw_cpu_ptr(&(pcp)) op val; \
73 #define raw_cpu_generic_add_return(pcp, val) \
75 raw_cpu_add(pcp, val); \
79 #define raw_cpu_generic_xchg(pcp, nval) \
82 __ret = raw_cpu_read(pcp); \
83 raw_cpu_write(pcp, nval); \
87 #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
90 __ret = raw_cpu_read(pcp); \
91 if (__ret == (oval)) \
92 raw_cpu_write(pcp, nval); \
96 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
99 if (raw_cpu_read(pcp1) == (oval1) && \
100 raw_cpu_read(pcp2) == (oval2)) { \
101 raw_cpu_write(pcp1, nval1); \
102 raw_cpu_write(pcp2, nval2); \
108 #define __this_cpu_generic_read_nopreempt(pcp) \
112 __ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \
117 #define __this_cpu_generic_read_noirq(pcp) \
120 unsigned long __flags; \
121 raw_local_irq_save(__flags); \
122 __ret = *raw_cpu_ptr(&(pcp)); \
123 raw_local_irq_restore(__flags); \
127 #define this_cpu_generic_read(pcp) \
130 if (__native_word(pcp)) \
131 __ret = __this_cpu_generic_read_nopreempt(pcp); \
133 __ret = __this_cpu_generic_read_noirq(pcp); \
137 #define this_cpu_generic_to_op(pcp, val, op) \
139 unsigned long __flags; \
140 raw_local_irq_save(__flags); \
141 *raw_cpu_ptr(&(pcp)) op val; \
142 raw_local_irq_restore(__flags); \
145 #define this_cpu_generic_add_return(pcp, val) \
148 unsigned long __flags; \
149 raw_local_irq_save(__flags); \
150 raw_cpu_add(pcp, val); \
151 __ret = raw_cpu_read(pcp); \
152 raw_local_irq_restore(__flags); \
156 #define this_cpu_generic_xchg(pcp, nval) \
159 unsigned long __flags; \
160 raw_local_irq_save(__flags); \
161 __ret = raw_cpu_read(pcp); \
162 raw_cpu_write(pcp, nval); \
163 raw_local_irq_restore(__flags); \
167 #define this_cpu_generic_cmpxchg(pcp, oval, nval) \
170 unsigned long __flags; \
171 raw_local_irq_save(__flags); \
172 __ret = raw_cpu_read(pcp); \
173 if (__ret == (oval)) \
174 raw_cpu_write(pcp, nval); \
175 raw_local_irq_restore(__flags); \
179 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
182 unsigned long __flags; \
183 raw_local_irq_save(__flags); \
184 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
185 oval1, oval2, nval1, nval2); \
186 raw_local_irq_restore(__flags); \
190 #ifndef raw_cpu_read_1
191 #define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
193 #ifndef raw_cpu_read_2
194 #define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
196 #ifndef raw_cpu_read_4
197 #define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
199 #ifndef raw_cpu_read_8
200 #define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
203 #ifndef raw_cpu_write_1
204 #define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
206 #ifndef raw_cpu_write_2
207 #define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
209 #ifndef raw_cpu_write_4
210 #define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
212 #ifndef raw_cpu_write_8
213 #define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
216 #ifndef raw_cpu_add_1
217 #define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
219 #ifndef raw_cpu_add_2
220 #define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
222 #ifndef raw_cpu_add_4
223 #define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
225 #ifndef raw_cpu_add_8
226 #define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
229 #ifndef raw_cpu_and_1
230 #define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
232 #ifndef raw_cpu_and_2
233 #define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
235 #ifndef raw_cpu_and_4
236 #define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
238 #ifndef raw_cpu_and_8
239 #define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
243 #define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
246 #define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
249 #define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
252 #define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
255 #ifndef raw_cpu_add_return_1
256 #define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
258 #ifndef raw_cpu_add_return_2
259 #define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
261 #ifndef raw_cpu_add_return_4
262 #define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
264 #ifndef raw_cpu_add_return_8
265 #define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
268 #ifndef raw_cpu_xchg_1
269 #define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
271 #ifndef raw_cpu_xchg_2
272 #define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
274 #ifndef raw_cpu_xchg_4
275 #define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
277 #ifndef raw_cpu_xchg_8
278 #define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
281 #ifndef raw_cpu_cmpxchg_1
282 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
283 raw_cpu_generic_cmpxchg(pcp, oval, nval)
285 #ifndef raw_cpu_cmpxchg_2
286 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
287 raw_cpu_generic_cmpxchg(pcp, oval, nval)
289 #ifndef raw_cpu_cmpxchg_4
290 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
291 raw_cpu_generic_cmpxchg(pcp, oval, nval)
293 #ifndef raw_cpu_cmpxchg_8
294 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
295 raw_cpu_generic_cmpxchg(pcp, oval, nval)
298 #ifndef raw_cpu_cmpxchg_double_1
299 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
300 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
302 #ifndef raw_cpu_cmpxchg_double_2
303 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
304 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
306 #ifndef raw_cpu_cmpxchg_double_4
307 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
308 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
310 #ifndef raw_cpu_cmpxchg_double_8
311 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
312 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
315 #ifndef this_cpu_read_1
316 #define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
318 #ifndef this_cpu_read_2
319 #define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
321 #ifndef this_cpu_read_4
322 #define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
324 #ifndef this_cpu_read_8
325 #define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
328 #ifndef this_cpu_write_1
329 #define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
331 #ifndef this_cpu_write_2
332 #define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
334 #ifndef this_cpu_write_4
335 #define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
337 #ifndef this_cpu_write_8
338 #define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
341 #ifndef this_cpu_add_1
342 #define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
344 #ifndef this_cpu_add_2
345 #define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
347 #ifndef this_cpu_add_4
348 #define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
350 #ifndef this_cpu_add_8
351 #define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
354 #ifndef this_cpu_and_1
355 #define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
357 #ifndef this_cpu_and_2
358 #define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
360 #ifndef this_cpu_and_4
361 #define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
363 #ifndef this_cpu_and_8
364 #define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
367 #ifndef this_cpu_or_1
368 #define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
370 #ifndef this_cpu_or_2
371 #define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
373 #ifndef this_cpu_or_4
374 #define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
376 #ifndef this_cpu_or_8
377 #define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
380 #ifndef this_cpu_add_return_1
381 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
383 #ifndef this_cpu_add_return_2
384 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
386 #ifndef this_cpu_add_return_4
387 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
389 #ifndef this_cpu_add_return_8
390 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
393 #ifndef this_cpu_xchg_1
394 #define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
396 #ifndef this_cpu_xchg_2
397 #define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
399 #ifndef this_cpu_xchg_4
400 #define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
402 #ifndef this_cpu_xchg_8
403 #define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
406 #ifndef this_cpu_cmpxchg_1
407 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
408 this_cpu_generic_cmpxchg(pcp, oval, nval)
410 #ifndef this_cpu_cmpxchg_2
411 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
412 this_cpu_generic_cmpxchg(pcp, oval, nval)
414 #ifndef this_cpu_cmpxchg_4
415 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
416 this_cpu_generic_cmpxchg(pcp, oval, nval)
418 #ifndef this_cpu_cmpxchg_8
419 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
420 this_cpu_generic_cmpxchg(pcp, oval, nval)
423 #ifndef this_cpu_cmpxchg_double_1
424 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
425 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
427 #ifndef this_cpu_cmpxchg_double_2
428 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
429 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
431 #ifndef this_cpu_cmpxchg_double_4
432 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
433 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
435 #ifndef this_cpu_cmpxchg_double_8
436 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
437 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
440 #endif /* _ASM_GENERIC_PERCPU_H_ */