1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_U64_STATS_SYNC_H
3 #define _LINUX_U64_STATS_SYNC_H
6 * Protect against 64-bit values tearing on 32-bit architectures. This is
7 * typically used for statistics read/update in different subsystems.
11 * - Use a seqcount on 32-bit SMP, only disable preemption for 32-bit UP.
12 * - The whole thing is a no-op on 64-bit architectures.
16 * 1) Write side must ensure mutual exclusion, or one seqcount update could
17 * be lost, thus blocking readers forever.
19 * 2) Write side must disable preemption, or a seqcount reader can preempt the
20 * writer and also spin forever.
22 * 3) Write side must use the _irqsave() variant if other writers, or a reader,
23 * can be invoked from an IRQ context.
25 * 4) If reader fetches several counters, there is no guarantee the whole values
26 * are consistent w.r.t. each other (remember point #2: seqcounts are not
27 * used for 64bit architectures).
29 * 5) Readers are allowed to sleep or be preempted/interrupted: they perform
32 * 6) Readers must use both u64_stats_fetch_{begin,retry}_irq() if the stats
33 * might be updated from a hardirq or softirq context (remember point #1:
34 * seqcounts are not used for UP kernels). 32-bit UP stat readers could read
35 * corrupted 64-bit values otherwise.
39 * Stats producer (writer) should use following template granted it already got
40 * an exclusive access to counters (a lock is already taken, or per cpu
41 * data is used [in a non preemptable context])
43 * spin_lock_bh(...) or other synchronization to get exclusive access
45 * u64_stats_update_begin(&stats->syncp);
46 * u64_stats_add(&stats->bytes64, len); // non atomic operation
47 * u64_stats_inc(&stats->packets64); // non atomic operation
48 * u64_stats_update_end(&stats->syncp);
50 * While a consumer (reader) should use following template to get consistent
51 * snapshot for each variable (but no guarantee on several ones)
53 * u64 tbytes, tpackets;
57 * start = u64_stats_fetch_begin(&stats->syncp);
58 * tbytes = u64_stats_read(&stats->bytes64); // non atomic operation
59 * tpackets = u64_stats_read(&stats->packets64); // non atomic operation
60 * } while (u64_stats_fetch_retry(&stats->syncp, start));
63 * Example of use in drivers/net/loopback.c, using per_cpu containers,
64 * in BH disabled context.
66 #include <linux/seqlock.h>
68 struct u64_stats_sync {
69 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
74 #if BITS_PER_LONG == 64
75 #include <asm/local64.h>
81 static inline u64 u64_stats_read(const u64_stats_t *p)
83 return local64_read(&p->v);
86 static inline void u64_stats_add(u64_stats_t *p, unsigned long val)
88 local64_add(val, &p->v);
91 static inline void u64_stats_inc(u64_stats_t *p)
102 static inline u64 u64_stats_read(const u64_stats_t *p)
107 static inline void u64_stats_add(u64_stats_t *p, unsigned long val)
112 static inline void u64_stats_inc(u64_stats_t *p)
118 #if BITS_PER_LONG == 32 && defined(CONFIG_SMP)
119 #define u64_stats_init(syncp) \
121 struct u64_stats_sync *__s = (syncp); \
122 seqcount_init(&__s->seq); \
125 static inline void u64_stats_init(struct u64_stats_sync *syncp)
130 static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)
132 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
133 write_seqcount_begin(&syncp->seq);
137 static inline void u64_stats_update_end(struct u64_stats_sync *syncp)
139 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
140 write_seqcount_end(&syncp->seq);
144 static inline unsigned long
145 u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)
147 unsigned long flags = 0;
149 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
150 local_irq_save(flags);
151 write_seqcount_begin(&syncp->seq);
157 u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
160 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
161 write_seqcount_end(&syncp->seq);
162 local_irq_restore(flags);
166 static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
168 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
169 return read_seqcount_begin(&syncp->seq);
175 static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
177 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
180 return __u64_stats_fetch_begin(syncp);
183 static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
186 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
187 return read_seqcount_retry(&syncp->seq, start);
193 static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
196 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
199 return __u64_stats_fetch_retry(syncp, start);
203 * In case irq handlers can update u64 counters, readers can use following helpers
204 * - SMP 32bit arches use seqcount protection, irq safe.
205 * - UP 32bit must disable irqs.
206 * - 64bit have no problem atomically reading u64 values, irq safe.
208 static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp)
210 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
213 return __u64_stats_fetch_begin(syncp);
216 static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp,
219 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
222 return __u64_stats_fetch_retry(syncp, start);
225 #endif /* _LINUX_U64_STATS_SYNC_H */