GNU Linux-libre 5.19-rc6-gnu
[releases.git] / drivers / perf / riscv_pmu.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * RISC-V performance counter support.
4  *
5  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
6  *
7  * This implementation is based on old RISC-V perf and ARM perf event code
8  * which are in turn based on sparc64 and x86 code.
9  */
10
11 #include <linux/cpumask.h>
12 #include <linux/irq.h>
13 #include <linux/irqdesc.h>
14 #include <linux/perf/riscv_pmu.h>
15 #include <linux/printk.h>
16 #include <linux/smp.h>
17
18 #include <asm/sbi.h>
19
20 static unsigned long csr_read_num(int csr_num)
21 {
22 #define switchcase_csr_read(__csr_num, __val)           {\
23         case __csr_num:                                 \
24                 __val = csr_read(__csr_num);            \
25                 break; }
26 #define switchcase_csr_read_2(__csr_num, __val)         {\
27         switchcase_csr_read(__csr_num + 0, __val)        \
28         switchcase_csr_read(__csr_num + 1, __val)}
29 #define switchcase_csr_read_4(__csr_num, __val)         {\
30         switchcase_csr_read_2(__csr_num + 0, __val)      \
31         switchcase_csr_read_2(__csr_num + 2, __val)}
32 #define switchcase_csr_read_8(__csr_num, __val)         {\
33         switchcase_csr_read_4(__csr_num + 0, __val)      \
34         switchcase_csr_read_4(__csr_num + 4, __val)}
35 #define switchcase_csr_read_16(__csr_num, __val)        {\
36         switchcase_csr_read_8(__csr_num + 0, __val)      \
37         switchcase_csr_read_8(__csr_num + 8, __val)}
38 #define switchcase_csr_read_32(__csr_num, __val)        {\
39         switchcase_csr_read_16(__csr_num + 0, __val)     \
40         switchcase_csr_read_16(__csr_num + 16, __val)}
41
42         unsigned long ret = 0;
43
44         switch (csr_num) {
45         switchcase_csr_read_32(CSR_CYCLE, ret)
46         switchcase_csr_read_32(CSR_CYCLEH, ret)
47         default :
48                 break;
49         }
50
51         return ret;
52 #undef switchcase_csr_read_32
53 #undef switchcase_csr_read_16
54 #undef switchcase_csr_read_8
55 #undef switchcase_csr_read_4
56 #undef switchcase_csr_read_2
57 #undef switchcase_csr_read
58 }
59
60 /*
61  * Read the CSR of a corresponding counter.
62  */
63 unsigned long riscv_pmu_ctr_read_csr(unsigned long csr)
64 {
65         if (csr < CSR_CYCLE || csr > CSR_HPMCOUNTER31H ||
66            (csr > CSR_HPMCOUNTER31 && csr < CSR_CYCLEH)) {
67                 pr_err("Invalid performance counter csr %lx\n", csr);
68                 return -EINVAL;
69         }
70
71         return csr_read_num(csr);
72 }
73
74 u64 riscv_pmu_ctr_get_width_mask(struct perf_event *event)
75 {
76         int cwidth;
77         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
78         struct hw_perf_event *hwc = &event->hw;
79
80         if (!rvpmu->ctr_get_width)
81         /**
82          * If the pmu driver doesn't support counter width, set it to default
83          * maximum allowed by the specification.
84          */
85                 cwidth = 63;
86         else {
87                 if (hwc->idx == -1)
88                         /* Handle init case where idx is not initialized yet */
89                         cwidth = rvpmu->ctr_get_width(0);
90                 else
91                         cwidth = rvpmu->ctr_get_width(hwc->idx);
92         }
93
94         return GENMASK_ULL(cwidth, 0);
95 }
96
97 u64 riscv_pmu_event_update(struct perf_event *event)
98 {
99         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
100         struct hw_perf_event *hwc = &event->hw;
101         u64 prev_raw_count, new_raw_count;
102         unsigned long cmask;
103         u64 oldval, delta;
104
105         if (!rvpmu->ctr_read)
106                 return 0;
107
108         cmask = riscv_pmu_ctr_get_width_mask(event);
109
110         do {
111                 prev_raw_count = local64_read(&hwc->prev_count);
112                 new_raw_count = rvpmu->ctr_read(event);
113                 oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count,
114                                          new_raw_count);
115         } while (oldval != prev_raw_count);
116
117         delta = (new_raw_count - prev_raw_count) & cmask;
118         local64_add(delta, &event->count);
119         local64_sub(delta, &hwc->period_left);
120
121         return delta;
122 }
123
124 static void riscv_pmu_stop(struct perf_event *event, int flags)
125 {
126         struct hw_perf_event *hwc = &event->hw;
127         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
128
129         WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
130
131         if (!(hwc->state & PERF_HES_STOPPED)) {
132                 if (rvpmu->ctr_stop) {
133                         rvpmu->ctr_stop(event, 0);
134                         hwc->state |= PERF_HES_STOPPED;
135                 }
136                 riscv_pmu_event_update(event);
137                 hwc->state |= PERF_HES_UPTODATE;
138         }
139 }
140
141 int riscv_pmu_event_set_period(struct perf_event *event)
142 {
143         struct hw_perf_event *hwc = &event->hw;
144         s64 left = local64_read(&hwc->period_left);
145         s64 period = hwc->sample_period;
146         int overflow = 0;
147         uint64_t max_period = riscv_pmu_ctr_get_width_mask(event);
148
149         if (unlikely(left <= -period)) {
150                 left = period;
151                 local64_set(&hwc->period_left, left);
152                 hwc->last_period = period;
153                 overflow = 1;
154         }
155
156         if (unlikely(left <= 0)) {
157                 left += period;
158                 local64_set(&hwc->period_left, left);
159                 hwc->last_period = period;
160                 overflow = 1;
161         }
162
163         /*
164          * Limit the maximum period to prevent the counter value
165          * from overtaking the one we are about to program. In
166          * effect we are reducing max_period to account for
167          * interrupt latency (and we are being very conservative).
168          */
169         if (left > (max_period >> 1))
170                 left = (max_period >> 1);
171
172         local64_set(&hwc->prev_count, (u64)-left);
173         perf_event_update_userpage(event);
174
175         return overflow;
176 }
177
178 static void riscv_pmu_start(struct perf_event *event, int flags)
179 {
180         struct hw_perf_event *hwc = &event->hw;
181         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
182         uint64_t max_period = riscv_pmu_ctr_get_width_mask(event);
183         u64 init_val;
184
185         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
186                 return;
187
188         if (flags & PERF_EF_RELOAD)
189                 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
190
191         hwc->state = 0;
192         riscv_pmu_event_set_period(event);
193         init_val = local64_read(&hwc->prev_count) & max_period;
194         rvpmu->ctr_start(event, init_val);
195         perf_event_update_userpage(event);
196 }
197
198 static int riscv_pmu_add(struct perf_event *event, int flags)
199 {
200         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
201         struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
202         struct hw_perf_event *hwc = &event->hw;
203         int idx;
204
205         idx = rvpmu->ctr_get_idx(event);
206         if (idx < 0)
207                 return idx;
208
209         hwc->idx = idx;
210         cpuc->events[idx] = event;
211         cpuc->n_events++;
212         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
213         if (flags & PERF_EF_START)
214                 riscv_pmu_start(event, PERF_EF_RELOAD);
215
216         /* Propagate our changes to the userspace mapping. */
217         perf_event_update_userpage(event);
218
219         return 0;
220 }
221
222 static void riscv_pmu_del(struct perf_event *event, int flags)
223 {
224         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
225         struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
226         struct hw_perf_event *hwc = &event->hw;
227
228         riscv_pmu_stop(event, PERF_EF_UPDATE);
229         cpuc->events[hwc->idx] = NULL;
230         /* The firmware need to reset the counter mapping */
231         if (rvpmu->ctr_stop)
232                 rvpmu->ctr_stop(event, RISCV_PMU_STOP_FLAG_RESET);
233         cpuc->n_events--;
234         if (rvpmu->ctr_clear_idx)
235                 rvpmu->ctr_clear_idx(event);
236         perf_event_update_userpage(event);
237         hwc->idx = -1;
238 }
239
240 static void riscv_pmu_read(struct perf_event *event)
241 {
242         riscv_pmu_event_update(event);
243 }
244
245 static int riscv_pmu_event_init(struct perf_event *event)
246 {
247         struct hw_perf_event *hwc = &event->hw;
248         struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
249         int mapped_event;
250         u64 event_config = 0;
251         uint64_t cmask;
252
253         hwc->flags = 0;
254         mapped_event = rvpmu->event_map(event, &event_config);
255         if (mapped_event < 0) {
256                 pr_debug("event %x:%llx not supported\n", event->attr.type,
257                          event->attr.config);
258                 return mapped_event;
259         }
260
261         /*
262          * idx is set to -1 because the index of a general event should not be
263          * decided until binding to some counter in pmu->add().
264          * config will contain the information about counter CSR
265          * the idx will contain the counter index
266          */
267         hwc->config = event_config;
268         hwc->idx = -1;
269         hwc->event_base = mapped_event;
270
271         if (!is_sampling_event(event)) {
272                 /*
273                  * For non-sampling runs, limit the sample_period to half
274                  * of the counter width. That way, the new counter value
275                  * is far less likely to overtake the previous one unless
276                  * you have some serious IRQ latency issues.
277                  */
278                 cmask = riscv_pmu_ctr_get_width_mask(event);
279                 hwc->sample_period  =  cmask >> 1;
280                 hwc->last_period    = hwc->sample_period;
281                 local64_set(&hwc->period_left, hwc->sample_period);
282         }
283
284         return 0;
285 }
286
287 struct riscv_pmu *riscv_pmu_alloc(void)
288 {
289         struct riscv_pmu *pmu;
290         int cpuid, i;
291         struct cpu_hw_events *cpuc;
292
293         pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
294         if (!pmu)
295                 goto out;
296
297         pmu->hw_events = alloc_percpu_gfp(struct cpu_hw_events, GFP_KERNEL);
298         if (!pmu->hw_events) {
299                 pr_info("failed to allocate per-cpu PMU data.\n");
300                 goto out_free_pmu;
301         }
302
303         for_each_possible_cpu(cpuid) {
304                 cpuc = per_cpu_ptr(pmu->hw_events, cpuid);
305                 cpuc->n_events = 0;
306                 for (i = 0; i < RISCV_MAX_COUNTERS; i++)
307                         cpuc->events[i] = NULL;
308         }
309         pmu->pmu = (struct pmu) {
310                 .event_init     = riscv_pmu_event_init,
311                 .add            = riscv_pmu_add,
312                 .del            = riscv_pmu_del,
313                 .start          = riscv_pmu_start,
314                 .stop           = riscv_pmu_stop,
315                 .read           = riscv_pmu_read,
316         };
317
318         return pmu;
319
320 out_free_pmu:
321         kfree(pmu);
322 out:
323         return NULL;
324 }