GNU Linux-libre 4.19.264-gnu1
[releases.git] / arch / arm / kernel / perf_event_v7.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
4  *
5  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
6  * 2010 (c) MontaVista Software, LLC.
7  *
8  * Copied from ARMv6 code, with the low level code inspired
9  *  by the ARMv7 Oprofile code.
10  *
11  * Cortex-A8 has up to 4 configurable performance counters and
12  *  a single cycle counter.
13  * Cortex-A9 has up to 31 configurable performance counters and
14  *  a single cycle counter.
15  *
16  * All counters can be enabled/disabled and IRQ masked separately. The cycle
17  *  counter and all 4 performance counters together can be reset separately.
18  */
19
20 #ifdef CONFIG_CPU_V7
21
22 #include <asm/cp15.h>
23 #include <asm/cputype.h>
24 #include <asm/irq_regs.h>
25 #include <asm/vfp.h>
26 #include "../vfp/vfpinstr.h"
27
28 #include <linux/of.h>
29 #include <linux/perf/arm_pmu.h>
30 #include <linux/platform_device.h>
31
32 /*
33  * Common ARMv7 event types
34  *
35  * Note: An implementation may not be able to count all of these events
36  * but the encodings are considered to be `reserved' in the case that
37  * they are not available.
38  */
39 #define ARMV7_PERFCTR_PMNC_SW_INCR                      0x00
40 #define ARMV7_PERFCTR_L1_ICACHE_REFILL                  0x01
41 #define ARMV7_PERFCTR_ITLB_REFILL                       0x02
42 #define ARMV7_PERFCTR_L1_DCACHE_REFILL                  0x03
43 #define ARMV7_PERFCTR_L1_DCACHE_ACCESS                  0x04
44 #define ARMV7_PERFCTR_DTLB_REFILL                       0x05
45 #define ARMV7_PERFCTR_MEM_READ                          0x06
46 #define ARMV7_PERFCTR_MEM_WRITE                         0x07
47 #define ARMV7_PERFCTR_INSTR_EXECUTED                    0x08
48 #define ARMV7_PERFCTR_EXC_TAKEN                         0x09
49 #define ARMV7_PERFCTR_EXC_EXECUTED                      0x0A
50 #define ARMV7_PERFCTR_CID_WRITE                         0x0B
51
52 /*
53  * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
54  * It counts:
55  *  - all (taken) branch instructions,
56  *  - instructions that explicitly write the PC,
57  *  - exception generating instructions.
58  */
59 #define ARMV7_PERFCTR_PC_WRITE                          0x0C
60 #define ARMV7_PERFCTR_PC_IMM_BRANCH                     0x0D
61 #define ARMV7_PERFCTR_PC_PROC_RETURN                    0x0E
62 #define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              0x0F
63 #define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                0x10
64 #define ARMV7_PERFCTR_CLOCK_CYCLES                      0x11
65 #define ARMV7_PERFCTR_PC_BRANCH_PRED                    0x12
66
67 /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
68 #define ARMV7_PERFCTR_MEM_ACCESS                        0x13
69 #define ARMV7_PERFCTR_L1_ICACHE_ACCESS                  0x14
70 #define ARMV7_PERFCTR_L1_DCACHE_WB                      0x15
71 #define ARMV7_PERFCTR_L2_CACHE_ACCESS                   0x16
72 #define ARMV7_PERFCTR_L2_CACHE_REFILL                   0x17
73 #define ARMV7_PERFCTR_L2_CACHE_WB                       0x18
74 #define ARMV7_PERFCTR_BUS_ACCESS                        0x19
75 #define ARMV7_PERFCTR_MEM_ERROR                         0x1A
76 #define ARMV7_PERFCTR_INSTR_SPEC                        0x1B
77 #define ARMV7_PERFCTR_TTBR_WRITE                        0x1C
78 #define ARMV7_PERFCTR_BUS_CYCLES                        0x1D
79
80 #define ARMV7_PERFCTR_CPU_CYCLES                        0xFF
81
82 /* ARMv7 Cortex-A8 specific event types */
83 #define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                0x43
84 #define ARMV7_A8_PERFCTR_L2_CACHE_REFILL                0x44
85 #define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               0x50
86 #define ARMV7_A8_PERFCTR_STALL_ISIDE                    0x56
87
88 /* ARMv7 Cortex-A9 specific event types */
89 #define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              0x68
90 #define ARMV7_A9_PERFCTR_STALL_ICACHE                   0x60
91 #define ARMV7_A9_PERFCTR_STALL_DISPATCH                 0x66
92
93 /* ARMv7 Cortex-A5 specific event types */
94 #define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              0xc2
95 #define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         0xc3
96
97 /* ARMv7 Cortex-A15 specific event types */
98 #define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         0x40
99 #define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        0x41
100 #define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         0x42
101 #define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        0x43
102
103 #define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           0x4C
104 #define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          0x4D
105
106 #define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          0x50
107 #define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         0x51
108 #define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          0x52
109 #define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         0x53
110
111 #define ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 0x76
112
113 /* ARMv7 Cortex-A12 specific event types */
114 #define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ         0x40
115 #define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE        0x41
116
117 #define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ          0x50
118 #define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE         0x51
119
120 #define ARMV7_A12_PERFCTR_PC_WRITE_SPEC                 0x76
121
122 #define ARMV7_A12_PERFCTR_PF_TLB_REFILL                 0xe7
123
124 /* ARMv7 Krait specific event types */
125 #define KRAIT_PMRESR0_GROUP0                            0xcc
126 #define KRAIT_PMRESR1_GROUP0                            0xd0
127 #define KRAIT_PMRESR2_GROUP0                            0xd4
128 #define KRAIT_VPMRESR0_GROUP0                           0xd8
129
130 #define KRAIT_PERFCTR_L1_ICACHE_ACCESS                  0x10011
131 #define KRAIT_PERFCTR_L1_ICACHE_MISS                    0x10010
132
133 #define KRAIT_PERFCTR_L1_ITLB_ACCESS                    0x12222
134 #define KRAIT_PERFCTR_L1_DTLB_ACCESS                    0x12210
135
136 /* ARMv7 Scorpion specific event types */
137 #define SCORPION_LPM0_GROUP0                            0x4c
138 #define SCORPION_LPM1_GROUP0                            0x50
139 #define SCORPION_LPM2_GROUP0                            0x54
140 #define SCORPION_L2LPM_GROUP0                           0x58
141 #define SCORPION_VLPM_GROUP0                            0x5c
142
143 #define SCORPION_ICACHE_ACCESS                          0x10053
144 #define SCORPION_ICACHE_MISS                            0x10052
145
146 #define SCORPION_DTLB_ACCESS                            0x12013
147 #define SCORPION_DTLB_MISS                              0x12012
148
149 #define SCORPION_ITLB_MISS                              0x12021
150
151 /*
152  * Cortex-A8 HW events mapping
153  *
154  * The hardware events that we support. We do support cache operations but
155  * we have harvard caches and no way to combine instruction and data
156  * accesses/misses in hardware.
157  */
158 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
159         PERF_MAP_ALL_UNSUPPORTED,
160         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
161         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
162         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
163         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
164         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
165         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
166         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
167 };
168
169 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
170                                           [PERF_COUNT_HW_CACHE_OP_MAX]
171                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
172         PERF_CACHE_MAP_ALL_UNSUPPORTED,
173
174         /*
175          * The performance counters don't differentiate between read and write
176          * accesses/misses so this isn't strictly correct, but it's the best we
177          * can do. Writes and reads get combined.
178          */
179         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
180         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
181         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
182         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
183
184         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
185         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
186
187         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
188         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
189         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
190         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
191
192         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
193         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
194
195         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
196         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
197
198         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
199         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
200         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
201         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
202 };
203
204 /*
205  * Cortex-A9 HW events mapping
206  */
207 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
208         PERF_MAP_ALL_UNSUPPORTED,
209         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
210         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
211         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
212         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
213         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
214         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
215         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
216         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
217 };
218
219 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
220                                           [PERF_COUNT_HW_CACHE_OP_MAX]
221                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
222         PERF_CACHE_MAP_ALL_UNSUPPORTED,
223
224         /*
225          * The performance counters don't differentiate between read and write
226          * accesses/misses so this isn't strictly correct, but it's the best we
227          * can do. Writes and reads get combined.
228          */
229         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
230         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
231         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
232         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
233
234         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
235
236         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
237         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
238
239         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
240         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
241
242         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
243         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
244         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
245         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
246 };
247
248 /*
249  * Cortex-A5 HW events mapping
250  */
251 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
252         PERF_MAP_ALL_UNSUPPORTED,
253         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
254         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
255         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
256         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
257         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
258         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
259 };
260
261 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
262                                         [PERF_COUNT_HW_CACHE_OP_MAX]
263                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
264         PERF_CACHE_MAP_ALL_UNSUPPORTED,
265
266         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
267         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
268         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
269         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
270         [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
271         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
272
273         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
274         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
275         /*
276          * The prefetch counters don't differentiate between the I side and the
277          * D side.
278          */
279         [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
280         [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
281
282         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
283         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
284
285         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
286         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
287
288         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
289         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
290         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
291         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
292 };
293
294 /*
295  * Cortex-A15 HW events mapping
296  */
297 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
298         PERF_MAP_ALL_UNSUPPORTED,
299         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
300         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
301         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
302         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
303         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
304         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
305         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
306 };
307
308 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
309                                         [PERF_COUNT_HW_CACHE_OP_MAX]
310                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
311         PERF_CACHE_MAP_ALL_UNSUPPORTED,
312
313         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
314         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
315         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
316         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
317
318         /*
319          * Not all performance counters differentiate between read and write
320          * accesses/misses so we're not always strictly correct, but it's the
321          * best we can do. Writes and reads get combined in these cases.
322          */
323         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
324         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
325
326         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
327         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
328         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
329         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
330
331         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
332         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
333
334         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
335         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
336
337         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
338         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
339         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
340         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
341 };
342
343 /*
344  * Cortex-A7 HW events mapping
345  */
346 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
347         PERF_MAP_ALL_UNSUPPORTED,
348         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
349         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
350         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
351         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
352         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
353         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
354         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
355 };
356
357 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
358                                         [PERF_COUNT_HW_CACHE_OP_MAX]
359                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
360         PERF_CACHE_MAP_ALL_UNSUPPORTED,
361
362         /*
363          * The performance counters don't differentiate between read and write
364          * accesses/misses so this isn't strictly correct, but it's the best we
365          * can do. Writes and reads get combined.
366          */
367         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
368         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
369         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
370         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
371
372         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
373         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
374
375         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_PERFCTR_L2_CACHE_ACCESS,
376         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
377         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L2_CACHE_ACCESS,
378         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
379
380         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
381         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
382
383         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
384         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
385
386         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
387         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
388         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
389         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
390 };
391
392 /*
393  * Cortex-A12 HW events mapping
394  */
395 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
396         PERF_MAP_ALL_UNSUPPORTED,
397         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
398         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
399         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
400         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
401         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
402         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
403         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
404 };
405
406 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
407                                         [PERF_COUNT_HW_CACHE_OP_MAX]
408                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
409         PERF_CACHE_MAP_ALL_UNSUPPORTED,
410
411         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
412         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
413         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
414         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
415
416         /*
417          * Not all performance counters differentiate between read and write
418          * accesses/misses so we're not always strictly correct, but it's the
419          * best we can do. Writes and reads get combined in these cases.
420          */
421         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
422         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
423
424         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
425         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
426         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
427         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
428
429         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
430         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
431         [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]       = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
432
433         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
434         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
435
436         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
437         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
438         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
439         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
440 };
441
442 /*
443  * Krait HW events mapping
444  */
445 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
446         PERF_MAP_ALL_UNSUPPORTED,
447         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
448         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
449         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
450         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
451         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
452 };
453
454 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
455         PERF_MAP_ALL_UNSUPPORTED,
456         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
457         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
458         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
459         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
460 };
461
462 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
463                                           [PERF_COUNT_HW_CACHE_OP_MAX]
464                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
465         PERF_CACHE_MAP_ALL_UNSUPPORTED,
466
467         /*
468          * The performance counters don't differentiate between read and write
469          * accesses/misses so this isn't strictly correct, but it's the best we
470          * can do. Writes and reads get combined.
471          */
472         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
473         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
474         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
475         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
476
477         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
478         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = KRAIT_PERFCTR_L1_ICACHE_MISS,
479
480         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
481         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_DTLB_ACCESS,
482
483         [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
484         [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_ITLB_ACCESS,
485
486         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
487         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
488         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
489         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
490 };
491
492 /*
493  * Scorpion HW events mapping
494  */
495 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
496         PERF_MAP_ALL_UNSUPPORTED,
497         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
498         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
499         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
500         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
501         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
502 };
503
504 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
505                                             [PERF_COUNT_HW_CACHE_OP_MAX]
506                                             [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
507         PERF_CACHE_MAP_ALL_UNSUPPORTED,
508         /*
509          * The performance counters don't differentiate between read and write
510          * accesses/misses so this isn't strictly correct, but it's the best we
511          * can do. Writes and reads get combined.
512          */
513         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
514         [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
515         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
516         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
517         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
518         [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
519         /*
520          * Only ITLB misses and DTLB refills are supported.  If users want the
521          * DTLB refills misses a raw counter must be used.
522          */
523         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
524         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
525         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
526         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
527         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
528         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
529         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
530         [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
531         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
532         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
533 };
534
535 PMU_FORMAT_ATTR(event, "config:0-7");
536
537 static struct attribute *armv7_pmu_format_attrs[] = {
538         &format_attr_event.attr,
539         NULL,
540 };
541
542 static struct attribute_group armv7_pmu_format_attr_group = {
543         .name = "format",
544         .attrs = armv7_pmu_format_attrs,
545 };
546
547 #define ARMV7_EVENT_ATTR_RESOLVE(m) #m
548 #define ARMV7_EVENT_ATTR(name, config) \
549         PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \
550                               "event=" ARMV7_EVENT_ATTR_RESOLVE(config))
551
552 ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR);
553 ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL);
554 ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL);
555 ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL);
556 ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS);
557 ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL);
558 ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ);
559 ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE);
560 ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED);
561 ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN);
562 ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED);
563 ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE);
564 ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE);
565 ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH);
566 ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN);
567 ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS);
568 ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED);
569 ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES);
570 ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED);
571
572 static struct attribute *armv7_pmuv1_event_attrs[] = {
573         &armv7_event_attr_sw_incr.attr.attr,
574         &armv7_event_attr_l1i_cache_refill.attr.attr,
575         &armv7_event_attr_l1i_tlb_refill.attr.attr,
576         &armv7_event_attr_l1d_cache_refill.attr.attr,
577         &armv7_event_attr_l1d_cache.attr.attr,
578         &armv7_event_attr_l1d_tlb_refill.attr.attr,
579         &armv7_event_attr_ld_retired.attr.attr,
580         &armv7_event_attr_st_retired.attr.attr,
581         &armv7_event_attr_inst_retired.attr.attr,
582         &armv7_event_attr_exc_taken.attr.attr,
583         &armv7_event_attr_exc_return.attr.attr,
584         &armv7_event_attr_cid_write_retired.attr.attr,
585         &armv7_event_attr_pc_write_retired.attr.attr,
586         &armv7_event_attr_br_immed_retired.attr.attr,
587         &armv7_event_attr_br_return_retired.attr.attr,
588         &armv7_event_attr_unaligned_ldst_retired.attr.attr,
589         &armv7_event_attr_br_mis_pred.attr.attr,
590         &armv7_event_attr_cpu_cycles.attr.attr,
591         &armv7_event_attr_br_pred.attr.attr,
592         NULL,
593 };
594
595 static struct attribute_group armv7_pmuv1_events_attr_group = {
596         .name = "events",
597         .attrs = armv7_pmuv1_event_attrs,
598 };
599
600 ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS);
601 ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS);
602 ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB);
603 ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS);
604 ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL);
605 ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB);
606 ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS);
607 ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR);
608 ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC);
609 ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE);
610 ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES);
611
612 static struct attribute *armv7_pmuv2_event_attrs[] = {
613         &armv7_event_attr_sw_incr.attr.attr,
614         &armv7_event_attr_l1i_cache_refill.attr.attr,
615         &armv7_event_attr_l1i_tlb_refill.attr.attr,
616         &armv7_event_attr_l1d_cache_refill.attr.attr,
617         &armv7_event_attr_l1d_cache.attr.attr,
618         &armv7_event_attr_l1d_tlb_refill.attr.attr,
619         &armv7_event_attr_ld_retired.attr.attr,
620         &armv7_event_attr_st_retired.attr.attr,
621         &armv7_event_attr_inst_retired.attr.attr,
622         &armv7_event_attr_exc_taken.attr.attr,
623         &armv7_event_attr_exc_return.attr.attr,
624         &armv7_event_attr_cid_write_retired.attr.attr,
625         &armv7_event_attr_pc_write_retired.attr.attr,
626         &armv7_event_attr_br_immed_retired.attr.attr,
627         &armv7_event_attr_br_return_retired.attr.attr,
628         &armv7_event_attr_unaligned_ldst_retired.attr.attr,
629         &armv7_event_attr_br_mis_pred.attr.attr,
630         &armv7_event_attr_cpu_cycles.attr.attr,
631         &armv7_event_attr_br_pred.attr.attr,
632         &armv7_event_attr_mem_access.attr.attr,
633         &armv7_event_attr_l1i_cache.attr.attr,
634         &armv7_event_attr_l1d_cache_wb.attr.attr,
635         &armv7_event_attr_l2d_cache.attr.attr,
636         &armv7_event_attr_l2d_cache_refill.attr.attr,
637         &armv7_event_attr_l2d_cache_wb.attr.attr,
638         &armv7_event_attr_bus_access.attr.attr,
639         &armv7_event_attr_memory_error.attr.attr,
640         &armv7_event_attr_inst_spec.attr.attr,
641         &armv7_event_attr_ttbr_write_retired.attr.attr,
642         &armv7_event_attr_bus_cycles.attr.attr,
643         NULL,
644 };
645
646 static struct attribute_group armv7_pmuv2_events_attr_group = {
647         .name = "events",
648         .attrs = armv7_pmuv2_event_attrs,
649 };
650
651 /*
652  * Perf Events' indices
653  */
654 #define ARMV7_IDX_CYCLE_COUNTER 0
655 #define ARMV7_IDX_COUNTER0      1
656 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
657         (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
658
659 #define ARMV7_MAX_COUNTERS      32
660 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
661
662 /*
663  * ARMv7 low level PMNC access
664  */
665
666 /*
667  * Perf Event to low level counters mapping
668  */
669 #define ARMV7_IDX_TO_COUNTER(x) \
670         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
671
672 /*
673  * Per-CPU PMNC: config reg
674  */
675 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
676 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
677 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
678 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
679 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
680 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
681 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
682 #define ARMV7_PMNC_N_MASK       0x1f
683 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
684
685 /*
686  * FLAG: counters overflow flag status reg
687  */
688 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
689 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
690
691 /*
692  * PMXEVTYPER: Event selection reg
693  */
694 #define ARMV7_EVTYPE_MASK       0xc80000ff      /* Mask for writable bits */
695 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
696
697 /*
698  * Event filters for PMUv2
699  */
700 #define ARMV7_EXCLUDE_PL1       (1 << 31)
701 #define ARMV7_EXCLUDE_USER      (1 << 30)
702 #define ARMV7_INCLUDE_HYP       (1 << 27)
703
704 /*
705  * Secure debug enable reg
706  */
707 #define ARMV7_SDER_SUNIDEN      BIT(1) /* Permit non-invasive debug */
708
709 static inline u32 armv7_pmnc_read(void)
710 {
711         u32 val;
712         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
713         return val;
714 }
715
716 static inline void armv7_pmnc_write(u32 val)
717 {
718         val &= ARMV7_PMNC_MASK;
719         isb();
720         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
721 }
722
723 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
724 {
725         return pmnc & ARMV7_OVERFLOWED_MASK;
726 }
727
728 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
729 {
730         return idx >= ARMV7_IDX_CYCLE_COUNTER &&
731                 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
732 }
733
734 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
735 {
736         return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
737 }
738
739 static inline void armv7_pmnc_select_counter(int idx)
740 {
741         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
742         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
743         isb();
744 }
745
746 static inline u64 armv7pmu_read_counter(struct perf_event *event)
747 {
748         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
749         struct hw_perf_event *hwc = &event->hw;
750         int idx = hwc->idx;
751         u32 value = 0;
752
753         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
754                 pr_err("CPU%u reading wrong counter %d\n",
755                         smp_processor_id(), idx);
756         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
757                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
758         } else {
759                 armv7_pmnc_select_counter(idx);
760                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
761         }
762
763         return value;
764 }
765
766 static inline void armv7pmu_write_counter(struct perf_event *event, u64 value)
767 {
768         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
769         struct hw_perf_event *hwc = &event->hw;
770         int idx = hwc->idx;
771
772         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
773                 pr_err("CPU%u writing wrong counter %d\n",
774                         smp_processor_id(), idx);
775         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
776                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value));
777         } else {
778                 armv7_pmnc_select_counter(idx);
779                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value));
780         }
781 }
782
783 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
784 {
785         armv7_pmnc_select_counter(idx);
786         val &= ARMV7_EVTYPE_MASK;
787         asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
788 }
789
790 static inline void armv7_pmnc_enable_counter(int idx)
791 {
792         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
793         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
794 }
795
796 static inline void armv7_pmnc_disable_counter(int idx)
797 {
798         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
799         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
800 }
801
802 static inline void armv7_pmnc_enable_intens(int idx)
803 {
804         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
805         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
806 }
807
808 static inline void armv7_pmnc_disable_intens(int idx)
809 {
810         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
811         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
812         isb();
813         /* Clear the overflow flag in case an interrupt is pending. */
814         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
815         isb();
816 }
817
818 static inline u32 armv7_pmnc_getreset_flags(void)
819 {
820         u32 val;
821
822         /* Read */
823         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
824
825         /* Write to clear flags */
826         val &= ARMV7_FLAG_MASK;
827         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
828
829         return val;
830 }
831
832 #ifdef DEBUG
833 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
834 {
835         u32 val;
836         unsigned int cnt;
837
838         pr_info("PMNC registers dump:\n");
839
840         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
841         pr_info("PMNC  =0x%08x\n", val);
842
843         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
844         pr_info("CNTENS=0x%08x\n", val);
845
846         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
847         pr_info("INTENS=0x%08x\n", val);
848
849         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
850         pr_info("FLAGS =0x%08x\n", val);
851
852         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
853         pr_info("SELECT=0x%08x\n", val);
854
855         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
856         pr_info("CCNT  =0x%08x\n", val);
857
858         for (cnt = ARMV7_IDX_COUNTER0;
859                         cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
860                 armv7_pmnc_select_counter(cnt);
861                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
862                 pr_info("CNT[%d] count =0x%08x\n",
863                         ARMV7_IDX_TO_COUNTER(cnt), val);
864                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
865                 pr_info("CNT[%d] evtsel=0x%08x\n",
866                         ARMV7_IDX_TO_COUNTER(cnt), val);
867         }
868 }
869 #endif
870
871 static void armv7pmu_enable_event(struct perf_event *event)
872 {
873         unsigned long flags;
874         struct hw_perf_event *hwc = &event->hw;
875         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
876         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
877         int idx = hwc->idx;
878
879         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
880                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
881                         smp_processor_id(), idx);
882                 return;
883         }
884
885         /*
886          * Enable counter and interrupt, and set the counter to count
887          * the event that we're interested in.
888          */
889         raw_spin_lock_irqsave(&events->pmu_lock, flags);
890
891         /*
892          * Disable counter
893          */
894         armv7_pmnc_disable_counter(idx);
895
896         /*
897          * Set event (if destined for PMNx counters)
898          * We only need to set the event for the cycle counter if we
899          * have the ability to perform event filtering.
900          */
901         if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
902                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
903
904         /*
905          * Enable interrupt for this counter
906          */
907         armv7_pmnc_enable_intens(idx);
908
909         /*
910          * Enable counter
911          */
912         armv7_pmnc_enable_counter(idx);
913
914         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
915 }
916
917 static void armv7pmu_disable_event(struct perf_event *event)
918 {
919         unsigned long flags;
920         struct hw_perf_event *hwc = &event->hw;
921         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
922         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
923         int idx = hwc->idx;
924
925         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
926                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
927                         smp_processor_id(), idx);
928                 return;
929         }
930
931         /*
932          * Disable counter and interrupt
933          */
934         raw_spin_lock_irqsave(&events->pmu_lock, flags);
935
936         /*
937          * Disable counter
938          */
939         armv7_pmnc_disable_counter(idx);
940
941         /*
942          * Disable interrupt for this counter
943          */
944         armv7_pmnc_disable_intens(idx);
945
946         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
947 }
948
949 static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu)
950 {
951         u32 pmnc;
952         struct perf_sample_data data;
953         struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
954         struct pt_regs *regs;
955         int idx;
956
957         /*
958          * Get and reset the IRQ flags
959          */
960         pmnc = armv7_pmnc_getreset_flags();
961
962         /*
963          * Did an overflow occur?
964          */
965         if (!armv7_pmnc_has_overflowed(pmnc))
966                 return IRQ_NONE;
967
968         /*
969          * Handle the counter(s) overflow(s)
970          */
971         regs = get_irq_regs();
972
973         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
974                 struct perf_event *event = cpuc->events[idx];
975                 struct hw_perf_event *hwc;
976
977                 /* Ignore if we don't have an event. */
978                 if (!event)
979                         continue;
980
981                 /*
982                  * We have a single interrupt for all counters. Check that
983                  * each counter has overflowed before we process it.
984                  */
985                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
986                         continue;
987
988                 hwc = &event->hw;
989                 armpmu_event_update(event);
990                 perf_sample_data_init(&data, 0, hwc->last_period);
991                 if (!armpmu_event_set_period(event))
992                         continue;
993
994                 if (perf_event_overflow(event, &data, regs))
995                         cpu_pmu->disable(event);
996         }
997
998         /*
999          * Handle the pending perf events.
1000          *
1001          * Note: this call *must* be run with interrupts disabled. For
1002          * platforms that can have the PMU interrupts raised as an NMI, this
1003          * will not work.
1004          */
1005         irq_work_run();
1006
1007         return IRQ_HANDLED;
1008 }
1009
1010 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
1011 {
1012         unsigned long flags;
1013         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1014
1015         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1016         /* Enable all counters */
1017         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1018         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1019 }
1020
1021 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
1022 {
1023         unsigned long flags;
1024         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1025
1026         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1027         /* Disable all counters */
1028         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1029         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1030 }
1031
1032 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1033                                   struct perf_event *event)
1034 {
1035         int idx;
1036         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1037         struct hw_perf_event *hwc = &event->hw;
1038         unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
1039
1040         /* Always place a cycle counter into the cycle counter. */
1041         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1042                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1043                         return -EAGAIN;
1044
1045                 return ARMV7_IDX_CYCLE_COUNTER;
1046         }
1047
1048         /*
1049          * For anything other than a cycle counter, try and use
1050          * the events counters
1051          */
1052         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1053                 if (!test_and_set_bit(idx, cpuc->used_mask))
1054                         return idx;
1055         }
1056
1057         /* The counters are all in use. */
1058         return -EAGAIN;
1059 }
1060
1061 static void armv7pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1062                                      struct perf_event *event)
1063 {
1064         clear_bit(event->hw.idx, cpuc->used_mask);
1065 }
1066
1067 /*
1068  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1069  */
1070 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1071                                      struct perf_event_attr *attr)
1072 {
1073         unsigned long config_base = 0;
1074
1075         if (attr->exclude_idle)
1076                 return -EPERM;
1077         if (attr->exclude_user)
1078                 config_base |= ARMV7_EXCLUDE_USER;
1079         if (attr->exclude_kernel)
1080                 config_base |= ARMV7_EXCLUDE_PL1;
1081         if (!attr->exclude_hv)
1082                 config_base |= ARMV7_INCLUDE_HYP;
1083
1084         /*
1085          * Install the filter into config_base as this is used to
1086          * construct the event type.
1087          */
1088         event->config_base = config_base;
1089
1090         return 0;
1091 }
1092
1093 static void armv7pmu_reset(void *info)
1094 {
1095         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1096         u32 idx, nb_cnt = cpu_pmu->num_events, val;
1097
1098         if (cpu_pmu->secure_access) {
1099                 asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val));
1100                 val |= ARMV7_SDER_SUNIDEN;
1101                 asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val));
1102         }
1103
1104         /* The counter and interrupt enable registers are unknown at reset. */
1105         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1106                 armv7_pmnc_disable_counter(idx);
1107                 armv7_pmnc_disable_intens(idx);
1108         }
1109
1110         /* Initialize & Reset PMNC: C and P bits */
1111         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1112 }
1113
1114 static int armv7_a8_map_event(struct perf_event *event)
1115 {
1116         return armpmu_map_event(event, &armv7_a8_perf_map,
1117                                 &armv7_a8_perf_cache_map, 0xFF);
1118 }
1119
1120 static int armv7_a9_map_event(struct perf_event *event)
1121 {
1122         return armpmu_map_event(event, &armv7_a9_perf_map,
1123                                 &armv7_a9_perf_cache_map, 0xFF);
1124 }
1125
1126 static int armv7_a5_map_event(struct perf_event *event)
1127 {
1128         return armpmu_map_event(event, &armv7_a5_perf_map,
1129                                 &armv7_a5_perf_cache_map, 0xFF);
1130 }
1131
1132 static int armv7_a15_map_event(struct perf_event *event)
1133 {
1134         return armpmu_map_event(event, &armv7_a15_perf_map,
1135                                 &armv7_a15_perf_cache_map, 0xFF);
1136 }
1137
1138 static int armv7_a7_map_event(struct perf_event *event)
1139 {
1140         return armpmu_map_event(event, &armv7_a7_perf_map,
1141                                 &armv7_a7_perf_cache_map, 0xFF);
1142 }
1143
1144 static int armv7_a12_map_event(struct perf_event *event)
1145 {
1146         return armpmu_map_event(event, &armv7_a12_perf_map,
1147                                 &armv7_a12_perf_cache_map, 0xFF);
1148 }
1149
1150 static int krait_map_event(struct perf_event *event)
1151 {
1152         return armpmu_map_event(event, &krait_perf_map,
1153                                 &krait_perf_cache_map, 0xFFFFF);
1154 }
1155
1156 static int krait_map_event_no_branch(struct perf_event *event)
1157 {
1158         return armpmu_map_event(event, &krait_perf_map_no_branch,
1159                                 &krait_perf_cache_map, 0xFFFFF);
1160 }
1161
1162 static int scorpion_map_event(struct perf_event *event)
1163 {
1164         return armpmu_map_event(event, &scorpion_perf_map,
1165                                 &scorpion_perf_cache_map, 0xFFFFF);
1166 }
1167
1168 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1169 {
1170         cpu_pmu->handle_irq     = armv7pmu_handle_irq;
1171         cpu_pmu->enable         = armv7pmu_enable_event;
1172         cpu_pmu->disable        = armv7pmu_disable_event;
1173         cpu_pmu->read_counter   = armv7pmu_read_counter;
1174         cpu_pmu->write_counter  = armv7pmu_write_counter;
1175         cpu_pmu->get_event_idx  = armv7pmu_get_event_idx;
1176         cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx;
1177         cpu_pmu->start          = armv7pmu_start;
1178         cpu_pmu->stop           = armv7pmu_stop;
1179         cpu_pmu->reset          = armv7pmu_reset;
1180 };
1181
1182 static void armv7_read_num_pmnc_events(void *info)
1183 {
1184         int *nb_cnt = info;
1185
1186         /* Read the nb of CNTx counters supported from PMNC */
1187         *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1188
1189         /* Add the CPU cycles counter */
1190         *nb_cnt += 1;
1191 }
1192
1193 static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1194 {
1195         return smp_call_function_any(&arm_pmu->supported_cpus,
1196                                      armv7_read_num_pmnc_events,
1197                                      &arm_pmu->num_events, 1);
1198 }
1199
1200 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1201 {
1202         armv7pmu_init(cpu_pmu);
1203         cpu_pmu->name           = "armv7_cortex_a8";
1204         cpu_pmu->map_event      = armv7_a8_map_event;
1205         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1206                 &armv7_pmuv1_events_attr_group;
1207         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1208                 &armv7_pmu_format_attr_group;
1209         return armv7_probe_num_events(cpu_pmu);
1210 }
1211
1212 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1213 {
1214         armv7pmu_init(cpu_pmu);
1215         cpu_pmu->name           = "armv7_cortex_a9";
1216         cpu_pmu->map_event      = armv7_a9_map_event;
1217         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1218                 &armv7_pmuv1_events_attr_group;
1219         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1220                 &armv7_pmu_format_attr_group;
1221         return armv7_probe_num_events(cpu_pmu);
1222 }
1223
1224 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1225 {
1226         armv7pmu_init(cpu_pmu);
1227         cpu_pmu->name           = "armv7_cortex_a5";
1228         cpu_pmu->map_event      = armv7_a5_map_event;
1229         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1230                 &armv7_pmuv1_events_attr_group;
1231         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1232                 &armv7_pmu_format_attr_group;
1233         return armv7_probe_num_events(cpu_pmu);
1234 }
1235
1236 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1237 {
1238         armv7pmu_init(cpu_pmu);
1239         cpu_pmu->name           = "armv7_cortex_a15";
1240         cpu_pmu->map_event      = armv7_a15_map_event;
1241         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1242         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1243                 &armv7_pmuv2_events_attr_group;
1244         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1245                 &armv7_pmu_format_attr_group;
1246         return armv7_probe_num_events(cpu_pmu);
1247 }
1248
1249 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1250 {
1251         armv7pmu_init(cpu_pmu);
1252         cpu_pmu->name           = "armv7_cortex_a7";
1253         cpu_pmu->map_event      = armv7_a7_map_event;
1254         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1255         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1256                 &armv7_pmuv2_events_attr_group;
1257         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1258                 &armv7_pmu_format_attr_group;
1259         return armv7_probe_num_events(cpu_pmu);
1260 }
1261
1262 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1263 {
1264         armv7pmu_init(cpu_pmu);
1265         cpu_pmu->name           = "armv7_cortex_a12";
1266         cpu_pmu->map_event      = armv7_a12_map_event;
1267         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1268         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1269                 &armv7_pmuv2_events_attr_group;
1270         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1271                 &armv7_pmu_format_attr_group;
1272         return armv7_probe_num_events(cpu_pmu);
1273 }
1274
1275 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1276 {
1277         int ret = armv7_a12_pmu_init(cpu_pmu);
1278         cpu_pmu->name = "armv7_cortex_a17";
1279         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1280                 &armv7_pmuv2_events_attr_group;
1281         cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1282                 &armv7_pmu_format_attr_group;
1283         return ret;
1284 }
1285
1286 /*
1287  * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1288  *
1289  *            31   30     24     16     8      0
1290  *            +--------------------------------+
1291  *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1292  *            +--------------------------------+
1293  *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1294  *            +--------------------------------+
1295  *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1296  *            +--------------------------------+
1297  *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1298  *            +--------------------------------+
1299  *              EN | G=3  | G=2  | G=1  | G=0
1300  *
1301  *  Event Encoding:
1302  *
1303  *      hwc->config_base = 0xNRCCG
1304  *
1305  *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1306  *      R  = region register
1307  *      CC = class of events the group G is choosing from
1308  *      G  = group or particular event
1309  *
1310  *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1311  *
1312  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1313  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1314  *  events (interrupts for example). An event code is broken down into
1315  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1316  *  example).
1317  */
1318
1319 #define KRAIT_EVENT             (1 << 16)
1320 #define VENUM_EVENT             (2 << 16)
1321 #define KRAIT_EVENT_MASK        (KRAIT_EVENT | VENUM_EVENT)
1322 #define PMRESRn_EN              BIT(31)
1323
1324 #define EVENT_REGION(event)     (((event) >> 12) & 0xf)         /* R */
1325 #define EVENT_GROUP(event)      ((event) & 0xf)                 /* G */
1326 #define EVENT_CODE(event)       (((event) >> 4) & 0xff)         /* CC */
1327 #define EVENT_VENUM(event)      (!!(event & VENUM_EVENT))       /* N=2 */
1328 #define EVENT_CPU(event)        (!!(event & KRAIT_EVENT))       /* N=1 */
1329
1330 static u32 krait_read_pmresrn(int n)
1331 {
1332         u32 val;
1333
1334         switch (n) {
1335         case 0:
1336                 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1337                 break;
1338         case 1:
1339                 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1340                 break;
1341         case 2:
1342                 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1343                 break;
1344         default:
1345                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1346         }
1347
1348         return val;
1349 }
1350
1351 static void krait_write_pmresrn(int n, u32 val)
1352 {
1353         switch (n) {
1354         case 0:
1355                 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1356                 break;
1357         case 1:
1358                 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1359                 break;
1360         case 2:
1361                 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1362                 break;
1363         default:
1364                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1365         }
1366 }
1367
1368 static u32 venum_read_pmresr(void)
1369 {
1370         u32 val;
1371         asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1372         return val;
1373 }
1374
1375 static void venum_write_pmresr(u32 val)
1376 {
1377         asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1378 }
1379
1380 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1381 {
1382         u32 venum_new_val;
1383         u32 fp_new_val;
1384
1385         BUG_ON(preemptible());
1386         /* CPACR Enable CP10 and CP11 access */
1387         *venum_orig_val = get_copro_access();
1388         venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1389         set_copro_access(venum_new_val);
1390
1391         /* Enable FPEXC */
1392         *fp_orig_val = fmrx(FPEXC);
1393         fp_new_val = *fp_orig_val | FPEXC_EN;
1394         fmxr(FPEXC, fp_new_val);
1395 }
1396
1397 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1398 {
1399         BUG_ON(preemptible());
1400         /* Restore FPEXC */
1401         fmxr(FPEXC, fp_orig_val);
1402         isb();
1403         /* Restore CPACR */
1404         set_copro_access(venum_orig_val);
1405 }
1406
1407 static u32 krait_get_pmresrn_event(unsigned int region)
1408 {
1409         static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1410                                              KRAIT_PMRESR1_GROUP0,
1411                                              KRAIT_PMRESR2_GROUP0 };
1412         return pmresrn_table[region];
1413 }
1414
1415 static void krait_evt_setup(int idx, u32 config_base)
1416 {
1417         u32 val;
1418         u32 mask;
1419         u32 vval, fval;
1420         unsigned int region = EVENT_REGION(config_base);
1421         unsigned int group = EVENT_GROUP(config_base);
1422         unsigned int code = EVENT_CODE(config_base);
1423         unsigned int group_shift;
1424         bool venum_event = EVENT_VENUM(config_base);
1425
1426         group_shift = group * 8;
1427         mask = 0xff << group_shift;
1428
1429         /* Configure evtsel for the region and group */
1430         if (venum_event)
1431                 val = KRAIT_VPMRESR0_GROUP0;
1432         else
1433                 val = krait_get_pmresrn_event(region);
1434         val += group;
1435         /* Mix in mode-exclusion bits */
1436         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1437         armv7_pmnc_write_evtsel(idx, val);
1438
1439         if (venum_event) {
1440                 venum_pre_pmresr(&vval, &fval);
1441                 val = venum_read_pmresr();
1442                 val &= ~mask;
1443                 val |= code << group_shift;
1444                 val |= PMRESRn_EN;
1445                 venum_write_pmresr(val);
1446                 venum_post_pmresr(vval, fval);
1447         } else {
1448                 val = krait_read_pmresrn(region);
1449                 val &= ~mask;
1450                 val |= code << group_shift;
1451                 val |= PMRESRn_EN;
1452                 krait_write_pmresrn(region, val);
1453         }
1454 }
1455
1456 static u32 clear_pmresrn_group(u32 val, int group)
1457 {
1458         u32 mask;
1459         int group_shift;
1460
1461         group_shift = group * 8;
1462         mask = 0xff << group_shift;
1463         val &= ~mask;
1464
1465         /* Don't clear enable bit if entire region isn't disabled */
1466         if (val & ~PMRESRn_EN)
1467                 return val |= PMRESRn_EN;
1468
1469         return 0;
1470 }
1471
1472 static void krait_clearpmu(u32 config_base)
1473 {
1474         u32 val;
1475         u32 vval, fval;
1476         unsigned int region = EVENT_REGION(config_base);
1477         unsigned int group = EVENT_GROUP(config_base);
1478         bool venum_event = EVENT_VENUM(config_base);
1479
1480         if (venum_event) {
1481                 venum_pre_pmresr(&vval, &fval);
1482                 val = venum_read_pmresr();
1483                 val = clear_pmresrn_group(val, group);
1484                 venum_write_pmresr(val);
1485                 venum_post_pmresr(vval, fval);
1486         } else {
1487                 val = krait_read_pmresrn(region);
1488                 val = clear_pmresrn_group(val, group);
1489                 krait_write_pmresrn(region, val);
1490         }
1491 }
1492
1493 static void krait_pmu_disable_event(struct perf_event *event)
1494 {
1495         unsigned long flags;
1496         struct hw_perf_event *hwc = &event->hw;
1497         int idx = hwc->idx;
1498         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1499         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1500
1501         /* Disable counter and interrupt */
1502         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1503
1504         /* Disable counter */
1505         armv7_pmnc_disable_counter(idx);
1506
1507         /*
1508          * Clear pmresr code (if destined for PMNx counters)
1509          */
1510         if (hwc->config_base & KRAIT_EVENT_MASK)
1511                 krait_clearpmu(hwc->config_base);
1512
1513         /* Disable interrupt for this counter */
1514         armv7_pmnc_disable_intens(idx);
1515
1516         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1517 }
1518
1519 static void krait_pmu_enable_event(struct perf_event *event)
1520 {
1521         unsigned long flags;
1522         struct hw_perf_event *hwc = &event->hw;
1523         int idx = hwc->idx;
1524         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1525         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1526
1527         /*
1528          * Enable counter and interrupt, and set the counter to count
1529          * the event that we're interested in.
1530          */
1531         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1532
1533         /* Disable counter */
1534         armv7_pmnc_disable_counter(idx);
1535
1536         /*
1537          * Set event (if destined for PMNx counters)
1538          * We set the event for the cycle counter because we
1539          * have the ability to perform event filtering.
1540          */
1541         if (hwc->config_base & KRAIT_EVENT_MASK)
1542                 krait_evt_setup(idx, hwc->config_base);
1543         else
1544                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1545
1546         /* Enable interrupt for this counter */
1547         armv7_pmnc_enable_intens(idx);
1548
1549         /* Enable counter */
1550         armv7_pmnc_enable_counter(idx);
1551
1552         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1553 }
1554
1555 static void krait_pmu_reset(void *info)
1556 {
1557         u32 vval, fval;
1558         struct arm_pmu *cpu_pmu = info;
1559         u32 idx, nb_cnt = cpu_pmu->num_events;
1560
1561         armv7pmu_reset(info);
1562
1563         /* Clear all pmresrs */
1564         krait_write_pmresrn(0, 0);
1565         krait_write_pmresrn(1, 0);
1566         krait_write_pmresrn(2, 0);
1567
1568         venum_pre_pmresr(&vval, &fval);
1569         venum_write_pmresr(0);
1570         venum_post_pmresr(vval, fval);
1571
1572         /* Reset PMxEVNCTCR to sane default */
1573         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1574                 armv7_pmnc_select_counter(idx);
1575                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1576         }
1577
1578 }
1579
1580 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1581                               unsigned int group)
1582 {
1583         int bit;
1584         struct hw_perf_event *hwc = &event->hw;
1585         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1586
1587         if (hwc->config_base & VENUM_EVENT)
1588                 bit = KRAIT_VPMRESR0_GROUP0;
1589         else
1590                 bit = krait_get_pmresrn_event(region);
1591         bit -= krait_get_pmresrn_event(0);
1592         bit += group;
1593         /*
1594          * Lower bits are reserved for use by the counters (see
1595          * armv7pmu_get_event_idx() for more info)
1596          */
1597         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1598
1599         return bit;
1600 }
1601
1602 /*
1603  * We check for column exclusion constraints here.
1604  * Two events cant use the same group within a pmresr register.
1605  */
1606 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1607                                    struct perf_event *event)
1608 {
1609         int idx;
1610         int bit = -1;
1611         struct hw_perf_event *hwc = &event->hw;
1612         unsigned int region = EVENT_REGION(hwc->config_base);
1613         unsigned int code = EVENT_CODE(hwc->config_base);
1614         unsigned int group = EVENT_GROUP(hwc->config_base);
1615         bool venum_event = EVENT_VENUM(hwc->config_base);
1616         bool krait_event = EVENT_CPU(hwc->config_base);
1617
1618         if (venum_event || krait_event) {
1619                 /* Ignore invalid events */
1620                 if (group > 3 || region > 2)
1621                         return -EINVAL;
1622                 if (venum_event && (code & 0xe0))
1623                         return -EINVAL;
1624
1625                 bit = krait_event_to_bit(event, region, group);
1626                 if (test_and_set_bit(bit, cpuc->used_mask))
1627                         return -EAGAIN;
1628         }
1629
1630         idx = armv7pmu_get_event_idx(cpuc, event);
1631         if (idx < 0 && bit >= 0)
1632                 clear_bit(bit, cpuc->used_mask);
1633
1634         return idx;
1635 }
1636
1637 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1638                                       struct perf_event *event)
1639 {
1640         int bit;
1641         struct hw_perf_event *hwc = &event->hw;
1642         unsigned int region = EVENT_REGION(hwc->config_base);
1643         unsigned int group = EVENT_GROUP(hwc->config_base);
1644         bool venum_event = EVENT_VENUM(hwc->config_base);
1645         bool krait_event = EVENT_CPU(hwc->config_base);
1646
1647         armv7pmu_clear_event_idx(cpuc, event);
1648         if (venum_event || krait_event) {
1649                 bit = krait_event_to_bit(event, region, group);
1650                 clear_bit(bit, cpuc->used_mask);
1651         }
1652 }
1653
1654 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1655 {
1656         armv7pmu_init(cpu_pmu);
1657         cpu_pmu->name           = "armv7_krait";
1658         /* Some early versions of Krait don't support PC write events */
1659         if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1660                                   "qcom,no-pc-write"))
1661                 cpu_pmu->map_event = krait_map_event_no_branch;
1662         else
1663                 cpu_pmu->map_event = krait_map_event;
1664         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1665         cpu_pmu->reset          = krait_pmu_reset;
1666         cpu_pmu->enable         = krait_pmu_enable_event;
1667         cpu_pmu->disable        = krait_pmu_disable_event;
1668         cpu_pmu->get_event_idx  = krait_pmu_get_event_idx;
1669         cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1670         return armv7_probe_num_events(cpu_pmu);
1671 }
1672
1673 /*
1674  * Scorpion Local Performance Monitor Register (LPMn)
1675  *
1676  *            31   30     24     16     8      0
1677  *            +--------------------------------+
1678  *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1679  *            +--------------------------------+
1680  *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1681  *            +--------------------------------+
1682  *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1683  *            +--------------------------------+
1684  *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1685  *            +--------------------------------+
1686  *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1687  *            +--------------------------------+
1688  *              EN | G=3  | G=2  | G=1  | G=0
1689  *
1690  *
1691  *  Event Encoding:
1692  *
1693  *      hwc->config_base = 0xNRCCG
1694  *
1695  *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1696  *      R  = region register
1697  *      CC = class of events the group G is choosing from
1698  *      G  = group or particular event
1699  *
1700  *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1701  *
1702  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1703  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1704  *  events (interrupts for example). An event code is broken down into
1705  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1706  *  example).
1707  */
1708
1709 static u32 scorpion_read_pmresrn(int n)
1710 {
1711         u32 val;
1712
1713         switch (n) {
1714         case 0:
1715                 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1716                 break;
1717         case 1:
1718                 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1719                 break;
1720         case 2:
1721                 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1722                 break;
1723         case 3:
1724                 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1725                 break;
1726         default:
1727                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1728         }
1729
1730         return val;
1731 }
1732
1733 static void scorpion_write_pmresrn(int n, u32 val)
1734 {
1735         switch (n) {
1736         case 0:
1737                 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1738                 break;
1739         case 1:
1740                 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1741                 break;
1742         case 2:
1743                 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1744                 break;
1745         case 3:
1746                 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1747                 break;
1748         default:
1749                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1750         }
1751 }
1752
1753 static u32 scorpion_get_pmresrn_event(unsigned int region)
1754 {
1755         static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1756                                              SCORPION_LPM1_GROUP0,
1757                                              SCORPION_LPM2_GROUP0,
1758                                              SCORPION_L2LPM_GROUP0 };
1759         return pmresrn_table[region];
1760 }
1761
1762 static void scorpion_evt_setup(int idx, u32 config_base)
1763 {
1764         u32 val;
1765         u32 mask;
1766         u32 vval, fval;
1767         unsigned int region = EVENT_REGION(config_base);
1768         unsigned int group = EVENT_GROUP(config_base);
1769         unsigned int code = EVENT_CODE(config_base);
1770         unsigned int group_shift;
1771         bool venum_event = EVENT_VENUM(config_base);
1772
1773         group_shift = group * 8;
1774         mask = 0xff << group_shift;
1775
1776         /* Configure evtsel for the region and group */
1777         if (venum_event)
1778                 val = SCORPION_VLPM_GROUP0;
1779         else
1780                 val = scorpion_get_pmresrn_event(region);
1781         val += group;
1782         /* Mix in mode-exclusion bits */
1783         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1784         armv7_pmnc_write_evtsel(idx, val);
1785
1786         asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1787
1788         if (venum_event) {
1789                 venum_pre_pmresr(&vval, &fval);
1790                 val = venum_read_pmresr();
1791                 val &= ~mask;
1792                 val |= code << group_shift;
1793                 val |= PMRESRn_EN;
1794                 venum_write_pmresr(val);
1795                 venum_post_pmresr(vval, fval);
1796         } else {
1797                 val = scorpion_read_pmresrn(region);
1798                 val &= ~mask;
1799                 val |= code << group_shift;
1800                 val |= PMRESRn_EN;
1801                 scorpion_write_pmresrn(region, val);
1802         }
1803 }
1804
1805 static void scorpion_clearpmu(u32 config_base)
1806 {
1807         u32 val;
1808         u32 vval, fval;
1809         unsigned int region = EVENT_REGION(config_base);
1810         unsigned int group = EVENT_GROUP(config_base);
1811         bool venum_event = EVENT_VENUM(config_base);
1812
1813         if (venum_event) {
1814                 venum_pre_pmresr(&vval, &fval);
1815                 val = venum_read_pmresr();
1816                 val = clear_pmresrn_group(val, group);
1817                 venum_write_pmresr(val);
1818                 venum_post_pmresr(vval, fval);
1819         } else {
1820                 val = scorpion_read_pmresrn(region);
1821                 val = clear_pmresrn_group(val, group);
1822                 scorpion_write_pmresrn(region, val);
1823         }
1824 }
1825
1826 static void scorpion_pmu_disable_event(struct perf_event *event)
1827 {
1828         unsigned long flags;
1829         struct hw_perf_event *hwc = &event->hw;
1830         int idx = hwc->idx;
1831         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1832         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1833
1834         /* Disable counter and interrupt */
1835         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1836
1837         /* Disable counter */
1838         armv7_pmnc_disable_counter(idx);
1839
1840         /*
1841          * Clear pmresr code (if destined for PMNx counters)
1842          */
1843         if (hwc->config_base & KRAIT_EVENT_MASK)
1844                 scorpion_clearpmu(hwc->config_base);
1845
1846         /* Disable interrupt for this counter */
1847         armv7_pmnc_disable_intens(idx);
1848
1849         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1850 }
1851
1852 static void scorpion_pmu_enable_event(struct perf_event *event)
1853 {
1854         unsigned long flags;
1855         struct hw_perf_event *hwc = &event->hw;
1856         int idx = hwc->idx;
1857         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1858         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1859
1860         /*
1861          * Enable counter and interrupt, and set the counter to count
1862          * the event that we're interested in.
1863          */
1864         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1865
1866         /* Disable counter */
1867         armv7_pmnc_disable_counter(idx);
1868
1869         /*
1870          * Set event (if destined for PMNx counters)
1871          * We don't set the event for the cycle counter because we
1872          * don't have the ability to perform event filtering.
1873          */
1874         if (hwc->config_base & KRAIT_EVENT_MASK)
1875                 scorpion_evt_setup(idx, hwc->config_base);
1876         else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1877                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1878
1879         /* Enable interrupt for this counter */
1880         armv7_pmnc_enable_intens(idx);
1881
1882         /* Enable counter */
1883         armv7_pmnc_enable_counter(idx);
1884
1885         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1886 }
1887
1888 static void scorpion_pmu_reset(void *info)
1889 {
1890         u32 vval, fval;
1891         struct arm_pmu *cpu_pmu = info;
1892         u32 idx, nb_cnt = cpu_pmu->num_events;
1893
1894         armv7pmu_reset(info);
1895
1896         /* Clear all pmresrs */
1897         scorpion_write_pmresrn(0, 0);
1898         scorpion_write_pmresrn(1, 0);
1899         scorpion_write_pmresrn(2, 0);
1900         scorpion_write_pmresrn(3, 0);
1901
1902         venum_pre_pmresr(&vval, &fval);
1903         venum_write_pmresr(0);
1904         venum_post_pmresr(vval, fval);
1905
1906         /* Reset PMxEVNCTCR to sane default */
1907         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1908                 armv7_pmnc_select_counter(idx);
1909                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1910         }
1911 }
1912
1913 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1914                               unsigned int group)
1915 {
1916         int bit;
1917         struct hw_perf_event *hwc = &event->hw;
1918         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1919
1920         if (hwc->config_base & VENUM_EVENT)
1921                 bit = SCORPION_VLPM_GROUP0;
1922         else
1923                 bit = scorpion_get_pmresrn_event(region);
1924         bit -= scorpion_get_pmresrn_event(0);
1925         bit += group;
1926         /*
1927          * Lower bits are reserved for use by the counters (see
1928          * armv7pmu_get_event_idx() for more info)
1929          */
1930         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1931
1932         return bit;
1933 }
1934
1935 /*
1936  * We check for column exclusion constraints here.
1937  * Two events cant use the same group within a pmresr register.
1938  */
1939 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1940                                    struct perf_event *event)
1941 {
1942         int idx;
1943         int bit = -1;
1944         struct hw_perf_event *hwc = &event->hw;
1945         unsigned int region = EVENT_REGION(hwc->config_base);
1946         unsigned int group = EVENT_GROUP(hwc->config_base);
1947         bool venum_event = EVENT_VENUM(hwc->config_base);
1948         bool scorpion_event = EVENT_CPU(hwc->config_base);
1949
1950         if (venum_event || scorpion_event) {
1951                 /* Ignore invalid events */
1952                 if (group > 3 || region > 3)
1953                         return -EINVAL;
1954
1955                 bit = scorpion_event_to_bit(event, region, group);
1956                 if (test_and_set_bit(bit, cpuc->used_mask))
1957                         return -EAGAIN;
1958         }
1959
1960         idx = armv7pmu_get_event_idx(cpuc, event);
1961         if (idx < 0 && bit >= 0)
1962                 clear_bit(bit, cpuc->used_mask);
1963
1964         return idx;
1965 }
1966
1967 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1968                                       struct perf_event *event)
1969 {
1970         int bit;
1971         struct hw_perf_event *hwc = &event->hw;
1972         unsigned int region = EVENT_REGION(hwc->config_base);
1973         unsigned int group = EVENT_GROUP(hwc->config_base);
1974         bool venum_event = EVENT_VENUM(hwc->config_base);
1975         bool scorpion_event = EVENT_CPU(hwc->config_base);
1976
1977         armv7pmu_clear_event_idx(cpuc, event);
1978         if (venum_event || scorpion_event) {
1979                 bit = scorpion_event_to_bit(event, region, group);
1980                 clear_bit(bit, cpuc->used_mask);
1981         }
1982 }
1983
1984 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1985 {
1986         armv7pmu_init(cpu_pmu);
1987         cpu_pmu->name           = "armv7_scorpion";
1988         cpu_pmu->map_event      = scorpion_map_event;
1989         cpu_pmu->reset          = scorpion_pmu_reset;
1990         cpu_pmu->enable         = scorpion_pmu_enable_event;
1991         cpu_pmu->disable        = scorpion_pmu_disable_event;
1992         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1993         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1994         return armv7_probe_num_events(cpu_pmu);
1995 }
1996
1997 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1998 {
1999         armv7pmu_init(cpu_pmu);
2000         cpu_pmu->name           = "armv7_scorpion_mp";
2001         cpu_pmu->map_event      = scorpion_map_event;
2002         cpu_pmu->reset          = scorpion_pmu_reset;
2003         cpu_pmu->enable         = scorpion_pmu_enable_event;
2004         cpu_pmu->disable        = scorpion_pmu_disable_event;
2005         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
2006         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
2007         return armv7_probe_num_events(cpu_pmu);
2008 }
2009
2010 static const struct of_device_id armv7_pmu_of_device_ids[] = {
2011         {.compatible = "arm,cortex-a17-pmu",    .data = armv7_a17_pmu_init},
2012         {.compatible = "arm,cortex-a15-pmu",    .data = armv7_a15_pmu_init},
2013         {.compatible = "arm,cortex-a12-pmu",    .data = armv7_a12_pmu_init},
2014         {.compatible = "arm,cortex-a9-pmu",     .data = armv7_a9_pmu_init},
2015         {.compatible = "arm,cortex-a8-pmu",     .data = armv7_a8_pmu_init},
2016         {.compatible = "arm,cortex-a7-pmu",     .data = armv7_a7_pmu_init},
2017         {.compatible = "arm,cortex-a5-pmu",     .data = armv7_a5_pmu_init},
2018         {.compatible = "qcom,krait-pmu",        .data = krait_pmu_init},
2019         {.compatible = "qcom,scorpion-pmu",     .data = scorpion_pmu_init},
2020         {.compatible = "qcom,scorpion-mp-pmu",  .data = scorpion_mp_pmu_init},
2021         {},
2022 };
2023
2024 static const struct pmu_probe_info armv7_pmu_probe_table[] = {
2025         ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
2026         ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
2027         { /* sentinel value */ }
2028 };
2029
2030
2031 static int armv7_pmu_device_probe(struct platform_device *pdev)
2032 {
2033         return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
2034                                     armv7_pmu_probe_table);
2035 }
2036
2037 static struct platform_driver armv7_pmu_driver = {
2038         .driver         = {
2039                 .name   = "armv7-pmu",
2040                 .of_match_table = armv7_pmu_of_device_ids,
2041                 .suppress_bind_attrs = true,
2042         },
2043         .probe          = armv7_pmu_device_probe,
2044 };
2045
2046 builtin_platform_driver(armv7_pmu_driver);
2047 #endif  /* CONFIG_CPU_V7 */