arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20
  21 #include <asm/cp15.h>
  22 #include <asm/vfp.h>
  23 #include "../vfp/vfpinstr.h"
  24
  25 /*
  26  * Common ARMv7 event types
  27  *
  28  * Note: An implementation may not be able to count all of these events
  29  * but the encodings are considered to be `reserved' in the case that
  30  * they are not available.
  31  */
  32 enum armv7_perf_types {
  33         ARMV7_PERFCTR_PMNC_SW_INCR                      = 0x00,
  34         ARMV7_PERFCTR_L1_ICACHE_REFILL                  = 0x01,
  35         ARMV7_PERFCTR_ITLB_REFILL                       = 0x02,
  36         ARMV7_PERFCTR_L1_DCACHE_REFILL                  = 0x03,
  37         ARMV7_PERFCTR_L1_DCACHE_ACCESS                  = 0x04,
  38         ARMV7_PERFCTR_DTLB_REFILL                       = 0x05,
  39         ARMV7_PERFCTR_MEM_READ                          = 0x06,
  40         ARMV7_PERFCTR_MEM_WRITE                         = 0x07,
  41         ARMV7_PERFCTR_INSTR_EXECUTED                    = 0x08,
  42         ARMV7_PERFCTR_EXC_TAKEN                         = 0x09,
  43         ARMV7_PERFCTR_EXC_EXECUTED                      = 0x0A,
  44         ARMV7_PERFCTR_CID_WRITE                         = 0x0B,
  45
  46         /*
  47          * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  48          * It counts:
  49          *  - all (taken) branch instructions,
  50          *  - instructions that explicitly write the PC,
  51          *  - exception generating instructions.
  52          */
  53         ARMV7_PERFCTR_PC_WRITE                          = 0x0C,
  54         ARMV7_PERFCTR_PC_IMM_BRANCH                     = 0x0D,
  55         ARMV7_PERFCTR_PC_PROC_RETURN                    = 0x0E,
  56         ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              = 0x0F,
  57         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                = 0x10,
  58         ARMV7_PERFCTR_CLOCK_CYCLES                      = 0x11,
  59         ARMV7_PERFCTR_PC_BRANCH_PRED                    = 0x12,
  60
  61         /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  62         ARMV7_PERFCTR_MEM_ACCESS                        = 0x13,
  63         ARMV7_PERFCTR_L1_ICACHE_ACCESS                  = 0x14,
  64         ARMV7_PERFCTR_L1_DCACHE_WB                      = 0x15,
  65         ARMV7_PERFCTR_L2_CACHE_ACCESS                   = 0x16,
  66         ARMV7_PERFCTR_L2_CACHE_REFILL                   = 0x17,
  67         ARMV7_PERFCTR_L2_CACHE_WB                       = 0x18,
  68         ARMV7_PERFCTR_BUS_ACCESS                        = 0x19,
  69         ARMV7_PERFCTR_MEM_ERROR                         = 0x1A,
  70         ARMV7_PERFCTR_INSTR_SPEC                        = 0x1B,
  71         ARMV7_PERFCTR_TTBR_WRITE                        = 0x1C,
  72         ARMV7_PERFCTR_BUS_CYCLES                        = 0x1D,
  73
  74         ARMV7_PERFCTR_CPU_CYCLES                        = 0xFF
  75 };
  76
  77 /* ARMv7 Cortex-A8 specific event types */
  78 enum armv7_a8_perf_types {
  79         ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                = 0x43,
  80         ARMV7_A8_PERFCTR_L2_CACHE_REFILL                = 0x44,
  81         ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               = 0x50,
  82         ARMV7_A8_PERFCTR_STALL_ISIDE                    = 0x56,
  83 };
  84
  85 /* ARMv7 Cortex-A9 specific event types */
  86 enum armv7_a9_perf_types {
  87         ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              = 0x68,
  88         ARMV7_A9_PERFCTR_STALL_ICACHE                   = 0x60,
  89         ARMV7_A9_PERFCTR_STALL_DISPATCH                 = 0x66,
  90 };
  91
  92 /* ARMv7 Cortex-A5 specific event types */
  93 enum armv7_a5_perf_types {
  94         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              = 0xc2,
  95         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         = 0xc3,
  96 };
  97
  98 /* ARMv7 Cortex-A15 specific event types */
  99 enum armv7_a15_perf_types {
 100         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
 101         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 102         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         = 0x42,
 103         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        = 0x43,
 104
 105         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           = 0x4C,
 106         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          = 0x4D,
 107
 108         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 109         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 110         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          = 0x52,
 111         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         = 0x53,
 112
 113         ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 114 };
 115
 116 /* ARMv7 Cortex-A12 specific event types */
 117 enum armv7_a12_perf_types {
 118         ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
 119         ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 120
 121         ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 122         ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 123
 124         ARMV7_A12_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 125
 126         ARMV7_A12_PERFCTR_PF_TLB_REFILL                 = 0xe7,
 127 };
 128
 129 /* ARMv7 Krait specific event types */
 130 enum krait_perf_types {
 131         KRAIT_PMRESR0_GROUP0                            = 0xcc,
 132         KRAIT_PMRESR1_GROUP0                            = 0xd0,
 133         KRAIT_PMRESR2_GROUP0                            = 0xd4,
 134         KRAIT_VPMRESR0_GROUP0                           = 0xd8,
 135
 136         KRAIT_PERFCTR_L1_ICACHE_ACCESS                  = 0x10011,
 137         KRAIT_PERFCTR_L1_ICACHE_MISS                    = 0x10010,
 138
 139         KRAIT_PERFCTR_L1_ITLB_ACCESS                    = 0x12222,
 140         KRAIT_PERFCTR_L1_DTLB_ACCESS                    = 0x12210,
 141 };
 142
 143 /* ARMv7 Scorpion specific event types */
 144 enum scorpion_perf_types {
 145         SCORPION_LPM0_GROUP0                            = 0x4c,
 146         SCORPION_LPM1_GROUP0                            = 0x50,
 147         SCORPION_LPM2_GROUP0                            = 0x54,
 148         SCORPION_L2LPM_GROUP0                           = 0x58,
 149         SCORPION_VLPM_GROUP0                            = 0x5c,
 150
 151         SCORPION_ICACHE_ACCESS                          = 0x10053,
 152         SCORPION_ICACHE_MISS                            = 0x10052,
 153
 154         SCORPION_DTLB_ACCESS                            = 0x12013,
 155         SCORPION_DTLB_MISS                              = 0x12012,
 156
 157         SCORPION_ITLB_MISS                              = 0x12021,
 158 };
 159
 160 /*
 161  * Cortex-A8 HW events mapping
 162  *
 163  * The hardware events that we support. We do support cache operations but
 164  * we have harvard caches and no way to combine instruction and data
 165  * accesses/misses in hardware.
 166  */
 167 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 168         PERF_MAP_ALL_UNSUPPORTED,
 169         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 170         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 171         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 172         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 173         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 174         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 175         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
 176 };
 177
 178 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 179                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 180                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 181         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 182
 183         /*
 184          * The performance counters don't differentiate between read and write
 185          * accesses/misses so this isn't strictly correct, but it's the best we
 186          * can do. Writes and reads get combined.
 187          */
 188         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 189         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 190         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 191         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 192
 193         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 194         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 195
 196         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 197         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 198         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 199         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 200
 201         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 202         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 203
 204         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 205         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 206
 207         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 208         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 209         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 210         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 211 };
 212
 213 /*
 214  * Cortex-A9 HW events mapping
 215  */
 216 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 217         PERF_MAP_ALL_UNSUPPORTED,
 218         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 219         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 220         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 221         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 222         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 223         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 224         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
 225         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
 226 };
 227
 228 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 229                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 230                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 231         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 232
 233         /*
 234          * The performance counters don't differentiate between read and write
 235          * accesses/misses so this isn't strictly correct, but it's the best we
 236          * can do. Writes and reads get combined.
 237          */
 238         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 239         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 240         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 241         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 242
 243         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 244
 245         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 246         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 247
 248         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 249         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 250
 251         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 252         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 253         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 254         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 255 };
 256
 257 /*
 258  * Cortex-A5 HW events mapping
 259  */
 260 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 261         PERF_MAP_ALL_UNSUPPORTED,
 262         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 263         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 264         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 265         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 266         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 267         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 268 };
 269
 270 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 271                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 272                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 273         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 274
 275         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 276         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 277         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 278         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 279         [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 280         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 281
 282         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 283         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 284         /*
 285          * The prefetch counters don't differentiate between the I side and the
 286          * D side.
 287          */
 288         [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 289         [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 290
 291         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 292         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 293
 294         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 295         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 296
 297         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 298         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 299         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 300         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 301 };
 302
 303 /*
 304  * Cortex-A15 HW events mapping
 305  */
 306 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 307         PERF_MAP_ALL_UNSUPPORTED,
 308         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 309         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 310         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 311         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 312         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 313         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 314         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 315 };
 316
 317 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 318                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 319                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 320         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 321
 322         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 323         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 324         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 325         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 326
 327         /*
 328          * Not all performance counters differentiate between read and write
 329          * accesses/misses so we're not always strictly correct, but it's the
 330          * best we can do. Writes and reads get combined in these cases.
 331          */
 332         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 333         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 334
 335         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 336         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 337         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 338         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 339
 340         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 341         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 342
 343         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 344         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 345
 346         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 347         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 348         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 349         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 350 };
 351
 352 /*
 353  * Cortex-A7 HW events mapping
 354  */
 355 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
 356         PERF_MAP_ALL_UNSUPPORTED,
 357         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 358         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 359         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 360         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 361         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 362         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 363         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 364 };
 365
 366 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 367                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 368                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 369         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 370
 371         /*
 372          * The performance counters don't differentiate between read and write
 373          * accesses/misses so this isn't strictly correct, but it's the best we
 374          * can do. Writes and reads get combined.
 375          */
 376         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 377         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 378         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 379         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 380
 381         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 382         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 383
 384         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 385         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 386         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 387         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 388
 389         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 390         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 391
 392         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 393         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 394
 395         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 396         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 397         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 398         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 399 };
 400
 401 /*
 402  * Cortex-A12 HW events mapping
 403  */
 404 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
 405         PERF_MAP_ALL_UNSUPPORTED,
 406         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 407         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 408         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 409         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 410         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
 411         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 412         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 413 };
 414
 415 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 416                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 417                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 418         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 419
 420         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
 421         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 422         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 423         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 424
 425         /*
 426          * Not all performance counters differentiate between read and write
 427          * accesses/misses so we're not always strictly correct, but it's the
 428          * best we can do. Writes and reads get combined in these cases.
 429          */
 430         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 431         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 432
 433         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
 434         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 435         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
 436         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 437
 438         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 439         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 440         [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]       = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
 441
 442         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 443         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 444
 445         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 446         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 447         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 448         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 449 };
 450
 451 /*
 452  * Krait HW events mapping
 453  */
 454 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
 455         PERF_MAP_ALL_UNSUPPORTED,
 456         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 457         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 458         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 459         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 460         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 461 };
 462
 463 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
 464         PERF_MAP_ALL_UNSUPPORTED,
 465         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 466         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 467         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 468         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 469 };
 470
 471 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 472                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 473                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 474         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 475
 476         /*
 477          * The performance counters don't differentiate between read and write
 478          * accesses/misses so this isn't strictly correct, but it's the best we
 479          * can do. Writes and reads get combined.
 480          */
 481         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 482         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 483         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 484         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 485
 486         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
 487         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = KRAIT_PERFCTR_L1_ICACHE_MISS,
 488
 489         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 490         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 491
 492         [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 493         [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 494
 495         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 496         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 497         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 498         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 499 };
 500
 501 /*
 502  * Scorpion HW events mapping
 503  */
 504 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
 505         PERF_MAP_ALL_UNSUPPORTED,
 506         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 507         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 508         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 509         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 510         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 511 };
 512
 513 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 514                                             [PERF_COUNT_HW_CACHE_OP_MAX]
 515                                             [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 516         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 517         /*
 518          * The performance counters don't differentiate between read and write
 519          * accesses/misses so this isn't strictly correct, but it's the best we
 520          * can do. Writes and reads get combined.
 521          */
 522         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 523         [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 524         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 525         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 526         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
 527         [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
 528         /*
 529          * Only ITLB misses and DTLB refills are supported.  If users want the
 530          * DTLB refills misses a raw counter must be used.
 531          */
 532         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 533         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 534         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 535         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 536         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 537         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 538         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 539         [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 540         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 541         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 542 };
 543
 544 /*
 545  * Perf Events' indices
 546  */
 547 #define ARMV7_IDX_CYCLE_COUNTER 0
 548 #define ARMV7_IDX_COUNTER0      1
 549 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
 550         (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 551
 552 #define ARMV7_MAX_COUNTERS      32
 553 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
 554
 555 /*
 556  * ARMv7 low level PMNC access
 557  */
 558
 559 /*
 560  * Perf Event to low level counters mapping
 561  */
 562 #define ARMV7_IDX_TO_COUNTER(x) \
 563         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 564
 565 /*
 566  * Per-CPU PMNC: config reg
 567  */
 568 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 569 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 570 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 571 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 572 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 573 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 574 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 575 #define ARMV7_PMNC_N_MASK       0x1f
 576 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 577
 578 /*
 579  * FLAG: counters overflow flag status reg
 580  */
 581 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 582 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 583
 584 /*
 585  * PMXEVTYPER: Event selection reg
 586  */
 587 #define ARMV7_EVTYPE_MASK       0xc80000ff      /* Mask for writable bits */
 588 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
 589
 590 /*
 591  * Event filters for PMUv2
 592  */
 593 #define ARMV7_EXCLUDE_PL1       (1 << 31)
 594 #define ARMV7_EXCLUDE_USER      (1 << 30)
 595 #define ARMV7_INCLUDE_HYP       (1 << 27)
 596
 597 static inline u32 armv7_pmnc_read(void)
 598 {
 599         u32 val;
 600         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 601         return val;
 602 }
 603
 604 static inline void armv7_pmnc_write(u32 val)
 605 {
 606         val &= ARMV7_PMNC_MASK;
 607         isb();
 608         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 609 }
 610
 611 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 612 {
 613         return pmnc & ARMV7_OVERFLOWED_MASK;
 614 }
 615
 616 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
 617 {
 618         return idx >= ARMV7_IDX_CYCLE_COUNTER &&
 619                 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
 620 }
 621
 622 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 623 {
 624         return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
 625 }
 626
 627 static inline void armv7_pmnc_select_counter(int idx)
 628 {
 629         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 630         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 631         isb();
 632 }
 633
 634 static inline u32 armv7pmu_read_counter(struct perf_event *event)
 635 {
 636         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 637         struct hw_perf_event *hwc = &event->hw;
 638         int idx = hwc->idx;
 639         u32 value = 0;
 640
 641         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 642                 pr_err("CPU%u reading wrong counter %d\n",
 643                         smp_processor_id(), idx);
 644         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 645                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 646         } else {
 647                 armv7_pmnc_select_counter(idx);
 648                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 649         }
 650
 651         return value;
 652 }
 653
 654 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
 655 {
 656         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 657         struct hw_perf_event *hwc = &event->hw;
 658         int idx = hwc->idx;
 659
 660         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 661                 pr_err("CPU%u writing wrong counter %d\n",
 662                         smp_processor_id(), idx);
 663         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 664                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 665         } else {
 666                 armv7_pmnc_select_counter(idx);
 667                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 668         }
 669 }
 670
 671 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 672 {
 673         armv7_pmnc_select_counter(idx);
 674         val &= ARMV7_EVTYPE_MASK;
 675         asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 676 }
 677
 678 static inline void armv7_pmnc_enable_counter(int idx)
 679 {
 680         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 681         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 682 }
 683
 684 static inline void armv7_pmnc_disable_counter(int idx)
 685 {
 686         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 687         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 688 }
 689
 690 static inline void armv7_pmnc_enable_intens(int idx)
 691 {
 692         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 693         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 694 }
 695
 696 static inline void armv7_pmnc_disable_intens(int idx)
 697 {
 698         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 699         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 700         isb();
 701         /* Clear the overflow flag in case an interrupt is pending. */
 702         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
 703         isb();
 704 }
 705
 706 static inline u32 armv7_pmnc_getreset_flags(void)
 707 {
 708         u32 val;
 709
 710         /* Read */
 711         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 712
 713         /* Write to clear flags */
 714         val &= ARMV7_FLAG_MASK;
 715         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 716
 717         return val;
 718 }
 719
 720 #ifdef DEBUG
 721 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
 722 {
 723         u32 val;
 724         unsigned int cnt;
 725
 726         pr_info("PMNC registers dump:\n");
 727
 728         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 729         pr_info("PMNC  =0x%08x\n", val);
 730
 731         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 732         pr_info("CNTENS=0x%08x\n", val);
 733
 734         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 735         pr_info("INTENS=0x%08x\n", val);
 736
 737         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 738         pr_info("FLAGS =0x%08x\n", val);
 739
 740         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 741         pr_info("SELECT=0x%08x\n", val);
 742
 743         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 744         pr_info("CCNT  =0x%08x\n", val);
 745
 746         for (cnt = ARMV7_IDX_COUNTER0;
 747                         cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
 748                 armv7_pmnc_select_counter(cnt);
 749                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 750                 pr_info("CNT[%d] count =0x%08x\n",
 751                         ARMV7_IDX_TO_COUNTER(cnt), val);
 752                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 753                 pr_info("CNT[%d] evtsel=0x%08x\n",
 754                         ARMV7_IDX_TO_COUNTER(cnt), val);
 755         }
 756 }
 757 #endif
 758
 759 static void armv7pmu_enable_event(struct perf_event *event)
 760 {
 761         unsigned long flags;
 762         struct hw_perf_event *hwc = &event->hw;
 763         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 764         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 765         int idx = hwc->idx;
 766
 767         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 768                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 769                         smp_processor_id(), idx);
 770                 return;
 771         }
 772
 773         /*
 774          * Enable counter and interrupt, and set the counter to count
 775          * the event that we're interested in.
 776          */
 777         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 778
 779         /*
 780          * Disable counter
 781          */
 782         armv7_pmnc_disable_counter(idx);
 783
 784         /*
 785          * Set event (if destined for PMNx counters)
 786          * We only need to set the event for the cycle counter if we
 787          * have the ability to perform event filtering.
 788          */
 789         if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
 790                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 791
 792         /*
 793          * Enable interrupt for this counter
 794          */
 795         armv7_pmnc_enable_intens(idx);
 796
 797         /*
 798          * Enable counter
 799          */
 800         armv7_pmnc_enable_counter(idx);
 801
 802         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 803 }
 804
 805 static void armv7pmu_disable_event(struct perf_event *event)
 806 {
 807         unsigned long flags;
 808         struct hw_perf_event *hwc = &event->hw;
 809         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 810         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 811         int idx = hwc->idx;
 812
 813         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 814                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 815                         smp_processor_id(), idx);
 816                 return;
 817         }
 818
 819         /*
 820          * Disable counter and interrupt
 821          */
 822         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 823
 824         /*
 825          * Disable counter
 826          */
 827         armv7_pmnc_disable_counter(idx);
 828
 829         /*
 830          * Disable interrupt for this counter
 831          */
 832         armv7_pmnc_disable_intens(idx);
 833
 834         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 835 }
 836
 837 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 838 {
 839         u32 pmnc;
 840         struct perf_sample_data data;
 841         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 842         struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 843         struct pt_regs *regs;
 844         int idx;
 845
 846         /*
 847          * Get and reset the IRQ flags
 848          */
 849         pmnc = armv7_pmnc_getreset_flags();
 850
 851         /*
 852          * Did an overflow occur?
 853          */
 854         if (!armv7_pmnc_has_overflowed(pmnc))
 855                 return IRQ_NONE;
 856
 857         /*
 858          * Handle the counter(s) overflow(s)
 859          */
 860         regs = get_irq_regs();
 861
 862         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 863                 struct perf_event *event = cpuc->events[idx];
 864                 struct hw_perf_event *hwc;
 865
 866                 /* Ignore if we don't have an event. */
 867                 if (!event)
 868                         continue;
 869
 870                 /*
 871                  * We have a single interrupt for all counters. Check that
 872                  * each counter has overflowed before we process it.
 873                  */
 874                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 875                         continue;
 876
 877                 hwc = &event->hw;
 878                 armpmu_event_update(event);
 879                 perf_sample_data_init(&data, 0, hwc->last_period);
 880                 if (!armpmu_event_set_period(event))
 881                         continue;
 882
 883                 if (perf_event_overflow(event, &data, regs))
 884                         cpu_pmu->disable(event);
 885         }
 886
 887         /*
 888          * Handle the pending perf events.
 889          *
 890          * Note: this call *must* be run with interrupts disabled. For
 891          * platforms that can have the PMU interrupts raised as an NMI, this
 892          * will not work.
 893          */
 894         irq_work_run();
 895
 896         return IRQ_HANDLED;
 897 }
 898
 899 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
 900 {
 901         unsigned long flags;
 902         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 903
 904         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 905         /* Enable all counters */
 906         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
 907         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 908 }
 909
 910 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
 911 {
 912         unsigned long flags;
 913         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 914
 915         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 916         /* Disable all counters */
 917         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
 918         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 919 }
 920
 921 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
 922                                   struct perf_event *event)
 923 {
 924         int idx;
 925         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 926         struct hw_perf_event *hwc = &event->hw;
 927         unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
 928
 929         /* Always place a cycle counter into the cycle counter. */
 930         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
 931                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
 932                         return -EAGAIN;
 933
 934                 return ARMV7_IDX_CYCLE_COUNTER;
 935         }
 936
 937         /*
 938          * For anything other than a cycle counter, try and use
 939          * the events counters
 940          */
 941         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
 942                 if (!test_and_set_bit(idx, cpuc->used_mask))
 943                         return idx;
 944         }
 945
 946         /* The counters are all in use. */
 947         return -EAGAIN;
 948 }
 949
 950 /*
 951  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
 952  */
 953 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
 954                                      struct perf_event_attr *attr)
 955 {
 956         unsigned long config_base = 0;
 957
 958         if (attr->exclude_idle)
 959                 return -EPERM;
 960         if (attr->exclude_user)
 961                 config_base |= ARMV7_EXCLUDE_USER;
 962         if (attr->exclude_kernel)
 963                 config_base |= ARMV7_EXCLUDE_PL1;
 964         if (!attr->exclude_hv)
 965                 config_base |= ARMV7_INCLUDE_HYP;
 966
 967         /*
 968          * Install the filter into config_base as this is used to
 969          * construct the event type.
 970          */
 971         event->config_base = config_base;
 972
 973         return 0;
 974 }
 975
 976 static void armv7pmu_reset(void *info)
 977 {
 978         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
 979         u32 idx, nb_cnt = cpu_pmu->num_events;
 980
 981         /* The counter and interrupt enable registers are unknown at reset. */
 982         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
 983                 armv7_pmnc_disable_counter(idx);
 984                 armv7_pmnc_disable_intens(idx);
 985         }
 986
 987         /* Initialize & Reset PMNC: C and P bits */
 988         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
 989 }
 990
 991 static int armv7_a8_map_event(struct perf_event *event)
 992 {
 993         return armpmu_map_event(event, &armv7_a8_perf_map,
 994                                 &armv7_a8_perf_cache_map, 0xFF);
 995 }
 996
 997 static int armv7_a9_map_event(struct perf_event *event)
 998 {
 999         return armpmu_map_event(event, &armv7_a9_perf_map,
1000                                 &armv7_a9_perf_cache_map, 0xFF);
1001 }
1002
1003 static int armv7_a5_map_event(struct perf_event *event)
1004 {
1005         return armpmu_map_event(event, &armv7_a5_perf_map,
1006                                 &armv7_a5_perf_cache_map, 0xFF);
1007 }
1008
1009 static int armv7_a15_map_event(struct perf_event *event)
1010 {
1011         return armpmu_map_event(event, &armv7_a15_perf_map,
1012                                 &armv7_a15_perf_cache_map, 0xFF);
1013 }
1014
1015 static int armv7_a7_map_event(struct perf_event *event)
1016 {
1017         return armpmu_map_event(event, &armv7_a7_perf_map,
1018                                 &armv7_a7_perf_cache_map, 0xFF);
1019 }
1020
1021 static int armv7_a12_map_event(struct perf_event *event)
1022 {
1023         return armpmu_map_event(event, &armv7_a12_perf_map,
1024                                 &armv7_a12_perf_cache_map, 0xFF);
1025 }
1026
1027 static int krait_map_event(struct perf_event *event)
1028 {
1029         return armpmu_map_event(event, &krait_perf_map,
1030                                 &krait_perf_cache_map, 0xFFFFF);
1031 }
1032
1033 static int krait_map_event_no_branch(struct perf_event *event)
1034 {
1035         return armpmu_map_event(event, &krait_perf_map_no_branch,
1036                                 &krait_perf_cache_map, 0xFFFFF);
1037 }
1038
1039 static int scorpion_map_event(struct perf_event *event)
1040 {
1041         return armpmu_map_event(event, &scorpion_perf_map,
1042                                 &scorpion_perf_cache_map, 0xFFFFF);
1043 }
1044
1045 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1046 {
1047         cpu_pmu->handle_irq     = armv7pmu_handle_irq;
1048         cpu_pmu->enable         = armv7pmu_enable_event;
1049         cpu_pmu->disable        = armv7pmu_disable_event;
1050         cpu_pmu->read_counter   = armv7pmu_read_counter;
1051         cpu_pmu->write_counter  = armv7pmu_write_counter;
1052         cpu_pmu->get_event_idx  = armv7pmu_get_event_idx;
1053         cpu_pmu->start          = armv7pmu_start;
1054         cpu_pmu->stop           = armv7pmu_stop;
1055         cpu_pmu->reset          = armv7pmu_reset;
1056         cpu_pmu->max_period     = (1LLU << 32) - 1;
1057 };
1058
1059 static void armv7_read_num_pmnc_events(void *info)
1060 {
1061         int *nb_cnt = info;
1062
1063         /* Read the nb of CNTx counters supported from PMNC */
1064         *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1065
1066         /* Add the CPU cycles counter */
1067         *nb_cnt += 1;
1068 }
1069
1070 static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1071 {
1072         return smp_call_function_any(&arm_pmu->supported_cpus,
1073                                      armv7_read_num_pmnc_events,
1074                                      &arm_pmu->num_events, 1);
1075 }
1076
1077 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1078 {
1079         armv7pmu_init(cpu_pmu);
1080         cpu_pmu->name           = "armv7_cortex_a8";
1081         cpu_pmu->map_event      = armv7_a8_map_event;
1082         return armv7_probe_num_events(cpu_pmu);
1083 }
1084
1085 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1086 {
1087         armv7pmu_init(cpu_pmu);
1088         cpu_pmu->name           = "armv7_cortex_a9";
1089         cpu_pmu->map_event      = armv7_a9_map_event;
1090         return armv7_probe_num_events(cpu_pmu);
1091 }
1092
1093 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1094 {
1095         armv7pmu_init(cpu_pmu);
1096         cpu_pmu->name           = "armv7_cortex_a5";
1097         cpu_pmu->map_event      = armv7_a5_map_event;
1098         return armv7_probe_num_events(cpu_pmu);
1099 }
1100
1101 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1102 {
1103         armv7pmu_init(cpu_pmu);
1104         cpu_pmu->name           = "armv7_cortex_a15";
1105         cpu_pmu->map_event      = armv7_a15_map_event;
1106         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1107         return armv7_probe_num_events(cpu_pmu);
1108 }
1109
1110 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1111 {
1112         armv7pmu_init(cpu_pmu);
1113         cpu_pmu->name           = "armv7_cortex_a7";
1114         cpu_pmu->map_event      = armv7_a7_map_event;
1115         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1116         return armv7_probe_num_events(cpu_pmu);
1117 }
1118
1119 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1120 {
1121         armv7pmu_init(cpu_pmu);
1122         cpu_pmu->name           = "armv7_cortex_a12";
1123         cpu_pmu->map_event      = armv7_a12_map_event;
1124         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1125         return armv7_probe_num_events(cpu_pmu);
1126 }
1127
1128 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1129 {
1130         int ret = armv7_a12_pmu_init(cpu_pmu);
1131         cpu_pmu->name = "armv7_cortex_a17";
1132         return ret;
1133 }
1134
1135 /*
1136  * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1137  *
1138  *            31   30     24     16     8      0
1139  *            +--------------------------------+
1140  *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1141  *            +--------------------------------+
1142  *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1143  *            +--------------------------------+
1144  *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1145  *            +--------------------------------+
1146  *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1147  *            +--------------------------------+
1148  *              EN | G=3  | G=2  | G=1  | G=0
1149  *
1150  *  Event Encoding:
1151  *
1152  *      hwc->config_base = 0xNRCCG
1153  *
1154  *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1155  *      R  = region register
1156  *      CC = class of events the group G is choosing from
1157  *      G  = group or particular event
1158  *
1159  *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1160  *
1161  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1162  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1163  *  events (interrupts for example). An event code is broken down into
1164  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1165  *  example).
1166  */
1167
1168 #define KRAIT_EVENT             (1 << 16)
1169 #define VENUM_EVENT             (2 << 16)
1170 #define KRAIT_EVENT_MASK        (KRAIT_EVENT | VENUM_EVENT)
1171 #define PMRESRn_EN              BIT(31)
1172
1173 #define EVENT_REGION(event)     (((event) >> 12) & 0xf)         /* R */
1174 #define EVENT_GROUP(event)      ((event) & 0xf)                 /* G */
1175 #define EVENT_CODE(event)       (((event) >> 4) & 0xff)         /* CC */
1176 #define EVENT_VENUM(event)      (!!(event & VENUM_EVENT))       /* N=2 */
1177 #define EVENT_CPU(event)        (!!(event & KRAIT_EVENT))       /* N=1 */
1178
1179 static u32 krait_read_pmresrn(int n)
1180 {
1181         u32 val;
1182
1183         switch (n) {
1184         case 0:
1185                 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1186                 break;
1187         case 1:
1188                 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1189                 break;
1190         case 2:
1191                 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1192                 break;
1193         default:
1194                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1195         }
1196
1197         return val;
1198 }
1199
1200 static void krait_write_pmresrn(int n, u32 val)
1201 {
1202         switch (n) {
1203         case 0:
1204                 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1205                 break;
1206         case 1:
1207                 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1208                 break;
1209         case 2:
1210                 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1211                 break;
1212         default:
1213                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1214         }
1215 }
1216
1217 static u32 venum_read_pmresr(void)
1218 {
1219         u32 val;
1220         asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1221         return val;
1222 }
1223
1224 static void venum_write_pmresr(u32 val)
1225 {
1226         asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1227 }
1228
1229 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1230 {
1231         u32 venum_new_val;
1232         u32 fp_new_val;
1233
1234         BUG_ON(preemptible());
1235         /* CPACR Enable CP10 and CP11 access */
1236         *venum_orig_val = get_copro_access();
1237         venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1238         set_copro_access(venum_new_val);
1239
1240         /* Enable FPEXC */
1241         *fp_orig_val = fmrx(FPEXC);
1242         fp_new_val = *fp_orig_val | FPEXC_EN;
1243         fmxr(FPEXC, fp_new_val);
1244 }
1245
1246 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1247 {
1248         BUG_ON(preemptible());
1249         /* Restore FPEXC */
1250         fmxr(FPEXC, fp_orig_val);
1251         isb();
1252         /* Restore CPACR */
1253         set_copro_access(venum_orig_val);
1254 }
1255
1256 static u32 krait_get_pmresrn_event(unsigned int region)
1257 {
1258         static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1259                                              KRAIT_PMRESR1_GROUP0,
1260                                              KRAIT_PMRESR2_GROUP0 };
1261         return pmresrn_table[region];
1262 }
1263
1264 static void krait_evt_setup(int idx, u32 config_base)
1265 {
1266         u32 val;
1267         u32 mask;
1268         u32 vval, fval;
1269         unsigned int region = EVENT_REGION(config_base);
1270         unsigned int group = EVENT_GROUP(config_base);
1271         unsigned int code = EVENT_CODE(config_base);
1272         unsigned int group_shift;
1273         bool venum_event = EVENT_VENUM(config_base);
1274
1275         group_shift = group * 8;
1276         mask = 0xff << group_shift;
1277
1278         /* Configure evtsel for the region and group */
1279         if (venum_event)
1280                 val = KRAIT_VPMRESR0_GROUP0;
1281         else
1282                 val = krait_get_pmresrn_event(region);
1283         val += group;
1284         /* Mix in mode-exclusion bits */
1285         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1286         armv7_pmnc_write_evtsel(idx, val);
1287
1288         if (venum_event) {
1289                 venum_pre_pmresr(&vval, &fval);
1290                 val = venum_read_pmresr();
1291                 val &= ~mask;
1292                 val |= code << group_shift;
1293                 val |= PMRESRn_EN;
1294                 venum_write_pmresr(val);
1295                 venum_post_pmresr(vval, fval);
1296         } else {
1297                 val = krait_read_pmresrn(region);
1298                 val &= ~mask;
1299                 val |= code << group_shift;
1300                 val |= PMRESRn_EN;
1301                 krait_write_pmresrn(region, val);
1302         }
1303 }
1304
1305 static u32 clear_pmresrn_group(u32 val, int group)
1306 {
1307         u32 mask;
1308         int group_shift;
1309
1310         group_shift = group * 8;
1311         mask = 0xff << group_shift;
1312         val &= ~mask;
1313
1314         /* Don't clear enable bit if entire region isn't disabled */
1315         if (val & ~PMRESRn_EN)
1316                 return val |= PMRESRn_EN;
1317
1318         return 0;
1319 }
1320
1321 static void krait_clearpmu(u32 config_base)
1322 {
1323         u32 val;
1324         u32 vval, fval;
1325         unsigned int region = EVENT_REGION(config_base);
1326         unsigned int group = EVENT_GROUP(config_base);
1327         bool venum_event = EVENT_VENUM(config_base);
1328
1329         if (venum_event) {
1330                 venum_pre_pmresr(&vval, &fval);
1331                 val = venum_read_pmresr();
1332                 val = clear_pmresrn_group(val, group);
1333                 venum_write_pmresr(val);
1334                 venum_post_pmresr(vval, fval);
1335         } else {
1336                 val = krait_read_pmresrn(region);
1337                 val = clear_pmresrn_group(val, group);
1338                 krait_write_pmresrn(region, val);
1339         }
1340 }
1341
1342 static void krait_pmu_disable_event(struct perf_event *event)
1343 {
1344         unsigned long flags;
1345         struct hw_perf_event *hwc = &event->hw;
1346         int idx = hwc->idx;
1347         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1348         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1349
1350         /* Disable counter and interrupt */
1351         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1352
1353         /* Disable counter */
1354         armv7_pmnc_disable_counter(idx);
1355
1356         /*
1357          * Clear pmresr code (if destined for PMNx counters)
1358          */
1359         if (hwc->config_base & KRAIT_EVENT_MASK)
1360                 krait_clearpmu(hwc->config_base);
1361
1362         /* Disable interrupt for this counter */
1363         armv7_pmnc_disable_intens(idx);
1364
1365         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1366 }
1367
1368 static void krait_pmu_enable_event(struct perf_event *event)
1369 {
1370         unsigned long flags;
1371         struct hw_perf_event *hwc = &event->hw;
1372         int idx = hwc->idx;
1373         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1374         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1375
1376         /*
1377          * Enable counter and interrupt, and set the counter to count
1378          * the event that we're interested in.
1379          */
1380         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1381
1382         /* Disable counter */
1383         armv7_pmnc_disable_counter(idx);
1384
1385         /*
1386          * Set event (if destined for PMNx counters)
1387          * We set the event for the cycle counter because we
1388          * have the ability to perform event filtering.
1389          */
1390         if (hwc->config_base & KRAIT_EVENT_MASK)
1391                 krait_evt_setup(idx, hwc->config_base);
1392         else
1393                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1394
1395         /* Enable interrupt for this counter */
1396         armv7_pmnc_enable_intens(idx);
1397
1398         /* Enable counter */
1399         armv7_pmnc_enable_counter(idx);
1400
1401         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1402 }
1403
1404 static void krait_pmu_reset(void *info)
1405 {
1406         u32 vval, fval;
1407         struct arm_pmu *cpu_pmu = info;
1408         u32 idx, nb_cnt = cpu_pmu->num_events;
1409
1410         armv7pmu_reset(info);
1411
1412         /* Clear all pmresrs */
1413         krait_write_pmresrn(0, 0);
1414         krait_write_pmresrn(1, 0);
1415         krait_write_pmresrn(2, 0);
1416
1417         venum_pre_pmresr(&vval, &fval);
1418         venum_write_pmresr(0);
1419         venum_post_pmresr(vval, fval);
1420
1421         /* Reset PMxEVNCTCR to sane default */
1422         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1423                 armv7_pmnc_select_counter(idx);
1424                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1425         }
1426
1427 }
1428
1429 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1430                               unsigned int group)
1431 {
1432         int bit;
1433         struct hw_perf_event *hwc = &event->hw;
1434         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1435
1436         if (hwc->config_base & VENUM_EVENT)
1437                 bit = KRAIT_VPMRESR0_GROUP0;
1438         else
1439                 bit = krait_get_pmresrn_event(region);
1440         bit -= krait_get_pmresrn_event(0);
1441         bit += group;
1442         /*
1443          * Lower bits are reserved for use by the counters (see
1444          * armv7pmu_get_event_idx() for more info)
1445          */
1446         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1447
1448         return bit;
1449 }
1450
1451 /*
1452  * We check for column exclusion constraints here.
1453  * Two events cant use the same group within a pmresr register.
1454  */
1455 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1456                                    struct perf_event *event)
1457 {
1458         int idx;
1459         int bit = -1;
1460         struct hw_perf_event *hwc = &event->hw;
1461         unsigned int region = EVENT_REGION(hwc->config_base);
1462         unsigned int code = EVENT_CODE(hwc->config_base);
1463         unsigned int group = EVENT_GROUP(hwc->config_base);
1464         bool venum_event = EVENT_VENUM(hwc->config_base);
1465         bool krait_event = EVENT_CPU(hwc->config_base);
1466
1467         if (venum_event || krait_event) {
1468                 /* Ignore invalid events */
1469                 if (group > 3 || region > 2)
1470                         return -EINVAL;
1471                 if (venum_event && (code & 0xe0))
1472                         return -EINVAL;
1473
1474                 bit = krait_event_to_bit(event, region, group);
1475                 if (test_and_set_bit(bit, cpuc->used_mask))
1476                         return -EAGAIN;
1477         }
1478
1479         idx = armv7pmu_get_event_idx(cpuc, event);
1480         if (idx < 0 && bit >= 0)
1481                 clear_bit(bit, cpuc->used_mask);
1482
1483         return idx;
1484 }
1485
1486 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1487                                       struct perf_event *event)
1488 {
1489         int bit;
1490         struct hw_perf_event *hwc = &event->hw;
1491         unsigned int region = EVENT_REGION(hwc->config_base);
1492         unsigned int group = EVENT_GROUP(hwc->config_base);
1493         bool venum_event = EVENT_VENUM(hwc->config_base);
1494         bool krait_event = EVENT_CPU(hwc->config_base);
1495
1496         if (venum_event || krait_event) {
1497                 bit = krait_event_to_bit(event, region, group);
1498                 clear_bit(bit, cpuc->used_mask);
1499         }
1500 }
1501
1502 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1503 {
1504         armv7pmu_init(cpu_pmu);
1505         cpu_pmu->name           = "armv7_krait";
1506         /* Some early versions of Krait don't support PC write events */
1507         if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1508                                   "qcom,no-pc-write"))
1509                 cpu_pmu->map_event = krait_map_event_no_branch;
1510         else
1511                 cpu_pmu->map_event = krait_map_event;
1512         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1513         cpu_pmu->reset          = krait_pmu_reset;
1514         cpu_pmu->enable         = krait_pmu_enable_event;
1515         cpu_pmu->disable        = krait_pmu_disable_event;
1516         cpu_pmu->get_event_idx  = krait_pmu_get_event_idx;
1517         cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1518         return armv7_probe_num_events(cpu_pmu);
1519 }
1520
1521 /*
1522  * Scorpion Local Performance Monitor Register (LPMn)
1523  *
1524  *            31   30     24     16     8      0
1525  *            +--------------------------------+
1526  *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1527  *            +--------------------------------+
1528  *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1529  *            +--------------------------------+
1530  *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1531  *            +--------------------------------+
1532  *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1533  *            +--------------------------------+
1534  *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1535  *            +--------------------------------+
1536  *              EN | G=3  | G=2  | G=1  | G=0
1537  *
1538  *
1539  *  Event Encoding:
1540  *
1541  *      hwc->config_base = 0xNRCCG
1542  *
1543  *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1544  *      R  = region register
1545  *      CC = class of events the group G is choosing from
1546  *      G  = group or particular event
1547  *
1548  *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1549  *
1550  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1551  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1552  *  events (interrupts for example). An event code is broken down into
1553  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1554  *  example).
1555  */
1556
1557 static u32 scorpion_read_pmresrn(int n)
1558 {
1559         u32 val;
1560
1561         switch (n) {
1562         case 0:
1563                 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1564                 break;
1565         case 1:
1566                 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1567                 break;
1568         case 2:
1569                 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1570                 break;
1571         case 3:
1572                 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1573                 break;
1574         default:
1575                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1576         }
1577
1578         return val;
1579 }
1580
1581 static void scorpion_write_pmresrn(int n, u32 val)
1582 {
1583         switch (n) {
1584         case 0:
1585                 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1586                 break;
1587         case 1:
1588                 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1589                 break;
1590         case 2:
1591                 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1592                 break;
1593         case 3:
1594                 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1595                 break;
1596         default:
1597                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1598         }
1599 }
1600
1601 static u32 scorpion_get_pmresrn_event(unsigned int region)
1602 {
1603         static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1604                                              SCORPION_LPM1_GROUP0,
1605                                              SCORPION_LPM2_GROUP0,
1606                                              SCORPION_L2LPM_GROUP0 };
1607         return pmresrn_table[region];
1608 }
1609
1610 static void scorpion_evt_setup(int idx, u32 config_base)
1611 {
1612         u32 val;
1613         u32 mask;
1614         u32 vval, fval;
1615         unsigned int region = EVENT_REGION(config_base);
1616         unsigned int group = EVENT_GROUP(config_base);
1617         unsigned int code = EVENT_CODE(config_base);
1618         unsigned int group_shift;
1619         bool venum_event = EVENT_VENUM(config_base);
1620
1621         group_shift = group * 8;
1622         mask = 0xff << group_shift;
1623
1624         /* Configure evtsel for the region and group */
1625         if (venum_event)
1626                 val = SCORPION_VLPM_GROUP0;
1627         else
1628                 val = scorpion_get_pmresrn_event(region);
1629         val += group;
1630         /* Mix in mode-exclusion bits */
1631         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1632         armv7_pmnc_write_evtsel(idx, val);
1633
1634         asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1635
1636         if (venum_event) {
1637                 venum_pre_pmresr(&vval, &fval);
1638                 val = venum_read_pmresr();
1639                 val &= ~mask;
1640                 val |= code << group_shift;
1641                 val |= PMRESRn_EN;
1642                 venum_write_pmresr(val);
1643                 venum_post_pmresr(vval, fval);
1644         } else {
1645                 val = scorpion_read_pmresrn(region);
1646                 val &= ~mask;
1647                 val |= code << group_shift;
1648                 val |= PMRESRn_EN;
1649                 scorpion_write_pmresrn(region, val);
1650         }
1651 }
1652
1653 static void scorpion_clearpmu(u32 config_base)
1654 {
1655         u32 val;
1656         u32 vval, fval;
1657         unsigned int region = EVENT_REGION(config_base);
1658         unsigned int group = EVENT_GROUP(config_base);
1659         bool venum_event = EVENT_VENUM(config_base);
1660
1661         if (venum_event) {
1662                 venum_pre_pmresr(&vval, &fval);
1663                 val = venum_read_pmresr();
1664                 val = clear_pmresrn_group(val, group);
1665                 venum_write_pmresr(val);
1666                 venum_post_pmresr(vval, fval);
1667         } else {
1668                 val = scorpion_read_pmresrn(region);
1669                 val = clear_pmresrn_group(val, group);
1670                 scorpion_write_pmresrn(region, val);
1671         }
1672 }
1673
1674 static void scorpion_pmu_disable_event(struct perf_event *event)
1675 {
1676         unsigned long flags;
1677         struct hw_perf_event *hwc = &event->hw;
1678         int idx = hwc->idx;
1679         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1680         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1681
1682         /* Disable counter and interrupt */
1683         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1684
1685         /* Disable counter */
1686         armv7_pmnc_disable_counter(idx);
1687
1688         /*
1689          * Clear pmresr code (if destined for PMNx counters)
1690          */
1691         if (hwc->config_base & KRAIT_EVENT_MASK)
1692                 scorpion_clearpmu(hwc->config_base);
1693
1694         /* Disable interrupt for this counter */
1695         armv7_pmnc_disable_intens(idx);
1696
1697         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1698 }
1699
1700 static void scorpion_pmu_enable_event(struct perf_event *event)
1701 {
1702         unsigned long flags;
1703         struct hw_perf_event *hwc = &event->hw;
1704         int idx = hwc->idx;
1705         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1706         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1707
1708         /*
1709          * Enable counter and interrupt, and set the counter to count
1710          * the event that we're interested in.
1711          */
1712         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1713
1714         /* Disable counter */
1715         armv7_pmnc_disable_counter(idx);
1716
1717         /*
1718          * Set event (if destined for PMNx counters)
1719          * We don't set the event for the cycle counter because we
1720          * don't have the ability to perform event filtering.
1721          */
1722         if (hwc->config_base & KRAIT_EVENT_MASK)
1723                 scorpion_evt_setup(idx, hwc->config_base);
1724         else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1725                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1726
1727         /* Enable interrupt for this counter */
1728         armv7_pmnc_enable_intens(idx);
1729
1730         /* Enable counter */
1731         armv7_pmnc_enable_counter(idx);
1732
1733         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1734 }
1735
1736 static void scorpion_pmu_reset(void *info)
1737 {
1738         u32 vval, fval;
1739         struct arm_pmu *cpu_pmu = info;
1740         u32 idx, nb_cnt = cpu_pmu->num_events;
1741
1742         armv7pmu_reset(info);
1743
1744         /* Clear all pmresrs */
1745         scorpion_write_pmresrn(0, 0);
1746         scorpion_write_pmresrn(1, 0);
1747         scorpion_write_pmresrn(2, 0);
1748         scorpion_write_pmresrn(3, 0);
1749
1750         venum_pre_pmresr(&vval, &fval);
1751         venum_write_pmresr(0);
1752         venum_post_pmresr(vval, fval);
1753
1754         /* Reset PMxEVNCTCR to sane default */
1755         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1756                 armv7_pmnc_select_counter(idx);
1757                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1758         }
1759 }
1760
1761 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1762                               unsigned int group)
1763 {
1764         int bit;
1765         struct hw_perf_event *hwc = &event->hw;
1766         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1767
1768         if (hwc->config_base & VENUM_EVENT)
1769                 bit = SCORPION_VLPM_GROUP0;
1770         else
1771                 bit = scorpion_get_pmresrn_event(region);
1772         bit -= scorpion_get_pmresrn_event(0);
1773         bit += group;
1774         /*
1775          * Lower bits are reserved for use by the counters (see
1776          * armv7pmu_get_event_idx() for more info)
1777          */
1778         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1779
1780         return bit;
1781 }
1782
1783 /*
1784  * We check for column exclusion constraints here.
1785  * Two events cant use the same group within a pmresr register.
1786  */
1787 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1788                                    struct perf_event *event)
1789 {
1790         int idx;
1791         int bit = -1;
1792         struct hw_perf_event *hwc = &event->hw;
1793         unsigned int region = EVENT_REGION(hwc->config_base);
1794         unsigned int group = EVENT_GROUP(hwc->config_base);
1795         bool venum_event = EVENT_VENUM(hwc->config_base);
1796         bool scorpion_event = EVENT_CPU(hwc->config_base);
1797
1798         if (venum_event || scorpion_event) {
1799                 /* Ignore invalid events */
1800                 if (group > 3 || region > 3)
1801                         return -EINVAL;
1802
1803                 bit = scorpion_event_to_bit(event, region, group);
1804                 if (test_and_set_bit(bit, cpuc->used_mask))
1805                         return -EAGAIN;
1806         }
1807
1808         idx = armv7pmu_get_event_idx(cpuc, event);
1809         if (idx < 0 && bit >= 0)
1810                 clear_bit(bit, cpuc->used_mask);
1811
1812         return idx;
1813 }
1814
1815 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1816                                       struct perf_event *event)
1817 {
1818         int bit;
1819         struct hw_perf_event *hwc = &event->hw;
1820         unsigned int region = EVENT_REGION(hwc->config_base);
1821         unsigned int group = EVENT_GROUP(hwc->config_base);
1822         bool venum_event = EVENT_VENUM(hwc->config_base);
1823         bool scorpion_event = EVENT_CPU(hwc->config_base);
1824
1825         if (venum_event || scorpion_event) {
1826                 bit = scorpion_event_to_bit(event, region, group);
1827                 clear_bit(bit, cpuc->used_mask);
1828         }
1829 }
1830
1831 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1832 {
1833         armv7pmu_init(cpu_pmu);
1834         cpu_pmu->name           = "armv7_scorpion";
1835         cpu_pmu->map_event      = scorpion_map_event;
1836         cpu_pmu->reset          = scorpion_pmu_reset;
1837         cpu_pmu->enable         = scorpion_pmu_enable_event;
1838         cpu_pmu->disable        = scorpion_pmu_disable_event;
1839         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1840         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1841         return armv7_probe_num_events(cpu_pmu);
1842 }
1843
1844 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1845 {
1846         armv7pmu_init(cpu_pmu);
1847         cpu_pmu->name           = "armv7_scorpion_mp";
1848         cpu_pmu->map_event      = scorpion_map_event;
1849         cpu_pmu->reset          = scorpion_pmu_reset;
1850         cpu_pmu->enable         = scorpion_pmu_enable_event;
1851         cpu_pmu->disable        = scorpion_pmu_disable_event;
1852         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1853         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1854         return armv7_probe_num_events(cpu_pmu);
1855 }
1856 #else
1857 static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1858 {
1859         return -ENODEV;
1860 }
1861
1862 static inline int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1863 {
1864         return -ENODEV;
1865 }
1866
1867 static inline int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1868 {
1869         return -ENODEV;
1870 }
1871
1872 static inline int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1873 {
1874         return -ENODEV;
1875 }
1876
1877 static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1878 {
1879         return -ENODEV;
1880 }
1881
1882 static inline int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1883 {
1884         return -ENODEV;
1885 }
1886
1887 static inline int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1888 {
1889         return -ENODEV;
1890 }
1891
1892 static inline int krait_pmu_init(struct arm_pmu *cpu_pmu)
1893 {
1894         return -ENODEV;
1895 }
1896
1897 static inline int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1898 {
1899         return -ENODEV;
1900 }
1901
1902 static inline int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1903 {
1904         return -ENODEV;
1905 }
1906 #endif  /* CONFIG_CPU_V7 */