arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *
  10  *  For licencing details see kernel-base/COPYING
  11  */
  12
  13 #include <linux/perf_counter.h>
  14 #include <linux/capability.h>
  15 #include <linux/notifier.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/module.h>
  19 #include <linux/kdebug.h>
  20 #include <linux/sched.h>
  21 #include <linux/uaccess.h>
  22
  23 #include <asm/apic.h>
  24 #include <asm/stacktrace.h>
  25 #include <asm/nmi.h>
  26
  27 static u64 perf_counter_mask __read_mostly;
  28
  29 struct cpu_hw_counters {
  30         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  31         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  32         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  33         unsigned long           interrupts;
  34         int                     enabled;
  35 };
  36
  37 /*
  38  * struct x86_pmu - generic x86 pmu
  39  */
  40 struct x86_pmu {
  41         const char      *name;
  42         int             version;
  43         int             (*handle_irq)(struct pt_regs *);
  44         void            (*disable_all)(void);
  45         void            (*enable_all)(void);
  46         void            (*enable)(struct hw_perf_counter *, int);
  47         void            (*disable)(struct hw_perf_counter *, int);
  48         unsigned        eventsel;
  49         unsigned        perfctr;
  50         u64             (*event_map)(int);
  51         u64             (*raw_event)(u64);
  52         int             max_events;
  53         int             num_counters;
  54         int             num_counters_fixed;
  55         int             counter_bits;
  56         u64             counter_mask;
  57         u64             max_period;
  58         u64             intel_ctrl;
  59 };
  60
  61 static struct x86_pmu x86_pmu __read_mostly;
  62
  63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  64         .enabled = 1,
  65 };
  66
  67 /*
  68  * Intel PerfMon v3. Used on Core2 and later.
  69  */
  70 static const u64 intel_perfmon_event_map[] =
  71 {
  72   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
  73   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
  74   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
  75   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
  76   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
  77   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
  78   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
  79 };
  80
  81 static u64 intel_pmu_event_map(int event)
  82 {
  83         return intel_perfmon_event_map[event];
  84 }
  85
  86 /*
  87  * Generalized hw caching related event table, filled
  88  * in on a per model basis. A value of 0 means
  89  * 'not supported', -1 means 'event makes no sense on
  90  * this CPU', any other value means the raw event
  91  * ID.
  92  */
  93
  94 #define C(x) PERF_COUNT_HW_CACHE_##x
  95
  96 static u64 __read_mostly hw_cache_event_ids
  97                                 [PERF_COUNT_HW_CACHE_MAX]
  98                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  99                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 100
 101 static const u64 nehalem_hw_cache_event_ids
 102                                 [PERF_COUNT_HW_CACHE_MAX]
 103                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 104                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 105 {
 106  [ C(L1D) ] = {
 107         [ C(OP_READ) ] = {
 108                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 109                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 110         },
 111         [ C(OP_WRITE) ] = {
 112                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 113                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 114         },
 115         [ C(OP_PREFETCH) ] = {
 116                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 117                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 118         },
 119  },
 120  [ C(L1I ) ] = {
 121         [ C(OP_READ) ] = {
 122                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 123                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 124         },
 125         [ C(OP_WRITE) ] = {
 126                 [ C(RESULT_ACCESS) ] = -1,
 127                 [ C(RESULT_MISS)   ] = -1,
 128         },
 129         [ C(OP_PREFETCH) ] = {
 130                 [ C(RESULT_ACCESS) ] = 0x0,
 131                 [ C(RESULT_MISS)   ] = 0x0,
 132         },
 133  },
 134  [ C(LL  ) ] = {
 135         [ C(OP_READ) ] = {
 136                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 137                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 138         },
 139         [ C(OP_WRITE) ] = {
 140                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 141                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 142         },
 143         [ C(OP_PREFETCH) ] = {
 144                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
 145                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
 146         },
 147  },
 148  [ C(DTLB) ] = {
 149         [ C(OP_READ) ] = {
 150                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 151                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 152         },
 153         [ C(OP_WRITE) ] = {
 154                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 155                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 156         },
 157         [ C(OP_PREFETCH) ] = {
 158                 [ C(RESULT_ACCESS) ] = 0x0,
 159                 [ C(RESULT_MISS)   ] = 0x0,
 160         },
 161  },
 162  [ C(ITLB) ] = {
 163         [ C(OP_READ) ] = {
 164                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 165                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 166         },
 167         [ C(OP_WRITE) ] = {
 168                 [ C(RESULT_ACCESS) ] = -1,
 169                 [ C(RESULT_MISS)   ] = -1,
 170         },
 171         [ C(OP_PREFETCH) ] = {
 172                 [ C(RESULT_ACCESS) ] = -1,
 173                 [ C(RESULT_MISS)   ] = -1,
 174         },
 175  },
 176  [ C(BPU ) ] = {
 177         [ C(OP_READ) ] = {
 178                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 179                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 180         },
 181         [ C(OP_WRITE) ] = {
 182                 [ C(RESULT_ACCESS) ] = -1,
 183                 [ C(RESULT_MISS)   ] = -1,
 184         },
 185         [ C(OP_PREFETCH) ] = {
 186                 [ C(RESULT_ACCESS) ] = -1,
 187                 [ C(RESULT_MISS)   ] = -1,
 188         },
 189  },
 190 };
 191
 192 static const u64 core2_hw_cache_event_ids
 193                                 [PERF_COUNT_HW_CACHE_MAX]
 194                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 195                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 196 {
 197  [ C(L1D) ] = {
 198         [ C(OP_READ) ] = {
 199                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 200                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 201         },
 202         [ C(OP_WRITE) ] = {
 203                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 204                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 205         },
 206         [ C(OP_PREFETCH) ] = {
 207                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 208                 [ C(RESULT_MISS)   ] = 0,
 209         },
 210  },
 211  [ C(L1I ) ] = {
 212         [ C(OP_READ) ] = {
 213                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 214                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 215         },
 216         [ C(OP_WRITE) ] = {
 217                 [ C(RESULT_ACCESS) ] = -1,
 218                 [ C(RESULT_MISS)   ] = -1,
 219         },
 220         [ C(OP_PREFETCH) ] = {
 221                 [ C(RESULT_ACCESS) ] = 0,
 222                 [ C(RESULT_MISS)   ] = 0,
 223         },
 224  },
 225  [ C(LL  ) ] = {
 226         [ C(OP_READ) ] = {
 227                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 228                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 229         },
 230         [ C(OP_WRITE) ] = {
 231                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 232                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 233         },
 234         [ C(OP_PREFETCH) ] = {
 235                 [ C(RESULT_ACCESS) ] = 0,
 236                 [ C(RESULT_MISS)   ] = 0,
 237         },
 238  },
 239  [ C(DTLB) ] = {
 240         [ C(OP_READ) ] = {
 241                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 242                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 243         },
 244         [ C(OP_WRITE) ] = {
 245                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 246                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 247         },
 248         [ C(OP_PREFETCH) ] = {
 249                 [ C(RESULT_ACCESS) ] = 0,
 250                 [ C(RESULT_MISS)   ] = 0,
 251         },
 252  },
 253  [ C(ITLB) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 256                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = -1,
 260                 [ C(RESULT_MISS)   ] = -1,
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = -1,
 264                 [ C(RESULT_MISS)   ] = -1,
 265         },
 266  },
 267  [ C(BPU ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 270                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = -1,
 278                 [ C(RESULT_MISS)   ] = -1,
 279         },
 280  },
 281 };
 282
 283 static const u64 atom_hw_cache_event_ids
 284                                 [PERF_COUNT_HW_CACHE_MAX]
 285                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 286                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 287 {
 288  [ C(L1D) ] = {
 289         [ C(OP_READ) ] = {
 290                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 291                 [ C(RESULT_MISS)   ] = 0,
 292         },
 293         [ C(OP_WRITE) ] = {
 294                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 295                 [ C(RESULT_MISS)   ] = 0,
 296         },
 297         [ C(OP_PREFETCH) ] = {
 298                 [ C(RESULT_ACCESS) ] = 0x0,
 299                 [ C(RESULT_MISS)   ] = 0,
 300         },
 301  },
 302  [ C(L1I ) ] = {
 303         [ C(OP_READ) ] = {
 304                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 305                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 306         },
 307         [ C(OP_WRITE) ] = {
 308                 [ C(RESULT_ACCESS) ] = -1,
 309                 [ C(RESULT_MISS)   ] = -1,
 310         },
 311         [ C(OP_PREFETCH) ] = {
 312                 [ C(RESULT_ACCESS) ] = 0,
 313                 [ C(RESULT_MISS)   ] = 0,
 314         },
 315  },
 316  [ C(LL  ) ] = {
 317         [ C(OP_READ) ] = {
 318                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 319                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 320         },
 321         [ C(OP_WRITE) ] = {
 322                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 323                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 324         },
 325         [ C(OP_PREFETCH) ] = {
 326                 [ C(RESULT_ACCESS) ] = 0,
 327                 [ C(RESULT_MISS)   ] = 0,
 328         },
 329  },
 330  [ C(DTLB) ] = {
 331         [ C(OP_READ) ] = {
 332                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 333                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 334         },
 335         [ C(OP_WRITE) ] = {
 336                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 337                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 338         },
 339         [ C(OP_PREFETCH) ] = {
 340                 [ C(RESULT_ACCESS) ] = 0,
 341                 [ C(RESULT_MISS)   ] = 0,
 342         },
 343  },
 344  [ C(ITLB) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 347                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = -1,
 351                 [ C(RESULT_MISS)   ] = -1,
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = -1,
 355                 [ C(RESULT_MISS)   ] = -1,
 356         },
 357  },
 358  [ C(BPU ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 361                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = -1,
 369                 [ C(RESULT_MISS)   ] = -1,
 370         },
 371  },
 372 };
 373
 374 static u64 intel_pmu_raw_event(u64 event)
 375 {
 376 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 377 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 378 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 379 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 380 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
 381
 382 #define CORE_EVNTSEL_MASK               \
 383         (CORE_EVNTSEL_EVENT_MASK |      \
 384          CORE_EVNTSEL_UNIT_MASK  |      \
 385          CORE_EVNTSEL_EDGE_MASK  |      \
 386          CORE_EVNTSEL_INV_MASK  |       \
 387          CORE_EVNTSEL_COUNTER_MASK)
 388
 389         return event & CORE_EVNTSEL_MASK;
 390 }
 391
 392 static const u64 amd_0f_hw_cache_event_ids
 393                                 [PERF_COUNT_HW_CACHE_MAX]
 394                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 395                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 396 {
 397  [ C(L1D) ] = {
 398         [ C(OP_READ) ] = {
 399                 [ C(RESULT_ACCESS) ] = 0,
 400                 [ C(RESULT_MISS)   ] = 0,
 401         },
 402         [ C(OP_WRITE) ] = {
 403                 [ C(RESULT_ACCESS) ] = 0,
 404                 [ C(RESULT_MISS)   ] = 0,
 405         },
 406         [ C(OP_PREFETCH) ] = {
 407                 [ C(RESULT_ACCESS) ] = 0,
 408                 [ C(RESULT_MISS)   ] = 0,
 409         },
 410  },
 411  [ C(L1I ) ] = {
 412         [ C(OP_READ) ] = {
 413                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 414                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 415         },
 416         [ C(OP_WRITE) ] = {
 417                 [ C(RESULT_ACCESS) ] = -1,
 418                 [ C(RESULT_MISS)   ] = -1,
 419         },
 420         [ C(OP_PREFETCH) ] = {
 421                 [ C(RESULT_ACCESS) ] = 0,
 422                 [ C(RESULT_MISS)   ] = 0,
 423         },
 424  },
 425  [ C(LL  ) ] = {
 426         [ C(OP_READ) ] = {
 427                 [ C(RESULT_ACCESS) ] = 0,
 428                 [ C(RESULT_MISS)   ] = 0,
 429         },
 430         [ C(OP_WRITE) ] = {
 431                 [ C(RESULT_ACCESS) ] = 0,
 432                 [ C(RESULT_MISS)   ] = 0,
 433         },
 434         [ C(OP_PREFETCH) ] = {
 435                 [ C(RESULT_ACCESS) ] = 0,
 436                 [ C(RESULT_MISS)   ] = 0,
 437         },
 438  },
 439  [ C(DTLB) ] = {
 440         [ C(OP_READ) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0,
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_WRITE) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448         [ C(OP_PREFETCH) ] = {
 449                 [ C(RESULT_ACCESS) ] = 0,
 450                 [ C(RESULT_MISS)   ] = 0,
 451         },
 452  },
 453  [ C(ITLB) ] = {
 454         [ C(OP_READ) ] = {
 455                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 456                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 457         },
 458         [ C(OP_WRITE) ] = {
 459                 [ C(RESULT_ACCESS) ] = -1,
 460                 [ C(RESULT_MISS)   ] = -1,
 461         },
 462         [ C(OP_PREFETCH) ] = {
 463                 [ C(RESULT_ACCESS) ] = -1,
 464                 [ C(RESULT_MISS)   ] = -1,
 465         },
 466  },
 467  [ C(BPU ) ] = {
 468         [ C(OP_READ) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 470                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 471         },
 472         [ C(OP_WRITE) ] = {
 473                 [ C(RESULT_ACCESS) ] = -1,
 474                 [ C(RESULT_MISS)   ] = -1,
 475         },
 476         [ C(OP_PREFETCH) ] = {
 477                 [ C(RESULT_ACCESS) ] = -1,
 478                 [ C(RESULT_MISS)   ] = -1,
 479         },
 480  },
 481 };
 482
 483 /*
 484  * AMD Performance Monitor K7 and later.
 485  */
 486 static const u64 amd_perfmon_event_map[] =
 487 {
 488   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
 489   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 490   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
 491   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
 492   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 493   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 494 };
 495
 496 static u64 amd_pmu_event_map(int event)
 497 {
 498         return amd_perfmon_event_map[event];
 499 }
 500
 501 static u64 amd_pmu_raw_event(u64 event)
 502 {
 503 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 504 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 505 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 506 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 507 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 508
 509 #define K7_EVNTSEL_MASK                 \
 510         (K7_EVNTSEL_EVENT_MASK |        \
 511          K7_EVNTSEL_UNIT_MASK  |        \
 512          K7_EVNTSEL_EDGE_MASK  |        \
 513          K7_EVNTSEL_INV_MASK   |        \
 514          K7_EVNTSEL_COUNTER_MASK)
 515
 516         return event & K7_EVNTSEL_MASK;
 517 }
 518
 519 /*
 520  * Propagate counter elapsed time into the generic counter.
 521  * Can only be executed on the CPU where the counter is active.
 522  * Returns the delta events processed.
 523  */
 524 static u64
 525 x86_perf_counter_update(struct perf_counter *counter,
 526                         struct hw_perf_counter *hwc, int idx)
 527 {
 528         int shift = 64 - x86_pmu.counter_bits;
 529         u64 prev_raw_count, new_raw_count;
 530         s64 delta;
 531
 532         /*
 533          * Careful: an NMI might modify the previous counter value.
 534          *
 535          * Our tactic to handle this is to first atomically read and
 536          * exchange a new raw count - then add that new-prev delta
 537          * count to the generic counter atomically:
 538          */
 539 again:
 540         prev_raw_count = atomic64_read(&hwc->prev_count);
 541         rdmsrl(hwc->counter_base + idx, new_raw_count);
 542
 543         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 544                                         new_raw_count) != prev_raw_count)
 545                 goto again;
 546
 547         /*
 548          * Now we have the new raw value and have updated the prev
 549          * timestamp already. We can now calculate the elapsed delta
 550          * (counter-)time and add that to the generic counter.
 551          *
 552          * Careful, not all hw sign-extends above the physical width
 553          * of the count.
 554          */
 555         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 556         delta >>= shift;
 557
 558         atomic64_add(delta, &counter->count);
 559         atomic64_sub(delta, &hwc->period_left);
 560
 561         return new_raw_count;
 562 }
 563
 564 static atomic_t active_counters;
 565 static DEFINE_MUTEX(pmc_reserve_mutex);
 566
 567 static bool reserve_pmc_hardware(void)
 568 {
 569         int i;
 570
 571         if (nmi_watchdog == NMI_LOCAL_APIC)
 572                 disable_lapic_nmi_watchdog();
 573
 574         for (i = 0; i < x86_pmu.num_counters; i++) {
 575                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 576                         goto perfctr_fail;
 577         }
 578
 579         for (i = 0; i < x86_pmu.num_counters; i++) {
 580                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 581                         goto eventsel_fail;
 582         }
 583
 584         return true;
 585
 586 eventsel_fail:
 587         for (i--; i >= 0; i--)
 588                 release_evntsel_nmi(x86_pmu.eventsel + i);
 589
 590         i = x86_pmu.num_counters;
 591
 592 perfctr_fail:
 593         for (i--; i >= 0; i--)
 594                 release_perfctr_nmi(x86_pmu.perfctr + i);
 595
 596         if (nmi_watchdog == NMI_LOCAL_APIC)
 597                 enable_lapic_nmi_watchdog();
 598
 599         return false;
 600 }
 601
 602 static void release_pmc_hardware(void)
 603 {
 604         int i;
 605
 606         for (i = 0; i < x86_pmu.num_counters; i++) {
 607                 release_perfctr_nmi(x86_pmu.perfctr + i);
 608                 release_evntsel_nmi(x86_pmu.eventsel + i);
 609         }
 610
 611         if (nmi_watchdog == NMI_LOCAL_APIC)
 612                 enable_lapic_nmi_watchdog();
 613 }
 614
 615 static void hw_perf_counter_destroy(struct perf_counter *counter)
 616 {
 617         if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
 618                 release_pmc_hardware();
 619                 mutex_unlock(&pmc_reserve_mutex);
 620         }
 621 }
 622
 623 static inline int x86_pmu_initialized(void)
 624 {
 625         return x86_pmu.handle_irq != NULL;
 626 }
 627
 628 static inline int
 629 set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
 630 {
 631         unsigned int cache_type, cache_op, cache_result;
 632         u64 config, val;
 633
 634         config = attr->config;
 635
 636         cache_type = (config >>  0) & 0xff;
 637         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 638                 return -EINVAL;
 639
 640         cache_op = (config >>  8) & 0xff;
 641         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 642                 return -EINVAL;
 643
 644         cache_result = (config >> 16) & 0xff;
 645         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 646                 return -EINVAL;
 647
 648         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 649
 650         if (val == 0)
 651                 return -ENOENT;
 652
 653         if (val == -1)
 654                 return -EINVAL;
 655
 656         hwc->config |= val;
 657
 658         return 0;
 659 }
 660
 661 /*
 662  * Setup the hardware configuration for a given attr_type
 663  */
 664 static int __hw_perf_counter_init(struct perf_counter *counter)
 665 {
 666         struct perf_counter_attr *attr = &counter->attr;
 667         struct hw_perf_counter *hwc = &counter->hw;
 668         int err;
 669
 670         if (!x86_pmu_initialized())
 671                 return -ENODEV;
 672
 673         err = 0;
 674         if (!atomic_inc_not_zero(&active_counters)) {
 675                 mutex_lock(&pmc_reserve_mutex);
 676                 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
 677                         err = -EBUSY;
 678                 else
 679                         atomic_inc(&active_counters);
 680                 mutex_unlock(&pmc_reserve_mutex);
 681         }
 682         if (err)
 683                 return err;
 684
 685         /*
 686          * Generate PMC IRQs:
 687          * (keep 'enabled' bit clear for now)
 688          */
 689         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 690
 691         /*
 692          * Count user and OS events unless requested not to.
 693          */
 694         if (!attr->exclude_user)
 695                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 696         if (!attr->exclude_kernel)
 697                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 698
 699         if (!hwc->sample_period) {
 700                 hwc->sample_period = x86_pmu.max_period;
 701                 hwc->last_period = hwc->sample_period;
 702                 atomic64_set(&hwc->period_left, hwc->sample_period);
 703         }
 704
 705         counter->destroy = hw_perf_counter_destroy;
 706
 707         /*
 708          * Raw event type provide the config in the event structure
 709          */
 710         if (attr->type == PERF_TYPE_RAW) {
 711                 hwc->config |= x86_pmu.raw_event(attr->config);
 712                 return 0;
 713         }
 714
 715         if (attr->type == PERF_TYPE_HW_CACHE)
 716                 return set_ext_hw_attr(hwc, attr);
 717
 718         if (attr->config >= x86_pmu.max_events)
 719                 return -EINVAL;
 720         /*
 721          * The generic map:
 722          */
 723         hwc->config |= x86_pmu.event_map(attr->config);
 724
 725         return 0;
 726 }
 727
 728 static void intel_pmu_disable_all(void)
 729 {
 730         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 731 }
 732
 733 static void amd_pmu_disable_all(void)
 734 {
 735         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 736         int idx;
 737
 738         if (!cpuc->enabled)
 739                 return;
 740
 741         cpuc->enabled = 0;
 742         /*
 743          * ensure we write the disable before we start disabling the
 744          * counters proper, so that amd_pmu_enable_counter() does the
 745          * right thing.
 746          */
 747         barrier();
 748
 749         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 750                 u64 val;
 751
 752                 if (!test_bit(idx, cpuc->active_mask))
 753                         continue;
 754                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 755                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
 756                         continue;
 757                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 758                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 759         }
 760 }
 761
 762 void hw_perf_disable(void)
 763 {
 764         if (!x86_pmu_initialized())
 765                 return;
 766         return x86_pmu.disable_all();
 767 }
 768
 769 static void intel_pmu_enable_all(void)
 770 {
 771         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 772 }
 773
 774 static void amd_pmu_enable_all(void)
 775 {
 776         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 777         int idx;
 778
 779         if (cpuc->enabled)
 780                 return;
 781
 782         cpuc->enabled = 1;
 783         barrier();
 784
 785         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 786                 u64 val;
 787
 788                 if (!test_bit(idx, cpuc->active_mask))
 789                         continue;
 790                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 791                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
 792                         continue;
 793                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 794                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 795         }
 796 }
 797
 798 void hw_perf_enable(void)
 799 {
 800         if (!x86_pmu_initialized())
 801                 return;
 802         x86_pmu.enable_all();
 803 }
 804
 805 static inline u64 intel_pmu_get_status(void)
 806 {
 807         u64 status;
 808
 809         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 810
 811         return status;
 812 }
 813
 814 static inline void intel_pmu_ack_status(u64 ack)
 815 {
 816         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 817 }
 818
 819 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 820 {
 821         int err;
 822         err = checking_wrmsrl(hwc->config_base + idx,
 823                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 824 }
 825
 826 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 827 {
 828         int err;
 829         err = checking_wrmsrl(hwc->config_base + idx,
 830                               hwc->config);
 831 }
 832
 833 static inline void
 834 intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 835 {
 836         int idx = __idx - X86_PMC_IDX_FIXED;
 837         u64 ctrl_val, mask;
 838         int err;
 839
 840         mask = 0xfULL << (idx * 4);
 841
 842         rdmsrl(hwc->config_base, ctrl_val);
 843         ctrl_val &= ~mask;
 844         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 845 }
 846
 847 static inline void
 848 intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 849 {
 850         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 851                 intel_pmu_disable_fixed(hwc, idx);
 852                 return;
 853         }
 854
 855         x86_pmu_disable_counter(hwc, idx);
 856 }
 857
 858 static inline void
 859 amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 860 {
 861         x86_pmu_disable_counter(hwc, idx);
 862 }
 863
 864 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 865
 866 /*
 867  * Set the next IRQ period, based on the hwc->period_left value.
 868  * To be called with the counter disabled in hw:
 869  */
 870 static int
 871 x86_perf_counter_set_period(struct perf_counter *counter,
 872                              struct hw_perf_counter *hwc, int idx)
 873 {
 874         s64 left = atomic64_read(&hwc->period_left);
 875         s64 period = hwc->sample_period;
 876         int err, ret = 0;
 877
 878         /*
 879          * If we are way outside a reasoable range then just skip forward:
 880          */
 881         if (unlikely(left <= -period)) {
 882                 left = period;
 883                 atomic64_set(&hwc->period_left, left);
 884                 hwc->last_period = period;
 885                 ret = 1;
 886         }
 887
 888         if (unlikely(left <= 0)) {
 889                 left += period;
 890                 atomic64_set(&hwc->period_left, left);
 891                 hwc->last_period = period;
 892                 ret = 1;
 893         }
 894         /*
 895          * Quirk: certain CPUs dont like it if just 1 event is left:
 896          */
 897         if (unlikely(left < 2))
 898                 left = 2;
 899
 900         if (left > x86_pmu.max_period)
 901                 left = x86_pmu.max_period;
 902
 903         per_cpu(prev_left[idx], smp_processor_id()) = left;
 904
 905         /*
 906          * The hw counter starts counting from this counter offset,
 907          * mark it to be able to extra future deltas:
 908          */
 909         atomic64_set(&hwc->prev_count, (u64)-left);
 910
 911         err = checking_wrmsrl(hwc->counter_base + idx,
 912                              (u64)(-left) & x86_pmu.counter_mask);
 913
 914         return ret;
 915 }
 916
 917 static inline void
 918 intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
 919 {
 920         int idx = __idx - X86_PMC_IDX_FIXED;
 921         u64 ctrl_val, bits, mask;
 922         int err;
 923
 924         /*
 925          * Enable IRQ generation (0x8),
 926          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 927          * if requested:
 928          */
 929         bits = 0x8ULL;
 930         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 931                 bits |= 0x2;
 932         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 933                 bits |= 0x1;
 934         bits <<= (idx * 4);
 935         mask = 0xfULL << (idx * 4);
 936
 937         rdmsrl(hwc->config_base, ctrl_val);
 938         ctrl_val &= ~mask;
 939         ctrl_val |= bits;
 940         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 941 }
 942
 943 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 944 {
 945         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 946                 intel_pmu_enable_fixed(hwc, idx);
 947                 return;
 948         }
 949
 950         x86_pmu_enable_counter(hwc, idx);
 951 }
 952
 953 static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 954 {
 955         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 956
 957         if (cpuc->enabled)
 958                 x86_pmu_enable_counter(hwc, idx);
 959         else
 960                 x86_pmu_disable_counter(hwc, idx);
 961 }
 962
 963 static int
 964 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 965 {
 966         unsigned int event;
 967
 968         if (!x86_pmu.num_counters_fixed)
 969                 return -1;
 970
 971         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 972
 973         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
 974                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 975         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
 976                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 977         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
 978                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 979
 980         return -1;
 981 }
 982
 983 /*
 984  * Find a PMC slot for the freshly enabled / scheduled in counter:
 985  */
 986 static int x86_pmu_enable(struct perf_counter *counter)
 987 {
 988         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 989         struct hw_perf_counter *hwc = &counter->hw;
 990         int idx;
 991
 992         idx = fixed_mode_idx(counter, hwc);
 993         if (idx >= 0) {
 994                 /*
 995                  * Try to get the fixed counter, if that is already taken
 996                  * then try to get a generic counter:
 997                  */
 998                 if (test_and_set_bit(idx, cpuc->used_mask))
 999                         goto try_generic;
1000
1001                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1002                 /*
1003                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
1004                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1005                  */
1006                 hwc->counter_base =
1007                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1008                 hwc->idx = idx;
1009         } else {
1010                 idx = hwc->idx;
1011                 /* Try to get the previous generic counter again */
1012                 if (test_and_set_bit(idx, cpuc->used_mask)) {
1013 try_generic:
1014                         idx = find_first_zero_bit(cpuc->used_mask,
1015                                                   x86_pmu.num_counters);
1016                         if (idx == x86_pmu.num_counters)
1017                                 return -EAGAIN;
1018
1019                         set_bit(idx, cpuc->used_mask);
1020                         hwc->idx = idx;
1021                 }
1022                 hwc->config_base  = x86_pmu.eventsel;
1023                 hwc->counter_base = x86_pmu.perfctr;
1024         }
1025
1026         perf_counters_lapic_init();
1027
1028         x86_pmu.disable(hwc, idx);
1029
1030         cpuc->counters[idx] = counter;
1031         set_bit(idx, cpuc->active_mask);
1032
1033         x86_perf_counter_set_period(counter, hwc, idx);
1034         x86_pmu.enable(hwc, idx);
1035
1036         return 0;
1037 }
1038
1039 static void x86_pmu_unthrottle(struct perf_counter *counter)
1040 {
1041         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1042         struct hw_perf_counter *hwc = &counter->hw;
1043
1044         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1045                                 cpuc->counters[hwc->idx] != counter))
1046                 return;
1047
1048         x86_pmu.enable(hwc, hwc->idx);
1049 }
1050
1051 void perf_counter_print_debug(void)
1052 {
1053         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1054         struct cpu_hw_counters *cpuc;
1055         unsigned long flags;
1056         int cpu, idx;
1057
1058         if (!x86_pmu.num_counters)
1059                 return;
1060
1061         local_irq_save(flags);
1062
1063         cpu = smp_processor_id();
1064         cpuc = &per_cpu(cpu_hw_counters, cpu);
1065
1066         if (x86_pmu.version >= 2) {
1067                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1068                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1069                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1070                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1071
1072                 pr_info("\n");
1073                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1074                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1075                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1076                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1077         }
1078         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1079
1080         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1081                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1082                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1083
1084                 prev_left = per_cpu(prev_left[idx], cpu);
1085
1086                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1087                         cpu, idx, pmc_ctrl);
1088                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1089                         cpu, idx, pmc_count);
1090                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1091                         cpu, idx, prev_left);
1092         }
1093         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1094                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1095
1096                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1097                         cpu, idx, pmc_count);
1098         }
1099         local_irq_restore(flags);
1100 }
1101
1102 static void x86_pmu_disable(struct perf_counter *counter)
1103 {
1104         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1105         struct hw_perf_counter *hwc = &counter->hw;
1106         int idx = hwc->idx;
1107
1108         /*
1109          * Must be done before we disable, otherwise the nmi handler
1110          * could reenable again:
1111          */
1112         clear_bit(idx, cpuc->active_mask);
1113         x86_pmu.disable(hwc, idx);
1114
1115         /*
1116          * Make sure the cleared pointer becomes visible before we
1117          * (potentially) free the counter:
1118          */
1119         barrier();
1120
1121         /*
1122          * Drain the remaining delta count out of a counter
1123          * that we are disabling:
1124          */
1125         x86_perf_counter_update(counter, hwc, idx);
1126         cpuc->counters[idx] = NULL;
1127         clear_bit(idx, cpuc->used_mask);
1128 }
1129
1130 /*
1131  * Save and restart an expired counter. Called by NMI contexts,
1132  * so it has to be careful about preempting normal counter ops:
1133  */
1134 static int intel_pmu_save_and_restart(struct perf_counter *counter)
1135 {
1136         struct hw_perf_counter *hwc = &counter->hw;
1137         int idx = hwc->idx;
1138         int ret;
1139
1140         x86_perf_counter_update(counter, hwc, idx);
1141         ret = x86_perf_counter_set_period(counter, hwc, idx);
1142
1143         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1144                 intel_pmu_enable_counter(hwc, idx);
1145
1146         return ret;
1147 }
1148
1149 static void intel_pmu_reset(void)
1150 {
1151         unsigned long flags;
1152         int idx;
1153
1154         if (!x86_pmu.num_counters)
1155                 return;
1156
1157         local_irq_save(flags);
1158
1159         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1160
1161         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1162                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1163                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1164         }
1165         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1166                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1167         }
1168
1169         local_irq_restore(flags);
1170 }
1171
1172
1173 /*
1174  * This handler is triggered by the local APIC, so the APIC IRQ handling
1175  * rules apply:
1176  */
1177 static int intel_pmu_handle_irq(struct pt_regs *regs)
1178 {
1179         struct perf_sample_data data;
1180         struct cpu_hw_counters *cpuc;
1181         int bit, cpu, loops;
1182         u64 ack, status;
1183
1184         data.regs = regs;
1185         data.addr = 0;
1186
1187         cpu = smp_processor_id();
1188         cpuc = &per_cpu(cpu_hw_counters, cpu);
1189
1190         perf_disable();
1191         status = intel_pmu_get_status();
1192         if (!status) {
1193                 perf_enable();
1194                 return 0;
1195         }
1196
1197         loops = 0;
1198 again:
1199         if (++loops > 100) {
1200                 WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
1201                 perf_counter_print_debug();
1202                 intel_pmu_reset();
1203                 perf_enable();
1204                 return 1;
1205         }
1206
1207         inc_irq_stat(apic_perf_irqs);
1208         ack = status;
1209         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1210                 struct perf_counter *counter = cpuc->counters[bit];
1211
1212                 clear_bit(bit, (unsigned long *) &status);
1213                 if (!test_bit(bit, cpuc->active_mask))
1214                         continue;
1215
1216                 if (!intel_pmu_save_and_restart(counter))
1217                         continue;
1218
1219                 if (perf_counter_overflow(counter, 1, &data))
1220                         intel_pmu_disable_counter(&counter->hw, bit);
1221         }
1222
1223         intel_pmu_ack_status(ack);
1224
1225         /*
1226          * Repeat if there is more work to be done:
1227          */
1228         status = intel_pmu_get_status();
1229         if (status)
1230                 goto again;
1231
1232         perf_enable();
1233
1234         return 1;
1235 }
1236
1237 static int amd_pmu_handle_irq(struct pt_regs *regs)
1238 {
1239         struct perf_sample_data data;
1240         struct cpu_hw_counters *cpuc;
1241         struct perf_counter *counter;
1242         struct hw_perf_counter *hwc;
1243         int cpu, idx, handled = 0;
1244         u64 val;
1245
1246         data.regs = regs;
1247         data.addr = 0;
1248
1249         cpu = smp_processor_id();
1250         cpuc = &per_cpu(cpu_hw_counters, cpu);
1251
1252         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1253                 if (!test_bit(idx, cpuc->active_mask))
1254                         continue;
1255
1256                 counter = cpuc->counters[idx];
1257                 hwc = &counter->hw;
1258
1259                 val = x86_perf_counter_update(counter, hwc, idx);
1260                 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
1261                         continue;
1262
1263                 /*
1264                  * counter overflow
1265                  */
1266                 handled         = 1;
1267                 data.period     = counter->hw.last_period;
1268
1269                 if (!x86_perf_counter_set_period(counter, hwc, idx))
1270                         continue;
1271
1272                 if (perf_counter_overflow(counter, 1, &data))
1273                         amd_pmu_disable_counter(hwc, idx);
1274         }
1275
1276         if (handled)
1277                 inc_irq_stat(apic_perf_irqs);
1278
1279         return handled;
1280 }
1281
1282 void smp_perf_pending_interrupt(struct pt_regs *regs)
1283 {
1284         irq_enter();
1285         ack_APIC_irq();
1286         inc_irq_stat(apic_pending_irqs);
1287         perf_counter_do_pending();
1288         irq_exit();
1289 }
1290
1291 void set_perf_counter_pending(void)
1292 {
1293         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1294 }
1295
1296 void perf_counters_lapic_init(void)
1297 {
1298         if (!x86_pmu_initialized())
1299                 return;
1300
1301         /*
1302          * Always use NMI for PMU
1303          */
1304         apic_write(APIC_LVTPC, APIC_DM_NMI);
1305 }
1306
1307 static int __kprobes
1308 perf_counter_nmi_handler(struct notifier_block *self,
1309                          unsigned long cmd, void *__args)
1310 {
1311         struct die_args *args = __args;
1312         struct pt_regs *regs;
1313
1314         if (!atomic_read(&active_counters))
1315                 return NOTIFY_DONE;
1316
1317         switch (cmd) {
1318         case DIE_NMI:
1319         case DIE_NMI_IPI:
1320                 break;
1321
1322         default:
1323                 return NOTIFY_DONE;
1324         }
1325
1326         regs = args->regs;
1327
1328         apic_write(APIC_LVTPC, APIC_DM_NMI);
1329         /*
1330          * Can't rely on the handled return value to say it was our NMI, two
1331          * counters could trigger 'simultaneously' raising two back-to-back NMIs.
1332          *
1333          * If the first NMI handles both, the latter will be empty and daze
1334          * the CPU.
1335          */
1336         x86_pmu.handle_irq(regs);
1337
1338         return NOTIFY_STOP;
1339 }
1340
1341 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
1342         .notifier_call          = perf_counter_nmi_handler,
1343         .next                   = NULL,
1344         .priority               = 1
1345 };
1346
1347 static struct x86_pmu intel_pmu = {
1348         .name                   = "Intel",
1349         .handle_irq             = intel_pmu_handle_irq,
1350         .disable_all            = intel_pmu_disable_all,
1351         .enable_all             = intel_pmu_enable_all,
1352         .enable                 = intel_pmu_enable_counter,
1353         .disable                = intel_pmu_disable_counter,
1354         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
1355         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
1356         .event_map              = intel_pmu_event_map,
1357         .raw_event              = intel_pmu_raw_event,
1358         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
1359         /*
1360          * Intel PMCs cannot be accessed sanely above 32 bit width,
1361          * so we install an artificial 1<<31 period regardless of
1362          * the generic counter period:
1363          */
1364         .max_period             = (1ULL << 31) - 1,
1365 };
1366
1367 static struct x86_pmu amd_pmu = {
1368         .name                   = "AMD",
1369         .handle_irq             = amd_pmu_handle_irq,
1370         .disable_all            = amd_pmu_disable_all,
1371         .enable_all             = amd_pmu_enable_all,
1372         .enable                 = amd_pmu_enable_counter,
1373         .disable                = amd_pmu_disable_counter,
1374         .eventsel               = MSR_K7_EVNTSEL0,
1375         .perfctr                = MSR_K7_PERFCTR0,
1376         .event_map              = amd_pmu_event_map,
1377         .raw_event              = amd_pmu_raw_event,
1378         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1379         .num_counters           = 4,
1380         .counter_bits           = 48,
1381         .counter_mask           = (1ULL << 48) - 1,
1382         /* use highest bit to detect overflow */
1383         .max_period             = (1ULL << 47) - 1,
1384 };
1385
1386 static int intel_pmu_init(void)
1387 {
1388         union cpuid10_edx edx;
1389         union cpuid10_eax eax;
1390         unsigned int unused;
1391         unsigned int ebx;
1392         int version;
1393
1394         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
1395                 return -ENODEV;
1396
1397         /*
1398          * Check whether the Architectural PerfMon supports
1399          * Branch Misses Retired Event or not.
1400          */
1401         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
1402         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
1403                 return -ENODEV;
1404
1405         version = eax.split.version_id;
1406         if (version < 2)
1407                 return -ENODEV;
1408
1409         x86_pmu                         = intel_pmu;
1410         x86_pmu.version                 = version;
1411         x86_pmu.num_counters            = eax.split.num_counters;
1412         x86_pmu.counter_bits            = eax.split.bit_width;
1413         x86_pmu.counter_mask            = (1ULL << eax.split.bit_width) - 1;
1414
1415         /*
1416          * Quirk: v2 perfmon does not report fixed-purpose counters, so
1417          * assume at least 3 counters:
1418          */
1419         x86_pmu.num_counters_fixed      = max((int)edx.split.num_counters_fixed, 3);
1420
1421         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1422
1423         /*
1424          * Install the hw-cache-events table:
1425          */
1426         switch (boot_cpu_data.x86_model) {
1427         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
1428         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
1429         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
1430         case 29: /* six-core 45 nm xeon "Dunnington" */
1431                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
1432                        sizeof(hw_cache_event_ids));
1433
1434                 pr_cont("Core2 events, ");
1435                 break;
1436         default:
1437         case 26:
1438                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
1439                        sizeof(hw_cache_event_ids));
1440
1441                 pr_cont("Nehalem/Corei7 events, ");
1442                 break;
1443         case 28:
1444                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
1445                        sizeof(hw_cache_event_ids));
1446
1447                 pr_cont("Atom events, ");
1448                 break;
1449         }
1450         return 0;
1451 }
1452
1453 static int amd_pmu_init(void)
1454 {
1455         x86_pmu = amd_pmu;
1456
1457         switch (boot_cpu_data.x86) {
1458         case 0x0f:
1459         case 0x10:
1460         case 0x11:
1461                 memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
1462                        sizeof(hw_cache_event_ids));
1463
1464                 pr_cont("AMD Family 0f/10/11 events, ");
1465                 break;
1466         }
1467         return 0;
1468 }
1469
1470 void __init init_hw_perf_counters(void)
1471 {
1472         int err;
1473
1474         pr_info("Performance Counters: ");
1475
1476         switch (boot_cpu_data.x86_vendor) {
1477         case X86_VENDOR_INTEL:
1478                 err = intel_pmu_init();
1479                 break;
1480         case X86_VENDOR_AMD:
1481                 err = amd_pmu_init();
1482                 break;
1483         default:
1484                 return;
1485         }
1486         if (err != 0) {
1487                 pr_cont("no PMU driver, software counters only.\n");
1488                 return;
1489         }
1490
1491         pr_cont("%s PMU driver.\n", x86_pmu.name);
1492
1493         if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1494                 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1495                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1496                      x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1497         }
1498         perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1499         perf_max_counters = x86_pmu.num_counters;
1500
1501         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1502                 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1503                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1504                      x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1505         }
1506
1507         perf_counter_mask |=
1508                 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1509
1510         perf_counters_lapic_init();
1511         register_die_notifier(&perf_counter_nmi_notifier);
1512
1513         pr_info("... version:                 %d\n",     x86_pmu.version);
1514         pr_info("... bit width:               %d\n",     x86_pmu.counter_bits);
1515         pr_info("... generic counters:        %d\n",     x86_pmu.num_counters);
1516         pr_info("... value mask:              %016Lx\n", x86_pmu.counter_mask);
1517         pr_info("... max period:              %016Lx\n", x86_pmu.max_period);
1518         pr_info("... fixed-purpose counters:  %d\n",     x86_pmu.num_counters_fixed);
1519         pr_info("... counter mask:            %016Lx\n", perf_counter_mask);
1520 }
1521
1522 static inline void x86_pmu_read(struct perf_counter *counter)
1523 {
1524         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1525 }
1526
1527 static const struct pmu pmu = {
1528         .enable         = x86_pmu_enable,
1529         .disable        = x86_pmu_disable,
1530         .read           = x86_pmu_read,
1531         .unthrottle     = x86_pmu_unthrottle,
1532 };
1533
1534 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1535 {
1536         int err;
1537
1538         err = __hw_perf_counter_init(counter);
1539         if (err)
1540                 return ERR_PTR(err);
1541
1542         return &pmu;
1543 }
1544
1545 /*
1546  * callchain support
1547  */
1548
1549 static inline
1550 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1551 {
1552         if (entry->nr < MAX_STACK_DEPTH)
1553                 entry->ip[entry->nr++] = ip;
1554 }
1555
1556 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1557 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1558
1559
1560 static void
1561 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1562 {
1563         /* Ignore warnings */
1564 }
1565
1566 static void backtrace_warning(void *data, char *msg)
1567 {
1568         /* Ignore warnings */
1569 }
1570
1571 static int backtrace_stack(void *data, char *name)
1572 {
1573         /* Don't bother with IRQ stacks for now */
1574         return -1;
1575 }
1576
1577 static void backtrace_address(void *data, unsigned long addr, int reliable)
1578 {
1579         struct perf_callchain_entry *entry = data;
1580
1581         if (reliable)
1582                 callchain_store(entry, addr);
1583 }
1584
1585 static const struct stacktrace_ops backtrace_ops = {
1586         .warning                = backtrace_warning,
1587         .warning_symbol         = backtrace_warning_symbol,
1588         .stack                  = backtrace_stack,
1589         .address                = backtrace_address,
1590 };
1591
1592 static void
1593 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1594 {
1595         unsigned long bp;
1596         char *stack;
1597         int nr = entry->nr;
1598
1599         callchain_store(entry, instruction_pointer(regs));
1600
1601         stack = ((char *)regs + sizeof(struct pt_regs));
1602 #ifdef CONFIG_FRAME_POINTER
1603         bp = frame_pointer(regs);
1604 #else
1605         bp = 0;
1606 #endif
1607
1608         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1609
1610         entry->kernel = entry->nr - nr;
1611 }
1612
1613
1614 struct stack_frame {
1615         const void __user       *next_fp;
1616         unsigned long           return_address;
1617 };
1618
1619 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1620 {
1621         int ret;
1622
1623         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1624                 return 0;
1625
1626         ret = 1;
1627         pagefault_disable();
1628         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1629                 ret = 0;
1630         pagefault_enable();
1631
1632         return ret;
1633 }
1634
1635 static void
1636 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1637 {
1638         struct stack_frame frame;
1639         const void __user *fp;
1640         int nr = entry->nr;
1641
1642         regs = (struct pt_regs *)current->thread.sp0 - 1;
1643         fp   = (void __user *)regs->bp;
1644
1645         callchain_store(entry, regs->ip);
1646
1647         while (entry->nr < MAX_STACK_DEPTH) {
1648                 frame.next_fp        = NULL;
1649                 frame.return_address = 0;
1650
1651                 if (!copy_stack_frame(fp, &frame))
1652                         break;
1653
1654                 if ((unsigned long)fp < user_stack_pointer(regs))
1655                         break;
1656
1657                 callchain_store(entry, frame.return_address);
1658                 fp = frame.next_fp;
1659         }
1660
1661         entry->user = entry->nr - nr;
1662 }
1663
1664 static void
1665 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1666 {
1667         int is_user;
1668
1669         if (!regs)
1670                 return;
1671
1672         is_user = user_mode(regs);
1673
1674         if (!current || current->pid == 0)
1675                 return;
1676
1677         if (is_user && current->state != TASK_RUNNING)
1678                 return;
1679
1680         if (!is_user)
1681                 perf_callchain_kernel(regs, entry);
1682
1683         if (current->mm)
1684                 perf_callchain_user(regs, entry);
1685 }
1686
1687 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1688 {
1689         struct perf_callchain_entry *entry;
1690
1691         if (in_nmi())
1692                 entry = &__get_cpu_var(nmi_entry);
1693         else
1694                 entry = &__get_cpu_var(irq_entry);
1695
1696         entry->nr = 0;
1697         entry->hv = 0;
1698         entry->kernel = 0;
1699         entry->user = 0;
1700
1701         perf_do_callchain(regs, entry);
1702
1703         return entry;
1704 }