arch/x86/kernel/cpu/perf_event_intel_uncore.c

   1 #include "perf_event_intel_uncore.h"
   2
   3 static struct intel_uncore_type *empty_uncore[] = { NULL, };
   4 static struct intel_uncore_type **msr_uncores = empty_uncore;
   5 static struct intel_uncore_type **pci_uncores = empty_uncore;
   6 /* pci bus to socket mapping */
   7 static int pcibus_to_physid[256] = { [0 ... 255] = -1, };
   8
   9 static DEFINE_RAW_SPINLOCK(uncore_box_lock);
  10
  11 /* mask of cpus that collect uncore events */
  12 static cpumask_t uncore_cpu_mask;
  13
  14 /* constraint for the fixed counter */
  15 static struct event_constraint constraint_fixed =
  16         EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
  17 static struct event_constraint constraint_empty =
  18         EVENT_CONSTRAINT(0, 0, 0);
  19
  20 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
  21 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
  22 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
  23 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
  24 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
  25 DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
  26 DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
  27 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
  28 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
  29 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
  30 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
  31 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
  32 DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4");
  33 DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17");
  34 DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22");
  35 DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31");
  36 DEFINE_UNCORE_FORMAT_ATTR(filter_brand0, filter_brand0, "config1:0-7");
  37 DEFINE_UNCORE_FORMAT_ATTR(filter_brand1, filter_brand1, "config1:8-15");
  38 DEFINE_UNCORE_FORMAT_ATTR(filter_brand2, filter_brand2, "config1:16-23");
  39 DEFINE_UNCORE_FORMAT_ATTR(filter_brand3, filter_brand3, "config1:24-31");
  40
  41 /* Sandy Bridge-EP uncore support */
  42 static struct intel_uncore_type snbep_uncore_cbox;
  43 static struct intel_uncore_type snbep_uncore_pcu;
  44
  45 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
  46 {
  47         struct pci_dev *pdev = box->pci_dev;
  48         int box_ctl = uncore_pci_box_ctl(box);
  49         u32 config;
  50
  51         pci_read_config_dword(pdev, box_ctl, &config);
  52         config |= SNBEP_PMON_BOX_CTL_FRZ;
  53         pci_write_config_dword(pdev, box_ctl, config);
  54 }
  55
  56 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
  57 {
  58         struct pci_dev *pdev = box->pci_dev;
  59         int box_ctl = uncore_pci_box_ctl(box);
  60         u32 config;
  61
  62         pci_read_config_dword(pdev, box_ctl, &config);
  63         config &= ~SNBEP_PMON_BOX_CTL_FRZ;
  64         pci_write_config_dword(pdev, box_ctl, config);
  65 }
  66
  67 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box,
  68                                         struct perf_event *event)
  69 {
  70         struct pci_dev *pdev = box->pci_dev;
  71         struct hw_perf_event *hwc = &event->hw;
  72
  73         pci_write_config_dword(pdev, hwc->config_base, hwc->config |
  74                                 SNBEP_PMON_CTL_EN);
  75 }
  76
  77 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box,
  78                                         struct perf_event *event)
  79 {
  80         struct pci_dev *pdev = box->pci_dev;
  81         struct hw_perf_event *hwc = &event->hw;
  82
  83         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
  84 }
  85
  86 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box,
  87                                         struct perf_event *event)
  88 {
  89         struct pci_dev *pdev = box->pci_dev;
  90         struct hw_perf_event *hwc = &event->hw;
  91         u64 count;
  92
  93         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
  94         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
  95         return count;
  96 }
  97
  98 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
  99 {
 100         struct pci_dev *pdev = box->pci_dev;
 101         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL,
 102                                 SNBEP_PMON_BOX_CTL_INT);
 103 }
 104
 105 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
 106 {
 107         u64 config;
 108         unsigned msr;
 109
 110         msr = uncore_msr_box_ctl(box);
 111         if (msr) {
 112                 rdmsrl(msr, config);
 113                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 114                 wrmsrl(msr, config);
 115                 return;
 116         }
 117 }
 118
 119 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
 120 {
 121         u64 config;
 122         unsigned msr;
 123
 124         msr = uncore_msr_box_ctl(box);
 125         if (msr) {
 126                 rdmsrl(msr, config);
 127                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 128                 wrmsrl(msr, config);
 129                 return;
 130         }
 131 }
 132
 133 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box,
 134                                         struct perf_event *event)
 135 {
 136         struct hw_perf_event *hwc = &event->hw;
 137         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 138
 139         if (reg1->idx != EXTRA_REG_NONE)
 140                 wrmsrl(reg1->reg, reg1->config);
 141
 142         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 143 }
 144
 145 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 146                                         struct perf_event *event)
 147 {
 148         struct hw_perf_event *hwc = &event->hw;
 149
 150         wrmsrl(hwc->config_base, hwc->config);
 151 }
 152
 153 static u64 snbep_uncore_msr_read_counter(struct intel_uncore_box *box,
 154                                         struct perf_event *event)
 155 {
 156         struct hw_perf_event *hwc = &event->hw;
 157         u64 count;
 158
 159         rdmsrl(hwc->event_base, count);
 160         return count;
 161 }
 162
 163 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 164 {
 165         unsigned msr = uncore_msr_box_ctl(box);
 166         if (msr)
 167                 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
 168 }
 169
 170 static struct event_constraint *
 171 snbep_uncore_get_constraint(struct intel_uncore_box *box,
 172                             struct perf_event *event)
 173 {
 174         struct intel_uncore_extra_reg *er;
 175         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 176         unsigned long flags;
 177         bool ok = false;
 178
 179         if (reg1->idx == EXTRA_REG_NONE || (box->phys_id >= 0 && reg1->alloc))
 180                 return NULL;
 181
 182         er = &box->shared_regs[reg1->idx];
 183         raw_spin_lock_irqsave(&er->lock, flags);
 184         if (!atomic_read(&er->ref) || er->config1 == reg1->config) {
 185                 atomic_inc(&er->ref);
 186                 er->config1 = reg1->config;
 187                 ok = true;
 188         }
 189         raw_spin_unlock_irqrestore(&er->lock, flags);
 190
 191         if (ok) {
 192                 if (box->phys_id >= 0)
 193                         reg1->alloc = 1;
 194                 return NULL;
 195         }
 196         return &constraint_empty;
 197 }
 198
 199 static void snbep_uncore_put_constraint(struct intel_uncore_box *box,
 200                                         struct perf_event *event)
 201 {
 202         struct intel_uncore_extra_reg *er;
 203         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 204
 205         if (box->phys_id < 0 || !reg1->alloc)
 206                 return;
 207
 208         er = &box->shared_regs[reg1->idx];
 209         atomic_dec(&er->ref);
 210         reg1->alloc = 0;
 211 }
 212
 213 static int snbep_uncore_hw_config(struct intel_uncore_box *box,
 214                                   struct perf_event *event)
 215 {
 216         struct hw_perf_event *hwc = &event->hw;
 217         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 218
 219         if (box->pmu->type == &snbep_uncore_cbox) {
 220                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
 221                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
 222                 reg1->config = event->attr.config1 &
 223                         SNBEP_CB0_MSR_PMON_BOX_FILTER_MASK;
 224         } else if (box->pmu->type == &snbep_uncore_pcu) {
 225                 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
 226                 reg1->config = event->attr.config1 &
 227                         SNBEP_PCU_MSR_PMON_BOX_FILTER_MASK;
 228         } else {
 229                 return 0;
 230         }
 231         reg1->idx = 0;
 232         return 0;
 233 }
 234
 235 static struct attribute *snbep_uncore_formats_attr[] = {
 236         &format_attr_event.attr,
 237         &format_attr_umask.attr,
 238         &format_attr_edge.attr,
 239         &format_attr_inv.attr,
 240         &format_attr_thresh8.attr,
 241         NULL,
 242 };
 243
 244 static struct attribute *snbep_uncore_ubox_formats_attr[] = {
 245         &format_attr_event.attr,
 246         &format_attr_umask.attr,
 247         &format_attr_edge.attr,
 248         &format_attr_inv.attr,
 249         &format_attr_thresh5.attr,
 250         NULL,
 251 };
 252
 253 static struct attribute *snbep_uncore_cbox_formats_attr[] = {
 254         &format_attr_event.attr,
 255         &format_attr_umask.attr,
 256         &format_attr_edge.attr,
 257         &format_attr_tid_en.attr,
 258         &format_attr_inv.attr,
 259         &format_attr_thresh8.attr,
 260         &format_attr_filter_tid.attr,
 261         &format_attr_filter_nid.attr,
 262         &format_attr_filter_state.attr,
 263         &format_attr_filter_opc.attr,
 264         NULL,
 265 };
 266
 267 static struct attribute *snbep_uncore_pcu_formats_attr[] = {
 268         &format_attr_event.attr,
 269         &format_attr_occ_sel.attr,
 270         &format_attr_edge.attr,
 271         &format_attr_inv.attr,
 272         &format_attr_thresh5.attr,
 273         &format_attr_occ_invert.attr,
 274         &format_attr_occ_edge.attr,
 275         &format_attr_filter_brand0.attr,
 276         &format_attr_filter_brand1.attr,
 277         &format_attr_filter_brand2.attr,
 278         &format_attr_filter_brand3.attr,
 279         NULL,
 280 };
 281
 282 static struct uncore_event_desc snbep_uncore_imc_events[] = {
 283         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0xff,umask=0x00"),
 284         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x03"),
 285         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
 286         { /* end: all zeroes */ },
 287 };
 288
 289 static struct uncore_event_desc snbep_uncore_qpi_events[] = {
 290         INTEL_UNCORE_EVENT_DESC(clockticks,       "event=0x14"),
 291         INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
 292         INTEL_UNCORE_EVENT_DESC(drs_data,         "event=0x02,umask=0x08"),
 293         INTEL_UNCORE_EVENT_DESC(ncb_data,         "event=0x03,umask=0x04"),
 294         { /* end: all zeroes */ },
 295 };
 296
 297 static struct attribute_group snbep_uncore_format_group = {
 298         .name = "format",
 299         .attrs = snbep_uncore_formats_attr,
 300 };
 301
 302 static struct attribute_group snbep_uncore_ubox_format_group = {
 303         .name = "format",
 304         .attrs = snbep_uncore_ubox_formats_attr,
 305 };
 306
 307 static struct attribute_group snbep_uncore_cbox_format_group = {
 308         .name = "format",
 309         .attrs = snbep_uncore_cbox_formats_attr,
 310 };
 311
 312 static struct attribute_group snbep_uncore_pcu_format_group = {
 313         .name = "format",
 314         .attrs = snbep_uncore_pcu_formats_attr,
 315 };
 316
 317 static struct intel_uncore_ops snbep_uncore_msr_ops = {
 318         .init_box       = snbep_uncore_msr_init_box,
 319         .disable_box    = snbep_uncore_msr_disable_box,
 320         .enable_box     = snbep_uncore_msr_enable_box,
 321         .disable_event  = snbep_uncore_msr_disable_event,
 322         .enable_event   = snbep_uncore_msr_enable_event,
 323         .read_counter   = snbep_uncore_msr_read_counter,
 324         .get_constraint = snbep_uncore_get_constraint,
 325         .put_constraint = snbep_uncore_put_constraint,
 326         .hw_config      = snbep_uncore_hw_config,
 327 };
 328
 329 static struct intel_uncore_ops snbep_uncore_pci_ops = {
 330         .init_box       = snbep_uncore_pci_init_box,
 331         .disable_box    = snbep_uncore_pci_disable_box,
 332         .enable_box     = snbep_uncore_pci_enable_box,
 333         .disable_event  = snbep_uncore_pci_disable_event,
 334         .enable_event   = snbep_uncore_pci_enable_event,
 335         .read_counter   = snbep_uncore_pci_read_counter,
 336 };
 337
 338 static struct event_constraint snbep_uncore_cbox_constraints[] = {
 339         UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
 340         UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
 341         UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
 342         UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
 343         UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
 344         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
 345         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 346         UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
 347         UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
 348         UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
 349         UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
 350         UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
 351         EVENT_CONSTRAINT_OVERLAP(0x1f, 0xe, 0xff),
 352         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 353         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 354         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 355         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 356         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 357         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 358         UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
 359         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
 360         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 361         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 362         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 363         UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
 364         EVENT_CONSTRAINT_END
 365 };
 366
 367 static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
 368         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 369         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 370         UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
 371         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 372         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 373         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 374         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 375         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 376         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 377         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 378         EVENT_CONSTRAINT_END
 379 };
 380
 381 static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
 382         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 383         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 384         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 385         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
 386         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
 387         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 388         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
 389         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 390         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 391         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 392         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 393         UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
 394         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 395         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 396         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 397         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 398         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
 399         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 400         EVENT_CONSTRAINT_END
 401 };
 402
 403 static struct intel_uncore_type snbep_uncore_ubox = {
 404         .name           = "ubox",
 405         .num_counters   = 2,
 406         .num_boxes      = 1,
 407         .perf_ctr_bits  = 44,
 408         .fixed_ctr_bits = 48,
 409         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
 410         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
 411         .event_mask     = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
 412         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
 413         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
 414         .ops            = &snbep_uncore_msr_ops,
 415         .format_group   = &snbep_uncore_ubox_format_group,
 416 };
 417
 418 static struct intel_uncore_type snbep_uncore_cbox = {
 419         .name                   = "cbox",
 420         .num_counters           = 4,
 421         .num_boxes              = 8,
 422         .perf_ctr_bits          = 44,
 423         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
 424         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
 425         .event_mask             = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
 426         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
 427         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
 428         .num_shared_regs        = 1,
 429         .constraints            = snbep_uncore_cbox_constraints,
 430         .ops                    = &snbep_uncore_msr_ops,
 431         .format_group           = &snbep_uncore_cbox_format_group,
 432 };
 433
 434 static struct intel_uncore_type snbep_uncore_pcu = {
 435         .name                   = "pcu",
 436         .num_counters           = 4,
 437         .num_boxes              = 1,
 438         .perf_ctr_bits          = 48,
 439         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
 440         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
 441         .event_mask             = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
 442         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
 443         .num_shared_regs        = 1,
 444         .ops                    = &snbep_uncore_msr_ops,
 445         .format_group           = &snbep_uncore_pcu_format_group,
 446 };
 447
 448 static struct intel_uncore_type *snbep_msr_uncores[] = {
 449         &snbep_uncore_ubox,
 450         &snbep_uncore_cbox,
 451         &snbep_uncore_pcu,
 452         NULL,
 453 };
 454
 455 #define SNBEP_UNCORE_PCI_COMMON_INIT()                          \
 456         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
 457         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
 458         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,            \
 459         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
 460         .ops            = &snbep_uncore_pci_ops,                \
 461         .format_group   = &snbep_uncore_format_group
 462
 463 static struct intel_uncore_type snbep_uncore_ha = {
 464         .name           = "ha",
 465         .num_counters   = 4,
 466         .num_boxes      = 1,
 467         .perf_ctr_bits  = 48,
 468         SNBEP_UNCORE_PCI_COMMON_INIT(),
 469 };
 470
 471 static struct intel_uncore_type snbep_uncore_imc = {
 472         .name           = "imc",
 473         .num_counters   = 4,
 474         .num_boxes      = 4,
 475         .perf_ctr_bits  = 48,
 476         .fixed_ctr_bits = 48,
 477         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
 478         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
 479         .event_descs    = snbep_uncore_imc_events,
 480         SNBEP_UNCORE_PCI_COMMON_INIT(),
 481 };
 482
 483 static struct intel_uncore_type snbep_uncore_qpi = {
 484         .name           = "qpi",
 485         .num_counters   = 4,
 486         .num_boxes      = 2,
 487         .perf_ctr_bits  = 48,
 488         .event_descs    = snbep_uncore_qpi_events,
 489         SNBEP_UNCORE_PCI_COMMON_INIT(),
 490 };
 491
 492
 493 static struct intel_uncore_type snbep_uncore_r2pcie = {
 494         .name           = "r2pcie",
 495         .num_counters   = 4,
 496         .num_boxes      = 1,
 497         .perf_ctr_bits  = 44,
 498         .constraints    = snbep_uncore_r2pcie_constraints,
 499         SNBEP_UNCORE_PCI_COMMON_INIT(),
 500 };
 501
 502 static struct intel_uncore_type snbep_uncore_r3qpi = {
 503         .name           = "r3qpi",
 504         .num_counters   = 3,
 505         .num_boxes      = 2,
 506         .perf_ctr_bits  = 44,
 507         .constraints    = snbep_uncore_r3qpi_constraints,
 508         SNBEP_UNCORE_PCI_COMMON_INIT(),
 509 };
 510
 511 static struct intel_uncore_type *snbep_pci_uncores[] = {
 512         &snbep_uncore_ha,
 513         &snbep_uncore_imc,
 514         &snbep_uncore_qpi,
 515         &snbep_uncore_r2pcie,
 516         &snbep_uncore_r3qpi,
 517         NULL,
 518 };
 519
 520 static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = {
 521         { /* Home Agent */
 522                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
 523                 .driver_data = (unsigned long)&snbep_uncore_ha,
 524         },
 525         { /* MC Channel 0 */
 526                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
 527                 .driver_data = (unsigned long)&snbep_uncore_imc,
 528         },
 529         { /* MC Channel 1 */
 530                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
 531                 .driver_data = (unsigned long)&snbep_uncore_imc,
 532         },
 533         { /* MC Channel 2 */
 534                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
 535                 .driver_data = (unsigned long)&snbep_uncore_imc,
 536         },
 537         { /* MC Channel 3 */
 538                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
 539                 .driver_data = (unsigned long)&snbep_uncore_imc,
 540         },
 541         { /* QPI Port 0 */
 542                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
 543                 .driver_data = (unsigned long)&snbep_uncore_qpi,
 544         },
 545         { /* QPI Port 1 */
 546                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
 547                 .driver_data = (unsigned long)&snbep_uncore_qpi,
 548         },
 549         { /* P2PCIe */
 550                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
 551                 .driver_data = (unsigned long)&snbep_uncore_r2pcie,
 552         },
 553         { /* R3QPI Link 0 */
 554                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
 555                 .driver_data = (unsigned long)&snbep_uncore_r3qpi,
 556         },
 557         { /* R3QPI Link 1 */
 558                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
 559                 .driver_data = (unsigned long)&snbep_uncore_r3qpi,
 560         },
 561         { /* end: all zeroes */ }
 562 };
 563
 564 static struct pci_driver snbep_uncore_pci_driver = {
 565         .name           = "snbep_uncore",
 566         .id_table       = snbep_uncore_pci_ids,
 567 };
 568
 569 /*
 570  * build pci bus to socket mapping
 571  */
 572 static void snbep_pci2phy_map_init(void)
 573 {
 574         struct pci_dev *ubox_dev = NULL;
 575         int i, bus, nodeid;
 576         u32 config;
 577
 578         while (1) {
 579                 /* find the UBOX device */
 580                 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL,
 581                                         PCI_DEVICE_ID_INTEL_JAKETOWN_UBOX,
 582                                         ubox_dev);
 583                 if (!ubox_dev)
 584                         break;
 585                 bus = ubox_dev->bus->number;
 586                 /* get the Node ID of the local register */
 587                 pci_read_config_dword(ubox_dev, 0x40, &config);
 588                 nodeid = config;
 589                 /* get the Node ID mapping */
 590                 pci_read_config_dword(ubox_dev, 0x54, &config);
 591                 /*
 592                  * every three bits in the Node ID mapping register maps
 593                  * to a particular node.
 594                  */
 595                 for (i = 0; i < 8; i++) {
 596                         if (nodeid == ((config >> (3 * i)) & 0x7)) {
 597                                 pcibus_to_physid[bus] = i;
 598                                 break;
 599                         }
 600                 }
 601         };
 602         return;
 603 }
 604 /* end of Sandy Bridge-EP uncore support */
 605
 606
 607 /* Sandy Bridge uncore support */
 608 static void snb_uncore_msr_enable_event(struct intel_uncore_box *box,
 609                                         struct perf_event *event)
 610 {
 611         struct hw_perf_event *hwc = &event->hw;
 612
 613         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 614                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 615         else
 616                 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
 617 }
 618
 619 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box,
 620                                         struct perf_event *event)
 621 {
 622         wrmsrl(event->hw.config_base, 0);
 623 }
 624
 625 static u64 snb_uncore_msr_read_counter(struct intel_uncore_box *box,
 626                                         struct perf_event *event)
 627 {
 628         u64 count;
 629         rdmsrl(event->hw.event_base, count);
 630         return count;
 631 }
 632
 633 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
 634 {
 635         if (box->pmu->pmu_idx == 0) {
 636                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
 637                         SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 638         }
 639 }
 640
 641 static struct attribute *snb_uncore_formats_attr[] = {
 642         &format_attr_event.attr,
 643         &format_attr_umask.attr,
 644         &format_attr_edge.attr,
 645         &format_attr_inv.attr,
 646         &format_attr_cmask5.attr,
 647         NULL,
 648 };
 649
 650 static struct attribute_group snb_uncore_format_group = {
 651         .name = "format",
 652         .attrs = snb_uncore_formats_attr,
 653 };
 654
 655 static struct intel_uncore_ops snb_uncore_msr_ops = {
 656         .init_box       = snb_uncore_msr_init_box,
 657         .disable_event  = snb_uncore_msr_disable_event,
 658         .enable_event   = snb_uncore_msr_enable_event,
 659         .read_counter   = snb_uncore_msr_read_counter,
 660 };
 661
 662 static struct event_constraint snb_uncore_cbox_constraints[] = {
 663         UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
 664         UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
 665         EVENT_CONSTRAINT_END
 666 };
 667
 668 static struct intel_uncore_type snb_uncore_cbox = {
 669         .name           = "cbox",
 670         .num_counters   = 2,
 671         .num_boxes      = 4,
 672         .perf_ctr_bits  = 44,
 673         .fixed_ctr_bits = 48,
 674         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
 675         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 676         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 677         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 678         .single_fixed   = 1,
 679         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 680         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
 681         .constraints    = snb_uncore_cbox_constraints,
 682         .ops            = &snb_uncore_msr_ops,
 683         .format_group   = &snb_uncore_format_group,
 684 };
 685
 686 static struct intel_uncore_type *snb_msr_uncores[] = {
 687         &snb_uncore_cbox,
 688         NULL,
 689 };
 690 /* end of Sandy Bridge uncore support */
 691
 692 /* Nehalem uncore support */
 693 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
 694 {
 695         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
 696 }
 697
 698 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
 699 {
 700         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL,
 701                 NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
 702 }
 703
 704 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box,
 705                                         struct perf_event *event)
 706 {
 707         struct hw_perf_event *hwc = &event->hw;
 708
 709         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 710                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 711         else
 712                 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
 713 }
 714
 715 static struct attribute *nhm_uncore_formats_attr[] = {
 716         &format_attr_event.attr,
 717         &format_attr_umask.attr,
 718         &format_attr_edge.attr,
 719         &format_attr_inv.attr,
 720         &format_attr_cmask8.attr,
 721         NULL,
 722 };
 723
 724 static struct attribute_group nhm_uncore_format_group = {
 725         .name = "format",
 726         .attrs = nhm_uncore_formats_attr,
 727 };
 728
 729 static struct uncore_event_desc nhm_uncore_events[] = {
 730         INTEL_UNCORE_EVENT_DESC(clockticks,                "event=0xff,umask=0x00"),
 731         INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any,       "event=0x2f,umask=0x0f"),
 732         INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any,      "event=0x2c,umask=0x0f"),
 733         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads,     "event=0x20,umask=0x01"),
 734         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes,    "event=0x20,umask=0x02"),
 735         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads,  "event=0x20,umask=0x04"),
 736         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
 737         INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads,   "event=0x20,umask=0x10"),
 738         INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes,  "event=0x20,umask=0x20"),
 739         { /* end: all zeroes */ },
 740 };
 741
 742 static struct intel_uncore_ops nhm_uncore_msr_ops = {
 743         .disable_box    = nhm_uncore_msr_disable_box,
 744         .enable_box     = nhm_uncore_msr_enable_box,
 745         .disable_event  = snb_uncore_msr_disable_event,
 746         .enable_event   = nhm_uncore_msr_enable_event,
 747         .read_counter   = snb_uncore_msr_read_counter,
 748 };
 749
 750 static struct intel_uncore_type nhm_uncore = {
 751         .name           = "",
 752         .num_counters   = 8,
 753         .num_boxes      = 1,
 754         .perf_ctr_bits  = 48,
 755         .fixed_ctr_bits = 48,
 756         .event_ctl      = NHM_UNC_PERFEVTSEL0,
 757         .perf_ctr       = NHM_UNC_UNCORE_PMC0,
 758         .fixed_ctr      = NHM_UNC_FIXED_CTR,
 759         .fixed_ctl      = NHM_UNC_FIXED_CTR_CTRL,
 760         .event_mask     = NHM_UNC_RAW_EVENT_MASK,
 761         .event_descs    = nhm_uncore_events,
 762         .ops            = &nhm_uncore_msr_ops,
 763         .format_group   = &nhm_uncore_format_group,
 764 };
 765
 766 static struct intel_uncore_type *nhm_msr_uncores[] = {
 767         &nhm_uncore,
 768         NULL,
 769 };
 770 /* end of Nehalem uncore support */
 771
 772 static void uncore_assign_hw_event(struct intel_uncore_box *box,
 773                                 struct perf_event *event, int idx)
 774 {
 775         struct hw_perf_event *hwc = &event->hw;
 776
 777         hwc->idx = idx;
 778         hwc->last_tag = ++box->tags[idx];
 779
 780         if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
 781                 hwc->event_base = uncore_fixed_ctr(box);
 782                 hwc->config_base = uncore_fixed_ctl(box);
 783                 return;
 784         }
 785
 786         hwc->config_base = uncore_event_ctl(box, hwc->idx);
 787         hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
 788 }
 789
 790 static void uncore_perf_event_update(struct intel_uncore_box *box,
 791                                         struct perf_event *event)
 792 {
 793         u64 prev_count, new_count, delta;
 794         int shift;
 795
 796         if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
 797                 shift = 64 - uncore_fixed_ctr_bits(box);
 798         else
 799                 shift = 64 - uncore_perf_ctr_bits(box);
 800
 801         /* the hrtimer might modify the previous event value */
 802 again:
 803         prev_count = local64_read(&event->hw.prev_count);
 804         new_count = uncore_read_counter(box, event);
 805         if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
 806                 goto again;
 807
 808         delta = (new_count << shift) - (prev_count << shift);
 809         delta >>= shift;
 810
 811         local64_add(delta, &event->count);
 812 }
 813
 814 /*
 815  * The overflow interrupt is unavailable for SandyBridge-EP, is broken
 816  * for SandyBridge. So we use hrtimer to periodically poll the counter
 817  * to avoid overflow.
 818  */
 819 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
 820 {
 821         struct intel_uncore_box *box;
 822         unsigned long flags;
 823         int bit;
 824
 825         box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
 826         if (!box->n_active || box->cpu != smp_processor_id())
 827                 return HRTIMER_NORESTART;
 828         /*
 829          * disable local interrupt to prevent uncore_pmu_event_start/stop
 830          * to interrupt the update process
 831          */
 832         local_irq_save(flags);
 833
 834         for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
 835                 uncore_perf_event_update(box, box->events[bit]);
 836
 837         local_irq_restore(flags);
 838
 839         hrtimer_forward_now(hrtimer, ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL));
 840         return HRTIMER_RESTART;
 841 }
 842
 843 static void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
 844 {
 845         __hrtimer_start_range_ns(&box->hrtimer,
 846                         ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL), 0,
 847                         HRTIMER_MODE_REL_PINNED, 0);
 848 }
 849
 850 static void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
 851 {
 852         hrtimer_cancel(&box->hrtimer);
 853 }
 854
 855 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
 856 {
 857         hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 858         box->hrtimer.function = uncore_pmu_hrtimer;
 859 }
 860
 861 struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
 862                                           int cpu)
 863 {
 864         struct intel_uncore_box *box;
 865         int i, size;
 866
 867         size = sizeof(*box) + type->num_shared_regs *
 868                 sizeof(struct intel_uncore_extra_reg);
 869
 870         box = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO, cpu_to_node(cpu));
 871         if (!box)
 872                 return NULL;
 873
 874         for (i = 0; i < type->num_shared_regs; i++)
 875                 raw_spin_lock_init(&box->shared_regs[i].lock);
 876
 877         uncore_pmu_init_hrtimer(box);
 878         atomic_set(&box->refcnt, 1);
 879         box->cpu = -1;
 880         box->phys_id = -1;
 881
 882         return box;
 883 }
 884
 885 static struct intel_uncore_box *
 886 uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
 887 {
 888         static struct intel_uncore_box *box;
 889
 890         box = *per_cpu_ptr(pmu->box, cpu);
 891         if (box)
 892                 return box;
 893
 894         raw_spin_lock(&uncore_box_lock);
 895         list_for_each_entry(box, &pmu->box_list, list) {
 896                 if (box->phys_id == topology_physical_package_id(cpu)) {
 897                         atomic_inc(&box->refcnt);
 898                         *per_cpu_ptr(pmu->box, cpu) = box;
 899                         break;
 900                 }
 901         }
 902         raw_spin_unlock(&uncore_box_lock);
 903
 904         return *per_cpu_ptr(pmu->box, cpu);
 905 }
 906
 907 static struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event)
 908 {
 909         return container_of(event->pmu, struct intel_uncore_pmu, pmu);
 910 }
 911
 912 static struct intel_uncore_box *uncore_event_to_box(struct perf_event *event)
 913 {
 914         /*
 915          * perf core schedules event on the basis of cpu, uncore events are
 916          * collected by one of the cpus inside a physical package.
 917          */
 918         return uncore_pmu_to_box(uncore_event_to_pmu(event),
 919                                  smp_processor_id());
 920 }
 921
 922 static int uncore_collect_events(struct intel_uncore_box *box,
 923                                 struct perf_event *leader, bool dogrp)
 924 {
 925         struct perf_event *event;
 926         int n, max_count;
 927
 928         max_count = box->pmu->type->num_counters;
 929         if (box->pmu->type->fixed_ctl)
 930                 max_count++;
 931
 932         if (box->n_events >= max_count)
 933                 return -EINVAL;
 934
 935         n = box->n_events;
 936         box->event_list[n] = leader;
 937         n++;
 938         if (!dogrp)
 939                 return n;
 940
 941         list_for_each_entry(event, &leader->sibling_list, group_entry) {
 942                 if (event->state <= PERF_EVENT_STATE_OFF)
 943                         continue;
 944
 945                 if (n >= max_count)
 946                         return -EINVAL;
 947
 948                 box->event_list[n] = event;
 949                 n++;
 950         }
 951         return n;
 952 }
 953
 954 static struct event_constraint *
 955 uncore_get_event_constraint(struct intel_uncore_box *box,
 956                             struct perf_event *event)
 957 {
 958         struct intel_uncore_type *type = box->pmu->type;
 959         struct event_constraint *c;
 960
 961         if (type->ops->get_constraint) {
 962                 c = type->ops->get_constraint(box, event);
 963                 if (c)
 964                         return c;
 965         }
 966
 967         if (event->hw.config == ~0ULL)
 968                 return &constraint_fixed;
 969
 970         if (type->constraints) {
 971                 for_each_event_constraint(c, type->constraints) {
 972                         if ((event->hw.config & c->cmask) == c->code)
 973                                 return c;
 974                 }
 975         }
 976
 977         return &type->unconstrainted;
 978 }
 979
 980 static void uncore_put_event_constraint(struct intel_uncore_box *box,
 981                                         struct perf_event *event)
 982 {
 983         if (box->pmu->type->ops->put_constraint)
 984                 box->pmu->type->ops->put_constraint(box, event);
 985 }
 986
 987 static int uncore_assign_events(struct intel_uncore_box *box,
 988                                 int assign[], int n)
 989 {
 990         unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
 991         struct event_constraint *c, *constraints[UNCORE_PMC_IDX_MAX];
 992         int i, wmin, wmax, ret = 0;
 993         struct hw_perf_event *hwc;
 994
 995         bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
 996
 997         for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
 998                 c = uncore_get_event_constraint(box, box->event_list[i]);
 999                 constraints[i] = c;
1000                 wmin = min(wmin, c->weight);
1001                 wmax = max(wmax, c->weight);
1002         }
1003
1004         /* fastpath, try to reuse previous register */
1005         for (i = 0; i < n; i++) {
1006                 hwc = &box->event_list[i]->hw;
1007                 c = constraints[i];
1008
1009                 /* never assigned */
1010                 if (hwc->idx == -1)
1011                         break;
1012
1013                 /* constraint still honored */
1014                 if (!test_bit(hwc->idx, c->idxmsk))
1015                         break;
1016
1017                 /* not already used */
1018                 if (test_bit(hwc->idx, used_mask))
1019                         break;
1020
1021                 __set_bit(hwc->idx, used_mask);
1022                 if (assign)
1023                         assign[i] = hwc->idx;
1024         }
1025         /* slow path */
1026         if (i != n)
1027                 ret = perf_assign_events(constraints, n, wmin, wmax, assign);
1028
1029         if (!assign || ret) {
1030                 for (i = 0; i < n; i++)
1031                         uncore_put_event_constraint(box, box->event_list[i]);
1032         }
1033         return ret ? -EINVAL : 0;
1034 }
1035
1036 static void uncore_pmu_event_start(struct perf_event *event, int flags)
1037 {
1038         struct intel_uncore_box *box = uncore_event_to_box(event);
1039         int idx = event->hw.idx;
1040
1041         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1042                 return;
1043
1044         if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
1045                 return;
1046
1047         event->hw.state = 0;
1048         box->events[idx] = event;
1049         box->n_active++;
1050         __set_bit(idx, box->active_mask);
1051
1052         local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
1053         uncore_enable_event(box, event);
1054
1055         if (box->n_active == 1) {
1056                 uncore_enable_box(box);
1057                 uncore_pmu_start_hrtimer(box);
1058         }
1059 }
1060
1061 static void uncore_pmu_event_stop(struct perf_event *event, int flags)
1062 {
1063         struct intel_uncore_box *box = uncore_event_to_box(event);
1064         struct hw_perf_event *hwc = &event->hw;
1065
1066         if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
1067                 uncore_disable_event(box, event);
1068                 box->n_active--;
1069                 box->events[hwc->idx] = NULL;
1070                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1071                 hwc->state |= PERF_HES_STOPPED;
1072
1073                 if (box->n_active == 0) {
1074                         uncore_disable_box(box);
1075                         uncore_pmu_cancel_hrtimer(box);
1076                 }
1077         }
1078
1079         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1080                 /*
1081                  * Drain the remaining delta count out of a event
1082                  * that we are disabling:
1083                  */
1084                 uncore_perf_event_update(box, event);
1085                 hwc->state |= PERF_HES_UPTODATE;
1086         }
1087 }
1088
1089 static int uncore_pmu_event_add(struct perf_event *event, int flags)
1090 {
1091         struct intel_uncore_box *box = uncore_event_to_box(event);
1092         struct hw_perf_event *hwc = &event->hw;
1093         int assign[UNCORE_PMC_IDX_MAX];
1094         int i, n, ret;
1095
1096         if (!box)
1097                 return -ENODEV;
1098
1099         ret = n = uncore_collect_events(box, event, false);
1100         if (ret < 0)
1101                 return ret;
1102
1103         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1104         if (!(flags & PERF_EF_START))
1105                 hwc->state |= PERF_HES_ARCH;
1106
1107         ret = uncore_assign_events(box, assign, n);
1108         if (ret)
1109                 return ret;
1110
1111         /* save events moving to new counters */
1112         for (i = 0; i < box->n_events; i++) {
1113                 event = box->event_list[i];
1114                 hwc = &event->hw;
1115
1116                 if (hwc->idx == assign[i] &&
1117                         hwc->last_tag == box->tags[assign[i]])
1118                         continue;
1119                 /*
1120                  * Ensure we don't accidentally enable a stopped
1121                  * counter simply because we rescheduled.
1122                  */
1123                 if (hwc->state & PERF_HES_STOPPED)
1124                         hwc->state |= PERF_HES_ARCH;
1125
1126                 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
1127         }
1128
1129         /* reprogram moved events into new counters */
1130         for (i = 0; i < n; i++) {
1131                 event = box->event_list[i];
1132                 hwc = &event->hw;
1133
1134                 if (hwc->idx != assign[i] ||
1135                         hwc->last_tag != box->tags[assign[i]])
1136                         uncore_assign_hw_event(box, event, assign[i]);
1137                 else if (i < box->n_events)
1138                         continue;
1139
1140                 if (hwc->state & PERF_HES_ARCH)
1141                         continue;
1142
1143                 uncore_pmu_event_start(event, 0);
1144         }
1145         box->n_events = n;
1146
1147         return 0;
1148 }
1149
1150 static void uncore_pmu_event_del(struct perf_event *event, int flags)
1151 {
1152         struct intel_uncore_box *box = uncore_event_to_box(event);
1153         int i;
1154
1155         uncore_pmu_event_stop(event, PERF_EF_UPDATE);
1156
1157         for (i = 0; i < box->n_events; i++) {
1158                 if (event == box->event_list[i]) {
1159                         uncore_put_event_constraint(box, event);
1160
1161                         while (++i < box->n_events)
1162                                 box->event_list[i - 1] = box->event_list[i];
1163
1164                         --box->n_events;
1165                         break;
1166                 }
1167         }
1168
1169         event->hw.idx = -1;
1170         event->hw.last_tag = ~0ULL;
1171 }
1172
1173 static void uncore_pmu_event_read(struct perf_event *event)
1174 {
1175         struct intel_uncore_box *box = uncore_event_to_box(event);
1176         uncore_perf_event_update(box, event);
1177 }
1178
1179 /*
1180  * validation ensures the group can be loaded onto the
1181  * PMU if it was the only group available.
1182  */
1183 static int uncore_validate_group(struct intel_uncore_pmu *pmu,
1184                                 struct perf_event *event)
1185 {
1186         struct perf_event *leader = event->group_leader;
1187         struct intel_uncore_box *fake_box;
1188         int ret = -EINVAL, n;
1189
1190         fake_box = uncore_alloc_box(pmu->type, smp_processor_id());
1191         if (!fake_box)
1192                 return -ENOMEM;
1193
1194         fake_box->pmu = pmu;
1195         /*
1196          * the event is not yet connected with its
1197          * siblings therefore we must first collect
1198          * existing siblings, then add the new event
1199          * before we can simulate the scheduling
1200          */
1201         n = uncore_collect_events(fake_box, leader, true);
1202         if (n < 0)
1203                 goto out;
1204
1205         fake_box->n_events = n;
1206         n = uncore_collect_events(fake_box, event, false);
1207         if (n < 0)
1208                 goto out;
1209
1210         fake_box->n_events = n;
1211
1212         ret = uncore_assign_events(fake_box, NULL, n);
1213 out:
1214         kfree(fake_box);
1215         return ret;
1216 }
1217
1218 int uncore_pmu_event_init(struct perf_event *event)
1219 {
1220         struct intel_uncore_pmu *pmu;
1221         struct intel_uncore_box *box;
1222         struct hw_perf_event *hwc = &event->hw;
1223         int ret;
1224
1225         if (event->attr.type != event->pmu->type)
1226                 return -ENOENT;
1227
1228         pmu = uncore_event_to_pmu(event);
1229         /* no device found for this pmu */
1230         if (pmu->func_id < 0)
1231                 return -ENOENT;
1232
1233         /*
1234          * Uncore PMU does measure at all privilege level all the time.
1235          * So it doesn't make sense to specify any exclude bits.
1236          */
1237         if (event->attr.exclude_user || event->attr.exclude_kernel ||
1238                         event->attr.exclude_hv || event->attr.exclude_idle)
1239                 return -EINVAL;
1240
1241         /* Sampling not supported yet */
1242         if (hwc->sample_period)
1243                 return -EINVAL;
1244
1245         /*
1246          * Place all uncore events for a particular physical package
1247          * onto a single cpu
1248          */
1249         if (event->cpu < 0)
1250                 return -EINVAL;
1251         box = uncore_pmu_to_box(pmu, event->cpu);
1252         if (!box || box->cpu < 0)
1253                 return -EINVAL;
1254         event->cpu = box->cpu;
1255
1256         event->hw.idx = -1;
1257         event->hw.last_tag = ~0ULL;
1258         event->hw.extra_reg.idx = EXTRA_REG_NONE;
1259
1260         if (event->attr.config == UNCORE_FIXED_EVENT) {
1261                 /* no fixed counter */
1262                 if (!pmu->type->fixed_ctl)
1263                         return -EINVAL;
1264                 /*
1265                  * if there is only one fixed counter, only the first pmu
1266                  * can access the fixed counter
1267                  */
1268                 if (pmu->type->single_fixed && pmu->pmu_idx > 0)
1269                         return -EINVAL;
1270                 hwc->config = ~0ULL;
1271         } else {
1272                 hwc->config = event->attr.config & pmu->type->event_mask;
1273                 if (pmu->type->ops->hw_config) {
1274                         ret = pmu->type->ops->hw_config(box, event);
1275                         if (ret)
1276                                 return ret;
1277                 }
1278         }
1279
1280         if (event->group_leader != event)
1281                 ret = uncore_validate_group(pmu, event);
1282         else
1283                 ret = 0;
1284
1285         return ret;
1286 }
1287
1288 static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
1289 {
1290         int ret;
1291
1292         pmu->pmu = (struct pmu) {
1293                 .attr_groups    = pmu->type->attr_groups,
1294                 .task_ctx_nr    = perf_invalid_context,
1295                 .event_init     = uncore_pmu_event_init,
1296                 .add            = uncore_pmu_event_add,
1297                 .del            = uncore_pmu_event_del,
1298                 .start          = uncore_pmu_event_start,
1299                 .stop           = uncore_pmu_event_stop,
1300                 .read           = uncore_pmu_event_read,
1301         };
1302
1303         if (pmu->type->num_boxes == 1) {
1304                 if (strlen(pmu->type->name) > 0)
1305                         sprintf(pmu->name, "uncore_%s", pmu->type->name);
1306                 else
1307                         sprintf(pmu->name, "uncore");
1308         } else {
1309                 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
1310                         pmu->pmu_idx);
1311         }
1312
1313         ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
1314         return ret;
1315 }
1316
1317 static void __init uncore_type_exit(struct intel_uncore_type *type)
1318 {
1319         int i;
1320
1321         for (i = 0; i < type->num_boxes; i++)
1322                 free_percpu(type->pmus[i].box);
1323         kfree(type->pmus);
1324         type->pmus = NULL;
1325         kfree(type->attr_groups[1]);
1326         type->attr_groups[1] = NULL;
1327 }
1328
1329 static void uncore_types_exit(struct intel_uncore_type **types)
1330 {
1331         int i;
1332         for (i = 0; types[i]; i++)
1333                 uncore_type_exit(types[i]);
1334 }
1335
1336 static int __init uncore_type_init(struct intel_uncore_type *type)
1337 {
1338         struct intel_uncore_pmu *pmus;
1339         struct attribute_group *events_group;
1340         struct attribute **attrs;
1341         int i, j;
1342
1343         pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
1344         if (!pmus)
1345                 return -ENOMEM;
1346
1347         type->unconstrainted = (struct event_constraint)
1348                 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
1349                                 0, type->num_counters, 0);
1350
1351         for (i = 0; i < type->num_boxes; i++) {
1352                 pmus[i].func_id = -1;
1353                 pmus[i].pmu_idx = i;
1354                 pmus[i].type = type;
1355                 INIT_LIST_HEAD(&pmus[i].box_list);
1356                 pmus[i].box = alloc_percpu(struct intel_uncore_box *);
1357                 if (!pmus[i].box)
1358                         goto fail;
1359         }
1360
1361         if (type->event_descs) {
1362                 i = 0;
1363                 while (type->event_descs[i].attr.attr.name)
1364                         i++;
1365
1366                 events_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
1367                                         sizeof(*events_group), GFP_KERNEL);
1368                 if (!events_group)
1369                         goto fail;
1370
1371                 attrs = (struct attribute **)(events_group + 1);
1372                 events_group->name = "events";
1373                 events_group->attrs = attrs;
1374
1375                 for (j = 0; j < i; j++)
1376                         attrs[j] = &type->event_descs[j].attr.attr;
1377
1378                 type->attr_groups[1] = events_group;
1379         }
1380
1381         type->pmus = pmus;
1382         return 0;
1383 fail:
1384         uncore_type_exit(type);
1385         return -ENOMEM;
1386 }
1387
1388 static int __init uncore_types_init(struct intel_uncore_type **types)
1389 {
1390         int i, ret;
1391
1392         for (i = 0; types[i]; i++) {
1393                 ret = uncore_type_init(types[i]);
1394                 if (ret)
1395                         goto fail;
1396         }
1397         return 0;
1398 fail:
1399         while (--i >= 0)
1400                 uncore_type_exit(types[i]);
1401         return ret;
1402 }
1403
1404 static struct pci_driver *uncore_pci_driver;
1405 static bool pcidrv_registered;
1406
1407 /*
1408  * add a pci uncore device
1409  */
1410 static int __devinit uncore_pci_add(struct intel_uncore_type *type,
1411                                     struct pci_dev *pdev)
1412 {
1413         struct intel_uncore_pmu *pmu;
1414         struct intel_uncore_box *box;
1415         int i, phys_id;
1416
1417         phys_id = pcibus_to_physid[pdev->bus->number];
1418         if (phys_id < 0)
1419                 return -ENODEV;
1420
1421         box = uncore_alloc_box(type, 0);
1422         if (!box)
1423                 return -ENOMEM;
1424
1425         /*
1426          * for performance monitoring unit with multiple boxes,
1427          * each box has a different function id.
1428          */
1429         for (i = 0; i < type->num_boxes; i++) {
1430                 pmu = &type->pmus[i];
1431                 if (pmu->func_id == pdev->devfn)
1432                         break;
1433                 if (pmu->func_id < 0) {
1434                         pmu->func_id = pdev->devfn;
1435                         break;
1436                 }
1437                 pmu = NULL;
1438         }
1439
1440         if (!pmu) {
1441                 kfree(box);
1442                 return -EINVAL;
1443         }
1444
1445         box->phys_id = phys_id;
1446         box->pci_dev = pdev;
1447         box->pmu = pmu;
1448         uncore_box_init(box);
1449         pci_set_drvdata(pdev, box);
1450
1451         raw_spin_lock(&uncore_box_lock);
1452         list_add_tail(&box->list, &pmu->box_list);
1453         raw_spin_unlock(&uncore_box_lock);
1454
1455         return 0;
1456 }
1457
1458 static void uncore_pci_remove(struct pci_dev *pdev)
1459 {
1460         struct intel_uncore_box *box = pci_get_drvdata(pdev);
1461         struct intel_uncore_pmu *pmu = box->pmu;
1462         int cpu, phys_id = pcibus_to_physid[pdev->bus->number];
1463
1464         if (WARN_ON_ONCE(phys_id != box->phys_id))
1465                 return;
1466
1467         raw_spin_lock(&uncore_box_lock);
1468         list_del(&box->list);
1469         raw_spin_unlock(&uncore_box_lock);
1470
1471         for_each_possible_cpu(cpu) {
1472                 if (*per_cpu_ptr(pmu->box, cpu) == box) {
1473                         *per_cpu_ptr(pmu->box, cpu) = NULL;
1474                         atomic_dec(&box->refcnt);
1475                 }
1476         }
1477
1478         WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
1479         kfree(box);
1480 }
1481
1482 static int __devinit uncore_pci_probe(struct pci_dev *pdev,
1483                                 const struct pci_device_id *id)
1484 {
1485         struct intel_uncore_type *type;
1486
1487         type = (struct intel_uncore_type *)id->driver_data;
1488         return uncore_pci_add(type, pdev);
1489 }
1490
1491 static int __init uncore_pci_init(void)
1492 {
1493         int ret;
1494
1495         switch (boot_cpu_data.x86_model) {
1496         case 45: /* Sandy Bridge-EP */
1497                 pci_uncores = snbep_pci_uncores;
1498                 uncore_pci_driver = &snbep_uncore_pci_driver;
1499                 snbep_pci2phy_map_init();
1500                 break;
1501         default:
1502                 return 0;
1503         }
1504
1505         ret = uncore_types_init(pci_uncores);
1506         if (ret)
1507                 return ret;
1508
1509         uncore_pci_driver->probe = uncore_pci_probe;
1510         uncore_pci_driver->remove = uncore_pci_remove;
1511
1512         ret = pci_register_driver(uncore_pci_driver);
1513         if (ret == 0)
1514                 pcidrv_registered = true;
1515         else
1516                 uncore_types_exit(pci_uncores);
1517
1518         return ret;
1519 }
1520
1521 static void __init uncore_pci_exit(void)
1522 {
1523         if (pcidrv_registered) {
1524                 pcidrv_registered = false;
1525                 pci_unregister_driver(uncore_pci_driver);
1526                 uncore_types_exit(pci_uncores);
1527         }
1528 }
1529
1530 static void __cpuinit uncore_cpu_dying(int cpu)
1531 {
1532         struct intel_uncore_type *type;
1533         struct intel_uncore_pmu *pmu;
1534         struct intel_uncore_box *box;
1535         int i, j;
1536
1537         for (i = 0; msr_uncores[i]; i++) {
1538                 type = msr_uncores[i];
1539                 for (j = 0; j < type->num_boxes; j++) {
1540                         pmu = &type->pmus[j];
1541                         box = *per_cpu_ptr(pmu->box, cpu);
1542                         *per_cpu_ptr(pmu->box, cpu) = NULL;
1543                         if (box && atomic_dec_and_test(&box->refcnt))
1544                                 kfree(box);
1545                 }
1546         }
1547 }
1548
1549 static int __cpuinit uncore_cpu_starting(int cpu)
1550 {
1551         struct intel_uncore_type *type;
1552         struct intel_uncore_pmu *pmu;
1553         struct intel_uncore_box *box, *exist;
1554         int i, j, k, phys_id;
1555
1556         phys_id = topology_physical_package_id(cpu);
1557
1558         for (i = 0; msr_uncores[i]; i++) {
1559                 type = msr_uncores[i];
1560                 for (j = 0; j < type->num_boxes; j++) {
1561                         pmu = &type->pmus[j];
1562                         box = *per_cpu_ptr(pmu->box, cpu);
1563                         /* called by uncore_cpu_init? */
1564                         if (box && box->phys_id >= 0) {
1565                                 uncore_box_init(box);
1566                                 continue;
1567                         }
1568
1569                         for_each_online_cpu(k) {
1570                                 exist = *per_cpu_ptr(pmu->box, k);
1571                                 if (exist && exist->phys_id == phys_id) {
1572                                         atomic_inc(&exist->refcnt);
1573                                         *per_cpu_ptr(pmu->box, cpu) = exist;
1574                                         kfree(box);
1575                                         box = NULL;
1576                                         break;
1577                                 }
1578                         }
1579
1580                         if (box) {
1581                                 box->phys_id = phys_id;
1582                                 uncore_box_init(box);
1583                         }
1584                 }
1585         }
1586         return 0;
1587 }
1588
1589 static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id)
1590 {
1591         struct intel_uncore_type *type;
1592         struct intel_uncore_pmu *pmu;
1593         struct intel_uncore_box *box;
1594         int i, j;
1595
1596         for (i = 0; msr_uncores[i]; i++) {
1597                 type = msr_uncores[i];
1598                 for (j = 0; j < type->num_boxes; j++) {
1599                         pmu = &type->pmus[j];
1600                         if (pmu->func_id < 0)
1601                                 pmu->func_id = j;
1602
1603                         box = uncore_alloc_box(type, cpu);
1604                         if (!box)
1605                                 return -ENOMEM;
1606
1607                         box->pmu = pmu;
1608                         box->phys_id = phys_id;
1609                         *per_cpu_ptr(pmu->box, cpu) = box;
1610                 }
1611         }
1612         return 0;
1613 }
1614
1615 static void __cpuinit uncore_change_context(struct intel_uncore_type **uncores,
1616                                             int old_cpu, int new_cpu)
1617 {
1618         struct intel_uncore_type *type;
1619         struct intel_uncore_pmu *pmu;
1620         struct intel_uncore_box *box;
1621         int i, j;
1622
1623         for (i = 0; uncores[i]; i++) {
1624                 type = uncores[i];
1625                 for (j = 0; j < type->num_boxes; j++) {
1626                         pmu = &type->pmus[j];
1627                         if (old_cpu < 0)
1628                                 box = uncore_pmu_to_box(pmu, new_cpu);
1629                         else
1630                                 box = uncore_pmu_to_box(pmu, old_cpu);
1631                         if (!box)
1632                                 continue;
1633
1634                         if (old_cpu < 0) {
1635                                 WARN_ON_ONCE(box->cpu != -1);
1636                                 box->cpu = new_cpu;
1637                                 continue;
1638                         }
1639
1640                         WARN_ON_ONCE(box->cpu != old_cpu);
1641                         if (new_cpu >= 0) {
1642                                 uncore_pmu_cancel_hrtimer(box);
1643                                 perf_pmu_migrate_context(&pmu->pmu,
1644                                                 old_cpu, new_cpu);
1645                                 box->cpu = new_cpu;
1646                         } else {
1647                                 box->cpu = -1;
1648                         }
1649                 }
1650         }
1651 }
1652
1653 static void __cpuinit uncore_event_exit_cpu(int cpu)
1654 {
1655         int i, phys_id, target;
1656
1657         /* if exiting cpu is used for collecting uncore events */
1658         if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
1659                 return;
1660
1661         /* find a new cpu to collect uncore events */
1662         phys_id = topology_physical_package_id(cpu);
1663         target = -1;
1664         for_each_online_cpu(i) {
1665                 if (i == cpu)
1666                         continue;
1667                 if (phys_id == topology_physical_package_id(i)) {
1668                         target = i;
1669                         break;
1670                 }
1671         }
1672
1673         /* migrate uncore events to the new cpu */
1674         if (target >= 0)
1675                 cpumask_set_cpu(target, &uncore_cpu_mask);
1676
1677         uncore_change_context(msr_uncores, cpu, target);
1678         uncore_change_context(pci_uncores, cpu, target);
1679 }
1680
1681 static void __cpuinit uncore_event_init_cpu(int cpu)
1682 {
1683         int i, phys_id;
1684
1685         phys_id = topology_physical_package_id(cpu);
1686         for_each_cpu(i, &uncore_cpu_mask) {
1687                 if (phys_id == topology_physical_package_id(i))
1688                         return;
1689         }
1690
1691         cpumask_set_cpu(cpu, &uncore_cpu_mask);
1692
1693         uncore_change_context(msr_uncores, -1, cpu);
1694         uncore_change_context(pci_uncores, -1, cpu);
1695 }
1696
1697 static int __cpuinit uncore_cpu_notifier(struct notifier_block *self,
1698                                          unsigned long action, void *hcpu)
1699 {
1700         unsigned int cpu = (long)hcpu;
1701
1702         /* allocate/free data structure for uncore box */
1703         switch (action & ~CPU_TASKS_FROZEN) {
1704         case CPU_UP_PREPARE:
1705                 uncore_cpu_prepare(cpu, -1);
1706                 break;
1707         case CPU_STARTING:
1708                 uncore_cpu_starting(cpu);
1709                 break;
1710         case CPU_UP_CANCELED:
1711         case CPU_DYING:
1712                 uncore_cpu_dying(cpu);
1713                 break;
1714         default:
1715                 break;
1716         }
1717
1718         /* select the cpu that collects uncore events */
1719         switch (action & ~CPU_TASKS_FROZEN) {
1720         case CPU_DOWN_FAILED:
1721         case CPU_STARTING:
1722                 uncore_event_init_cpu(cpu);
1723                 break;
1724         case CPU_DOWN_PREPARE:
1725                 uncore_event_exit_cpu(cpu);
1726                 break;
1727         default:
1728                 break;
1729         }
1730
1731         return NOTIFY_OK;
1732 }
1733
1734 static struct notifier_block uncore_cpu_nb __cpuinitdata = {
1735         .notifier_call = uncore_cpu_notifier,
1736         /*
1737          * to migrate uncore events, our notifier should be executed
1738          * before perf core's notifier.
1739          */
1740         .priority = CPU_PRI_PERF + 1,
1741 };
1742
1743 static void __init uncore_cpu_setup(void *dummy)
1744 {
1745         uncore_cpu_starting(smp_processor_id());
1746 }
1747
1748 static int __init uncore_cpu_init(void)
1749 {
1750         int ret, cpu, max_cores;
1751
1752         max_cores = boot_cpu_data.x86_max_cores;
1753         switch (boot_cpu_data.x86_model) {
1754         case 26: /* Nehalem */
1755         case 30:
1756         case 37: /* Westmere */
1757         case 44:
1758                 msr_uncores = nhm_msr_uncores;
1759                 break;
1760         case 42: /* Sandy Bridge */
1761                 if (snb_uncore_cbox.num_boxes > max_cores)
1762                         snb_uncore_cbox.num_boxes = max_cores;
1763                 msr_uncores = snb_msr_uncores;
1764                 break;
1765         case 45: /* Sandy Birdge-EP */
1766                 if (snbep_uncore_cbox.num_boxes > max_cores)
1767                         snbep_uncore_cbox.num_boxes = max_cores;
1768                 msr_uncores = snbep_msr_uncores;
1769                 break;
1770         default:
1771                 return 0;
1772         }
1773
1774         ret = uncore_types_init(msr_uncores);
1775         if (ret)
1776                 return ret;
1777
1778         get_online_cpus();
1779
1780         for_each_online_cpu(cpu) {
1781                 int i, phys_id = topology_physical_package_id(cpu);
1782
1783                 for_each_cpu(i, &uncore_cpu_mask) {
1784                         if (phys_id == topology_physical_package_id(i)) {
1785                                 phys_id = -1;
1786                                 break;
1787                         }
1788                 }
1789                 if (phys_id < 0)
1790                         continue;
1791
1792                 uncore_cpu_prepare(cpu, phys_id);
1793                 uncore_event_init_cpu(cpu);
1794         }
1795         on_each_cpu(uncore_cpu_setup, NULL, 1);
1796
1797         register_cpu_notifier(&uncore_cpu_nb);
1798
1799         put_online_cpus();
1800
1801         return 0;
1802 }
1803
1804 static int __init uncore_pmus_register(void)
1805 {
1806         struct intel_uncore_pmu *pmu;
1807         struct intel_uncore_type *type;
1808         int i, j;
1809
1810         for (i = 0; msr_uncores[i]; i++) {
1811                 type = msr_uncores[i];
1812                 for (j = 0; j < type->num_boxes; j++) {
1813                         pmu = &type->pmus[j];
1814                         uncore_pmu_register(pmu);
1815                 }
1816         }
1817
1818         for (i = 0; pci_uncores[i]; i++) {
1819                 type = pci_uncores[i];
1820                 for (j = 0; j < type->num_boxes; j++) {
1821                         pmu = &type->pmus[j];
1822                         uncore_pmu_register(pmu);
1823                 }
1824         }
1825
1826         return 0;
1827 }
1828
1829 static int __init intel_uncore_init(void)
1830 {
1831         int ret;
1832
1833         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
1834                 return -ENODEV;
1835
1836         ret = uncore_pci_init();
1837         if (ret)
1838                 goto fail;
1839         ret = uncore_cpu_init();
1840         if (ret) {
1841                 uncore_pci_exit();
1842                 goto fail;
1843         }
1844
1845         uncore_pmus_register();
1846         return 0;
1847 fail:
1848         return ret;
1849 }
1850 device_initcall(intel_uncore_init);