arch/x86/kernel/cpu/perf_event_intel_uncore.c

   1 #include "perf_event_intel_uncore.h"
   2
   3 static struct intel_uncore_type *empty_uncore[] = { NULL, };
   4 static struct intel_uncore_type **msr_uncores = empty_uncore;
   5 static struct intel_uncore_type **pci_uncores = empty_uncore;
   6 /* pci bus to socket mapping */
   7 static int pcibus_to_physid[256] = { [0 ... 255] = -1, };
   8
   9 static DEFINE_RAW_SPINLOCK(uncore_box_lock);
  10
  11 /* mask of cpus that collect uncore events */
  12 static cpumask_t uncore_cpu_mask;
  13
  14 /* constraint for the fixed counter */
  15 static struct event_constraint constraint_fixed =
  16         EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
  17 static struct event_constraint constraint_empty =
  18         EVENT_CONSTRAINT(0, 0, 0);
  19
  20 #define __BITS_VALUE(x, i, n)  ((typeof(x))(((x) >> ((i) * (n))) & \
  21                                 ((1ULL << (n)) - 1)))
  22
  23 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
  24 DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
  25 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
  26 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
  27 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
  28 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
  29 DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
  30 DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
  31 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
  32 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
  33 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
  34 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
  35 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
  36 DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4");
  37 DEFINE_UNCORE_FORMAT_ATTR(filter_link, filter_link, "config1:5-8");
  38 DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17");
  39 DEFINE_UNCORE_FORMAT_ATTR(filter_nid2, filter_nid, "config1:32-47");
  40 DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22");
  41 DEFINE_UNCORE_FORMAT_ATTR(filter_state2, filter_state, "config1:17-22");
  42 DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31");
  43 DEFINE_UNCORE_FORMAT_ATTR(filter_opc2, filter_opc, "config1:52-60");
  44 DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7");
  45 DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15");
  46 DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23");
  47 DEFINE_UNCORE_FORMAT_ATTR(filter_band3, filter_band3, "config1:24-31");
  48
  49 static u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
  50 {
  51         u64 count;
  52
  53         rdmsrl(event->hw.event_base, count);
  54
  55         return count;
  56 }
  57
  58 /*
  59  * generic get constraint function for shared match/mask registers.
  60  */
  61 static struct event_constraint *
  62 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
  63 {
  64         struct intel_uncore_extra_reg *er;
  65         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
  66         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
  67         unsigned long flags;
  68         bool ok = false;
  69
  70         /*
  71          * reg->alloc can be set due to existing state, so for fake box we
  72          * need to ignore this, otherwise we might fail to allocate proper
  73          * fake state for this extra reg constraint.
  74          */
  75         if (reg1->idx == EXTRA_REG_NONE ||
  76             (!uncore_box_is_fake(box) && reg1->alloc))
  77                 return NULL;
  78
  79         er = &box->shared_regs[reg1->idx];
  80         raw_spin_lock_irqsave(&er->lock, flags);
  81         if (!atomic_read(&er->ref) ||
  82             (er->config1 == reg1->config && er->config2 == reg2->config)) {
  83                 atomic_inc(&er->ref);
  84                 er->config1 = reg1->config;
  85                 er->config2 = reg2->config;
  86                 ok = true;
  87         }
  88         raw_spin_unlock_irqrestore(&er->lock, flags);
  89
  90         if (ok) {
  91                 if (!uncore_box_is_fake(box))
  92                         reg1->alloc = 1;
  93                 return NULL;
  94         }
  95
  96         return &constraint_empty;
  97 }
  98
  99 static void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 100 {
 101         struct intel_uncore_extra_reg *er;
 102         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 103
 104         /*
 105          * Only put constraint if extra reg was actually allocated. Also
 106          * takes care of event which do not use an extra shared reg.
 107          *
 108          * Also, if this is a fake box we shouldn't touch any event state
 109          * (reg->alloc) and we don't care about leaving inconsistent box
 110          * state either since it will be thrown out.
 111          */
 112         if (uncore_box_is_fake(box) || !reg1->alloc)
 113                 return;
 114
 115         er = &box->shared_regs[reg1->idx];
 116         atomic_dec(&er->ref);
 117         reg1->alloc = 0;
 118 }
 119
 120 static u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
 121 {
 122         struct intel_uncore_extra_reg *er;
 123         unsigned long flags;
 124         u64 config;
 125
 126         er = &box->shared_regs[idx];
 127
 128         raw_spin_lock_irqsave(&er->lock, flags);
 129         config = er->config;
 130         raw_spin_unlock_irqrestore(&er->lock, flags);
 131
 132         return config;
 133 }
 134
 135 /* Sandy Bridge-EP uncore support */
 136 static struct intel_uncore_type snbep_uncore_cbox;
 137 static struct intel_uncore_type snbep_uncore_pcu;
 138
 139 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
 140 {
 141         struct pci_dev *pdev = box->pci_dev;
 142         int box_ctl = uncore_pci_box_ctl(box);
 143         u32 config = 0;
 144
 145         if (!pci_read_config_dword(pdev, box_ctl, &config)) {
 146                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 147                 pci_write_config_dword(pdev, box_ctl, config);
 148         }
 149 }
 150
 151 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
 152 {
 153         struct pci_dev *pdev = box->pci_dev;
 154         int box_ctl = uncore_pci_box_ctl(box);
 155         u32 config = 0;
 156
 157         if (!pci_read_config_dword(pdev, box_ctl, &config)) {
 158                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 159                 pci_write_config_dword(pdev, box_ctl, config);
 160         }
 161 }
 162
 163 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 164 {
 165         struct pci_dev *pdev = box->pci_dev;
 166         struct hw_perf_event *hwc = &event->hw;
 167
 168         pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 169 }
 170
 171 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 172 {
 173         struct pci_dev *pdev = box->pci_dev;
 174         struct hw_perf_event *hwc = &event->hw;
 175
 176         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
 177 }
 178
 179 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct perf_event *event)
 180 {
 181         struct pci_dev *pdev = box->pci_dev;
 182         struct hw_perf_event *hwc = &event->hw;
 183         u64 count = 0;
 184
 185         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
 186         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
 187
 188         return count;
 189 }
 190
 191 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
 192 {
 193         struct pci_dev *pdev = box->pci_dev;
 194
 195         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, SNBEP_PMON_BOX_CTL_INT);
 196 }
 197
 198 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
 199 {
 200         u64 config;
 201         unsigned msr;
 202
 203         msr = uncore_msr_box_ctl(box);
 204         if (msr) {
 205                 rdmsrl(msr, config);
 206                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 207                 wrmsrl(msr, config);
 208         }
 209 }
 210
 211 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
 212 {
 213         u64 config;
 214         unsigned msr;
 215
 216         msr = uncore_msr_box_ctl(box);
 217         if (msr) {
 218                 rdmsrl(msr, config);
 219                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 220                 wrmsrl(msr, config);
 221         }
 222 }
 223
 224 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 225 {
 226         struct hw_perf_event *hwc = &event->hw;
 227         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 228
 229         if (reg1->idx != EXTRA_REG_NONE)
 230                 wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0));
 231
 232         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 233 }
 234
 235 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 236                                         struct perf_event *event)
 237 {
 238         struct hw_perf_event *hwc = &event->hw;
 239
 240         wrmsrl(hwc->config_base, hwc->config);
 241 }
 242
 243 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 244 {
 245         unsigned msr = uncore_msr_box_ctl(box);
 246
 247         if (msr)
 248                 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
 249 }
 250
 251 static struct attribute *snbep_uncore_formats_attr[] = {
 252         &format_attr_event.attr,
 253         &format_attr_umask.attr,
 254         &format_attr_edge.attr,
 255         &format_attr_inv.attr,
 256         &format_attr_thresh8.attr,
 257         NULL,
 258 };
 259
 260 static struct attribute *snbep_uncore_ubox_formats_attr[] = {
 261         &format_attr_event.attr,
 262         &format_attr_umask.attr,
 263         &format_attr_edge.attr,
 264         &format_attr_inv.attr,
 265         &format_attr_thresh5.attr,
 266         NULL,
 267 };
 268
 269 static struct attribute *snbep_uncore_cbox_formats_attr[] = {
 270         &format_attr_event.attr,
 271         &format_attr_umask.attr,
 272         &format_attr_edge.attr,
 273         &format_attr_tid_en.attr,
 274         &format_attr_inv.attr,
 275         &format_attr_thresh8.attr,
 276         &format_attr_filter_tid.attr,
 277         &format_attr_filter_nid.attr,
 278         &format_attr_filter_state.attr,
 279         &format_attr_filter_opc.attr,
 280         NULL,
 281 };
 282
 283 static struct attribute *snbep_uncore_pcu_formats_attr[] = {
 284         &format_attr_event.attr,
 285         &format_attr_occ_sel.attr,
 286         &format_attr_edge.attr,
 287         &format_attr_inv.attr,
 288         &format_attr_thresh5.attr,
 289         &format_attr_occ_invert.attr,
 290         &format_attr_occ_edge.attr,
 291         &format_attr_filter_band0.attr,
 292         &format_attr_filter_band1.attr,
 293         &format_attr_filter_band2.attr,
 294         &format_attr_filter_band3.attr,
 295         NULL,
 296 };
 297
 298 static struct attribute *snbep_uncore_qpi_formats_attr[] = {
 299         &format_attr_event_ext.attr,
 300         &format_attr_umask.attr,
 301         &format_attr_edge.attr,
 302         &format_attr_inv.attr,
 303         &format_attr_thresh8.attr,
 304         NULL,
 305 };
 306
 307 static struct uncore_event_desc snbep_uncore_imc_events[] = {
 308         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0xff,umask=0x00"),
 309         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x03"),
 310         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
 311         { /* end: all zeroes */ },
 312 };
 313
 314 static struct uncore_event_desc snbep_uncore_qpi_events[] = {
 315         INTEL_UNCORE_EVENT_DESC(clockticks,       "event=0x14"),
 316         INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
 317         INTEL_UNCORE_EVENT_DESC(drs_data,         "event=0x02,umask=0x08"),
 318         INTEL_UNCORE_EVENT_DESC(ncb_data,         "event=0x03,umask=0x04"),
 319         { /* end: all zeroes */ },
 320 };
 321
 322 static struct attribute_group snbep_uncore_format_group = {
 323         .name = "format",
 324         .attrs = snbep_uncore_formats_attr,
 325 };
 326
 327 static struct attribute_group snbep_uncore_ubox_format_group = {
 328         .name = "format",
 329         .attrs = snbep_uncore_ubox_formats_attr,
 330 };
 331
 332 static struct attribute_group snbep_uncore_cbox_format_group = {
 333         .name = "format",
 334         .attrs = snbep_uncore_cbox_formats_attr,
 335 };
 336
 337 static struct attribute_group snbep_uncore_pcu_format_group = {
 338         .name = "format",
 339         .attrs = snbep_uncore_pcu_formats_attr,
 340 };
 341
 342 static struct attribute_group snbep_uncore_qpi_format_group = {
 343         .name = "format",
 344         .attrs = snbep_uncore_qpi_formats_attr,
 345 };
 346
 347 #define SNBEP_UNCORE_MSR_OPS_COMMON_INIT()                      \
 348         .init_box       = snbep_uncore_msr_init_box,            \
 349         .disable_box    = snbep_uncore_msr_disable_box,         \
 350         .enable_box     = snbep_uncore_msr_enable_box,          \
 351         .disable_event  = snbep_uncore_msr_disable_event,       \
 352         .enable_event   = snbep_uncore_msr_enable_event,        \
 353         .read_counter   = uncore_msr_read_counter
 354
 355 static struct intel_uncore_ops snbep_uncore_msr_ops = {
 356         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 357 };
 358
 359 static struct intel_uncore_ops snbep_uncore_pci_ops = {
 360         .init_box       = snbep_uncore_pci_init_box,
 361         .disable_box    = snbep_uncore_pci_disable_box,
 362         .enable_box     = snbep_uncore_pci_enable_box,
 363         .disable_event  = snbep_uncore_pci_disable_event,
 364         .enable_event   = snbep_uncore_pci_enable_event,
 365         .read_counter   = snbep_uncore_pci_read_counter,
 366 };
 367
 368 static struct event_constraint snbep_uncore_cbox_constraints[] = {
 369         UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
 370         UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
 371         UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
 372         UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
 373         UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
 374         UNCORE_EVENT_CONSTRAINT(0x09, 0x3),
 375         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
 376         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 377         UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
 378         UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
 379         UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
 380         UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
 381         UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
 382         EVENT_CONSTRAINT_OVERLAP(0x1f, 0xe, 0xff),
 383         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 384         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 385         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 386         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 387         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 388         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 389         UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
 390         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
 391         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 392         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 393         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 394         UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
 395         EVENT_CONSTRAINT_END
 396 };
 397
 398 static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
 399         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 400         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 401         UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
 402         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 403         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 404         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 405         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 406         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 407         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 408         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 409         EVENT_CONSTRAINT_END
 410 };
 411
 412 static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
 413         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 414         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 415         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 416         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
 417         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
 418         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 419         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
 420         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 421         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 422         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 423         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 424         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
 425         UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
 426         UNCORE_EVENT_CONSTRAINT(0x2a, 0x3),
 427         UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
 428         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
 429         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
 430         UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
 431         UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
 432         UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
 433         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 434         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 435         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 436         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 437         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
 438         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 439         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 440         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 441         EVENT_CONSTRAINT_END
 442 };
 443
 444 static struct intel_uncore_type snbep_uncore_ubox = {
 445         .name           = "ubox",
 446         .num_counters   = 2,
 447         .num_boxes      = 1,
 448         .perf_ctr_bits  = 44,
 449         .fixed_ctr_bits = 48,
 450         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
 451         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
 452         .event_mask     = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
 453         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
 454         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
 455         .ops            = &snbep_uncore_msr_ops,
 456         .format_group   = &snbep_uncore_ubox_format_group,
 457 };
 458
 459 static struct extra_reg snbep_uncore_cbox_extra_regs[] = {
 460         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
 461                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
 462         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
 463         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
 464         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
 465         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6),
 466         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8),
 467         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8),
 468         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0xc),
 469         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0xc),
 470         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x2),
 471         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x2),
 472         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x2),
 473         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x2),
 474         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x8),
 475         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x8),
 476         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0xc),
 477         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0xc),
 478         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x2),
 479         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x2),
 480         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x2),
 481         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x2),
 482         EVENT_EXTRA_END
 483 };
 484
 485 static void snbep_cbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 486 {
 487         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 488         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 489         int i;
 490
 491         if (uncore_box_is_fake(box))
 492                 return;
 493
 494         for (i = 0; i < 5; i++) {
 495                 if (reg1->alloc & (0x1 << i))
 496                         atomic_sub(1 << (i * 6), &er->ref);
 497         }
 498         reg1->alloc = 0;
 499 }
 500
 501 static struct event_constraint *
 502 __snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event,
 503                             u64 (*cbox_filter_mask)(int fields))
 504 {
 505         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 506         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 507         int i, alloc = 0;
 508         unsigned long flags;
 509         u64 mask;
 510
 511         if (reg1->idx == EXTRA_REG_NONE)
 512                 return NULL;
 513
 514         raw_spin_lock_irqsave(&er->lock, flags);
 515         for (i = 0; i < 5; i++) {
 516                 if (!(reg1->idx & (0x1 << i)))
 517                         continue;
 518                 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
 519                         continue;
 520
 521                 mask = cbox_filter_mask(0x1 << i);
 522                 if (!__BITS_VALUE(atomic_read(&er->ref), i, 6) ||
 523                     !((reg1->config ^ er->config) & mask)) {
 524                         atomic_add(1 << (i * 6), &er->ref);
 525                         er->config &= ~mask;
 526                         er->config |= reg1->config & mask;
 527                         alloc |= (0x1 << i);
 528                 } else {
 529                         break;
 530                 }
 531         }
 532         raw_spin_unlock_irqrestore(&er->lock, flags);
 533         if (i < 5)
 534                 goto fail;
 535
 536         if (!uncore_box_is_fake(box))
 537                 reg1->alloc |= alloc;
 538
 539         return 0;
 540 fail:
 541         for (; i >= 0; i--) {
 542                 if (alloc & (0x1 << i))
 543                         atomic_sub(1 << (i * 6), &er->ref);
 544         }
 545         return &constraint_empty;
 546 }
 547
 548 static u64 snbep_cbox_filter_mask(int fields)
 549 {
 550         u64 mask = 0;
 551
 552         if (fields & 0x1)
 553                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_TID;
 554         if (fields & 0x2)
 555                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_NID;
 556         if (fields & 0x4)
 557                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
 558         if (fields & 0x8)
 559                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
 560
 561         return mask;
 562 }
 563
 564 static struct event_constraint *
 565 snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
 566 {
 567         return __snbep_cbox_get_constraint(box, event, snbep_cbox_filter_mask);
 568 }
 569
 570 static int snbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
 571 {
 572         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 573         struct extra_reg *er;
 574         int idx = 0;
 575
 576         for (er = snbep_uncore_cbox_extra_regs; er->msr; er++) {
 577                 if (er->event != (event->hw.config & er->config_mask))
 578                         continue;
 579                 idx |= er->idx;
 580         }
 581
 582         if (idx) {
 583                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
 584                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
 585                 reg1->config = event->attr.config1 & snbep_cbox_filter_mask(idx);
 586                 reg1->idx = idx;
 587         }
 588         return 0;
 589 }
 590
 591 static struct intel_uncore_ops snbep_uncore_cbox_ops = {
 592         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 593         .hw_config              = snbep_cbox_hw_config,
 594         .get_constraint         = snbep_cbox_get_constraint,
 595         .put_constraint         = snbep_cbox_put_constraint,
 596 };
 597
 598 static struct intel_uncore_type snbep_uncore_cbox = {
 599         .name                   = "cbox",
 600         .num_counters           = 4,
 601         .num_boxes              = 8,
 602         .perf_ctr_bits          = 44,
 603         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
 604         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
 605         .event_mask             = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
 606         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
 607         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
 608         .num_shared_regs        = 1,
 609         .constraints            = snbep_uncore_cbox_constraints,
 610         .ops                    = &snbep_uncore_cbox_ops,
 611         .format_group           = &snbep_uncore_cbox_format_group,
 612 };
 613
 614 static u64 snbep_pcu_alter_er(struct perf_event *event, int new_idx, bool modify)
 615 {
 616         struct hw_perf_event *hwc = &event->hw;
 617         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 618         u64 config = reg1->config;
 619
 620         if (new_idx > reg1->idx)
 621                 config <<= 8 * (new_idx - reg1->idx);
 622         else
 623                 config >>= 8 * (reg1->idx - new_idx);
 624
 625         if (modify) {
 626                 hwc->config += new_idx - reg1->idx;
 627                 reg1->config = config;
 628                 reg1->idx = new_idx;
 629         }
 630         return config;
 631 }
 632
 633 static struct event_constraint *
 634 snbep_pcu_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
 635 {
 636         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 637         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 638         unsigned long flags;
 639         int idx = reg1->idx;
 640         u64 mask, config1 = reg1->config;
 641         bool ok = false;
 642
 643         if (reg1->idx == EXTRA_REG_NONE ||
 644             (!uncore_box_is_fake(box) && reg1->alloc))
 645                 return NULL;
 646 again:
 647         mask = 0xff << (idx * 8);
 648         raw_spin_lock_irqsave(&er->lock, flags);
 649         if (!__BITS_VALUE(atomic_read(&er->ref), idx, 8) ||
 650             !((config1 ^ er->config) & mask)) {
 651                 atomic_add(1 << (idx * 8), &er->ref);
 652                 er->config &= ~mask;
 653                 er->config |= config1 & mask;
 654                 ok = true;
 655         }
 656         raw_spin_unlock_irqrestore(&er->lock, flags);
 657
 658         if (!ok) {
 659                 idx = (idx + 1) % 4;
 660                 if (idx != reg1->idx) {
 661                         config1 = snbep_pcu_alter_er(event, idx, false);
 662                         goto again;
 663                 }
 664                 return &constraint_empty;
 665         }
 666
 667         if (!uncore_box_is_fake(box)) {
 668                 if (idx != reg1->idx)
 669                         snbep_pcu_alter_er(event, idx, true);
 670                 reg1->alloc = 1;
 671         }
 672         return NULL;
 673 }
 674
 675 static void snbep_pcu_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 676 {
 677         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 678         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 679
 680         if (uncore_box_is_fake(box) || !reg1->alloc)
 681                 return;
 682
 683         atomic_sub(1 << (reg1->idx * 8), &er->ref);
 684         reg1->alloc = 0;
 685 }
 686
 687 static int snbep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
 688 {
 689         struct hw_perf_event *hwc = &event->hw;
 690         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 691         int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
 692
 693         if (ev_sel >= 0xb && ev_sel <= 0xe) {
 694                 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
 695                 reg1->idx = ev_sel - 0xb;
 696                 reg1->config = event->attr.config1 & (0xff << reg1->idx);
 697         }
 698         return 0;
 699 }
 700
 701 static struct intel_uncore_ops snbep_uncore_pcu_ops = {
 702         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 703         .hw_config              = snbep_pcu_hw_config,
 704         .get_constraint         = snbep_pcu_get_constraint,
 705         .put_constraint         = snbep_pcu_put_constraint,
 706 };
 707
 708 static struct intel_uncore_type snbep_uncore_pcu = {
 709         .name                   = "pcu",
 710         .num_counters           = 4,
 711         .num_boxes              = 1,
 712         .perf_ctr_bits          = 48,
 713         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
 714         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
 715         .event_mask             = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
 716         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
 717         .num_shared_regs        = 1,
 718         .ops                    = &snbep_uncore_pcu_ops,
 719         .format_group           = &snbep_uncore_pcu_format_group,
 720 };
 721
 722 static struct intel_uncore_type *snbep_msr_uncores[] = {
 723         &snbep_uncore_ubox,
 724         &snbep_uncore_cbox,
 725         &snbep_uncore_pcu,
 726         NULL,
 727 };
 728
 729 #define SNBEP_UNCORE_PCI_COMMON_INIT()                          \
 730         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
 731         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
 732         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,            \
 733         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
 734         .ops            = &snbep_uncore_pci_ops,                \
 735         .format_group   = &snbep_uncore_format_group
 736
 737 static struct intel_uncore_type snbep_uncore_ha = {
 738         .name           = "ha",
 739         .num_counters   = 4,
 740         .num_boxes      = 1,
 741         .perf_ctr_bits  = 48,
 742         SNBEP_UNCORE_PCI_COMMON_INIT(),
 743 };
 744
 745 static struct intel_uncore_type snbep_uncore_imc = {
 746         .name           = "imc",
 747         .num_counters   = 4,
 748         .num_boxes      = 4,
 749         .perf_ctr_bits  = 48,
 750         .fixed_ctr_bits = 48,
 751         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
 752         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
 753         .event_descs    = snbep_uncore_imc_events,
 754         SNBEP_UNCORE_PCI_COMMON_INIT(),
 755 };
 756
 757 static struct intel_uncore_type snbep_uncore_qpi = {
 758         .name           = "qpi",
 759         .num_counters   = 4,
 760         .num_boxes      = 2,
 761         .perf_ctr_bits  = 48,
 762         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
 763         .event_ctl      = SNBEP_PCI_PMON_CTL0,
 764         .event_mask     = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
 765         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
 766         .ops            = &snbep_uncore_pci_ops,
 767         .event_descs    = snbep_uncore_qpi_events,
 768         .format_group   = &snbep_uncore_qpi_format_group,
 769 };
 770
 771
 772 static struct intel_uncore_type snbep_uncore_r2pcie = {
 773         .name           = "r2pcie",
 774         .num_counters   = 4,
 775         .num_boxes      = 1,
 776         .perf_ctr_bits  = 44,
 777         .constraints    = snbep_uncore_r2pcie_constraints,
 778         SNBEP_UNCORE_PCI_COMMON_INIT(),
 779 };
 780
 781 static struct intel_uncore_type snbep_uncore_r3qpi = {
 782         .name           = "r3qpi",
 783         .num_counters   = 3,
 784         .num_boxes      = 2,
 785         .perf_ctr_bits  = 44,
 786         .constraints    = snbep_uncore_r3qpi_constraints,
 787         SNBEP_UNCORE_PCI_COMMON_INIT(),
 788 };
 789
 790 enum {
 791         SNBEP_PCI_UNCORE_HA,
 792         SNBEP_PCI_UNCORE_IMC,
 793         SNBEP_PCI_UNCORE_QPI,
 794         SNBEP_PCI_UNCORE_R2PCIE,
 795         SNBEP_PCI_UNCORE_R3QPI,
 796 };
 797
 798 static struct intel_uncore_type *snbep_pci_uncores[] = {
 799         [SNBEP_PCI_UNCORE_HA]           = &snbep_uncore_ha,
 800         [SNBEP_PCI_UNCORE_IMC]          = &snbep_uncore_imc,
 801         [SNBEP_PCI_UNCORE_QPI]          = &snbep_uncore_qpi,
 802         [SNBEP_PCI_UNCORE_R2PCIE]       = &snbep_uncore_r2pcie,
 803         [SNBEP_PCI_UNCORE_R3QPI]        = &snbep_uncore_r3qpi,
 804         NULL,
 805 };
 806
 807 static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = {
 808         { /* Home Agent */
 809                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
 810                 .driver_data = SNBEP_PCI_UNCORE_HA,
 811         },
 812         { /* MC Channel 0 */
 813                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
 814                 .driver_data = SNBEP_PCI_UNCORE_IMC,
 815         },
 816         { /* MC Channel 1 */
 817                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
 818                 .driver_data = SNBEP_PCI_UNCORE_IMC,
 819         },
 820         { /* MC Channel 2 */
 821                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
 822                 .driver_data = SNBEP_PCI_UNCORE_IMC,
 823         },
 824         { /* MC Channel 3 */
 825                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
 826                 .driver_data = SNBEP_PCI_UNCORE_IMC,
 827         },
 828         { /* QPI Port 0 */
 829                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
 830                 .driver_data = SNBEP_PCI_UNCORE_QPI,
 831         },
 832         { /* QPI Port 1 */
 833                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
 834                 .driver_data = SNBEP_PCI_UNCORE_QPI,
 835         },
 836         { /* R2PCIe */
 837                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
 838                 .driver_data = SNBEP_PCI_UNCORE_R2PCIE,
 839         },
 840         { /* R3QPI Link 0 */
 841                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
 842                 .driver_data = SNBEP_PCI_UNCORE_R3QPI,
 843         },
 844         { /* R3QPI Link 1 */
 845                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
 846                 .driver_data = SNBEP_PCI_UNCORE_R3QPI,
 847         },
 848         { /* end: all zeroes */ }
 849 };
 850
 851 static struct pci_driver snbep_uncore_pci_driver = {
 852         .name           = "snbep_uncore",
 853         .id_table       = snbep_uncore_pci_ids,
 854 };
 855
 856 /*
 857  * build pci bus to socket mapping
 858  */
 859 static int snbep_pci2phy_map_init(int devid)
 860 {
 861         struct pci_dev *ubox_dev = NULL;
 862         int i, bus, nodeid;
 863         int err = 0;
 864         u32 config = 0;
 865
 866         while (1) {
 867                 /* find the UBOX device */
 868                 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, ubox_dev);
 869                 if (!ubox_dev)
 870                         break;
 871                 bus = ubox_dev->bus->number;
 872                 /* get the Node ID of the local register */
 873                 err = pci_read_config_dword(ubox_dev, 0x40, &config);
 874                 if (err)
 875                         break;
 876                 nodeid = config;
 877                 /* get the Node ID mapping */
 878                 err = pci_read_config_dword(ubox_dev, 0x54, &config);
 879                 if (err)
 880                         break;
 881                 /*
 882                  * every three bits in the Node ID mapping register maps
 883                  * to a particular node.
 884                  */
 885                 for (i = 0; i < 8; i++) {
 886                         if (nodeid == ((config >> (3 * i)) & 0x7)) {
 887                                 pcibus_to_physid[bus] = i;
 888                                 break;
 889                         }
 890                 }
 891         }
 892
 893         if (ubox_dev)
 894                 pci_dev_put(ubox_dev);
 895
 896         return err ? pcibios_err_to_errno(err) : 0;
 897 }
 898 /* end of Sandy Bridge-EP uncore support */
 899
 900 /* IvyTown uncore support */
 901 static void ivt_uncore_msr_init_box(struct intel_uncore_box *box)
 902 {
 903         unsigned msr = uncore_msr_box_ctl(box);
 904         if (msr)
 905                 wrmsrl(msr, IVT_PMON_BOX_CTL_INT);
 906 }
 907
 908 static void ivt_uncore_pci_init_box(struct intel_uncore_box *box)
 909 {
 910         struct pci_dev *pdev = box->pci_dev;
 911
 912         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVT_PMON_BOX_CTL_INT);
 913 }
 914
 915 #define IVT_UNCORE_MSR_OPS_COMMON_INIT()                        \
 916         .init_box       = ivt_uncore_msr_init_box,              \
 917         .disable_box    = snbep_uncore_msr_disable_box,         \
 918         .enable_box     = snbep_uncore_msr_enable_box,          \
 919         .disable_event  = snbep_uncore_msr_disable_event,       \
 920         .enable_event   = snbep_uncore_msr_enable_event,        \
 921         .read_counter   = uncore_msr_read_counter
 922
 923 static struct intel_uncore_ops ivt_uncore_msr_ops = {
 924         IVT_UNCORE_MSR_OPS_COMMON_INIT(),
 925 };
 926
 927 static struct intel_uncore_ops ivt_uncore_pci_ops = {
 928         .init_box       = ivt_uncore_pci_init_box,
 929         .disable_box    = snbep_uncore_pci_disable_box,
 930         .enable_box     = snbep_uncore_pci_enable_box,
 931         .disable_event  = snbep_uncore_pci_disable_event,
 932         .enable_event   = snbep_uncore_pci_enable_event,
 933         .read_counter   = snbep_uncore_pci_read_counter,
 934 };
 935
 936 #define IVT_UNCORE_PCI_COMMON_INIT()                            \
 937         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
 938         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
 939         .event_mask     = IVT_PMON_RAW_EVENT_MASK,              \
 940         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
 941         .ops            = &ivt_uncore_pci_ops,                  \
 942         .format_group   = &ivt_uncore_format_group
 943
 944 static struct attribute *ivt_uncore_formats_attr[] = {
 945         &format_attr_event.attr,
 946         &format_attr_umask.attr,
 947         &format_attr_edge.attr,
 948         &format_attr_inv.attr,
 949         &format_attr_thresh8.attr,
 950         NULL,
 951 };
 952
 953 static struct attribute *ivt_uncore_ubox_formats_attr[] = {
 954         &format_attr_event.attr,
 955         &format_attr_umask.attr,
 956         &format_attr_edge.attr,
 957         &format_attr_inv.attr,
 958         &format_attr_thresh5.attr,
 959         NULL,
 960 };
 961
 962 static struct attribute *ivt_uncore_cbox_formats_attr[] = {
 963         &format_attr_event.attr,
 964         &format_attr_umask.attr,
 965         &format_attr_edge.attr,
 966         &format_attr_tid_en.attr,
 967         &format_attr_thresh8.attr,
 968         &format_attr_filter_tid.attr,
 969         &format_attr_filter_link.attr,
 970         &format_attr_filter_state2.attr,
 971         &format_attr_filter_nid2.attr,
 972         &format_attr_filter_opc2.attr,
 973         NULL,
 974 };
 975
 976 static struct attribute *ivt_uncore_pcu_formats_attr[] = {
 977         &format_attr_event_ext.attr,
 978         &format_attr_occ_sel.attr,
 979         &format_attr_edge.attr,
 980         &format_attr_thresh5.attr,
 981         &format_attr_occ_invert.attr,
 982         &format_attr_occ_edge.attr,
 983         &format_attr_filter_band0.attr,
 984         &format_attr_filter_band1.attr,
 985         &format_attr_filter_band2.attr,
 986         &format_attr_filter_band3.attr,
 987         NULL,
 988 };
 989
 990 static struct attribute *ivt_uncore_qpi_formats_attr[] = {
 991         &format_attr_event_ext.attr,
 992         &format_attr_umask.attr,
 993         &format_attr_edge.attr,
 994         &format_attr_thresh8.attr,
 995         NULL,
 996 };
 997
 998 static struct attribute_group ivt_uncore_format_group = {
 999         .name = "format",
1000         .attrs = ivt_uncore_formats_attr,
1001 };
1002
1003 static struct attribute_group ivt_uncore_ubox_format_group = {
1004         .name = "format",
1005         .attrs = ivt_uncore_ubox_formats_attr,
1006 };
1007
1008 static struct attribute_group ivt_uncore_cbox_format_group = {
1009         .name = "format",
1010         .attrs = ivt_uncore_cbox_formats_attr,
1011 };
1012
1013 static struct attribute_group ivt_uncore_pcu_format_group = {
1014         .name = "format",
1015         .attrs = ivt_uncore_pcu_formats_attr,
1016 };
1017
1018 static struct attribute_group ivt_uncore_qpi_format_group = {
1019         .name = "format",
1020         .attrs = ivt_uncore_qpi_formats_attr,
1021 };
1022
1023 static struct intel_uncore_type ivt_uncore_ubox = {
1024         .name           = "ubox",
1025         .num_counters   = 2,
1026         .num_boxes      = 1,
1027         .perf_ctr_bits  = 44,
1028         .fixed_ctr_bits = 48,
1029         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
1030         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
1031         .event_mask     = IVT_U_MSR_PMON_RAW_EVENT_MASK,
1032         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
1033         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
1034         .ops            = &ivt_uncore_msr_ops,
1035         .format_group   = &ivt_uncore_ubox_format_group,
1036 };
1037
1038 static struct extra_reg ivt_uncore_cbox_extra_regs[] = {
1039         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1040                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1041         SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2),
1042         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1043         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1044         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1045         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc),
1046         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10),
1047         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1048         SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
1049         SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
1050         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
1051         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
1052         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
1053         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
1054         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
1055         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
1056         SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
1057         SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
1058         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
1059         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
1060         SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1061         SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1062         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
1063         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
1064         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
1065         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
1066         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
1067         SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
1068         SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
1069         SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
1070         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
1071         EVENT_EXTRA_END
1072 };
1073
1074 static u64 ivt_cbox_filter_mask(int fields)
1075 {
1076         u64 mask = 0;
1077
1078         if (fields & 0x1)
1079                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_TID;
1080         if (fields & 0x2)
1081                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_LINK;
1082         if (fields & 0x4)
1083                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_STATE;
1084         if (fields & 0x8)
1085                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_NID;
1086         if (fields & 0x10)
1087                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_OPC;
1088
1089         return mask;
1090 }
1091
1092 static struct event_constraint *
1093 ivt_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1094 {
1095         return __snbep_cbox_get_constraint(box, event, ivt_cbox_filter_mask);
1096 }
1097
1098 static int ivt_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1099 {
1100         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1101         struct extra_reg *er;
1102         int idx = 0;
1103
1104         for (er = ivt_uncore_cbox_extra_regs; er->msr; er++) {
1105                 if (er->event != (event->hw.config & er->config_mask))
1106                         continue;
1107                 idx |= er->idx;
1108         }
1109
1110         if (idx) {
1111                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
1112                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1113                 reg1->config = event->attr.config1 & ivt_cbox_filter_mask(idx);
1114                 reg1->idx = idx;
1115         }
1116         return 0;
1117 }
1118
1119 static void ivt_cbox_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1120 {
1121         struct hw_perf_event *hwc = &event->hw;
1122         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1123
1124         if (reg1->idx != EXTRA_REG_NONE) {
1125                 u64 filter = uncore_shared_reg_config(box, 0);
1126                 wrmsrl(reg1->reg, filter & 0xffffffff);
1127                 wrmsrl(reg1->reg + 6, filter >> 32);
1128         }
1129
1130         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1131 }
1132
1133 static struct intel_uncore_ops ivt_uncore_cbox_ops = {
1134         .init_box               = ivt_uncore_msr_init_box,
1135         .disable_box            = snbep_uncore_msr_disable_box,
1136         .enable_box             = snbep_uncore_msr_enable_box,
1137         .disable_event          = snbep_uncore_msr_disable_event,
1138         .enable_event           = ivt_cbox_enable_event,
1139         .read_counter           = uncore_msr_read_counter,
1140         .hw_config              = ivt_cbox_hw_config,
1141         .get_constraint         = ivt_cbox_get_constraint,
1142         .put_constraint         = snbep_cbox_put_constraint,
1143 };
1144
1145 static struct intel_uncore_type ivt_uncore_cbox = {
1146         .name                   = "cbox",
1147         .num_counters           = 4,
1148         .num_boxes              = 15,
1149         .perf_ctr_bits          = 44,
1150         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
1151         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
1152         .event_mask             = IVT_CBO_MSR_PMON_RAW_EVENT_MASK,
1153         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
1154         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
1155         .num_shared_regs        = 1,
1156         .constraints            = snbep_uncore_cbox_constraints,
1157         .ops                    = &ivt_uncore_cbox_ops,
1158         .format_group           = &ivt_uncore_cbox_format_group,
1159 };
1160
1161 static struct intel_uncore_ops ivt_uncore_pcu_ops = {
1162         IVT_UNCORE_MSR_OPS_COMMON_INIT(),
1163         .hw_config              = snbep_pcu_hw_config,
1164         .get_constraint         = snbep_pcu_get_constraint,
1165         .put_constraint         = snbep_pcu_put_constraint,
1166 };
1167
1168 static struct intel_uncore_type ivt_uncore_pcu = {
1169         .name                   = "pcu",
1170         .num_counters           = 4,
1171         .num_boxes              = 1,
1172         .perf_ctr_bits          = 48,
1173         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
1174         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
1175         .event_mask             = IVT_PCU_MSR_PMON_RAW_EVENT_MASK,
1176         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
1177         .num_shared_regs        = 1,
1178         .ops                    = &ivt_uncore_pcu_ops,
1179         .format_group           = &ivt_uncore_pcu_format_group,
1180 };
1181
1182 static struct intel_uncore_type *ivt_msr_uncores[] = {
1183         &ivt_uncore_ubox,
1184         &ivt_uncore_cbox,
1185         &ivt_uncore_pcu,
1186         NULL,
1187 };
1188
1189 static struct intel_uncore_type ivt_uncore_ha = {
1190         .name           = "ha",
1191         .num_counters   = 4,
1192         .num_boxes      = 2,
1193         .perf_ctr_bits  = 48,
1194         IVT_UNCORE_PCI_COMMON_INIT(),
1195 };
1196
1197 static struct intel_uncore_type ivt_uncore_imc = {
1198         .name           = "imc",
1199         .num_counters   = 4,
1200         .num_boxes      = 8,
1201         .perf_ctr_bits  = 48,
1202         .fixed_ctr_bits = 48,
1203         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
1204         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
1205         IVT_UNCORE_PCI_COMMON_INIT(),
1206 };
1207
1208 static struct intel_uncore_type ivt_uncore_qpi = {
1209         .name           = "qpi",
1210         .num_counters   = 4,
1211         .num_boxes      = 3,
1212         .perf_ctr_bits  = 48,
1213         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
1214         .event_ctl      = SNBEP_PCI_PMON_CTL0,
1215         .event_mask     = IVT_QPI_PCI_PMON_RAW_EVENT_MASK,
1216         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
1217         .ops            = &ivt_uncore_pci_ops,
1218         .format_group   = &ivt_uncore_qpi_format_group,
1219 };
1220
1221 static struct intel_uncore_type ivt_uncore_r2pcie = {
1222         .name           = "r2pcie",
1223         .num_counters   = 4,
1224         .num_boxes      = 1,
1225         .perf_ctr_bits  = 44,
1226         .constraints    = snbep_uncore_r2pcie_constraints,
1227         IVT_UNCORE_PCI_COMMON_INIT(),
1228 };
1229
1230 static struct intel_uncore_type ivt_uncore_r3qpi = {
1231         .name           = "r3qpi",
1232         .num_counters   = 3,
1233         .num_boxes      = 2,
1234         .perf_ctr_bits  = 44,
1235         .constraints    = snbep_uncore_r3qpi_constraints,
1236         IVT_UNCORE_PCI_COMMON_INIT(),
1237 };
1238
1239 enum {
1240         IVT_PCI_UNCORE_HA,
1241         IVT_PCI_UNCORE_IMC,
1242         IVT_PCI_UNCORE_QPI,
1243         IVT_PCI_UNCORE_R2PCIE,
1244         IVT_PCI_UNCORE_R3QPI,
1245 };
1246
1247 static struct intel_uncore_type *ivt_pci_uncores[] = {
1248         [IVT_PCI_UNCORE_HA]     = &ivt_uncore_ha,
1249         [IVT_PCI_UNCORE_IMC]    = &ivt_uncore_imc,
1250         [IVT_PCI_UNCORE_QPI]    = &ivt_uncore_qpi,
1251         [IVT_PCI_UNCORE_R2PCIE] = &ivt_uncore_r2pcie,
1252         [IVT_PCI_UNCORE_R3QPI]  = &ivt_uncore_r3qpi,
1253         NULL,
1254 };
1255
1256 static DEFINE_PCI_DEVICE_TABLE(ivt_uncore_pci_ids) = {
1257         { /* Home Agent 0 */
1258                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30),
1259                 .driver_data = IVT_PCI_UNCORE_HA,
1260         },
1261         { /* Home Agent 1 */
1262                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38),
1263                 .driver_data = IVT_PCI_UNCORE_HA,
1264         },
1265         { /* MC0 Channel 0 */
1266                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4),
1267                 .driver_data = IVT_PCI_UNCORE_IMC,
1268         },
1269         { /* MC0 Channel 1 */
1270                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5),
1271                 .driver_data = IVT_PCI_UNCORE_IMC,
1272         },
1273         { /* MC0 Channel 3 */
1274                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0),
1275                 .driver_data = IVT_PCI_UNCORE_IMC,
1276         },
1277         { /* MC0 Channel 4 */
1278                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1),
1279                 .driver_data = IVT_PCI_UNCORE_IMC,
1280         },
1281         { /* MC1 Channel 0 */
1282                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4),
1283                 .driver_data = IVT_PCI_UNCORE_IMC,
1284         },
1285         { /* MC1 Channel 1 */
1286                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5),
1287                 .driver_data = IVT_PCI_UNCORE_IMC,
1288         },
1289         { /* MC1 Channel 3 */
1290                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0),
1291                 .driver_data = IVT_PCI_UNCORE_IMC,
1292         },
1293         { /* MC1 Channel 4 */
1294                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1),
1295                 .driver_data = IVT_PCI_UNCORE_IMC,
1296         },
1297         { /* QPI0 Port 0 */
1298                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32),
1299                 .driver_data = IVT_PCI_UNCORE_QPI,
1300         },
1301         { /* QPI0 Port 1 */
1302                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33),
1303                 .driver_data = IVT_PCI_UNCORE_QPI,
1304         },
1305         { /* QPI1 Port 2 */
1306                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a),
1307                 .driver_data = IVT_PCI_UNCORE_QPI,
1308         },
1309         { /* R2PCIe */
1310                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34),
1311                 .driver_data = IVT_PCI_UNCORE_R2PCIE,
1312         },
1313         { /* R3QPI0 Link 0 */
1314                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36),
1315                 .driver_data = IVT_PCI_UNCORE_R3QPI,
1316         },
1317         { /* R3QPI0 Link 1 */
1318                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37),
1319                 .driver_data = IVT_PCI_UNCORE_R3QPI,
1320         },
1321         { /* R3QPI1 Link 2 */
1322                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e),
1323                 .driver_data = IVT_PCI_UNCORE_R3QPI,
1324         },
1325         { /* end: all zeroes */ }
1326 };
1327
1328 static struct pci_driver ivt_uncore_pci_driver = {
1329         .name           = "ivt_uncore",
1330         .id_table       = ivt_uncore_pci_ids,
1331 };
1332 /* end of IvyTown uncore support */
1333
1334 /* Sandy Bridge uncore support */
1335 static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1336 {
1337         struct hw_perf_event *hwc = &event->hw;
1338
1339         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
1340                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
1341         else
1342                 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
1343 }
1344
1345 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
1346 {
1347         wrmsrl(event->hw.config_base, 0);
1348 }
1349
1350 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
1351 {
1352         if (box->pmu->pmu_idx == 0) {
1353                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
1354                         SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
1355         }
1356 }
1357
1358 static struct uncore_event_desc snb_uncore_events[] = {
1359         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"),
1360         { /* end: all zeroes */ },
1361 };
1362
1363 static struct attribute *snb_uncore_formats_attr[] = {
1364         &format_attr_event.attr,
1365         &format_attr_umask.attr,
1366         &format_attr_edge.attr,
1367         &format_attr_inv.attr,
1368         &format_attr_cmask5.attr,
1369         NULL,
1370 };
1371
1372 static struct attribute_group snb_uncore_format_group = {
1373         .name           = "format",
1374         .attrs          = snb_uncore_formats_attr,
1375 };
1376
1377 static struct intel_uncore_ops snb_uncore_msr_ops = {
1378         .init_box       = snb_uncore_msr_init_box,
1379         .disable_event  = snb_uncore_msr_disable_event,
1380         .enable_event   = snb_uncore_msr_enable_event,
1381         .read_counter   = uncore_msr_read_counter,
1382 };
1383
1384 static struct event_constraint snb_uncore_cbox_constraints[] = {
1385         UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
1386         UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
1387         EVENT_CONSTRAINT_END
1388 };
1389
1390 static struct intel_uncore_type snb_uncore_cbox = {
1391         .name           = "cbox",
1392         .num_counters   = 2,
1393         .num_boxes      = 4,
1394         .perf_ctr_bits  = 44,
1395         .fixed_ctr_bits = 48,
1396         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
1397         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
1398         .fixed_ctr      = SNB_UNC_FIXED_CTR,
1399         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
1400         .single_fixed   = 1,
1401         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
1402         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
1403         .constraints    = snb_uncore_cbox_constraints,
1404         .ops            = &snb_uncore_msr_ops,
1405         .format_group   = &snb_uncore_format_group,
1406         .event_descs    = snb_uncore_events,
1407 };
1408
1409 static struct intel_uncore_type *snb_msr_uncores[] = {
1410         &snb_uncore_cbox,
1411         NULL,
1412 };
1413 /* end of Sandy Bridge uncore support */
1414
1415 /* Nehalem uncore support */
1416 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
1417 {
1418         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
1419 }
1420
1421 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
1422 {
1423         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
1424 }
1425
1426 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1427 {
1428         struct hw_perf_event *hwc = &event->hw;
1429
1430         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
1431                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
1432         else
1433                 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
1434 }
1435
1436 static struct attribute *nhm_uncore_formats_attr[] = {
1437         &format_attr_event.attr,
1438         &format_attr_umask.attr,
1439         &format_attr_edge.attr,
1440         &format_attr_inv.attr,
1441         &format_attr_cmask8.attr,
1442         NULL,
1443 };
1444
1445 static struct attribute_group nhm_uncore_format_group = {
1446         .name = "format",
1447         .attrs = nhm_uncore_formats_attr,
1448 };
1449
1450 static struct uncore_event_desc nhm_uncore_events[] = {
1451         INTEL_UNCORE_EVENT_DESC(clockticks,                "event=0xff,umask=0x00"),
1452         INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any,       "event=0x2f,umask=0x0f"),
1453         INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any,      "event=0x2c,umask=0x0f"),
1454         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads,     "event=0x20,umask=0x01"),
1455         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes,    "event=0x20,umask=0x02"),
1456         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads,  "event=0x20,umask=0x04"),
1457         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
1458         INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads,   "event=0x20,umask=0x10"),
1459         INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes,  "event=0x20,umask=0x20"),
1460         { /* end: all zeroes */ },
1461 };
1462
1463 static struct intel_uncore_ops nhm_uncore_msr_ops = {
1464         .disable_box    = nhm_uncore_msr_disable_box,
1465         .enable_box     = nhm_uncore_msr_enable_box,
1466         .disable_event  = snb_uncore_msr_disable_event,
1467         .enable_event   = nhm_uncore_msr_enable_event,
1468         .read_counter   = uncore_msr_read_counter,
1469 };
1470
1471 static struct intel_uncore_type nhm_uncore = {
1472         .name           = "",
1473         .num_counters   = 8,
1474         .num_boxes      = 1,
1475         .perf_ctr_bits  = 48,
1476         .fixed_ctr_bits = 48,
1477         .event_ctl      = NHM_UNC_PERFEVTSEL0,
1478         .perf_ctr       = NHM_UNC_UNCORE_PMC0,
1479         .fixed_ctr      = NHM_UNC_FIXED_CTR,
1480         .fixed_ctl      = NHM_UNC_FIXED_CTR_CTRL,
1481         .event_mask     = NHM_UNC_RAW_EVENT_MASK,
1482         .event_descs    = nhm_uncore_events,
1483         .ops            = &nhm_uncore_msr_ops,
1484         .format_group   = &nhm_uncore_format_group,
1485 };
1486
1487 static struct intel_uncore_type *nhm_msr_uncores[] = {
1488         &nhm_uncore,
1489         NULL,
1490 };
1491 /* end of Nehalem uncore support */
1492
1493 /* Nehalem-EX uncore support */
1494 DEFINE_UNCORE_FORMAT_ATTR(event5, event, "config:1-5");
1495 DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7");
1496 DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63");
1497 DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63");
1498
1499 static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box)
1500 {
1501         wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL);
1502 }
1503
1504 static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box)
1505 {
1506         unsigned msr = uncore_msr_box_ctl(box);
1507         u64 config;
1508
1509         if (msr) {
1510                 rdmsrl(msr, config);
1511                 config &= ~((1ULL << uncore_num_counters(box)) - 1);
1512                 /* WBox has a fixed counter */
1513                 if (uncore_msr_fixed_ctl(box))
1514                         config &= ~NHMEX_W_PMON_GLOBAL_FIXED_EN;
1515                 wrmsrl(msr, config);
1516         }
1517 }
1518
1519 static void nhmex_uncore_msr_enable_box(struct intel_uncore_box *box)
1520 {
1521         unsigned msr = uncore_msr_box_ctl(box);
1522         u64 config;
1523
1524         if (msr) {
1525                 rdmsrl(msr, config);
1526                 config |= (1ULL << uncore_num_counters(box)) - 1;
1527                 /* WBox has a fixed counter */
1528                 if (uncore_msr_fixed_ctl(box))
1529                         config |= NHMEX_W_PMON_GLOBAL_FIXED_EN;
1530                 wrmsrl(msr, config);
1531         }
1532 }
1533
1534 static void nhmex_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
1535 {
1536         wrmsrl(event->hw.config_base, 0);
1537 }
1538
1539 static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1540 {
1541         struct hw_perf_event *hwc = &event->hw;
1542
1543         if (hwc->idx >= UNCORE_PMC_IDX_FIXED)
1544                 wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0);
1545         else if (box->pmu->type->event_mask & NHMEX_PMON_CTL_EN_BIT0)
1546                 wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
1547         else
1548                 wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
1549 }
1550
1551 #define NHMEX_UNCORE_OPS_COMMON_INIT()                          \
1552         .init_box       = nhmex_uncore_msr_init_box,            \
1553         .disable_box    = nhmex_uncore_msr_disable_box,         \
1554         .enable_box     = nhmex_uncore_msr_enable_box,          \
1555         .disable_event  = nhmex_uncore_msr_disable_event,       \
1556         .read_counter   = uncore_msr_read_counter
1557
1558 static struct intel_uncore_ops nhmex_uncore_ops = {
1559         NHMEX_UNCORE_OPS_COMMON_INIT(),
1560         .enable_event   = nhmex_uncore_msr_enable_event,
1561 };
1562
1563 static struct attribute *nhmex_uncore_ubox_formats_attr[] = {
1564         &format_attr_event.attr,
1565         &format_attr_edge.attr,
1566         NULL,
1567 };
1568
1569 static struct attribute_group nhmex_uncore_ubox_format_group = {
1570         .name           = "format",
1571         .attrs          = nhmex_uncore_ubox_formats_attr,
1572 };
1573
1574 static struct intel_uncore_type nhmex_uncore_ubox = {
1575         .name           = "ubox",
1576         .num_counters   = 1,
1577         .num_boxes      = 1,
1578         .perf_ctr_bits  = 48,
1579         .event_ctl      = NHMEX_U_MSR_PMON_EV_SEL,
1580         .perf_ctr       = NHMEX_U_MSR_PMON_CTR,
1581         .event_mask     = NHMEX_U_PMON_RAW_EVENT_MASK,
1582         .box_ctl        = NHMEX_U_MSR_PMON_GLOBAL_CTL,
1583         .ops            = &nhmex_uncore_ops,
1584         .format_group   = &nhmex_uncore_ubox_format_group
1585 };
1586
1587 static struct attribute *nhmex_uncore_cbox_formats_attr[] = {
1588         &format_attr_event.attr,
1589         &format_attr_umask.attr,
1590         &format_attr_edge.attr,
1591         &format_attr_inv.attr,
1592         &format_attr_thresh8.attr,
1593         NULL,
1594 };
1595
1596 static struct attribute_group nhmex_uncore_cbox_format_group = {
1597         .name = "format",
1598         .attrs = nhmex_uncore_cbox_formats_attr,
1599 };
1600
1601 /* msr offset for each instance of cbox */
1602 static unsigned nhmex_cbox_msr_offsets[] = {
1603         0x0, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 0x240, 0x2c0,
1604 };
1605
1606 static struct intel_uncore_type nhmex_uncore_cbox = {
1607         .name                   = "cbox",
1608         .num_counters           = 6,
1609         .num_boxes              = 10,
1610         .perf_ctr_bits          = 48,
1611         .event_ctl              = NHMEX_C0_MSR_PMON_EV_SEL0,
1612         .perf_ctr               = NHMEX_C0_MSR_PMON_CTR0,
1613         .event_mask             = NHMEX_PMON_RAW_EVENT_MASK,
1614         .box_ctl                = NHMEX_C0_MSR_PMON_GLOBAL_CTL,
1615         .msr_offsets            = nhmex_cbox_msr_offsets,
1616         .pair_ctr_ctl           = 1,
1617         .ops                    = &nhmex_uncore_ops,
1618         .format_group           = &nhmex_uncore_cbox_format_group
1619 };
1620
1621 static struct uncore_event_desc nhmex_uncore_wbox_events[] = {
1622         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0"),
1623         { /* end: all zeroes */ },
1624 };
1625
1626 static struct intel_uncore_type nhmex_uncore_wbox = {
1627         .name                   = "wbox",
1628         .num_counters           = 4,
1629         .num_boxes              = 1,
1630         .perf_ctr_bits          = 48,
1631         .event_ctl              = NHMEX_W_MSR_PMON_CNT0,
1632         .perf_ctr               = NHMEX_W_MSR_PMON_EVT_SEL0,
1633         .fixed_ctr              = NHMEX_W_MSR_PMON_FIXED_CTR,
1634         .fixed_ctl              = NHMEX_W_MSR_PMON_FIXED_CTL,
1635         .event_mask             = NHMEX_PMON_RAW_EVENT_MASK,
1636         .box_ctl                = NHMEX_W_MSR_GLOBAL_CTL,
1637         .pair_ctr_ctl           = 1,
1638         .event_descs            = nhmex_uncore_wbox_events,
1639         .ops                    = &nhmex_uncore_ops,
1640         .format_group           = &nhmex_uncore_cbox_format_group
1641 };
1642
1643 static int nhmex_bbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1644 {
1645         struct hw_perf_event *hwc = &event->hw;
1646         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1647         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1648         int ctr, ev_sel;
1649
1650         ctr = (hwc->config & NHMEX_B_PMON_CTR_MASK) >>
1651                 NHMEX_B_PMON_CTR_SHIFT;
1652         ev_sel = (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK) >>
1653                   NHMEX_B_PMON_CTL_EV_SEL_SHIFT;
1654
1655         /* events that do not use the match/mask registers */
1656         if ((ctr == 0 && ev_sel > 0x3) || (ctr == 1 && ev_sel > 0x6) ||
1657             (ctr == 2 && ev_sel != 0x4) || ctr == 3)
1658                 return 0;
1659
1660         if (box->pmu->pmu_idx == 0)
1661                 reg1->reg = NHMEX_B0_MSR_MATCH;
1662         else
1663                 reg1->reg = NHMEX_B1_MSR_MATCH;
1664         reg1->idx = 0;
1665         reg1->config = event->attr.config1;
1666         reg2->config = event->attr.config2;
1667         return 0;
1668 }
1669
1670 static void nhmex_bbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1671 {
1672         struct hw_perf_event *hwc = &event->hw;
1673         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1674         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1675
1676         if (reg1->idx != EXTRA_REG_NONE) {
1677                 wrmsrl(reg1->reg, reg1->config);
1678                 wrmsrl(reg1->reg + 1, reg2->config);
1679         }
1680         wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
1681                 (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK));
1682 }
1683
1684 /*
1685  * The Bbox has 4 counters, but each counter monitors different events.
1686  * Use bits 6-7 in the event config to select counter.
1687  */
1688 static struct event_constraint nhmex_uncore_bbox_constraints[] = {
1689         EVENT_CONSTRAINT(0 , 1, 0xc0),
1690         EVENT_CONSTRAINT(0x40, 2, 0xc0),
1691         EVENT_CONSTRAINT(0x80, 4, 0xc0),
1692         EVENT_CONSTRAINT(0xc0, 8, 0xc0),
1693         EVENT_CONSTRAINT_END,
1694 };
1695
1696 static struct attribute *nhmex_uncore_bbox_formats_attr[] = {
1697         &format_attr_event5.attr,
1698         &format_attr_counter.attr,
1699         &format_attr_match.attr,
1700         &format_attr_mask.attr,
1701         NULL,
1702 };
1703
1704 static struct attribute_group nhmex_uncore_bbox_format_group = {
1705         .name = "format",
1706         .attrs = nhmex_uncore_bbox_formats_attr,
1707 };
1708
1709 static struct intel_uncore_ops nhmex_uncore_bbox_ops = {
1710         NHMEX_UNCORE_OPS_COMMON_INIT(),
1711         .enable_event           = nhmex_bbox_msr_enable_event,
1712         .hw_config              = nhmex_bbox_hw_config,
1713         .get_constraint         = uncore_get_constraint,
1714         .put_constraint         = uncore_put_constraint,
1715 };
1716
1717 static struct intel_uncore_type nhmex_uncore_bbox = {
1718         .name                   = "bbox",
1719         .num_counters           = 4,
1720         .num_boxes              = 2,
1721         .perf_ctr_bits          = 48,
1722         .event_ctl              = NHMEX_B0_MSR_PMON_CTL0,
1723         .perf_ctr               = NHMEX_B0_MSR_PMON_CTR0,
1724         .event_mask             = NHMEX_B_PMON_RAW_EVENT_MASK,
1725         .box_ctl                = NHMEX_B0_MSR_PMON_GLOBAL_CTL,
1726         .msr_offset             = NHMEX_B_MSR_OFFSET,
1727         .pair_ctr_ctl           = 1,
1728         .num_shared_regs        = 1,
1729         .constraints            = nhmex_uncore_bbox_constraints,
1730         .ops                    = &nhmex_uncore_bbox_ops,
1731         .format_group           = &nhmex_uncore_bbox_format_group
1732 };
1733
1734 static int nhmex_sbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1735 {
1736         struct hw_perf_event *hwc = &event->hw;
1737         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1738         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1739
1740         /* only TO_R_PROG_EV event uses the match/mask register */
1741         if ((hwc->config & NHMEX_PMON_CTL_EV_SEL_MASK) !=
1742             NHMEX_S_EVENT_TO_R_PROG_EV)
1743                 return 0;
1744
1745         if (box->pmu->pmu_idx == 0)
1746                 reg1->reg = NHMEX_S0_MSR_MM_CFG;
1747         else
1748                 reg1->reg = NHMEX_S1_MSR_MM_CFG;
1749         reg1->idx = 0;
1750         reg1->config = event->attr.config1;
1751         reg2->config = event->attr.config2;
1752         return 0;
1753 }
1754
1755 static void nhmex_sbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1756 {
1757         struct hw_perf_event *hwc = &event->hw;
1758         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1759         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1760
1761         if (reg1->idx != EXTRA_REG_NONE) {
1762                 wrmsrl(reg1->reg, 0);
1763                 wrmsrl(reg1->reg + 1, reg1->config);
1764                 wrmsrl(reg1->reg + 2, reg2->config);
1765                 wrmsrl(reg1->reg, NHMEX_S_PMON_MM_CFG_EN);
1766         }
1767         wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
1768 }
1769
1770 static struct attribute *nhmex_uncore_sbox_formats_attr[] = {
1771         &format_attr_event.attr,
1772         &format_attr_umask.attr,
1773         &format_attr_edge.attr,
1774         &format_attr_inv.attr,
1775         &format_attr_thresh8.attr,
1776         &format_attr_match.attr,
1777         &format_attr_mask.attr,
1778         NULL,
1779 };
1780
1781 static struct attribute_group nhmex_uncore_sbox_format_group = {
1782         .name                   = "format",
1783         .attrs                  = nhmex_uncore_sbox_formats_attr,
1784 };
1785
1786 static struct intel_uncore_ops nhmex_uncore_sbox_ops = {
1787         NHMEX_UNCORE_OPS_COMMON_INIT(),
1788         .enable_event           = nhmex_sbox_msr_enable_event,
1789         .hw_config              = nhmex_sbox_hw_config,
1790         .get_constraint         = uncore_get_constraint,
1791         .put_constraint         = uncore_put_constraint,
1792 };
1793
1794 static struct intel_uncore_type nhmex_uncore_sbox = {
1795         .name                   = "sbox",
1796         .num_counters           = 4,
1797         .num_boxes              = 2,
1798         .perf_ctr_bits          = 48,
1799         .event_ctl              = NHMEX_S0_MSR_PMON_CTL0,
1800         .perf_ctr               = NHMEX_S0_MSR_PMON_CTR0,
1801         .event_mask             = NHMEX_PMON_RAW_EVENT_MASK,
1802         .box_ctl                = NHMEX_S0_MSR_PMON_GLOBAL_CTL,
1803         .msr_offset             = NHMEX_S_MSR_OFFSET,
1804         .pair_ctr_ctl           = 1,
1805         .num_shared_regs        = 1,
1806         .ops                    = &nhmex_uncore_sbox_ops,
1807         .format_group           = &nhmex_uncore_sbox_format_group
1808 };
1809
1810 enum {
1811         EXTRA_REG_NHMEX_M_FILTER,
1812         EXTRA_REG_NHMEX_M_DSP,
1813         EXTRA_REG_NHMEX_M_ISS,
1814         EXTRA_REG_NHMEX_M_MAP,
1815         EXTRA_REG_NHMEX_M_MSC_THR,
1816         EXTRA_REG_NHMEX_M_PGT,
1817         EXTRA_REG_NHMEX_M_PLD,
1818         EXTRA_REG_NHMEX_M_ZDP_CTL_FVC,
1819 };
1820
1821 static struct extra_reg nhmex_uncore_mbox_extra_regs[] = {
1822         MBOX_INC_SEL_EXTAR_REG(0x0, DSP),
1823         MBOX_INC_SEL_EXTAR_REG(0x4, MSC_THR),
1824         MBOX_INC_SEL_EXTAR_REG(0x5, MSC_THR),
1825         MBOX_INC_SEL_EXTAR_REG(0x9, ISS),
1826         /* event 0xa uses two extra registers */
1827         MBOX_INC_SEL_EXTAR_REG(0xa, ISS),
1828         MBOX_INC_SEL_EXTAR_REG(0xa, PLD),
1829         MBOX_INC_SEL_EXTAR_REG(0xb, PLD),
1830         /* events 0xd ~ 0x10 use the same extra register */
1831         MBOX_INC_SEL_EXTAR_REG(0xd, ZDP_CTL_FVC),
1832         MBOX_INC_SEL_EXTAR_REG(0xe, ZDP_CTL_FVC),
1833         MBOX_INC_SEL_EXTAR_REG(0xf, ZDP_CTL_FVC),
1834         MBOX_INC_SEL_EXTAR_REG(0x10, ZDP_CTL_FVC),
1835         MBOX_INC_SEL_EXTAR_REG(0x16, PGT),
1836         MBOX_SET_FLAG_SEL_EXTRA_REG(0x0, DSP),
1837         MBOX_SET_FLAG_SEL_EXTRA_REG(0x1, ISS),
1838         MBOX_SET_FLAG_SEL_EXTRA_REG(0x5, PGT),
1839         MBOX_SET_FLAG_SEL_EXTRA_REG(0x6, MAP),
1840         EVENT_EXTRA_END
1841 };
1842
1843 /* Nehalem-EX or Westmere-EX ? */
1844 static bool uncore_nhmex;
1845
1846 static bool nhmex_mbox_get_shared_reg(struct intel_uncore_box *box, int idx, u64 config)
1847 {
1848         struct intel_uncore_extra_reg *er;
1849         unsigned long flags;
1850         bool ret = false;
1851         u64 mask;
1852
1853         if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
1854                 er = &box->shared_regs[idx];
1855                 raw_spin_lock_irqsave(&er->lock, flags);
1856                 if (!atomic_read(&er->ref) || er->config == config) {
1857                         atomic_inc(&er->ref);
1858                         er->config = config;
1859                         ret = true;
1860                 }
1861                 raw_spin_unlock_irqrestore(&er->lock, flags);
1862
1863                 return ret;
1864         }
1865         /*
1866          * The ZDP_CTL_FVC MSR has 4 fields which are used to control
1867          * events 0xd ~ 0x10. Besides these 4 fields, there are additional
1868          * fields which are shared.
1869          */
1870         idx -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
1871         if (WARN_ON_ONCE(idx >= 4))
1872                 return false;
1873
1874         /* mask of the shared fields */
1875         if (uncore_nhmex)
1876                 mask = NHMEX_M_PMON_ZDP_CTL_FVC_MASK;
1877         else
1878                 mask = WSMEX_M_PMON_ZDP_CTL_FVC_MASK;
1879         er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
1880
1881         raw_spin_lock_irqsave(&er->lock, flags);
1882         /* add mask of the non-shared field if it's in use */
1883         if (__BITS_VALUE(atomic_read(&er->ref), idx, 8)) {
1884                 if (uncore_nhmex)
1885                         mask |= NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1886                 else
1887                         mask |= WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1888         }
1889
1890         if (!atomic_read(&er->ref) || !((er->config ^ config) & mask)) {
1891                 atomic_add(1 << (idx * 8), &er->ref);
1892                 if (uncore_nhmex)
1893                         mask = NHMEX_M_PMON_ZDP_CTL_FVC_MASK |
1894                                 NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1895                 else
1896                         mask = WSMEX_M_PMON_ZDP_CTL_FVC_MASK |
1897                                 WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1898                 er->config &= ~mask;
1899                 er->config |= (config & mask);
1900                 ret = true;
1901         }
1902         raw_spin_unlock_irqrestore(&er->lock, flags);
1903
1904         return ret;
1905 }
1906
1907 static void nhmex_mbox_put_shared_reg(struct intel_uncore_box *box, int idx)
1908 {
1909         struct intel_uncore_extra_reg *er;
1910
1911         if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
1912                 er = &box->shared_regs[idx];
1913                 atomic_dec(&er->ref);
1914                 return;
1915         }
1916
1917         idx -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
1918         er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
1919         atomic_sub(1 << (idx * 8), &er->ref);
1920 }
1921
1922 static u64 nhmex_mbox_alter_er(struct perf_event *event, int new_idx, bool modify)
1923 {
1924         struct hw_perf_event *hwc = &event->hw;
1925         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1926         int idx, orig_idx = __BITS_VALUE(reg1->idx, 0, 8);
1927         u64 config = reg1->config;
1928
1929         /* get the non-shared control bits and shift them */
1930         idx = orig_idx - EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
1931         if (uncore_nhmex)
1932                 config &= NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1933         else
1934                 config &= WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1935         if (new_idx > orig_idx) {
1936                 idx = new_idx - orig_idx;
1937                 config <<= 3 * idx;
1938         } else {
1939                 idx = orig_idx - new_idx;
1940                 config >>= 3 * idx;
1941         }
1942
1943         /* add the shared control bits back */
1944         if (uncore_nhmex)
1945                 config |= NHMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
1946         else
1947                 config |= WSMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
1948         config |= NHMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
1949         if (modify) {
1950                 /* adjust the main event selector */
1951                 if (new_idx > orig_idx)
1952                         hwc->config += idx << NHMEX_M_PMON_CTL_INC_SEL_SHIFT;
1953                 else
1954                         hwc->config -= idx << NHMEX_M_PMON_CTL_INC_SEL_SHIFT;
1955                 reg1->config = config;
1956                 reg1->idx = ~0xff | new_idx;
1957         }
1958         return config;
1959 }
1960
1961 static struct event_constraint *
1962 nhmex_mbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1963 {
1964         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1965         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
1966         int i, idx[2], alloc = 0;
1967         u64 config1 = reg1->config;
1968
1969         idx[0] = __BITS_VALUE(reg1->idx, 0, 8);
1970         idx[1] = __BITS_VALUE(reg1->idx, 1, 8);
1971 again:
1972         for (i = 0; i < 2; i++) {
1973                 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
1974                         idx[i] = 0xff;
1975
1976                 if (idx[i] == 0xff)
1977                         continue;
1978
1979                 if (!nhmex_mbox_get_shared_reg(box, idx[i],
1980                                 __BITS_VALUE(config1, i, 32)))
1981                         goto fail;
1982                 alloc |= (0x1 << i);
1983         }
1984
1985         /* for the match/mask registers */
1986         if (reg2->idx != EXTRA_REG_NONE &&
1987             (uncore_box_is_fake(box) || !reg2->alloc) &&
1988             !nhmex_mbox_get_shared_reg(box, reg2->idx, reg2->config))
1989                 goto fail;
1990
1991         /*
1992          * If it's a fake box -- as per validate_{group,event}() we
1993          * shouldn't touch event state and we can avoid doing so
1994          * since both will only call get_event_constraints() once
1995          * on each event, this avoids the need for reg->alloc.
1996          */
1997         if (!uncore_box_is_fake(box)) {
1998                 if (idx[0] != 0xff && idx[0] != __BITS_VALUE(reg1->idx, 0, 8))
1999                         nhmex_mbox_alter_er(event, idx[0], true);
2000                 reg1->alloc |= alloc;
2001                 if (reg2->idx != EXTRA_REG_NONE)
2002                         reg2->alloc = 1;
2003         }
2004         return NULL;
2005 fail:
2006         if (idx[0] != 0xff && !(alloc & 0x1) &&
2007             idx[0] >= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
2008                 /*
2009                  * events 0xd ~ 0x10 are functional identical, but are
2010                  * controlled by different fields in the ZDP_CTL_FVC
2011                  * register. If we failed to take one field, try the
2012                  * rest 3 choices.
2013                  */
2014                 BUG_ON(__BITS_VALUE(reg1->idx, 1, 8) != 0xff);
2015                 idx[0] -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
2016                 idx[0] = (idx[0] + 1) % 4;
2017                 idx[0] += EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
2018                 if (idx[0] != __BITS_VALUE(reg1->idx, 0, 8)) {
2019                         config1 = nhmex_mbox_alter_er(event, idx[0], false);
2020                         goto again;
2021                 }
2022         }
2023
2024         if (alloc & 0x1)
2025                 nhmex_mbox_put_shared_reg(box, idx[0]);
2026         if (alloc & 0x2)
2027                 nhmex_mbox_put_shared_reg(box, idx[1]);
2028         return &constraint_empty;
2029 }
2030
2031 static void nhmex_mbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
2032 {
2033         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2034         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
2035
2036         if (uncore_box_is_fake(box))
2037                 return;
2038
2039         if (reg1->alloc & 0x1)
2040                 nhmex_mbox_put_shared_reg(box, __BITS_VALUE(reg1->idx, 0, 8));
2041         if (reg1->alloc & 0x2)
2042                 nhmex_mbox_put_shared_reg(box, __BITS_VALUE(reg1->idx, 1, 8));
2043         reg1->alloc = 0;
2044
2045         if (reg2->alloc) {
2046                 nhmex_mbox_put_shared_reg(box, reg2->idx);
2047                 reg2->alloc = 0;
2048         }
2049 }
2050
2051 static int nhmex_mbox_extra_reg_idx(struct extra_reg *er)
2052 {
2053         if (er->idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC)
2054                 return er->idx;
2055         return er->idx + (er->event >> NHMEX_M_PMON_CTL_INC_SEL_SHIFT) - 0xd;
2056 }
2057
2058 static int nhmex_mbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2059 {
2060         struct intel_uncore_type *type = box->pmu->type;
2061         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2062         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
2063         struct extra_reg *er;
2064         unsigned msr;
2065         int reg_idx = 0;
2066         /*
2067          * The mbox events may require 2 extra MSRs at the most. But only
2068          * the lower 32 bits in these MSRs are significant, so we can use
2069          * config1 to pass two MSRs' config.
2070          */
2071         for (er = nhmex_uncore_mbox_extra_regs; er->msr; er++) {
2072                 if (er->event != (event->hw.config & er->config_mask))
2073                         continue;
2074                 if (event->attr.config1 & ~er->valid_mask)
2075                         return -EINVAL;
2076
2077                 msr = er->msr + type->msr_offset * box->pmu->pmu_idx;
2078                 if (WARN_ON_ONCE(msr >= 0xffff || er->idx >= 0xff))
2079                         return -EINVAL;
2080
2081                 /* always use the 32~63 bits to pass the PLD config */
2082                 if (er->idx == EXTRA_REG_NHMEX_M_PLD)
2083                         reg_idx = 1;
2084                 else if (WARN_ON_ONCE(reg_idx > 0))
2085                         return -EINVAL;
2086
2087                 reg1->idx &= ~(0xff << (reg_idx * 8));
2088                 reg1->reg &= ~(0xffff << (reg_idx * 16));
2089                 reg1->idx |= nhmex_mbox_extra_reg_idx(er) << (reg_idx * 8);
2090                 reg1->reg |= msr << (reg_idx * 16);
2091                 reg1->config = event->attr.config1;
2092                 reg_idx++;
2093         }
2094         /*
2095          * The mbox only provides ability to perform address matching
2096          * for the PLD events.
2097          */
2098         if (reg_idx == 2) {
2099                 reg2->idx = EXTRA_REG_NHMEX_M_FILTER;
2100                 if (event->attr.config2 & NHMEX_M_PMON_MM_CFG_EN)
2101                         reg2->config = event->attr.config2;
2102                 else
2103                         reg2->config = ~0ULL;
2104                 if (box->pmu->pmu_idx == 0)
2105                         reg2->reg = NHMEX_M0_MSR_PMU_MM_CFG;
2106                 else
2107                         reg2->reg = NHMEX_M1_MSR_PMU_MM_CFG;
2108         }
2109         return 0;
2110 }
2111
2112 static u64 nhmex_mbox_shared_reg_config(struct intel_uncore_box *box, int idx)
2113 {
2114         struct intel_uncore_extra_reg *er;
2115         unsigned long flags;
2116         u64 config;
2117
2118         if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC)
2119                 return box->shared_regs[idx].config;
2120
2121         er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
2122         raw_spin_lock_irqsave(&er->lock, flags);
2123         config = er->config;
2124         raw_spin_unlock_irqrestore(&er->lock, flags);
2125         return config;
2126 }
2127
2128 static void nhmex_mbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
2129 {
2130         struct hw_perf_event *hwc = &event->hw;
2131         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2132         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
2133         int idx;
2134
2135         idx = __BITS_VALUE(reg1->idx, 0, 8);
2136         if (idx != 0xff)
2137                 wrmsrl(__BITS_VALUE(reg1->reg, 0, 16),
2138                         nhmex_mbox_shared_reg_config(box, idx));
2139         idx = __BITS_VALUE(reg1->idx, 1, 8);
2140         if (idx != 0xff)
2141                 wrmsrl(__BITS_VALUE(reg1->reg, 1, 16),
2142                         nhmex_mbox_shared_reg_config(box, idx));
2143
2144         if (reg2->idx != EXTRA_REG_NONE) {
2145                 wrmsrl(reg2->reg, 0);
2146                 if (reg2->config != ~0ULL) {
2147                         wrmsrl(reg2->reg + 1,
2148                                 reg2->config & NHMEX_M_PMON_ADDR_MATCH_MASK);
2149                         wrmsrl(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK &
2150                                 (reg2->config >> NHMEX_M_PMON_ADDR_MASK_SHIFT));
2151                         wrmsrl(reg2->reg, NHMEX_M_PMON_MM_CFG_EN);
2152                 }
2153         }
2154
2155         wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
2156 }
2157
2158 DEFINE_UNCORE_FORMAT_ATTR(count_mode,           count_mode,     "config:2-3");
2159 DEFINE_UNCORE_FORMAT_ATTR(storage_mode,         storage_mode,   "config:4-5");
2160 DEFINE_UNCORE_FORMAT_ATTR(wrap_mode,            wrap_mode,      "config:6");
2161 DEFINE_UNCORE_FORMAT_ATTR(flag_mode,            flag_mode,      "config:7");
2162 DEFINE_UNCORE_FORMAT_ATTR(inc_sel,              inc_sel,        "config:9-13");
2163 DEFINE_UNCORE_FORMAT_ATTR(set_flag_sel,         set_flag_sel,   "config:19-21");
2164 DEFINE_UNCORE_FORMAT_ATTR(filter_cfg_en,        filter_cfg_en,  "config2:63");
2165 DEFINE_UNCORE_FORMAT_ATTR(filter_match,         filter_match,   "config2:0-33");
2166 DEFINE_UNCORE_FORMAT_ATTR(filter_mask,          filter_mask,    "config2:34-61");
2167 DEFINE_UNCORE_FORMAT_ATTR(dsp,                  dsp,            "config1:0-31");
2168 DEFINE_UNCORE_FORMAT_ATTR(thr,                  thr,            "config1:0-31");
2169 DEFINE_UNCORE_FORMAT_ATTR(fvc,                  fvc,            "config1:0-31");
2170 DEFINE_UNCORE_FORMAT_ATTR(pgt,                  pgt,            "config1:0-31");
2171 DEFINE_UNCORE_FORMAT_ATTR(map,                  map,            "config1:0-31");
2172 DEFINE_UNCORE_FORMAT_ATTR(iss,                  iss,            "config1:0-31");
2173 DEFINE_UNCORE_FORMAT_ATTR(pld,                  pld,            "config1:32-63");
2174
2175 static struct attribute *nhmex_uncore_mbox_formats_attr[] = {
2176         &format_attr_count_mode.attr,
2177         &format_attr_storage_mode.attr,
2178         &format_attr_wrap_mode.attr,
2179         &format_attr_flag_mode.attr,
2180         &format_attr_inc_sel.attr,
2181         &format_attr_set_flag_sel.attr,
2182         &format_attr_filter_cfg_en.attr,
2183         &format_attr_filter_match.attr,
2184         &format_attr_filter_mask.attr,
2185         &format_attr_dsp.attr,
2186         &format_attr_thr.attr,
2187         &format_attr_fvc.attr,
2188         &format_attr_pgt.attr,
2189         &format_attr_map.attr,
2190         &format_attr_iss.attr,
2191         &format_attr_pld.attr,
2192         NULL,
2193 };
2194
2195 static struct attribute_group nhmex_uncore_mbox_format_group = {
2196         .name           = "format",
2197         .attrs          = nhmex_uncore_mbox_formats_attr,
2198 };
2199
2200 static struct uncore_event_desc nhmex_uncore_mbox_events[] = {
2201         INTEL_UNCORE_EVENT_DESC(bbox_cmds_read, "inc_sel=0xd,fvc=0x2800"),
2202         INTEL_UNCORE_EVENT_DESC(bbox_cmds_write, "inc_sel=0xd,fvc=0x2820"),
2203         { /* end: all zeroes */ },
2204 };
2205
2206 static struct uncore_event_desc wsmex_uncore_mbox_events[] = {
2207         INTEL_UNCORE_EVENT_DESC(bbox_cmds_read, "inc_sel=0xd,fvc=0x5000"),
2208         INTEL_UNCORE_EVENT_DESC(bbox_cmds_write, "inc_sel=0xd,fvc=0x5040"),
2209         { /* end: all zeroes */ },
2210 };
2211
2212 static struct intel_uncore_ops nhmex_uncore_mbox_ops = {
2213         NHMEX_UNCORE_OPS_COMMON_INIT(),
2214         .enable_event   = nhmex_mbox_msr_enable_event,
2215         .hw_config      = nhmex_mbox_hw_config,
2216         .get_constraint = nhmex_mbox_get_constraint,
2217         .put_constraint = nhmex_mbox_put_constraint,
2218 };
2219
2220 static struct intel_uncore_type nhmex_uncore_mbox = {
2221         .name                   = "mbox",
2222         .num_counters           = 6,
2223         .num_boxes              = 2,
2224         .perf_ctr_bits          = 48,
2225         .event_ctl              = NHMEX_M0_MSR_PMU_CTL0,
2226         .perf_ctr               = NHMEX_M0_MSR_PMU_CNT0,
2227         .event_mask             = NHMEX_M_PMON_RAW_EVENT_MASK,
2228         .box_ctl                = NHMEX_M0_MSR_GLOBAL_CTL,
2229         .msr_offset             = NHMEX_M_MSR_OFFSET,
2230         .pair_ctr_ctl           = 1,
2231         .num_shared_regs        = 8,
2232         .event_descs            = nhmex_uncore_mbox_events,
2233         .ops                    = &nhmex_uncore_mbox_ops,
2234         .format_group           = &nhmex_uncore_mbox_format_group,
2235 };
2236
2237 static void nhmex_rbox_alter_er(struct intel_uncore_box *box, struct perf_event *event)
2238 {
2239         struct hw_perf_event *hwc = &event->hw;
2240         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2241
2242         /* adjust the main event selector and extra register index */
2243         if (reg1->idx % 2) {
2244                 reg1->idx--;
2245                 hwc->config -= 1 << NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
2246         } else {
2247                 reg1->idx++;
2248                 hwc->config += 1 << NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
2249         }
2250
2251         /* adjust extra register config */
2252         switch (reg1->idx % 6) {
2253         case 2:
2254                 /* shift the 8~15 bits to the 0~7 bits */
2255                 reg1->config >>= 8;
2256                 break;
2257         case 3:
2258                 /* shift the 0~7 bits to the 8~15 bits */
2259                 reg1->config <<= 8;
2260                 break;
2261         };
2262 }
2263
2264 /*
2265  * Each rbox has 4 event set which monitor PQI port 0~3 or 4~7.
2266  * An event set consists of 6 events, the 3rd and 4th events in
2267  * an event set use the same extra register. So an event set uses
2268  * 5 extra registers.
2269  */
2270 static struct event_constraint *
2271 nhmex_rbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
2272 {
2273         struct hw_perf_event *hwc = &event->hw;
2274         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2275         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
2276         struct intel_uncore_extra_reg *er;
2277         unsigned long flags;
2278         int idx, er_idx;
2279         u64 config1;
2280         bool ok = false;
2281
2282         if (!uncore_box_is_fake(box) && reg1->alloc)
2283                 return NULL;
2284
2285         idx = reg1->idx % 6;
2286         config1 = reg1->config;
2287 again:
2288         er_idx = idx;
2289         /* the 3rd and 4th events use the same extra register */
2290         if (er_idx > 2)
2291                 er_idx--;
2292         er_idx += (reg1->idx / 6) * 5;
2293
2294         er = &box->shared_regs[er_idx];
2295         raw_spin_lock_irqsave(&er->lock, flags);
2296         if (idx < 2) {
2297                 if (!atomic_read(&er->ref) || er->config == reg1->config) {
2298                         atomic_inc(&er->ref);
2299                         er->config = reg1->config;
2300                         ok = true;
2301                 }
2302         } else if (idx == 2 || idx == 3) {
2303                 /*
2304                  * these two events use different fields in a extra register,
2305                  * the 0~7 bits and the 8~15 bits respectively.
2306                  */
2307                 u64 mask = 0xff << ((idx - 2) * 8);
2308                 if (!__BITS_VALUE(atomic_read(&er->ref), idx - 2, 8) ||
2309                                 !((er->config ^ config1) & mask)) {
2310                         atomic_add(1 << ((idx - 2) * 8), &er->ref);
2311                         er->config &= ~mask;
2312                         er->config |= config1 & mask;
2313                         ok = true;
2314                 }
2315         } else {
2316                 if (!atomic_read(&er->ref) ||
2317                                 (er->config == (hwc->config >> 32) &&
2318                                  er->config1 == reg1->config &&
2319                                  er->config2 == reg2->config)) {
2320                         atomic_inc(&er->ref);
2321                         er->config = (hwc->config >> 32);
2322                         er->config1 = reg1->config;
2323                         er->config2 = reg2->config;
2324                         ok = true;
2325                 }
2326         }
2327         raw_spin_unlock_irqrestore(&er->lock, flags);
2328
2329         if (!ok) {
2330                 /*
2331                  * The Rbox events are always in pairs. The paired
2332                  * events are functional identical, but use different
2333                  * extra registers. If we failed to take an extra
2334                  * register, try the alternative.
2335                  */
2336                 if (idx % 2)
2337                         idx--;
2338                 else
2339                         idx++;
2340                 if (idx != reg1->idx % 6) {
2341                         if (idx == 2)
2342                                 config1 >>= 8;
2343                         else if (idx == 3)
2344                                 config1 <<= 8;
2345                         goto again;
2346                 }
2347         } else {
2348                 if (!uncore_box_is_fake(box)) {
2349                         if (idx != reg1->idx % 6)
2350                                 nhmex_rbox_alter_er(box, event);
2351                         reg1->alloc = 1;
2352                 }
2353                 return NULL;
2354         }
2355         return &constraint_empty;
2356 }
2357
2358 static void nhmex_rbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
2359 {
2360         struct intel_uncore_extra_reg *er;
2361         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2362         int idx, er_idx;
2363
2364         if (uncore_box_is_fake(box) || !reg1->alloc)
2365                 return;
2366
2367         idx = reg1->idx % 6;
2368         er_idx = idx;
2369         if (er_idx > 2)
2370                 er_idx--;
2371         er_idx += (reg1->idx / 6) * 5;
2372
2373         er = &box->shared_regs[er_idx];
2374         if (idx == 2 || idx == 3)
2375                 atomic_sub(1 << ((idx - 2) * 8), &er->ref);
2376         else
2377                 atomic_dec(&er->ref);
2378
2379         reg1->alloc = 0;
2380 }
2381
2382 static int nhmex_rbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2383 {
2384         struct hw_perf_event *hwc = &event->hw;
2385         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2386         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
2387         int idx;
2388
2389         idx = (event->hw.config & NHMEX_R_PMON_CTL_EV_SEL_MASK) >>
2390                 NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
2391         if (idx >= 0x18)
2392                 return -EINVAL;
2393
2394         reg1->idx = idx;
2395         reg1->config = event->attr.config1;
2396
2397         switch (idx % 6) {
2398         case 4:
2399         case 5:
2400                 hwc->config |= event->attr.config & (~0ULL << 32);
2401                 reg2->config = event->attr.config2;
2402                 break;
2403         };
2404         return 0;
2405 }
2406
2407 static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
2408 {
2409         struct hw_perf_event *hwc = &event->hw;
2410         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2411         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
2412         int idx, port;
2413
2414         idx = reg1->idx;
2415         port = idx / 6 + box->pmu->pmu_idx * 4;
2416
2417         switch (idx % 6) {
2418         case 0:
2419                 wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config);
2420                 break;
2421         case 1:
2422                 wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config);
2423                 break;
2424         case 2:
2425         case 3:
2426                 wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port),
2427                         uncore_shared_reg_config(box, 2 + (idx / 6) * 5));
2428                 break;
2429         case 4:
2430                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port),
2431                         hwc->config >> 32);
2432                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config);
2433                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config);
2434                 break;
2435         case 5:
2436                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port),
2437                         hwc->config >> 32);
2438                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config);
2439                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config);
2440                 break;
2441         };
2442
2443         wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
2444                 (hwc->config & NHMEX_R_PMON_CTL_EV_SEL_MASK));
2445 }
2446
2447 DEFINE_UNCORE_FORMAT_ATTR(xbr_mm_cfg, xbr_mm_cfg, "config:32-63");
2448 DEFINE_UNCORE_FORMAT_ATTR(xbr_match, xbr_match, "config1:0-63");
2449 DEFINE_UNCORE_FORMAT_ATTR(xbr_mask, xbr_mask, "config2:0-63");
2450 DEFINE_UNCORE_FORMAT_ATTR(qlx_cfg, qlx_cfg, "config1:0-15");
2451 DEFINE_UNCORE_FORMAT_ATTR(iperf_cfg, iperf_cfg, "config1:0-31");
2452
2453 static struct attribute *nhmex_uncore_rbox_formats_attr[] = {
2454         &format_attr_event5.attr,
2455         &format_attr_xbr_mm_cfg.attr,
2456         &format_attr_xbr_match.attr,
2457         &format_attr_xbr_mask.attr,
2458         &format_attr_qlx_cfg.attr,
2459         &format_attr_iperf_cfg.attr,
2460         NULL,
2461 };
2462
2463 static struct attribute_group nhmex_uncore_rbox_format_group = {
2464         .name = "format",
2465         .attrs = nhmex_uncore_rbox_formats_attr,
2466 };
2467
2468 static struct uncore_event_desc nhmex_uncore_rbox_events[] = {
2469         INTEL_UNCORE_EVENT_DESC(qpi0_flit_send,         "event=0x0,iperf_cfg=0x80000000"),
2470         INTEL_UNCORE_EVENT_DESC(qpi1_filt_send,         "event=0x6,iperf_cfg=0x80000000"),
2471         INTEL_UNCORE_EVENT_DESC(qpi0_idle_filt,         "event=0x0,iperf_cfg=0x40000000"),
2472         INTEL_UNCORE_EVENT_DESC(qpi1_idle_filt,         "event=0x6,iperf_cfg=0x40000000"),
2473         INTEL_UNCORE_EVENT_DESC(qpi0_date_response,     "event=0x0,iperf_cfg=0xc4"),
2474         INTEL_UNCORE_EVENT_DESC(qpi1_date_response,     "event=0x6,iperf_cfg=0xc4"),
2475         { /* end: all zeroes */ },
2476 };
2477
2478 static struct intel_uncore_ops nhmex_uncore_rbox_ops = {
2479         NHMEX_UNCORE_OPS_COMMON_INIT(),
2480         .enable_event           = nhmex_rbox_msr_enable_event,
2481         .hw_config              = nhmex_rbox_hw_config,
2482         .get_constraint         = nhmex_rbox_get_constraint,
2483         .put_constraint         = nhmex_rbox_put_constraint,
2484 };
2485
2486 static struct intel_uncore_type nhmex_uncore_rbox = {
2487         .name                   = "rbox",
2488         .num_counters           = 8,
2489         .num_boxes              = 2,
2490         .perf_ctr_bits          = 48,
2491         .event_ctl              = NHMEX_R_MSR_PMON_CTL0,
2492         .perf_ctr               = NHMEX_R_MSR_PMON_CNT0,
2493         .event_mask             = NHMEX_R_PMON_RAW_EVENT_MASK,
2494         .box_ctl                = NHMEX_R_MSR_GLOBAL_CTL,
2495         .msr_offset             = NHMEX_R_MSR_OFFSET,
2496         .pair_ctr_ctl           = 1,
2497         .num_shared_regs        = 20,
2498         .event_descs            = nhmex_uncore_rbox_events,
2499         .ops                    = &nhmex_uncore_rbox_ops,
2500         .format_group           = &nhmex_uncore_rbox_format_group
2501 };
2502
2503 static struct intel_uncore_type *nhmex_msr_uncores[] = {
2504         &nhmex_uncore_ubox,
2505         &nhmex_uncore_cbox,
2506         &nhmex_uncore_bbox,
2507         &nhmex_uncore_sbox,
2508         &nhmex_uncore_mbox,
2509         &nhmex_uncore_rbox,
2510         &nhmex_uncore_wbox,
2511         NULL,
2512 };
2513 /* end of Nehalem-EX uncore support */
2514
2515 static void uncore_assign_hw_event(struct intel_uncore_box *box, struct perf_event *event, int idx)
2516 {
2517         struct hw_perf_event *hwc = &event->hw;
2518
2519         hwc->idx = idx;
2520         hwc->last_tag = ++box->tags[idx];
2521
2522         if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
2523                 hwc->event_base = uncore_fixed_ctr(box);
2524                 hwc->config_base = uncore_fixed_ctl(box);
2525                 return;
2526         }
2527
2528         hwc->config_base = uncore_event_ctl(box, hwc->idx);
2529         hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
2530 }
2531
2532 static void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
2533 {
2534         u64 prev_count, new_count, delta;
2535         int shift;
2536
2537         if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
2538                 shift = 64 - uncore_fixed_ctr_bits(box);
2539         else
2540                 shift = 64 - uncore_perf_ctr_bits(box);
2541
2542         /* the hrtimer might modify the previous event value */
2543 again:
2544         prev_count = local64_read(&event->hw.prev_count);
2545         new_count = uncore_read_counter(box, event);
2546         if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
2547                 goto again;
2548
2549         delta = (new_count << shift) - (prev_count << shift);
2550         delta >>= shift;
2551
2552         local64_add(delta, &event->count);
2553 }
2554
2555 /*
2556  * The overflow interrupt is unavailable for SandyBridge-EP, is broken
2557  * for SandyBridge. So we use hrtimer to periodically poll the counter
2558  * to avoid overflow.
2559  */
2560 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
2561 {
2562         struct intel_uncore_box *box;
2563         unsigned long flags;
2564         int bit;
2565
2566         box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
2567         if (!box->n_active || box->cpu != smp_processor_id())
2568                 return HRTIMER_NORESTART;
2569         /*
2570          * disable local interrupt to prevent uncore_pmu_event_start/stop
2571          * to interrupt the update process
2572          */
2573         local_irq_save(flags);
2574
2575         for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
2576                 uncore_perf_event_update(box, box->events[bit]);
2577
2578         local_irq_restore(flags);
2579
2580         hrtimer_forward_now(hrtimer, ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL));
2581         return HRTIMER_RESTART;
2582 }
2583
2584 static void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
2585 {
2586         __hrtimer_start_range_ns(&box->hrtimer,
2587                         ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL), 0,
2588                         HRTIMER_MODE_REL_PINNED, 0);
2589 }
2590
2591 static void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
2592 {
2593         hrtimer_cancel(&box->hrtimer);
2594 }
2595
2596 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
2597 {
2598         hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2599         box->hrtimer.function = uncore_pmu_hrtimer;
2600 }
2601
2602 struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, int cpu)
2603 {
2604         struct intel_uncore_box *box;
2605         int i, size;
2606
2607         size = sizeof(*box) + type->num_shared_regs * sizeof(struct intel_uncore_extra_reg);
2608
2609         box = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO, cpu_to_node(cpu));
2610         if (!box)
2611                 return NULL;
2612
2613         for (i = 0; i < type->num_shared_regs; i++)
2614                 raw_spin_lock_init(&box->shared_regs[i].lock);
2615
2616         uncore_pmu_init_hrtimer(box);
2617         atomic_set(&box->refcnt, 1);
2618         box->cpu = -1;
2619         box->phys_id = -1;
2620
2621         return box;
2622 }
2623
2624 static struct intel_uncore_box *
2625 uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
2626 {
2627         struct intel_uncore_box *box;
2628
2629         box = *per_cpu_ptr(pmu->box, cpu);
2630         if (box)
2631                 return box;
2632
2633         raw_spin_lock(&uncore_box_lock);
2634         list_for_each_entry(box, &pmu->box_list, list) {
2635                 if (box->phys_id == topology_physical_package_id(cpu)) {
2636                         atomic_inc(&box->refcnt);
2637                         *per_cpu_ptr(pmu->box, cpu) = box;
2638                         break;
2639                 }
2640         }
2641         raw_spin_unlock(&uncore_box_lock);
2642
2643         return *per_cpu_ptr(pmu->box, cpu);
2644 }
2645
2646 static struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event)
2647 {
2648         return container_of(event->pmu, struct intel_uncore_pmu, pmu);
2649 }
2650
2651 static struct intel_uncore_box *uncore_event_to_box(struct perf_event *event)
2652 {
2653         /*
2654          * perf core schedules event on the basis of cpu, uncore events are
2655          * collected by one of the cpus inside a physical package.
2656          */
2657         return uncore_pmu_to_box(uncore_event_to_pmu(event), smp_processor_id());
2658 }
2659
2660 static int
2661 uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader, bool dogrp)
2662 {
2663         struct perf_event *event;
2664         int n, max_count;
2665
2666         max_count = box->pmu->type->num_counters;
2667         if (box->pmu->type->fixed_ctl)
2668                 max_count++;
2669
2670         if (box->n_events >= max_count)
2671                 return -EINVAL;
2672
2673         n = box->n_events;
2674         box->event_list[n] = leader;
2675         n++;
2676         if (!dogrp)
2677                 return n;
2678
2679         list_for_each_entry(event, &leader->sibling_list, group_entry) {
2680                 if (event->state <= PERF_EVENT_STATE_OFF)
2681                         continue;
2682
2683                 if (n >= max_count)
2684                         return -EINVAL;
2685
2686                 box->event_list[n] = event;
2687                 n++;
2688         }
2689         return n;
2690 }
2691
2692 static struct event_constraint *
2693 uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
2694 {
2695         struct intel_uncore_type *type = box->pmu->type;
2696         struct event_constraint *c;
2697
2698         if (type->ops->get_constraint) {
2699                 c = type->ops->get_constraint(box, event);
2700                 if (c)
2701                         return c;
2702         }
2703
2704         if (event->hw.config == ~0ULL)
2705                 return &constraint_fixed;
2706
2707         if (type->constraints) {
2708                 for_each_event_constraint(c, type->constraints) {
2709                         if ((event->hw.config & c->cmask) == c->code)
2710                                 return c;
2711                 }
2712         }
2713
2714         return &type->unconstrainted;
2715 }
2716
2717 static void uncore_put_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
2718 {
2719         if (box->pmu->type->ops->put_constraint)
2720                 box->pmu->type->ops->put_constraint(box, event);
2721 }
2722
2723 static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
2724 {
2725         unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
2726         struct event_constraint *c, *constraints[UNCORE_PMC_IDX_MAX];
2727         int i, wmin, wmax, ret = 0;
2728         struct hw_perf_event *hwc;
2729
2730         bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
2731
2732         for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
2733                 c = uncore_get_event_constraint(box, box->event_list[i]);
2734                 constraints[i] = c;
2735                 wmin = min(wmin, c->weight);
2736                 wmax = max(wmax, c->weight);
2737         }
2738
2739         /* fastpath, try to reuse previous register */
2740         for (i = 0; i < n; i++) {
2741                 hwc = &box->event_list[i]->hw;
2742                 c = constraints[i];
2743
2744                 /* never assigned */
2745                 if (hwc->idx == -1)
2746                         break;
2747
2748                 /* constraint still honored */
2749                 if (!test_bit(hwc->idx, c->idxmsk))
2750                         break;
2751
2752                 /* not already used */
2753                 if (test_bit(hwc->idx, used_mask))
2754                         break;
2755
2756                 __set_bit(hwc->idx, used_mask);
2757                 if (assign)
2758                         assign[i] = hwc->idx;
2759         }
2760         /* slow path */
2761         if (i != n)
2762                 ret = perf_assign_events(constraints, n, wmin, wmax, assign);
2763
2764         if (!assign || ret) {
2765                 for (i = 0; i < n; i++)
2766                         uncore_put_event_constraint(box, box->event_list[i]);
2767         }
2768         return ret ? -EINVAL : 0;
2769 }
2770
2771 static void uncore_pmu_event_start(struct perf_event *event, int flags)
2772 {
2773         struct intel_uncore_box *box = uncore_event_to_box(event);
2774         int idx = event->hw.idx;
2775
2776         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
2777                 return;
2778
2779         if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
2780                 return;
2781
2782         event->hw.state = 0;
2783         box->events[idx] = event;
2784         box->n_active++;
2785         __set_bit(idx, box->active_mask);
2786
2787         local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
2788         uncore_enable_event(box, event);
2789
2790         if (box->n_active == 1) {
2791                 uncore_enable_box(box);
2792                 uncore_pmu_start_hrtimer(box);
2793         }
2794 }
2795
2796 static void uncore_pmu_event_stop(struct perf_event *event, int flags)
2797 {
2798         struct intel_uncore_box *box = uncore_event_to_box(event);
2799         struct hw_perf_event *hwc = &event->hw;
2800
2801         if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
2802                 uncore_disable_event(box, event);
2803                 box->n_active--;
2804                 box->events[hwc->idx] = NULL;
2805                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
2806                 hwc->state |= PERF_HES_STOPPED;
2807
2808                 if (box->n_active == 0) {
2809                         uncore_disable_box(box);
2810                         uncore_pmu_cancel_hrtimer(box);
2811                 }
2812         }
2813
2814         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
2815                 /*
2816                  * Drain the remaining delta count out of a event
2817                  * that we are disabling:
2818                  */
2819                 uncore_perf_event_update(box, event);
2820                 hwc->state |= PERF_HES_UPTODATE;
2821         }
2822 }
2823
2824 static int uncore_pmu_event_add(struct perf_event *event, int flags)
2825 {
2826         struct intel_uncore_box *box = uncore_event_to_box(event);
2827         struct hw_perf_event *hwc = &event->hw;
2828         int assign[UNCORE_PMC_IDX_MAX];
2829         int i, n, ret;
2830
2831         if (!box)
2832                 return -ENODEV;
2833
2834         ret = n = uncore_collect_events(box, event, false);
2835         if (ret < 0)
2836                 return ret;
2837
2838         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
2839         if (!(flags & PERF_EF_START))
2840                 hwc->state |= PERF_HES_ARCH;
2841
2842         ret = uncore_assign_events(box, assign, n);
2843         if (ret)
2844                 return ret;
2845
2846         /* save events moving to new counters */
2847         for (i = 0; i < box->n_events; i++) {
2848                 event = box->event_list[i];
2849                 hwc = &event->hw;
2850
2851                 if (hwc->idx == assign[i] &&
2852                         hwc->last_tag == box->tags[assign[i]])
2853                         continue;
2854                 /*
2855                  * Ensure we don't accidentally enable a stopped
2856                  * counter simply because we rescheduled.
2857                  */
2858                 if (hwc->state & PERF_HES_STOPPED)
2859                         hwc->state |= PERF_HES_ARCH;
2860
2861                 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
2862         }
2863
2864         /* reprogram moved events into new counters */
2865         for (i = 0; i < n; i++) {
2866                 event = box->event_list[i];
2867                 hwc = &event->hw;
2868
2869                 if (hwc->idx != assign[i] ||
2870                         hwc->last_tag != box->tags[assign[i]])
2871                         uncore_assign_hw_event(box, event, assign[i]);
2872                 else if (i < box->n_events)
2873                         continue;
2874
2875                 if (hwc->state & PERF_HES_ARCH)
2876                         continue;
2877
2878                 uncore_pmu_event_start(event, 0);
2879         }
2880         box->n_events = n;
2881
2882         return 0;
2883 }
2884
2885 static void uncore_pmu_event_del(struct perf_event *event, int flags)
2886 {
2887         struct intel_uncore_box *box = uncore_event_to_box(event);
2888         int i;
2889
2890         uncore_pmu_event_stop(event, PERF_EF_UPDATE);
2891
2892         for (i = 0; i < box->n_events; i++) {
2893                 if (event == box->event_list[i]) {
2894                         uncore_put_event_constraint(box, event);
2895
2896                         while (++i < box->n_events)
2897                                 box->event_list[i - 1] = box->event_list[i];
2898
2899                         --box->n_events;
2900                         break;
2901                 }
2902         }
2903
2904         event->hw.idx = -1;
2905         event->hw.last_tag = ~0ULL;
2906 }
2907
2908 static void uncore_pmu_event_read(struct perf_event *event)
2909 {
2910         struct intel_uncore_box *box = uncore_event_to_box(event);
2911         uncore_perf_event_update(box, event);
2912 }
2913
2914 /*
2915  * validation ensures the group can be loaded onto the
2916  * PMU if it was the only group available.
2917  */
2918 static int uncore_validate_group(struct intel_uncore_pmu *pmu,
2919                                 struct perf_event *event)
2920 {
2921         struct perf_event *leader = event->group_leader;
2922         struct intel_uncore_box *fake_box;
2923         int ret = -EINVAL, n;
2924
2925         fake_box = uncore_alloc_box(pmu->type, smp_processor_id());
2926         if (!fake_box)
2927                 return -ENOMEM;
2928
2929         fake_box->pmu = pmu;
2930         /*
2931          * the event is not yet connected with its
2932          * siblings therefore we must first collect
2933          * existing siblings, then add the new event
2934          * before we can simulate the scheduling
2935          */
2936         n = uncore_collect_events(fake_box, leader, true);
2937         if (n < 0)
2938                 goto out;
2939
2940         fake_box->n_events = n;
2941         n = uncore_collect_events(fake_box, event, false);
2942         if (n < 0)
2943                 goto out;
2944
2945         fake_box->n_events = n;
2946
2947         ret = uncore_assign_events(fake_box, NULL, n);
2948 out:
2949         kfree(fake_box);
2950         return ret;
2951 }
2952
2953 static int uncore_pmu_event_init(struct perf_event *event)
2954 {
2955         struct intel_uncore_pmu *pmu;
2956         struct intel_uncore_box *box;
2957         struct hw_perf_event *hwc = &event->hw;
2958         int ret;
2959
2960         if (event->attr.type != event->pmu->type)
2961                 return -ENOENT;
2962
2963         pmu = uncore_event_to_pmu(event);
2964         /* no device found for this pmu */
2965         if (pmu->func_id < 0)
2966                 return -ENOENT;
2967
2968         /*
2969          * Uncore PMU does measure at all privilege level all the time.
2970          * So it doesn't make sense to specify any exclude bits.
2971          */
2972         if (event->attr.exclude_user || event->attr.exclude_kernel ||
2973                         event->attr.exclude_hv || event->attr.exclude_idle)
2974                 return -EINVAL;
2975
2976         /* Sampling not supported yet */
2977         if (hwc->sample_period)
2978                 return -EINVAL;
2979
2980         /*
2981          * Place all uncore events for a particular physical package
2982          * onto a single cpu
2983          */
2984         if (event->cpu < 0)
2985                 return -EINVAL;
2986         box = uncore_pmu_to_box(pmu, event->cpu);
2987         if (!box || box->cpu < 0)
2988                 return -EINVAL;
2989         event->cpu = box->cpu;
2990
2991         event->hw.idx = -1;
2992         event->hw.last_tag = ~0ULL;
2993         event->hw.extra_reg.idx = EXTRA_REG_NONE;
2994         event->hw.branch_reg.idx = EXTRA_REG_NONE;
2995
2996         if (event->attr.config == UNCORE_FIXED_EVENT) {
2997                 /* no fixed counter */
2998                 if (!pmu->type->fixed_ctl)
2999                         return -EINVAL;
3000                 /*
3001                  * if there is only one fixed counter, only the first pmu
3002                  * can access the fixed counter
3003                  */
3004                 if (pmu->type->single_fixed && pmu->pmu_idx > 0)
3005                         return -EINVAL;
3006                 hwc->config = ~0ULL;
3007         } else {
3008                 hwc->config = event->attr.config & pmu->type->event_mask;
3009                 if (pmu->type->ops->hw_config) {
3010                         ret = pmu->type->ops->hw_config(box, event);
3011                         if (ret)
3012                                 return ret;
3013                 }
3014         }
3015
3016         if (event->group_leader != event)
3017                 ret = uncore_validate_group(pmu, event);
3018         else
3019                 ret = 0;
3020
3021         return ret;
3022 }
3023
3024 static ssize_t uncore_get_attr_cpumask(struct device *dev,
3025                                 struct device_attribute *attr, char *buf)
3026 {
3027         int n = cpulist_scnprintf(buf, PAGE_SIZE - 2, &uncore_cpu_mask);
3028
3029         buf[n++] = '\n';
3030         buf[n] = '\0';
3031         return n;
3032 }
3033
3034 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
3035
3036 static struct attribute *uncore_pmu_attrs[] = {
3037         &dev_attr_cpumask.attr,
3038         NULL,
3039 };
3040
3041 static struct attribute_group uncore_pmu_attr_group = {
3042         .attrs = uncore_pmu_attrs,
3043 };
3044
3045 static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
3046 {
3047         int ret;
3048
3049         pmu->pmu = (struct pmu) {
3050                 .attr_groups    = pmu->type->attr_groups,
3051                 .task_ctx_nr    = perf_invalid_context,
3052                 .event_init     = uncore_pmu_event_init,
3053                 .add            = uncore_pmu_event_add,
3054                 .del            = uncore_pmu_event_del,
3055                 .start          = uncore_pmu_event_start,
3056                 .stop           = uncore_pmu_event_stop,
3057                 .read           = uncore_pmu_event_read,
3058         };
3059
3060         if (pmu->type->num_boxes == 1) {
3061                 if (strlen(pmu->type->name) > 0)
3062                         sprintf(pmu->name, "uncore_%s", pmu->type->name);
3063                 else
3064                         sprintf(pmu->name, "uncore");
3065         } else {
3066                 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
3067                         pmu->pmu_idx);
3068         }
3069
3070         ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
3071         return ret;
3072 }
3073
3074 static void __init uncore_type_exit(struct intel_uncore_type *type)
3075 {
3076         int i;
3077
3078         for (i = 0; i < type->num_boxes; i++)
3079                 free_percpu(type->pmus[i].box);
3080         kfree(type->pmus);
3081         type->pmus = NULL;
3082         kfree(type->events_group);
3083         type->events_group = NULL;
3084 }
3085
3086 static void __init uncore_types_exit(struct intel_uncore_type **types)
3087 {
3088         int i;
3089         for (i = 0; types[i]; i++)
3090                 uncore_type_exit(types[i]);
3091 }
3092
3093 static int __init uncore_type_init(struct intel_uncore_type *type)
3094 {
3095         struct intel_uncore_pmu *pmus;
3096         struct attribute_group *attr_group;
3097         struct attribute **attrs;
3098         int i, j;
3099
3100         pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
3101         if (!pmus)
3102                 return -ENOMEM;
3103
3104         type->unconstrainted = (struct event_constraint)
3105                 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
3106                                 0, type->num_counters, 0, 0);
3107
3108         for (i = 0; i < type->num_boxes; i++) {
3109                 pmus[i].func_id = -1;
3110                 pmus[i].pmu_idx = i;
3111                 pmus[i].type = type;
3112                 INIT_LIST_HEAD(&pmus[i].box_list);
3113                 pmus[i].box = alloc_percpu(struct intel_uncore_box *);
3114                 if (!pmus[i].box)
3115                         goto fail;
3116         }
3117
3118         if (type->event_descs) {
3119                 i = 0;
3120                 while (type->event_descs[i].attr.attr.name)
3121                         i++;
3122
3123                 attr_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
3124                                         sizeof(*attr_group), GFP_KERNEL);
3125                 if (!attr_group)
3126                         goto fail;
3127
3128                 attrs = (struct attribute **)(attr_group + 1);
3129                 attr_group->name = "events";
3130                 attr_group->attrs = attrs;
3131
3132                 for (j = 0; j < i; j++)
3133                         attrs[j] = &type->event_descs[j].attr.attr;
3134
3135                 type->events_group = attr_group;
3136         }
3137
3138         type->pmu_group = &uncore_pmu_attr_group;
3139         type->pmus = pmus;
3140         return 0;
3141 fail:
3142         uncore_type_exit(type);
3143         return -ENOMEM;
3144 }
3145
3146 static int __init uncore_types_init(struct intel_uncore_type **types)
3147 {
3148         int i, ret;
3149
3150         for (i = 0; types[i]; i++) {
3151                 ret = uncore_type_init(types[i]);
3152                 if (ret)
3153                         goto fail;
3154         }
3155         return 0;
3156 fail:
3157         while (--i >= 0)
3158                 uncore_type_exit(types[i]);
3159         return ret;
3160 }
3161
3162 static struct pci_driver *uncore_pci_driver;
3163 static bool pcidrv_registered;
3164
3165 /*
3166  * add a pci uncore device
3167  */
3168 static int uncore_pci_add(struct intel_uncore_type *type, struct pci_dev *pdev)
3169 {
3170         struct intel_uncore_pmu *pmu;
3171         struct intel_uncore_box *box;
3172         int i, phys_id;
3173
3174         phys_id = pcibus_to_physid[pdev->bus->number];
3175         if (phys_id < 0)
3176                 return -ENODEV;
3177
3178         box = uncore_alloc_box(type, 0);
3179         if (!box)
3180                 return -ENOMEM;
3181
3182         /*
3183          * for performance monitoring unit with multiple boxes,
3184          * each box has a different function id.
3185          */
3186         for (i = 0; i < type->num_boxes; i++) {
3187                 pmu = &type->pmus[i];
3188                 if (pmu->func_id == pdev->devfn)
3189                         break;
3190                 if (pmu->func_id < 0) {
3191                         pmu->func_id = pdev->devfn;
3192                         break;
3193                 }
3194                 pmu = NULL;
3195         }
3196
3197         if (!pmu) {
3198                 kfree(box);
3199                 return -EINVAL;
3200         }
3201
3202         box->phys_id = phys_id;
3203         box->pci_dev = pdev;
3204         box->pmu = pmu;
3205         uncore_box_init(box);
3206         pci_set_drvdata(pdev, box);
3207
3208         raw_spin_lock(&uncore_box_lock);
3209         list_add_tail(&box->list, &pmu->box_list);
3210         raw_spin_unlock(&uncore_box_lock);
3211
3212         return 0;
3213 }
3214
3215 static void uncore_pci_remove(struct pci_dev *pdev)
3216 {
3217         struct intel_uncore_box *box = pci_get_drvdata(pdev);
3218         struct intel_uncore_pmu *pmu = box->pmu;
3219         int cpu, phys_id = pcibus_to_physid[pdev->bus->number];
3220
3221         if (WARN_ON_ONCE(phys_id != box->phys_id))
3222                 return;
3223
3224         pci_set_drvdata(pdev, NULL);
3225
3226         raw_spin_lock(&uncore_box_lock);
3227         list_del(&box->list);
3228         raw_spin_unlock(&uncore_box_lock);
3229
3230         for_each_possible_cpu(cpu) {
3231                 if (*per_cpu_ptr(pmu->box, cpu) == box) {
3232                         *per_cpu_ptr(pmu->box, cpu) = NULL;
3233                         atomic_dec(&box->refcnt);
3234                 }
3235         }
3236
3237         WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
3238         kfree(box);
3239 }
3240
3241 static int uncore_pci_probe(struct pci_dev *pdev,
3242                             const struct pci_device_id *id)
3243 {
3244         return uncore_pci_add(pci_uncores[id->driver_data], pdev);
3245 }
3246
3247 static int __init uncore_pci_init(void)
3248 {
3249         int ret;
3250
3251         switch (boot_cpu_data.x86_model) {
3252         case 45: /* Sandy Bridge-EP */
3253                 ret = snbep_pci2phy_map_init(0x3ce0);
3254                 if (ret)
3255                         return ret;
3256                 pci_uncores = snbep_pci_uncores;
3257                 uncore_pci_driver = &snbep_uncore_pci_driver;
3258                 break;
3259         case 62: /* IvyTown */
3260                 ret = snbep_pci2phy_map_init(0x0e1e);
3261                 if (ret)
3262                         return ret;
3263                 pci_uncores = ivt_pci_uncores;
3264                 uncore_pci_driver = &ivt_uncore_pci_driver;
3265                 break;
3266         default:
3267                 return 0;
3268         }
3269
3270         ret = uncore_types_init(pci_uncores);
3271         if (ret)
3272                 return ret;
3273
3274         uncore_pci_driver->probe = uncore_pci_probe;
3275         uncore_pci_driver->remove = uncore_pci_remove;
3276
3277         ret = pci_register_driver(uncore_pci_driver);
3278         if (ret == 0)
3279                 pcidrv_registered = true;
3280         else
3281                 uncore_types_exit(pci_uncores);
3282
3283         return ret;
3284 }
3285
3286 static void __init uncore_pci_exit(void)
3287 {
3288         if (pcidrv_registered) {
3289                 pcidrv_registered = false;
3290                 pci_unregister_driver(uncore_pci_driver);
3291                 uncore_types_exit(pci_uncores);
3292         }
3293 }
3294
3295 /* CPU hot plug/unplug are serialized by cpu_add_remove_lock mutex */
3296 static LIST_HEAD(boxes_to_free);
3297
3298 static void __cpuinit uncore_kfree_boxes(void)
3299 {
3300         struct intel_uncore_box *box;
3301
3302         while (!list_empty(&boxes_to_free)) {
3303                 box = list_entry(boxes_to_free.next,
3304                                  struct intel_uncore_box, list);
3305                 list_del(&box->list);
3306                 kfree(box);
3307         }
3308 }
3309
3310 static void __cpuinit uncore_cpu_dying(int cpu)
3311 {
3312         struct intel_uncore_type *type;
3313         struct intel_uncore_pmu *pmu;
3314         struct intel_uncore_box *box;
3315         int i, j;
3316
3317         for (i = 0; msr_uncores[i]; i++) {
3318                 type = msr_uncores[i];
3319                 for (j = 0; j < type->num_boxes; j++) {
3320                         pmu = &type->pmus[j];
3321                         box = *per_cpu_ptr(pmu->box, cpu);
3322                         *per_cpu_ptr(pmu->box, cpu) = NULL;
3323                         if (box && atomic_dec_and_test(&box->refcnt))
3324                                 list_add(&box->list, &boxes_to_free);
3325                 }
3326         }
3327 }
3328
3329 static int __cpuinit uncore_cpu_starting(int cpu)
3330 {
3331         struct intel_uncore_type *type;
3332         struct intel_uncore_pmu *pmu;
3333         struct intel_uncore_box *box, *exist;
3334         int i, j, k, phys_id;
3335
3336         phys_id = topology_physical_package_id(cpu);
3337
3338         for (i = 0; msr_uncores[i]; i++) {
3339                 type = msr_uncores[i];
3340                 for (j = 0; j < type->num_boxes; j++) {
3341                         pmu = &type->pmus[j];
3342                         box = *per_cpu_ptr(pmu->box, cpu);
3343                         /* called by uncore_cpu_init? */
3344                         if (box && box->phys_id >= 0) {
3345                                 uncore_box_init(box);
3346                                 continue;
3347                         }
3348
3349                         for_each_online_cpu(k) {
3350                                 exist = *per_cpu_ptr(pmu->box, k);
3351                                 if (exist && exist->phys_id == phys_id) {
3352                                         atomic_inc(&exist->refcnt);
3353                                         *per_cpu_ptr(pmu->box, cpu) = exist;
3354                                         if (box) {
3355                                                 list_add(&box->list,
3356                                                          &boxes_to_free);
3357                                                 box = NULL;
3358                                         }
3359                                         break;
3360                                 }
3361                         }
3362
3363                         if (box) {
3364                                 box->phys_id = phys_id;
3365                                 uncore_box_init(box);
3366                         }
3367                 }
3368         }
3369         return 0;
3370 }
3371
3372 static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id)
3373 {
3374         struct intel_uncore_type *type;
3375         struct intel_uncore_pmu *pmu;
3376         struct intel_uncore_box *box;
3377         int i, j;
3378
3379         for (i = 0; msr_uncores[i]; i++) {
3380                 type = msr_uncores[i];
3381                 for (j = 0; j < type->num_boxes; j++) {
3382                         pmu = &type->pmus[j];
3383                         if (pmu->func_id < 0)
3384                                 pmu->func_id = j;
3385
3386                         box = uncore_alloc_box(type, cpu);
3387                         if (!box)
3388                                 return -ENOMEM;
3389
3390                         box->pmu = pmu;
3391                         box->phys_id = phys_id;
3392                         *per_cpu_ptr(pmu->box, cpu) = box;
3393                 }
3394         }
3395         return 0;
3396 }
3397
3398 static void __cpuinit
3399 uncore_change_context(struct intel_uncore_type **uncores, int old_cpu, int new_cpu)
3400 {
3401         struct intel_uncore_type *type;
3402         struct intel_uncore_pmu *pmu;
3403         struct intel_uncore_box *box;
3404         int i, j;
3405
3406         for (i = 0; uncores[i]; i++) {
3407                 type = uncores[i];
3408                 for (j = 0; j < type->num_boxes; j++) {
3409                         pmu = &type->pmus[j];
3410                         if (old_cpu < 0)
3411                                 box = uncore_pmu_to_box(pmu, new_cpu);
3412                         else
3413                                 box = uncore_pmu_to_box(pmu, old_cpu);
3414                         if (!box)
3415                                 continue;
3416
3417                         if (old_cpu < 0) {
3418                                 WARN_ON_ONCE(box->cpu != -1);
3419                                 box->cpu = new_cpu;
3420                                 continue;
3421                         }
3422
3423                         WARN_ON_ONCE(box->cpu != old_cpu);
3424                         if (new_cpu >= 0) {
3425                                 uncore_pmu_cancel_hrtimer(box);
3426                                 perf_pmu_migrate_context(&pmu->pmu,
3427                                                 old_cpu, new_cpu);
3428                                 box->cpu = new_cpu;
3429                         } else {
3430                                 box->cpu = -1;
3431                         }
3432                 }
3433         }
3434 }
3435
3436 static void __cpuinit uncore_event_exit_cpu(int cpu)
3437 {
3438         int i, phys_id, target;
3439
3440         /* if exiting cpu is used for collecting uncore events */
3441         if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
3442                 return;
3443
3444         /* find a new cpu to collect uncore events */
3445         phys_id = topology_physical_package_id(cpu);
3446         target = -1;
3447         for_each_online_cpu(i) {
3448                 if (i == cpu)
3449                         continue;
3450                 if (phys_id == topology_physical_package_id(i)) {
3451                         target = i;
3452                         break;
3453                 }
3454         }
3455
3456         /* migrate uncore events to the new cpu */
3457         if (target >= 0)
3458                 cpumask_set_cpu(target, &uncore_cpu_mask);
3459
3460         uncore_change_context(msr_uncores, cpu, target);
3461         uncore_change_context(pci_uncores, cpu, target);
3462 }
3463
3464 static void __cpuinit uncore_event_init_cpu(int cpu)
3465 {
3466         int i, phys_id;
3467
3468         phys_id = topology_physical_package_id(cpu);
3469         for_each_cpu(i, &uncore_cpu_mask) {
3470                 if (phys_id == topology_physical_package_id(i))
3471                         return;
3472         }
3473
3474         cpumask_set_cpu(cpu, &uncore_cpu_mask);
3475
3476         uncore_change_context(msr_uncores, -1, cpu);
3477         uncore_change_context(pci_uncores, -1, cpu);
3478 }
3479
3480 static int
3481  __cpuinit uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
3482 {
3483         unsigned int cpu = (long)hcpu;
3484
3485         /* allocate/free data structure for uncore box */
3486         switch (action & ~CPU_TASKS_FROZEN) {
3487         case CPU_UP_PREPARE:
3488                 uncore_cpu_prepare(cpu, -1);
3489                 break;
3490         case CPU_STARTING:
3491                 uncore_cpu_starting(cpu);
3492                 break;
3493         case CPU_UP_CANCELED:
3494         case CPU_DYING:
3495                 uncore_cpu_dying(cpu);
3496                 break;
3497         case CPU_ONLINE:
3498         case CPU_DEAD:
3499                 uncore_kfree_boxes();
3500                 break;
3501         default:
3502                 break;
3503         }
3504
3505         /* select the cpu that collects uncore events */
3506         switch (action & ~CPU_TASKS_FROZEN) {
3507         case CPU_DOWN_FAILED:
3508         case CPU_STARTING:
3509                 uncore_event_init_cpu(cpu);
3510                 break;
3511         case CPU_DOWN_PREPARE:
3512                 uncore_event_exit_cpu(cpu);
3513                 break;
3514         default:
3515                 break;
3516         }
3517
3518         return NOTIFY_OK;
3519 }
3520
3521 static struct notifier_block uncore_cpu_nb __cpuinitdata = {
3522         .notifier_call  = uncore_cpu_notifier,
3523         /*
3524          * to migrate uncore events, our notifier should be executed
3525          * before perf core's notifier.
3526          */
3527         .priority       = CPU_PRI_PERF + 1,
3528 };
3529
3530 static void __init uncore_cpu_setup(void *dummy)
3531 {
3532         uncore_cpu_starting(smp_processor_id());
3533 }
3534
3535 static int __init uncore_cpu_init(void)
3536 {
3537         int ret, cpu, max_cores;
3538
3539         max_cores = boot_cpu_data.x86_max_cores;
3540         switch (boot_cpu_data.x86_model) {
3541         case 26: /* Nehalem */
3542         case 30:
3543         case 37: /* Westmere */
3544         case 44:
3545                 msr_uncores = nhm_msr_uncores;
3546                 break;
3547         case 42: /* Sandy Bridge */
3548         case 58: /* Ivy Bridge */
3549                 if (snb_uncore_cbox.num_boxes > max_cores)
3550                         snb_uncore_cbox.num_boxes = max_cores;
3551                 msr_uncores = snb_msr_uncores;
3552                 break;
3553         case 45: /* Sandy Bridge-EP */
3554                 if (snbep_uncore_cbox.num_boxes > max_cores)
3555                         snbep_uncore_cbox.num_boxes = max_cores;
3556                 msr_uncores = snbep_msr_uncores;
3557                 break;
3558         case 46: /* Nehalem-EX */
3559                 uncore_nhmex = true;
3560         case 47: /* Westmere-EX aka. Xeon E7 */
3561                 if (!uncore_nhmex)
3562                         nhmex_uncore_mbox.event_descs = wsmex_uncore_mbox_events;
3563                 if (nhmex_uncore_cbox.num_boxes > max_cores)
3564                         nhmex_uncore_cbox.num_boxes = max_cores;
3565                 msr_uncores = nhmex_msr_uncores;
3566                 break;
3567         case 62: /* IvyTown */
3568                 if (ivt_uncore_cbox.num_boxes > max_cores)
3569                         ivt_uncore_cbox.num_boxes = max_cores;
3570                 msr_uncores = ivt_msr_uncores;
3571                 break;
3572
3573         default:
3574                 return 0;
3575         }
3576
3577         ret = uncore_types_init(msr_uncores);
3578         if (ret)
3579                 return ret;
3580
3581         get_online_cpus();
3582
3583         for_each_online_cpu(cpu) {
3584                 int i, phys_id = topology_physical_package_id(cpu);
3585
3586                 for_each_cpu(i, &uncore_cpu_mask) {
3587                         if (phys_id == topology_physical_package_id(i)) {
3588                                 phys_id = -1;
3589                                 break;
3590                         }
3591                 }
3592                 if (phys_id < 0)
3593                         continue;
3594
3595                 uncore_cpu_prepare(cpu, phys_id);
3596                 uncore_event_init_cpu(cpu);
3597         }
3598         on_each_cpu(uncore_cpu_setup, NULL, 1);
3599
3600         register_cpu_notifier(&uncore_cpu_nb);
3601
3602         put_online_cpus();
3603
3604         return 0;
3605 }
3606
3607 static int __init uncore_pmus_register(void)
3608 {
3609         struct intel_uncore_pmu *pmu;
3610         struct intel_uncore_type *type;
3611         int i, j;
3612
3613         for (i = 0; msr_uncores[i]; i++) {
3614                 type = msr_uncores[i];
3615                 for (j = 0; j < type->num_boxes; j++) {
3616                         pmu = &type->pmus[j];
3617                         uncore_pmu_register(pmu);
3618                 }
3619         }
3620
3621         for (i = 0; pci_uncores[i]; i++) {
3622                 type = pci_uncores[i];
3623                 for (j = 0; j < type->num_boxes; j++) {
3624                         pmu = &type->pmus[j];
3625                         uncore_pmu_register(pmu);
3626                 }
3627         }
3628
3629         return 0;
3630 }
3631
3632 static int __init intel_uncore_init(void)
3633 {
3634         int ret;
3635
3636         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
3637                 return -ENODEV;
3638
3639         if (cpu_has_hypervisor)
3640                 return -ENODEV;
3641
3642         ret = uncore_pci_init();
3643         if (ret)
3644                 goto fail;
3645         ret = uncore_cpu_init();
3646         if (ret) {
3647                 uncore_pci_exit();
3648                 goto fail;
3649         }
3650
3651         uncore_pmus_register();
3652         return 0;
3653 fail:
3654         return ret;
3655 }
3656 device_initcall(intel_uncore_init);