2 * Copyright (C) ARM Limited 2010-2015. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
11 /* gator_events_armvX.c is used for Linux 2.6.x */
12 #if GATOR_PERF_PMU_SUPPORT
16 #include <linux/of_address.h>
18 #include <linux/perf_event.h>
19 #include <linux/slab.h>
21 extern bool event_based_sampling;
23 /* Maximum number of per-core counters - currently reserves enough space for two full hardware PMUs for big.LITTLE */
28 /* Maximum number of uncore counters */
29 /* + 1 for the cci-400 cycles counter */
30 /* cci-500 has no cycles counter */
31 /* + 1 for the CCN-5xx cycles counter */
32 #define UCCNT (CCI_400 + 1 + CCI_500 + CCN_5XX + 1)
34 /* Default to 0 if unable to probe the revision which was the previous behavior */
35 #define DEFAULT_CCI_REVISION 0
37 /* A gator_attr is needed for every counter */
39 /* Set once in gator_events_perf_pmu_*_init - the name of the event in the gatorfs */
41 /* Exposed in gatorfs - set by gatord to enable this counter */
42 unsigned long enabled;
43 /* Set once in gator_events_perf_pmu_*_init - the perf type to use, see perf_type_id in the perf_event.h header file. */
45 /* Exposed in gatorfs - set by gatord to select the event to collect */
47 /* Exposed in gatorfs - set by gatord with the sample period to use and enable EBS for this counter */
49 /* Exposed as read only in gatorfs - set once in __attr_init as the key to use in the APC data */
53 /* Per-core counter attributes */
54 static struct gator_attr attrs[CNTMAX];
55 /* Number of initialized per-core counters */
56 static int attr_count;
57 /* Uncore counter attributes */
58 static struct gator_attr uc_attrs[UCCNT];
59 /* Number of initialized uncore counters */
60 static int uc_attr_count;
67 struct perf_event *pevent;
68 struct perf_event_attr *pevent_attr;
71 static DEFINE_PER_CPU(struct gator_event[CNTMAX], events);
72 static struct gator_event uc_events[UCCNT];
73 static DEFINE_PER_CPU(int[(CNTMAX + UCCNT)*2], perf_cnt);
75 static void gator_events_perf_pmu_stop(void);
77 static int __create_files(struct super_block *sb, struct dentry *root, struct gator_attr *const attr)
81 if (attr->name[0] == '\0')
83 dir = gatorfs_mkdir(sb, root, attr->name);
86 gatorfs_create_ulong(sb, dir, "enabled", &attr->enabled);
87 gatorfs_create_ulong(sb, dir, "count", &attr->count);
88 gatorfs_create_ro_ulong(sb, dir, "key", &attr->key);
89 gatorfs_create_ulong(sb, dir, "event", &attr->event);
94 static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root)
98 for (cnt = 0; cnt < attr_count; cnt++) {
99 if (__create_files(sb, root, &attrs[cnt]) != 0)
103 for (cnt = 0; cnt < uc_attr_count; cnt++) {
104 if (__create_files(sb, root, &uc_attrs[cnt]) != 0)
111 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
112 static void ebs_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
114 static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
117 gator_backtrace_handler(regs);
120 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
121 static void dummy_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
123 static void dummy_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
126 /* Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll */
129 static int gator_events_perf_pmu_read(int **buffer, bool sched_switch);
131 static int gator_events_perf_pmu_online(int **buffer, bool migrate)
133 return gator_events_perf_pmu_read(buffer, false);
136 static void __online_dispatch(int cpu, bool migrate, struct gator_attr *const attr, struct gator_event *const event)
138 perf_overflow_handler_t handler;
142 if (event->pevent != NULL || event->pevent_attr == 0 || migrate)
146 handler = ebs_overflow_handler;
148 handler = dummy_handler;
150 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
151 event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler);
153 event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler, 0);
155 if (IS_ERR(event->pevent)) {
156 pr_debug("gator: unable to online a counter on cpu %d\n", cpu);
157 event->pevent = NULL;
161 if (event->pevent->state != PERF_EVENT_STATE_ACTIVE) {
162 pr_debug("gator: inactive counter on cpu %d\n", cpu);
163 perf_event_release_kernel(event->pevent);
164 event->pevent = NULL;
169 static void gator_events_perf_pmu_online_dispatch(int cpu, bool migrate)
173 cpu = pcpu_to_lcpu(cpu);
175 for (cnt = 0; cnt < attr_count; cnt++)
176 __online_dispatch(cpu, migrate, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
179 for (cnt = 0; cnt < uc_attr_count; cnt++)
180 __online_dispatch(cpu, migrate, &uc_attrs[cnt], &uc_events[cnt]);
184 static void __offline_dispatch(int cpu, struct gator_event *const event)
186 struct perf_event *pe = NULL;
190 event->pevent = NULL;
194 perf_event_release_kernel(pe);
197 static void gator_events_perf_pmu_offline_dispatch(int cpu, bool migrate)
203 cpu = pcpu_to_lcpu(cpu);
205 for (cnt = 0; cnt < attr_count; cnt++)
206 __offline_dispatch(cpu, &per_cpu(events, cpu)[cnt]);
209 for (cnt = 0; cnt < uc_attr_count; cnt++)
210 __offline_dispatch(cpu, &uc_events[cnt]);
214 static int __check_ebs(struct gator_attr *const attr)
216 if (attr->count > 0) {
217 if (!event_based_sampling) {
218 event_based_sampling = true;
220 pr_warning("gator: Only one ebs counter is allowed\n");
228 static int __start(struct gator_attr *const attr, struct gator_event *const event)
230 u32 size = sizeof(struct perf_event_attr);
232 event->pevent = NULL;
233 /* Skip disabled counters */
239 event->prev_delta = 0;
240 event->pevent_attr = kmalloc(size, GFP_KERNEL);
241 if (!event->pevent_attr) {
242 gator_events_perf_pmu_stop();
246 memset(event->pevent_attr, 0, size);
247 event->pevent_attr->type = attr->type;
248 event->pevent_attr->size = size;
249 event->pevent_attr->config = attr->event;
250 event->pevent_attr->sample_period = attr->count;
251 event->pevent_attr->pinned = 1;
256 static int gator_events_perf_pmu_start(void)
260 event_based_sampling = false;
261 for (cnt = 0; cnt < attr_count; cnt++) {
262 if (__check_ebs(&attrs[cnt]) != 0)
266 for (cnt = 0; cnt < uc_attr_count; cnt++) {
267 if (__check_ebs(&uc_attrs[cnt]) != 0)
271 for_each_present_cpu(cpu) {
272 for (cnt = 0; cnt < attr_count; cnt++) {
273 if (__start(&attrs[cnt], &per_cpu(events, cpu)[cnt]) != 0)
278 for (cnt = 0; cnt < uc_attr_count; cnt++) {
279 if (__start(&uc_attrs[cnt], &uc_events[cnt]) != 0)
286 static void __event_stop(struct gator_event *const event)
288 kfree(event->pevent_attr);
289 event->pevent_attr = NULL;
292 static void __attr_stop(struct gator_attr *const attr)
299 static void gator_events_perf_pmu_stop(void)
301 unsigned int cnt, cpu;
303 for_each_present_cpu(cpu) {
304 for (cnt = 0; cnt < attr_count; cnt++)
305 __event_stop(&per_cpu(events, cpu)[cnt]);
308 for (cnt = 0; cnt < uc_attr_count; cnt++)
309 __event_stop(&uc_events[cnt]);
311 for (cnt = 0; cnt < attr_count; cnt++)
312 __attr_stop(&attrs[cnt]);
314 for (cnt = 0; cnt < uc_attr_count; cnt++)
315 __attr_stop(&uc_attrs[cnt]);
318 static void __read(int *const len, int cpu, struct gator_attr *const attr, struct gator_event *const event)
321 struct perf_event *const ev = event->pevent;
323 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
324 /* After creating the perf counter in __online_dispatch, there
325 * is a race condition between gator_events_perf_pmu_online and
326 * gator_events_perf_pmu_read. So have
327 * gator_events_perf_pmu_online call gator_events_perf_pmu_read
328 * and in __read check to see if it's the first call after
329 * __online_dispatch and if so, run the online code.
333 event->prev = event->curr = local64_read(&ev->count);
334 event->prev_delta = 0;
335 per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
336 per_cpu(perf_cnt, cpu)[(*len)++] = 0;
340 event->curr = local64_read(&ev->count);
341 delta = event->curr - event->prev;
342 if (delta != 0 || delta != event->prev_delta) {
343 event->prev_delta = delta;
344 event->prev = event->curr;
345 per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
346 per_cpu(perf_cnt, cpu)[(*len)++] = delta;
352 static int gator_events_perf_pmu_read(int **buffer, bool sched_switch)
355 const int cpu = get_logical_cpu();
357 for (cnt = 0; cnt < attr_count; cnt++)
358 __read(&len, cpu, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
361 for (cnt = 0; cnt < uc_attr_count; cnt++)
362 __read(&len, cpu, &uc_attrs[cnt], &uc_events[cnt]);
366 *buffer = per_cpu(perf_cnt, cpu);
371 static struct gator_interface gator_events_perf_pmu_interface = {
372 .create_files = gator_events_perf_pmu_create_files,
373 .start = gator_events_perf_pmu_start,
374 .stop = gator_events_perf_pmu_stop,
375 .online = gator_events_perf_pmu_online,
376 .online_dispatch = gator_events_perf_pmu_online_dispatch,
377 .offline_dispatch = gator_events_perf_pmu_offline_dispatch,
378 .read = gator_events_perf_pmu_read,
381 static void __attr_init(struct gator_attr *const attr)
383 attr->name[0] = '\0';
388 attr->key = gator_events_get_key();
393 static const struct of_device_id arm_cci_matches[] = {
394 {.compatible = "arm,cci-400" },
398 static int probe_cci_revision(void)
400 struct device_node *np;
402 void __iomem *cci_ctrl_base;
404 int ret = DEFAULT_CCI_REVISION;
406 np = of_find_matching_node(NULL, arm_cci_matches);
410 if (of_address_to_resource(np, 0, &res))
413 cci_ctrl_base = ioremap(res.start, resource_size(&res));
415 rev = (readl_relaxed(cci_ctrl_base + 0xfe8) >> 4) & 0xf;
422 iounmap(cci_ctrl_base);
432 static int probe_cci_revision(void)
434 return DEFAULT_CCI_REVISION;
439 static void gator_events_perf_pmu_uncore_init(const char *const name, const int type, const int count, const bool has_cycles_counter)
443 if (has_cycles_counter) {
444 snprintf(uc_attrs[uc_attr_count].name, sizeof(uc_attrs[uc_attr_count].name), "%s_ccnt", name);
445 uc_attrs[uc_attr_count].type = type;
449 for (cnt = 0; cnt < count; ++cnt, ++uc_attr_count) {
450 struct gator_attr *const attr = &uc_attrs[uc_attr_count];
452 snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", name, cnt);
457 static void gator_events_perf_pmu_cci_400_init(const int type)
459 const char *cci_name;
461 switch (probe_cci_revision()) {
463 cci_name = "CCI_400";
466 cci_name = "CCI_400-r1";
469 pr_debug("gator: unrecognized cci-400 revision\n");
473 gator_events_perf_pmu_uncore_init(cci_name, type, CCI_400, true);
476 static void gator_events_perf_pmu_cpu_init(const struct gator_cpu *const gator_cpu, const int type)
480 snprintf(attrs[attr_count].name, sizeof(attrs[attr_count].name), "%s_ccnt", gator_cpu->pmnc_name);
481 attrs[attr_count].type = type;
484 for (cnt = 0; cnt < gator_cpu->pmnc_counters; ++cnt, ++attr_count) {
485 struct gator_attr *const attr = &attrs[attr_count];
487 snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", gator_cpu->pmnc_name, cnt);
492 int gator_events_perf_pmu_init(void)
494 struct perf_event_attr pea;
495 struct perf_event *pe;
496 const struct gator_cpu *gator_cpu;
500 bool found_cpu = false;
502 for (cnt = 0; cnt < CNTMAX; cnt++)
503 __attr_init(&attrs[cnt]);
504 for (cnt = 0; cnt < UCCNT; cnt++)
505 __attr_init(&uc_attrs[cnt]);
507 memset(&pea, 0, sizeof(pea));
508 pea.size = sizeof(pea);
512 for (type = PERF_TYPE_MAX; type < 0x20; ++type) {
515 /* A particular PMU may work on some but not all cores, so try on each core */
517 for_each_present_cpu(cpu) {
518 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
519 pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler);
521 pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler, 0);
526 /* Assume that valid PMUs are contiguous */
529 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
530 pe = perf_event_create_kernel_counter(&pea, 0, 0, dummy_handler);
532 pe = perf_event_create_kernel_counter(&pea, 0, 0, dummy_handler, 0);
538 if (pe->pmu != NULL && type == pe->pmu->type) {
539 if (strcmp("CCI", pe->pmu->name) == 0 || strcmp("CCI_400", pe->pmu->name) == 0 || strcmp("CCI_400-r1", pe->pmu->name) == 0) {
540 gator_events_perf_pmu_cci_400_init(type);
541 } else if (strcmp("CCI_500", pe->pmu->name) == 0) {
542 gator_events_perf_pmu_uncore_init("CCI_500", type, CCI_500, false);
543 } else if (strcmp("ccn", pe->pmu->name) == 0) {
544 gator_events_perf_pmu_uncore_init("ARM_CCN_5XX", type, CCN_5XX, true);
545 } else if ((gator_cpu = gator_find_cpu_by_pmu_name(pe->pmu->name)) != NULL) {
547 gator_events_perf_pmu_cpu_init(gator_cpu, type);
549 /* Initialize gator_attrs for dynamic PMUs here */
552 perf_event_release_kernel(pe);
556 const struct gator_cpu *gator_cpu = gator_find_cpu_by_cpuid(gator_cpuid());
558 if (gator_cpu == NULL) {
559 gator_cpu = gator_find_cpu_by_cpuid(OTHER);
560 if (gator_cpu == NULL) {
561 pr_err("gator: Didn't find cpu\n");
565 gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW);
568 /* Initialize gator_attrs for non-dynamic PMUs here */
570 if (attr_count > CNTMAX) {
571 pr_err("gator: Too many perf counters\n");
575 if (uc_attr_count > UCCNT) {
576 pr_err("gator: Too many perf uncore counters\n");
580 return gator_events_install(&gator_events_perf_pmu_interface);