2 * Performance counter core code
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/sysfs.h>
19 #include <linux/ptrace.h>
20 #include <linux/percpu.h>
21 #include <linux/vmstat.h>
22 #include <linux/hardirq.h>
23 #include <linux/rculist.h>
24 #include <linux/uaccess.h>
25 #include <linux/syscalls.h>
26 #include <linux/anon_inodes.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/perf_counter.h>
29 #include <linux/dcache.h>
31 #include <asm/irq_regs.h>
34 * Each CPU has a list of per CPU counters:
36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
38 int perf_max_counters __read_mostly = 1;
39 static int perf_reserved_percpu __read_mostly;
40 static int perf_overcommit __read_mostly = 1;
42 static atomic_t nr_counters __read_mostly;
43 static atomic_t nr_mmap_tracking __read_mostly;
44 static atomic_t nr_munmap_tracking __read_mostly;
45 static atomic_t nr_comm_tracking __read_mostly;
47 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
48 int sysctl_perf_counter_mlock __read_mostly = 512; /* 'free' kb per user */
49 int sysctl_perf_counter_limit __read_mostly = 100000; /* max NMIs per second */
52 * Lock for (sysadmin-configurable) counter reservations:
54 static DEFINE_SPINLOCK(perf_resource_lock);
57 * Architecture provided APIs - weak aliases:
59 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
64 void __weak hw_perf_disable(void) { barrier(); }
65 void __weak hw_perf_enable(void) { barrier(); }
67 void __weak hw_perf_counter_setup(int cpu) { barrier(); }
68 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
69 struct perf_cpu_context *cpuctx,
70 struct perf_counter_context *ctx, int cpu)
75 void __weak perf_counter_print_debug(void) { }
77 static DEFINE_PER_CPU(int, disable_count);
79 void __perf_disable(void)
81 __get_cpu_var(disable_count)++;
84 bool __perf_enable(void)
86 return !--__get_cpu_var(disable_count);
89 void perf_disable(void)
95 void perf_enable(void)
101 static void get_ctx(struct perf_counter_context *ctx)
103 atomic_inc(&ctx->refcount);
106 static void free_ctx(struct rcu_head *head)
108 struct perf_counter_context *ctx;
110 ctx = container_of(head, struct perf_counter_context, rcu_head);
114 static void put_ctx(struct perf_counter_context *ctx)
116 if (atomic_dec_and_test(&ctx->refcount)) {
118 put_ctx(ctx->parent_ctx);
120 put_task_struct(ctx->task);
121 call_rcu(&ctx->rcu_head, free_ctx);
126 * Get the perf_counter_context for a task and lock it.
127 * This has to cope with with the fact that until it is locked,
128 * the context could get moved to another task.
130 static struct perf_counter_context *perf_lock_task_context(
131 struct task_struct *task, unsigned long *flags)
133 struct perf_counter_context *ctx;
137 ctx = rcu_dereference(task->perf_counter_ctxp);
140 * If this context is a clone of another, it might
141 * get swapped for another underneath us by
142 * perf_counter_task_sched_out, though the
143 * rcu_read_lock() protects us from any context
144 * getting freed. Lock the context and check if it
145 * got swapped before we could get the lock, and retry
146 * if so. If we locked the right context, then it
147 * can't get swapped on us any more.
149 spin_lock_irqsave(&ctx->lock, *flags);
150 if (ctx != rcu_dereference(task->perf_counter_ctxp)) {
151 spin_unlock_irqrestore(&ctx->lock, *flags);
160 * Get the context for a task and increment its pin_count so it
161 * can't get swapped to another task. This also increments its
162 * reference count so that the context can't get freed.
164 static struct perf_counter_context *perf_pin_task_context(struct task_struct *task)
166 struct perf_counter_context *ctx;
169 ctx = perf_lock_task_context(task, &flags);
173 spin_unlock_irqrestore(&ctx->lock, flags);
178 static void perf_unpin_context(struct perf_counter_context *ctx)
182 spin_lock_irqsave(&ctx->lock, flags);
184 spin_unlock_irqrestore(&ctx->lock, flags);
189 * Add a counter from the lists for its context.
190 * Must be called with ctx->mutex and ctx->lock held.
193 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
195 struct perf_counter *group_leader = counter->group_leader;
198 * Depending on whether it is a standalone or sibling counter,
199 * add it straight to the context's counter list, or to the group
200 * leader's sibling list:
202 if (group_leader == counter)
203 list_add_tail(&counter->list_entry, &ctx->counter_list);
205 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
206 group_leader->nr_siblings++;
209 list_add_rcu(&counter->event_entry, &ctx->event_list);
214 * Remove a counter from the lists for its context.
215 * Must be called with ctx->mutex and ctx->lock held.
218 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
220 struct perf_counter *sibling, *tmp;
222 if (list_empty(&counter->list_entry))
226 list_del_init(&counter->list_entry);
227 list_del_rcu(&counter->event_entry);
229 if (counter->group_leader != counter)
230 counter->group_leader->nr_siblings--;
233 * If this was a group counter with sibling counters then
234 * upgrade the siblings to singleton counters by adding them
235 * to the context list directly:
237 list_for_each_entry_safe(sibling, tmp,
238 &counter->sibling_list, list_entry) {
240 list_move_tail(&sibling->list_entry, &ctx->counter_list);
241 sibling->group_leader = sibling;
246 counter_sched_out(struct perf_counter *counter,
247 struct perf_cpu_context *cpuctx,
248 struct perf_counter_context *ctx)
250 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
253 counter->state = PERF_COUNTER_STATE_INACTIVE;
254 counter->tstamp_stopped = ctx->time;
255 counter->pmu->disable(counter);
258 if (!is_software_counter(counter))
259 cpuctx->active_oncpu--;
261 if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
262 cpuctx->exclusive = 0;
266 group_sched_out(struct perf_counter *group_counter,
267 struct perf_cpu_context *cpuctx,
268 struct perf_counter_context *ctx)
270 struct perf_counter *counter;
272 if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
275 counter_sched_out(group_counter, cpuctx, ctx);
278 * Schedule out siblings (if any):
280 list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
281 counter_sched_out(counter, cpuctx, ctx);
283 if (group_counter->hw_event.exclusive)
284 cpuctx->exclusive = 0;
288 * Cross CPU call to remove a performance counter
290 * We disable the counter on the hardware level first. After that we
291 * remove it from the context list.
293 static void __perf_counter_remove_from_context(void *info)
295 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
296 struct perf_counter *counter = info;
297 struct perf_counter_context *ctx = counter->ctx;
300 * If this is a task context, we need to check whether it is
301 * the current task context of this cpu. If not it has been
302 * scheduled out before the smp call arrived.
304 if (ctx->task && cpuctx->task_ctx != ctx)
307 spin_lock(&ctx->lock);
309 * Protect the list operation against NMI by disabling the
310 * counters on a global level.
314 counter_sched_out(counter, cpuctx, ctx);
316 list_del_counter(counter, ctx);
320 * Allow more per task counters with respect to the
323 cpuctx->max_pertask =
324 min(perf_max_counters - ctx->nr_counters,
325 perf_max_counters - perf_reserved_percpu);
329 spin_unlock(&ctx->lock);
334 * Remove the counter from a task's (or a CPU's) list of counters.
336 * Must be called with ctx->mutex held.
338 * CPU counters are removed with a smp call. For task counters we only
339 * call when the task is on a CPU.
341 * If counter->ctx is a cloned context, callers must make sure that
342 * every task struct that counter->ctx->task could possibly point to
343 * remains valid. This is OK when called from perf_release since
344 * that only calls us on the top-level context, which can't be a clone.
345 * When called from perf_counter_exit_task, it's OK because the
346 * context has been detached from its task.
348 static void perf_counter_remove_from_context(struct perf_counter *counter)
350 struct perf_counter_context *ctx = counter->ctx;
351 struct task_struct *task = ctx->task;
355 * Per cpu counters are removed via an smp call and
356 * the removal is always sucessful.
358 smp_call_function_single(counter->cpu,
359 __perf_counter_remove_from_context,
365 task_oncpu_function_call(task, __perf_counter_remove_from_context,
368 spin_lock_irq(&ctx->lock);
370 * If the context is active we need to retry the smp call.
372 if (ctx->nr_active && !list_empty(&counter->list_entry)) {
373 spin_unlock_irq(&ctx->lock);
378 * The lock prevents that this context is scheduled in so we
379 * can remove the counter safely, if the call above did not
382 if (!list_empty(&counter->list_entry)) {
383 list_del_counter(counter, ctx);
385 spin_unlock_irq(&ctx->lock);
388 static inline u64 perf_clock(void)
390 return cpu_clock(smp_processor_id());
394 * Update the record of the current time in a context.
396 static void update_context_time(struct perf_counter_context *ctx)
398 u64 now = perf_clock();
400 ctx->time += now - ctx->timestamp;
401 ctx->timestamp = now;
405 * Update the total_time_enabled and total_time_running fields for a counter.
407 static void update_counter_times(struct perf_counter *counter)
409 struct perf_counter_context *ctx = counter->ctx;
412 if (counter->state < PERF_COUNTER_STATE_INACTIVE)
415 counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
417 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
418 run_end = counter->tstamp_stopped;
422 counter->total_time_running = run_end - counter->tstamp_running;
426 * Update total_time_enabled and total_time_running for all counters in a group.
428 static void update_group_times(struct perf_counter *leader)
430 struct perf_counter *counter;
432 update_counter_times(leader);
433 list_for_each_entry(counter, &leader->sibling_list, list_entry)
434 update_counter_times(counter);
438 * Cross CPU call to disable a performance counter
440 static void __perf_counter_disable(void *info)
442 struct perf_counter *counter = info;
443 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
444 struct perf_counter_context *ctx = counter->ctx;
447 * If this is a per-task counter, need to check whether this
448 * counter's task is the current task on this cpu.
450 if (ctx->task && cpuctx->task_ctx != ctx)
453 spin_lock(&ctx->lock);
456 * If the counter is on, turn it off.
457 * If it is in error state, leave it in error state.
459 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
460 update_context_time(ctx);
461 update_counter_times(counter);
462 if (counter == counter->group_leader)
463 group_sched_out(counter, cpuctx, ctx);
465 counter_sched_out(counter, cpuctx, ctx);
466 counter->state = PERF_COUNTER_STATE_OFF;
469 spin_unlock(&ctx->lock);
475 * If counter->ctx is a cloned context, callers must make sure that
476 * every task struct that counter->ctx->task could possibly point to
477 * remains valid. This condition is satisifed when called through
478 * perf_counter_for_each_child or perf_counter_for_each because they
479 * hold the top-level counter's child_mutex, so any descendant that
480 * goes to exit will block in sync_child_counter.
481 * When called from perf_pending_counter it's OK because counter->ctx
482 * is the current context on this CPU and preemption is disabled,
483 * hence we can't get into perf_counter_task_sched_out for this context.
485 static void perf_counter_disable(struct perf_counter *counter)
487 struct perf_counter_context *ctx = counter->ctx;
488 struct task_struct *task = ctx->task;
492 * Disable the counter on the cpu that it's on
494 smp_call_function_single(counter->cpu, __perf_counter_disable,
500 task_oncpu_function_call(task, __perf_counter_disable, counter);
502 spin_lock_irq(&ctx->lock);
504 * If the counter is still active, we need to retry the cross-call.
506 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
507 spin_unlock_irq(&ctx->lock);
512 * Since we have the lock this context can't be scheduled
513 * in, so we can change the state safely.
515 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
516 update_counter_times(counter);
517 counter->state = PERF_COUNTER_STATE_OFF;
520 spin_unlock_irq(&ctx->lock);
524 counter_sched_in(struct perf_counter *counter,
525 struct perf_cpu_context *cpuctx,
526 struct perf_counter_context *ctx,
529 if (counter->state <= PERF_COUNTER_STATE_OFF)
532 counter->state = PERF_COUNTER_STATE_ACTIVE;
533 counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */
535 * The new state must be visible before we turn it on in the hardware:
539 if (counter->pmu->enable(counter)) {
540 counter->state = PERF_COUNTER_STATE_INACTIVE;
545 counter->tstamp_running += ctx->time - counter->tstamp_stopped;
547 if (!is_software_counter(counter))
548 cpuctx->active_oncpu++;
551 if (counter->hw_event.exclusive)
552 cpuctx->exclusive = 1;
558 group_sched_in(struct perf_counter *group_counter,
559 struct perf_cpu_context *cpuctx,
560 struct perf_counter_context *ctx,
563 struct perf_counter *counter, *partial_group;
566 if (group_counter->state == PERF_COUNTER_STATE_OFF)
569 ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
571 return ret < 0 ? ret : 0;
573 group_counter->prev_state = group_counter->state;
574 if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
578 * Schedule in siblings as one group (if any):
580 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
581 counter->prev_state = counter->state;
582 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
583 partial_group = counter;
592 * Groups can be scheduled in as one unit only, so undo any
593 * partial group before returning:
595 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
596 if (counter == partial_group)
598 counter_sched_out(counter, cpuctx, ctx);
600 counter_sched_out(group_counter, cpuctx, ctx);
606 * Return 1 for a group consisting entirely of software counters,
607 * 0 if the group contains any hardware counters.
609 static int is_software_only_group(struct perf_counter *leader)
611 struct perf_counter *counter;
613 if (!is_software_counter(leader))
616 list_for_each_entry(counter, &leader->sibling_list, list_entry)
617 if (!is_software_counter(counter))
624 * Work out whether we can put this counter group on the CPU now.
626 static int group_can_go_on(struct perf_counter *counter,
627 struct perf_cpu_context *cpuctx,
631 * Groups consisting entirely of software counters can always go on.
633 if (is_software_only_group(counter))
636 * If an exclusive group is already on, no other hardware
637 * counters can go on.
639 if (cpuctx->exclusive)
642 * If this group is exclusive and there are already
643 * counters on the CPU, it can't go on.
645 if (counter->hw_event.exclusive && cpuctx->active_oncpu)
648 * Otherwise, try to add it if all previous groups were able
654 static void add_counter_to_ctx(struct perf_counter *counter,
655 struct perf_counter_context *ctx)
657 list_add_counter(counter, ctx);
658 counter->prev_state = PERF_COUNTER_STATE_OFF;
659 counter->tstamp_enabled = ctx->time;
660 counter->tstamp_running = ctx->time;
661 counter->tstamp_stopped = ctx->time;
665 * Cross CPU call to install and enable a performance counter
667 * Must be called with ctx->mutex held
669 static void __perf_install_in_context(void *info)
671 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
672 struct perf_counter *counter = info;
673 struct perf_counter_context *ctx = counter->ctx;
674 struct perf_counter *leader = counter->group_leader;
675 int cpu = smp_processor_id();
679 * If this is a task context, we need to check whether it is
680 * the current task context of this cpu. If not it has been
681 * scheduled out before the smp call arrived.
682 * Or possibly this is the right context but it isn't
683 * on this cpu because it had no counters.
685 if (ctx->task && cpuctx->task_ctx != ctx) {
686 if (cpuctx->task_ctx || ctx->task != current)
688 cpuctx->task_ctx = ctx;
691 spin_lock(&ctx->lock);
693 update_context_time(ctx);
696 * Protect the list operation against NMI by disabling the
697 * counters on a global level. NOP for non NMI based counters.
701 add_counter_to_ctx(counter, ctx);
704 * Don't put the counter on if it is disabled or if
705 * it is in a group and the group isn't on.
707 if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
708 (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
712 * An exclusive counter can't go on if there are already active
713 * hardware counters, and no hardware counter can go on if there
714 * is already an exclusive counter on.
716 if (!group_can_go_on(counter, cpuctx, 1))
719 err = counter_sched_in(counter, cpuctx, ctx, cpu);
723 * This counter couldn't go on. If it is in a group
724 * then we have to pull the whole group off.
725 * If the counter group is pinned then put it in error state.
727 if (leader != counter)
728 group_sched_out(leader, cpuctx, ctx);
729 if (leader->hw_event.pinned) {
730 update_group_times(leader);
731 leader->state = PERF_COUNTER_STATE_ERROR;
735 if (!err && !ctx->task && cpuctx->max_pertask)
736 cpuctx->max_pertask--;
741 spin_unlock(&ctx->lock);
745 * Attach a performance counter to a context
747 * First we add the counter to the list with the hardware enable bit
748 * in counter->hw_config cleared.
750 * If the counter is attached to a task which is on a CPU we use a smp
751 * call to enable it in the task context. The task might have been
752 * scheduled away, but we check this in the smp call again.
754 * Must be called with ctx->mutex held.
757 perf_install_in_context(struct perf_counter_context *ctx,
758 struct perf_counter *counter,
761 struct task_struct *task = ctx->task;
765 * Per cpu counters are installed via an smp call and
766 * the install is always sucessful.
768 smp_call_function_single(cpu, __perf_install_in_context,
774 task_oncpu_function_call(task, __perf_install_in_context,
777 spin_lock_irq(&ctx->lock);
779 * we need to retry the smp call.
781 if (ctx->is_active && list_empty(&counter->list_entry)) {
782 spin_unlock_irq(&ctx->lock);
787 * The lock prevents that this context is scheduled in so we
788 * can add the counter safely, if it the call above did not
791 if (list_empty(&counter->list_entry))
792 add_counter_to_ctx(counter, ctx);
793 spin_unlock_irq(&ctx->lock);
797 * Cross CPU call to enable a performance counter
799 static void __perf_counter_enable(void *info)
801 struct perf_counter *counter = info;
802 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
803 struct perf_counter_context *ctx = counter->ctx;
804 struct perf_counter *leader = counter->group_leader;
808 * If this is a per-task counter, need to check whether this
809 * counter's task is the current task on this cpu.
811 if (ctx->task && cpuctx->task_ctx != ctx) {
812 if (cpuctx->task_ctx || ctx->task != current)
814 cpuctx->task_ctx = ctx;
817 spin_lock(&ctx->lock);
819 update_context_time(ctx);
821 counter->prev_state = counter->state;
822 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
824 counter->state = PERF_COUNTER_STATE_INACTIVE;
825 counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
828 * If the counter is in a group and isn't the group leader,
829 * then don't put it on unless the group is on.
831 if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
834 if (!group_can_go_on(counter, cpuctx, 1)) {
838 if (counter == leader)
839 err = group_sched_in(counter, cpuctx, ctx,
842 err = counter_sched_in(counter, cpuctx, ctx,
849 * If this counter can't go on and it's part of a
850 * group, then the whole group has to come off.
852 if (leader != counter)
853 group_sched_out(leader, cpuctx, ctx);
854 if (leader->hw_event.pinned) {
855 update_group_times(leader);
856 leader->state = PERF_COUNTER_STATE_ERROR;
861 spin_unlock(&ctx->lock);
867 * If counter->ctx is a cloned context, callers must make sure that
868 * every task struct that counter->ctx->task could possibly point to
869 * remains valid. This condition is satisfied when called through
870 * perf_counter_for_each_child or perf_counter_for_each as described
871 * for perf_counter_disable.
873 static void perf_counter_enable(struct perf_counter *counter)
875 struct perf_counter_context *ctx = counter->ctx;
876 struct task_struct *task = ctx->task;
880 * Enable the counter on the cpu that it's on
882 smp_call_function_single(counter->cpu, __perf_counter_enable,
887 spin_lock_irq(&ctx->lock);
888 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
892 * If the counter is in error state, clear that first.
893 * That way, if we see the counter in error state below, we
894 * know that it has gone back into error state, as distinct
895 * from the task having been scheduled away before the
896 * cross-call arrived.
898 if (counter->state == PERF_COUNTER_STATE_ERROR)
899 counter->state = PERF_COUNTER_STATE_OFF;
902 spin_unlock_irq(&ctx->lock);
903 task_oncpu_function_call(task, __perf_counter_enable, counter);
905 spin_lock_irq(&ctx->lock);
908 * If the context is active and the counter is still off,
909 * we need to retry the cross-call.
911 if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
915 * Since we have the lock this context can't be scheduled
916 * in, so we can change the state safely.
918 if (counter->state == PERF_COUNTER_STATE_OFF) {
919 counter->state = PERF_COUNTER_STATE_INACTIVE;
920 counter->tstamp_enabled =
921 ctx->time - counter->total_time_enabled;
924 spin_unlock_irq(&ctx->lock);
927 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
930 * not supported on inherited counters
932 if (counter->hw_event.inherit)
935 atomic_add(refresh, &counter->event_limit);
936 perf_counter_enable(counter);
941 void __perf_counter_sched_out(struct perf_counter_context *ctx,
942 struct perf_cpu_context *cpuctx)
944 struct perf_counter *counter;
946 spin_lock(&ctx->lock);
948 if (likely(!ctx->nr_counters))
950 update_context_time(ctx);
953 if (ctx->nr_active) {
954 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
955 if (counter != counter->group_leader)
956 counter_sched_out(counter, cpuctx, ctx);
958 group_sched_out(counter, cpuctx, ctx);
963 spin_unlock(&ctx->lock);
967 * Test whether two contexts are equivalent, i.e. whether they
968 * have both been cloned from the same version of the same context
969 * and they both have the same number of enabled counters.
970 * If the number of enabled counters is the same, then the set
971 * of enabled counters should be the same, because these are both
972 * inherited contexts, therefore we can't access individual counters
973 * in them directly with an fd; we can only enable/disable all
974 * counters via prctl, or enable/disable all counters in a family
975 * via ioctl, which will have the same effect on both contexts.
977 static int context_equiv(struct perf_counter_context *ctx1,
978 struct perf_counter_context *ctx2)
980 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
981 && ctx1->parent_gen == ctx2->parent_gen
982 && !ctx1->pin_count && !ctx2->pin_count;
986 * Called from scheduler to remove the counters of the current task,
987 * with interrupts disabled.
989 * We stop each counter and update the counter value in counter->count.
991 * This does not protect us against NMI, but disable()
992 * sets the disabled bit in the control field of counter _before_
993 * accessing the counter control register. If a NMI hits, then it will
994 * not restart the counter.
996 void perf_counter_task_sched_out(struct task_struct *task,
997 struct task_struct *next, int cpu)
999 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1000 struct perf_counter_context *ctx = task->perf_counter_ctxp;
1001 struct perf_counter_context *next_ctx;
1002 struct perf_counter_context *parent;
1003 struct pt_regs *regs;
1006 regs = task_pt_regs(task);
1007 perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
1009 if (likely(!ctx || !cpuctx->task_ctx))
1012 update_context_time(ctx);
1015 parent = rcu_dereference(ctx->parent_ctx);
1016 next_ctx = next->perf_counter_ctxp;
1017 if (parent && next_ctx &&
1018 rcu_dereference(next_ctx->parent_ctx) == parent) {
1020 * Looks like the two contexts are clones, so we might be
1021 * able to optimize the context switch. We lock both
1022 * contexts and check that they are clones under the
1023 * lock (including re-checking that neither has been
1024 * uncloned in the meantime). It doesn't matter which
1025 * order we take the locks because no other cpu could
1026 * be trying to lock both of these tasks.
1028 spin_lock(&ctx->lock);
1029 spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1030 if (context_equiv(ctx, next_ctx)) {
1032 * XXX do we need a memory barrier of sorts
1033 * wrt to rcu_dereference() of perf_counter_ctxp
1035 task->perf_counter_ctxp = next_ctx;
1036 next->perf_counter_ctxp = ctx;
1038 next_ctx->task = task;
1041 spin_unlock(&next_ctx->lock);
1042 spin_unlock(&ctx->lock);
1047 __perf_counter_sched_out(ctx, cpuctx);
1048 cpuctx->task_ctx = NULL;
1053 * Called with IRQs disabled
1055 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
1057 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1059 if (!cpuctx->task_ctx)
1062 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1065 __perf_counter_sched_out(ctx, cpuctx);
1066 cpuctx->task_ctx = NULL;
1070 * Called with IRQs disabled
1072 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
1074 __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
1078 __perf_counter_sched_in(struct perf_counter_context *ctx,
1079 struct perf_cpu_context *cpuctx, int cpu)
1081 struct perf_counter *counter;
1084 spin_lock(&ctx->lock);
1086 if (likely(!ctx->nr_counters))
1089 ctx->timestamp = perf_clock();
1094 * First go through the list and put on any pinned groups
1095 * in order to give them the best chance of going on.
1097 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1098 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1099 !counter->hw_event.pinned)
1101 if (counter->cpu != -1 && counter->cpu != cpu)
1104 if (counter != counter->group_leader)
1105 counter_sched_in(counter, cpuctx, ctx, cpu);
1107 if (group_can_go_on(counter, cpuctx, 1))
1108 group_sched_in(counter, cpuctx, ctx, cpu);
1112 * If this pinned group hasn't been scheduled,
1113 * put it in error state.
1115 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1116 update_group_times(counter);
1117 counter->state = PERF_COUNTER_STATE_ERROR;
1121 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1123 * Ignore counters in OFF or ERROR state, and
1124 * ignore pinned counters since we did them already.
1126 if (counter->state <= PERF_COUNTER_STATE_OFF ||
1127 counter->hw_event.pinned)
1131 * Listen to the 'cpu' scheduling filter constraint
1134 if (counter->cpu != -1 && counter->cpu != cpu)
1137 if (counter != counter->group_leader) {
1138 if (counter_sched_in(counter, cpuctx, ctx, cpu))
1141 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
1142 if (group_sched_in(counter, cpuctx, ctx, cpu))
1149 spin_unlock(&ctx->lock);
1153 * Called from scheduler to add the counters of the current task
1154 * with interrupts disabled.
1156 * We restore the counter value and then enable it.
1158 * This does not protect us against NMI, but enable()
1159 * sets the enabled bit in the control field of counter _before_
1160 * accessing the counter control register. If a NMI hits, then it will
1161 * keep the counter running.
1163 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
1165 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1166 struct perf_counter_context *ctx = task->perf_counter_ctxp;
1170 if (cpuctx->task_ctx == ctx)
1172 __perf_counter_sched_in(ctx, cpuctx, cpu);
1173 cpuctx->task_ctx = ctx;
1176 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1178 struct perf_counter_context *ctx = &cpuctx->ctx;
1180 __perf_counter_sched_in(ctx, cpuctx, cpu);
1183 #define MAX_INTERRUPTS (~0ULL)
1185 static void perf_log_throttle(struct perf_counter *counter, int enable);
1186 static void perf_log_period(struct perf_counter *counter, u64 period);
1188 static void perf_adjust_freq(struct perf_counter_context *ctx)
1190 struct perf_counter *counter;
1191 u64 interrupts, irq_period;
1195 spin_lock(&ctx->lock);
1196 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1197 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1200 interrupts = counter->hw.interrupts;
1201 counter->hw.interrupts = 0;
1203 if (interrupts == MAX_INTERRUPTS) {
1204 perf_log_throttle(counter, 1);
1205 counter->pmu->unthrottle(counter);
1206 interrupts = 2*sysctl_perf_counter_limit/HZ;
1209 if (!counter->hw_event.freq || !counter->hw_event.irq_freq)
1212 events = HZ * interrupts * counter->hw.irq_period;
1213 period = div64_u64(events, counter->hw_event.irq_freq);
1215 delta = (s64)(1 + period - counter->hw.irq_period);
1218 irq_period = counter->hw.irq_period + delta;
1223 perf_log_period(counter, irq_period);
1225 counter->hw.irq_period = irq_period;
1227 spin_unlock(&ctx->lock);
1231 * Round-robin a context's counters:
1233 static void rotate_ctx(struct perf_counter_context *ctx)
1235 struct perf_counter *counter;
1237 if (!ctx->nr_counters)
1240 spin_lock(&ctx->lock);
1242 * Rotate the first entry last (works just fine for group counters too):
1245 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1246 list_move_tail(&counter->list_entry, &ctx->counter_list);
1251 spin_unlock(&ctx->lock);
1254 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1256 struct perf_cpu_context *cpuctx;
1257 struct perf_counter_context *ctx;
1259 if (!atomic_read(&nr_counters))
1262 cpuctx = &per_cpu(perf_cpu_context, cpu);
1263 ctx = curr->perf_counter_ctxp;
1265 perf_adjust_freq(&cpuctx->ctx);
1267 perf_adjust_freq(ctx);
1269 perf_counter_cpu_sched_out(cpuctx);
1271 __perf_counter_task_sched_out(ctx);
1273 rotate_ctx(&cpuctx->ctx);
1277 perf_counter_cpu_sched_in(cpuctx, cpu);
1279 perf_counter_task_sched_in(curr, cpu);
1283 * Cross CPU call to read the hardware counter
1285 static void __read(void *info)
1287 struct perf_counter *counter = info;
1288 struct perf_counter_context *ctx = counter->ctx;
1289 unsigned long flags;
1291 local_irq_save(flags);
1293 update_context_time(ctx);
1294 counter->pmu->read(counter);
1295 update_counter_times(counter);
1296 local_irq_restore(flags);
1299 static u64 perf_counter_read(struct perf_counter *counter)
1302 * If counter is enabled and currently active on a CPU, update the
1303 * value in the counter structure:
1305 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1306 smp_call_function_single(counter->oncpu,
1307 __read, counter, 1);
1308 } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1309 update_counter_times(counter);
1312 return atomic64_read(&counter->count);
1316 * Initialize the perf_counter context in a task_struct:
1319 __perf_counter_init_context(struct perf_counter_context *ctx,
1320 struct task_struct *task)
1322 memset(ctx, 0, sizeof(*ctx));
1323 spin_lock_init(&ctx->lock);
1324 mutex_init(&ctx->mutex);
1325 INIT_LIST_HEAD(&ctx->counter_list);
1326 INIT_LIST_HEAD(&ctx->event_list);
1327 atomic_set(&ctx->refcount, 1);
1331 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1333 struct perf_cpu_context *cpuctx;
1334 struct perf_counter_context *ctx;
1335 struct perf_counter_context *parent_ctx;
1336 struct task_struct *task;
1337 unsigned long flags;
1341 * If cpu is not a wildcard then this is a percpu counter:
1344 /* Must be root to operate on a CPU counter: */
1345 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1346 return ERR_PTR(-EACCES);
1348 if (cpu < 0 || cpu > num_possible_cpus())
1349 return ERR_PTR(-EINVAL);
1352 * We could be clever and allow to attach a counter to an
1353 * offline CPU and activate it when the CPU comes up, but
1356 if (!cpu_isset(cpu, cpu_online_map))
1357 return ERR_PTR(-ENODEV);
1359 cpuctx = &per_cpu(perf_cpu_context, cpu);
1370 task = find_task_by_vpid(pid);
1372 get_task_struct(task);
1376 return ERR_PTR(-ESRCH);
1379 * Can't attach counters to a dying task.
1382 if (task->flags & PF_EXITING)
1385 /* Reuse ptrace permission checks for now. */
1387 if (!ptrace_may_access(task, PTRACE_MODE_READ))
1391 ctx = perf_lock_task_context(task, &flags);
1393 parent_ctx = ctx->parent_ctx;
1395 put_ctx(parent_ctx);
1396 ctx->parent_ctx = NULL; /* no longer a clone */
1399 * Get an extra reference before dropping the lock so that
1400 * this context won't get freed if the task exits.
1403 spin_unlock_irqrestore(&ctx->lock, flags);
1407 ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
1411 __perf_counter_init_context(ctx, task);
1413 if (cmpxchg(&task->perf_counter_ctxp, NULL, ctx)) {
1415 * We raced with some other task; use
1416 * the context they set.
1421 get_task_struct(task);
1424 put_task_struct(task);
1428 put_task_struct(task);
1429 return ERR_PTR(err);
1432 static void free_counter_rcu(struct rcu_head *head)
1434 struct perf_counter *counter;
1436 counter = container_of(head, struct perf_counter, rcu_head);
1440 static void perf_pending_sync(struct perf_counter *counter);
1442 static void free_counter(struct perf_counter *counter)
1444 perf_pending_sync(counter);
1446 atomic_dec(&nr_counters);
1447 if (counter->hw_event.mmap)
1448 atomic_dec(&nr_mmap_tracking);
1449 if (counter->hw_event.munmap)
1450 atomic_dec(&nr_munmap_tracking);
1451 if (counter->hw_event.comm)
1452 atomic_dec(&nr_comm_tracking);
1454 if (counter->destroy)
1455 counter->destroy(counter);
1457 put_ctx(counter->ctx);
1458 call_rcu(&counter->rcu_head, free_counter_rcu);
1462 * Called when the last reference to the file is gone.
1464 static int perf_release(struct inode *inode, struct file *file)
1466 struct perf_counter *counter = file->private_data;
1467 struct perf_counter_context *ctx = counter->ctx;
1469 file->private_data = NULL;
1471 WARN_ON_ONCE(ctx->parent_ctx);
1472 mutex_lock(&ctx->mutex);
1473 perf_counter_remove_from_context(counter);
1474 mutex_unlock(&ctx->mutex);
1476 mutex_lock(&counter->owner->perf_counter_mutex);
1477 list_del_init(&counter->owner_entry);
1478 mutex_unlock(&counter->owner->perf_counter_mutex);
1479 put_task_struct(counter->owner);
1481 free_counter(counter);
1487 * Read the performance counter - simple non blocking version for now
1490 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1496 * Return end-of-file for a read on a counter that is in
1497 * error state (i.e. because it was pinned but it couldn't be
1498 * scheduled on to the CPU at some point).
1500 if (counter->state == PERF_COUNTER_STATE_ERROR)
1503 WARN_ON_ONCE(counter->ctx->parent_ctx);
1504 mutex_lock(&counter->child_mutex);
1505 values[0] = perf_counter_read(counter);
1507 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1508 values[n++] = counter->total_time_enabled +
1509 atomic64_read(&counter->child_total_time_enabled);
1510 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1511 values[n++] = counter->total_time_running +
1512 atomic64_read(&counter->child_total_time_running);
1513 mutex_unlock(&counter->child_mutex);
1515 if (count < n * sizeof(u64))
1517 count = n * sizeof(u64);
1519 if (copy_to_user(buf, values, count))
1526 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1528 struct perf_counter *counter = file->private_data;
1530 return perf_read_hw(counter, buf, count);
1533 static unsigned int perf_poll(struct file *file, poll_table *wait)
1535 struct perf_counter *counter = file->private_data;
1536 struct perf_mmap_data *data;
1537 unsigned int events = POLL_HUP;
1540 data = rcu_dereference(counter->data);
1542 events = atomic_xchg(&data->poll, 0);
1545 poll_wait(file, &counter->waitq, wait);
1550 static void perf_counter_reset(struct perf_counter *counter)
1552 (void)perf_counter_read(counter);
1553 atomic64_set(&counter->count, 0);
1554 perf_counter_update_userpage(counter);
1557 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1558 void (*func)(struct perf_counter *))
1560 struct perf_counter_context *ctx = counter->ctx;
1561 struct perf_counter *sibling;
1563 WARN_ON_ONCE(ctx->parent_ctx);
1564 mutex_lock(&ctx->mutex);
1565 counter = counter->group_leader;
1568 list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1570 mutex_unlock(&ctx->mutex);
1574 * Holding the top-level counter's child_mutex means that any
1575 * descendant process that has inherited this counter will block
1576 * in sync_child_counter if it goes to exit, thus satisfying the
1577 * task existence requirements of perf_counter_enable/disable.
1579 static void perf_counter_for_each_child(struct perf_counter *counter,
1580 void (*func)(struct perf_counter *))
1582 struct perf_counter *child;
1584 WARN_ON_ONCE(counter->ctx->parent_ctx);
1585 mutex_lock(&counter->child_mutex);
1587 list_for_each_entry(child, &counter->child_list, child_list)
1589 mutex_unlock(&counter->child_mutex);
1592 static void perf_counter_for_each(struct perf_counter *counter,
1593 void (*func)(struct perf_counter *))
1595 struct perf_counter *child;
1597 WARN_ON_ONCE(counter->ctx->parent_ctx);
1598 mutex_lock(&counter->child_mutex);
1599 perf_counter_for_each_sibling(counter, func);
1600 list_for_each_entry(child, &counter->child_list, child_list)
1601 perf_counter_for_each_sibling(child, func);
1602 mutex_unlock(&counter->child_mutex);
1605 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1607 struct perf_counter *counter = file->private_data;
1608 void (*func)(struct perf_counter *);
1612 case PERF_COUNTER_IOC_ENABLE:
1613 func = perf_counter_enable;
1615 case PERF_COUNTER_IOC_DISABLE:
1616 func = perf_counter_disable;
1618 case PERF_COUNTER_IOC_RESET:
1619 func = perf_counter_reset;
1622 case PERF_COUNTER_IOC_REFRESH:
1623 return perf_counter_refresh(counter, arg);
1628 if (flags & PERF_IOC_FLAG_GROUP)
1629 perf_counter_for_each(counter, func);
1631 perf_counter_for_each_child(counter, func);
1636 int perf_counter_task_enable(void)
1638 struct perf_counter *counter;
1640 mutex_lock(¤t->perf_counter_mutex);
1641 list_for_each_entry(counter, ¤t->perf_counter_list, owner_entry)
1642 perf_counter_for_each_child(counter, perf_counter_enable);
1643 mutex_unlock(¤t->perf_counter_mutex);
1648 int perf_counter_task_disable(void)
1650 struct perf_counter *counter;
1652 mutex_lock(¤t->perf_counter_mutex);
1653 list_for_each_entry(counter, ¤t->perf_counter_list, owner_entry)
1654 perf_counter_for_each_child(counter, perf_counter_disable);
1655 mutex_unlock(¤t->perf_counter_mutex);
1661 * Callers need to ensure there can be no nesting of this function, otherwise
1662 * the seqlock logic goes bad. We can not serialize this because the arch
1663 * code calls this from NMI context.
1665 void perf_counter_update_userpage(struct perf_counter *counter)
1667 struct perf_mmap_data *data;
1668 struct perf_counter_mmap_page *userpg;
1671 data = rcu_dereference(counter->data);
1675 userpg = data->user_page;
1678 * Disable preemption so as to not let the corresponding user-space
1679 * spin too long if we get preempted.
1684 userpg->index = counter->hw.idx;
1685 userpg->offset = atomic64_read(&counter->count);
1686 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1687 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1696 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1698 struct perf_counter *counter = vma->vm_file->private_data;
1699 struct perf_mmap_data *data;
1700 int ret = VM_FAULT_SIGBUS;
1703 data = rcu_dereference(counter->data);
1707 if (vmf->pgoff == 0) {
1708 vmf->page = virt_to_page(data->user_page);
1710 int nr = vmf->pgoff - 1;
1712 if ((unsigned)nr > data->nr_pages)
1715 vmf->page = virt_to_page(data->data_pages[nr]);
1717 get_page(vmf->page);
1725 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1727 struct perf_mmap_data *data;
1731 WARN_ON(atomic_read(&counter->mmap_count));
1733 size = sizeof(struct perf_mmap_data);
1734 size += nr_pages * sizeof(void *);
1736 data = kzalloc(size, GFP_KERNEL);
1740 data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1741 if (!data->user_page)
1742 goto fail_user_page;
1744 for (i = 0; i < nr_pages; i++) {
1745 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1746 if (!data->data_pages[i])
1747 goto fail_data_pages;
1750 data->nr_pages = nr_pages;
1751 atomic_set(&data->lock, -1);
1753 rcu_assign_pointer(counter->data, data);
1758 for (i--; i >= 0; i--)
1759 free_page((unsigned long)data->data_pages[i]);
1761 free_page((unsigned long)data->user_page);
1770 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1772 struct perf_mmap_data *data = container_of(rcu_head,
1773 struct perf_mmap_data, rcu_head);
1776 free_page((unsigned long)data->user_page);
1777 for (i = 0; i < data->nr_pages; i++)
1778 free_page((unsigned long)data->data_pages[i]);
1782 static void perf_mmap_data_free(struct perf_counter *counter)
1784 struct perf_mmap_data *data = counter->data;
1786 WARN_ON(atomic_read(&counter->mmap_count));
1788 rcu_assign_pointer(counter->data, NULL);
1789 call_rcu(&data->rcu_head, __perf_mmap_data_free);
1792 static void perf_mmap_open(struct vm_area_struct *vma)
1794 struct perf_counter *counter = vma->vm_file->private_data;
1796 atomic_inc(&counter->mmap_count);
1799 static void perf_mmap_close(struct vm_area_struct *vma)
1801 struct perf_counter *counter = vma->vm_file->private_data;
1803 WARN_ON_ONCE(counter->ctx->parent_ctx);
1804 if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1805 &counter->mmap_mutex)) {
1806 struct user_struct *user = current_user();
1808 atomic_long_sub(counter->data->nr_pages + 1, &user->locked_vm);
1809 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1810 perf_mmap_data_free(counter);
1811 mutex_unlock(&counter->mmap_mutex);
1815 static struct vm_operations_struct perf_mmap_vmops = {
1816 .open = perf_mmap_open,
1817 .close = perf_mmap_close,
1818 .fault = perf_mmap_fault,
1821 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1823 struct perf_counter *counter = file->private_data;
1824 struct user_struct *user = current_user();
1825 unsigned long vma_size;
1826 unsigned long nr_pages;
1827 unsigned long user_locked, user_lock_limit;
1828 unsigned long locked, lock_limit;
1829 long user_extra, extra;
1832 if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1835 vma_size = vma->vm_end - vma->vm_start;
1836 nr_pages = (vma_size / PAGE_SIZE) - 1;
1839 * If we have data pages ensure they're a power-of-two number, so we
1840 * can do bitmasks instead of modulo.
1842 if (nr_pages != 0 && !is_power_of_2(nr_pages))
1845 if (vma_size != PAGE_SIZE * (1 + nr_pages))
1848 if (vma->vm_pgoff != 0)
1851 WARN_ON_ONCE(counter->ctx->parent_ctx);
1852 mutex_lock(&counter->mmap_mutex);
1853 if (atomic_inc_not_zero(&counter->mmap_count)) {
1854 if (nr_pages != counter->data->nr_pages)
1859 user_extra = nr_pages + 1;
1860 user_lock_limit = sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1863 * Increase the limit linearly with more CPUs:
1865 user_lock_limit *= num_online_cpus();
1867 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
1870 if (user_locked > user_lock_limit)
1871 extra = user_locked - user_lock_limit;
1873 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1874 lock_limit >>= PAGE_SHIFT;
1875 locked = vma->vm_mm->locked_vm + extra;
1877 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1882 WARN_ON(counter->data);
1883 ret = perf_mmap_data_alloc(counter, nr_pages);
1887 atomic_set(&counter->mmap_count, 1);
1888 atomic_long_add(user_extra, &user->locked_vm);
1889 vma->vm_mm->locked_vm += extra;
1890 counter->data->nr_locked = extra;
1892 mutex_unlock(&counter->mmap_mutex);
1894 vma->vm_flags &= ~VM_MAYWRITE;
1895 vma->vm_flags |= VM_RESERVED;
1896 vma->vm_ops = &perf_mmap_vmops;
1901 static int perf_fasync(int fd, struct file *filp, int on)
1903 struct perf_counter *counter = filp->private_data;
1904 struct inode *inode = filp->f_path.dentry->d_inode;
1907 mutex_lock(&inode->i_mutex);
1908 retval = fasync_helper(fd, filp, on, &counter->fasync);
1909 mutex_unlock(&inode->i_mutex);
1917 static const struct file_operations perf_fops = {
1918 .release = perf_release,
1921 .unlocked_ioctl = perf_ioctl,
1922 .compat_ioctl = perf_ioctl,
1924 .fasync = perf_fasync,
1928 * Perf counter wakeup
1930 * If there's data, ensure we set the poll() state and publish everything
1931 * to user-space before waking everybody up.
1934 void perf_counter_wakeup(struct perf_counter *counter)
1936 wake_up_all(&counter->waitq);
1938 if (counter->pending_kill) {
1939 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1940 counter->pending_kill = 0;
1947 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1949 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1950 * single linked list and use cmpxchg() to add entries lockless.
1953 static void perf_pending_counter(struct perf_pending_entry *entry)
1955 struct perf_counter *counter = container_of(entry,
1956 struct perf_counter, pending);
1958 if (counter->pending_disable) {
1959 counter->pending_disable = 0;
1960 perf_counter_disable(counter);
1963 if (counter->pending_wakeup) {
1964 counter->pending_wakeup = 0;
1965 perf_counter_wakeup(counter);
1969 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1971 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1975 static void perf_pending_queue(struct perf_pending_entry *entry,
1976 void (*func)(struct perf_pending_entry *))
1978 struct perf_pending_entry **head;
1980 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1985 head = &get_cpu_var(perf_pending_head);
1988 entry->next = *head;
1989 } while (cmpxchg(head, entry->next, entry) != entry->next);
1991 set_perf_counter_pending();
1993 put_cpu_var(perf_pending_head);
1996 static int __perf_pending_run(void)
1998 struct perf_pending_entry *list;
2001 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2002 while (list != PENDING_TAIL) {
2003 void (*func)(struct perf_pending_entry *);
2004 struct perf_pending_entry *entry = list;
2011 * Ensure we observe the unqueue before we issue the wakeup,
2012 * so that we won't be waiting forever.
2013 * -- see perf_not_pending().
2024 static inline int perf_not_pending(struct perf_counter *counter)
2027 * If we flush on whatever cpu we run, there is a chance we don't
2031 __perf_pending_run();
2035 * Ensure we see the proper queue state before going to sleep
2036 * so that we do not miss the wakeup. -- see perf_pending_handle()
2039 return counter->pending.next == NULL;
2042 static void perf_pending_sync(struct perf_counter *counter)
2044 wait_event(counter->waitq, perf_not_pending(counter));
2047 void perf_counter_do_pending(void)
2049 __perf_pending_run();
2053 * Callchain support -- arch specific
2056 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2065 struct perf_output_handle {
2066 struct perf_counter *counter;
2067 struct perf_mmap_data *data;
2068 unsigned int offset;
2073 unsigned long flags;
2076 static void perf_output_wakeup(struct perf_output_handle *handle)
2078 atomic_set(&handle->data->poll, POLL_IN);
2081 handle->counter->pending_wakeup = 1;
2082 perf_pending_queue(&handle->counter->pending,
2083 perf_pending_counter);
2085 perf_counter_wakeup(handle->counter);
2089 * Curious locking construct.
2091 * We need to ensure a later event doesn't publish a head when a former
2092 * event isn't done writing. However since we need to deal with NMIs we
2093 * cannot fully serialize things.
2095 * What we do is serialize between CPUs so we only have to deal with NMI
2096 * nesting on a single CPU.
2098 * We only publish the head (and generate a wakeup) when the outer-most
2101 static void perf_output_lock(struct perf_output_handle *handle)
2103 struct perf_mmap_data *data = handle->data;
2108 local_irq_save(handle->flags);
2109 cpu = smp_processor_id();
2111 if (in_nmi() && atomic_read(&data->lock) == cpu)
2114 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2120 static void perf_output_unlock(struct perf_output_handle *handle)
2122 struct perf_mmap_data *data = handle->data;
2125 data->done_head = data->head;
2127 if (!handle->locked)
2132 * The xchg implies a full barrier that ensures all writes are done
2133 * before we publish the new head, matched by a rmb() in userspace when
2134 * reading this position.
2136 while ((head = atomic_xchg(&data->done_head, 0)))
2137 data->user_page->data_head = head;
2140 * NMI can happen here, which means we can miss a done_head update.
2143 cpu = atomic_xchg(&data->lock, -1);
2144 WARN_ON_ONCE(cpu != smp_processor_id());
2147 * Therefore we have to validate we did not indeed do so.
2149 if (unlikely(atomic_read(&data->done_head))) {
2151 * Since we had it locked, we can lock it again.
2153 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2159 if (atomic_xchg(&data->wakeup, 0))
2160 perf_output_wakeup(handle);
2162 local_irq_restore(handle->flags);
2165 static int perf_output_begin(struct perf_output_handle *handle,
2166 struct perf_counter *counter, unsigned int size,
2167 int nmi, int overflow)
2169 struct perf_mmap_data *data;
2170 unsigned int offset, head;
2173 * For inherited counters we send all the output towards the parent.
2175 if (counter->parent)
2176 counter = counter->parent;
2179 data = rcu_dereference(counter->data);
2183 handle->data = data;
2184 handle->counter = counter;
2186 handle->overflow = overflow;
2188 if (!data->nr_pages)
2191 perf_output_lock(handle);
2194 offset = head = atomic_read(&data->head);
2196 } while (atomic_cmpxchg(&data->head, offset, head) != offset);
2198 handle->offset = offset;
2199 handle->head = head;
2201 if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
2202 atomic_set(&data->wakeup, 1);
2207 perf_output_wakeup(handle);
2214 static void perf_output_copy(struct perf_output_handle *handle,
2215 void *buf, unsigned int len)
2217 unsigned int pages_mask;
2218 unsigned int offset;
2222 offset = handle->offset;
2223 pages_mask = handle->data->nr_pages - 1;
2224 pages = handle->data->data_pages;
2227 unsigned int page_offset;
2230 nr = (offset >> PAGE_SHIFT) & pages_mask;
2231 page_offset = offset & (PAGE_SIZE - 1);
2232 size = min_t(unsigned int, PAGE_SIZE - page_offset, len);
2234 memcpy(pages[nr] + page_offset, buf, size);
2241 handle->offset = offset;
2244 * Check we didn't copy past our reservation window, taking the
2245 * possible unsigned int wrap into account.
2247 WARN_ON_ONCE(((int)(handle->head - handle->offset)) < 0);
2250 #define perf_output_put(handle, x) \
2251 perf_output_copy((handle), &(x), sizeof(x))
2253 static void perf_output_end(struct perf_output_handle *handle)
2255 struct perf_counter *counter = handle->counter;
2256 struct perf_mmap_data *data = handle->data;
2258 int wakeup_events = counter->hw_event.wakeup_events;
2260 if (handle->overflow && wakeup_events) {
2261 int events = atomic_inc_return(&data->events);
2262 if (events >= wakeup_events) {
2263 atomic_sub(wakeup_events, &data->events);
2264 atomic_set(&data->wakeup, 1);
2268 perf_output_unlock(handle);
2272 static void perf_counter_output(struct perf_counter *counter,
2273 int nmi, struct pt_regs *regs, u64 addr)
2276 u64 record_type = counter->hw_event.record_type;
2277 struct perf_output_handle handle;
2278 struct perf_event_header header;
2287 struct perf_callchain_entry *callchain = NULL;
2288 int callchain_size = 0;
2295 header.size = sizeof(header);
2297 header.misc = PERF_EVENT_MISC_OVERFLOW;
2298 header.misc |= perf_misc_flags(regs);
2300 if (record_type & PERF_RECORD_IP) {
2301 ip = perf_instruction_pointer(regs);
2302 header.type |= PERF_RECORD_IP;
2303 header.size += sizeof(ip);
2306 if (record_type & PERF_RECORD_TID) {
2307 /* namespace issues */
2308 tid_entry.pid = current->group_leader->pid;
2309 tid_entry.tid = current->pid;
2311 header.type |= PERF_RECORD_TID;
2312 header.size += sizeof(tid_entry);
2315 if (record_type & PERF_RECORD_TIME) {
2317 * Maybe do better on x86 and provide cpu_clock_nmi()
2319 time = sched_clock();
2321 header.type |= PERF_RECORD_TIME;
2322 header.size += sizeof(u64);
2325 if (record_type & PERF_RECORD_ADDR) {
2326 header.type |= PERF_RECORD_ADDR;
2327 header.size += sizeof(u64);
2330 if (record_type & PERF_RECORD_CONFIG) {
2331 header.type |= PERF_RECORD_CONFIG;
2332 header.size += sizeof(u64);
2335 if (record_type & PERF_RECORD_CPU) {
2336 header.type |= PERF_RECORD_CPU;
2337 header.size += sizeof(cpu_entry);
2339 cpu_entry.cpu = raw_smp_processor_id();
2342 if (record_type & PERF_RECORD_GROUP) {
2343 header.type |= PERF_RECORD_GROUP;
2344 header.size += sizeof(u64) +
2345 counter->nr_siblings * sizeof(group_entry);
2348 if (record_type & PERF_RECORD_CALLCHAIN) {
2349 callchain = perf_callchain(regs);
2352 callchain_size = (1 + callchain->nr) * sizeof(u64);
2354 header.type |= PERF_RECORD_CALLCHAIN;
2355 header.size += callchain_size;
2359 ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2363 perf_output_put(&handle, header);
2365 if (record_type & PERF_RECORD_IP)
2366 perf_output_put(&handle, ip);
2368 if (record_type & PERF_RECORD_TID)
2369 perf_output_put(&handle, tid_entry);
2371 if (record_type & PERF_RECORD_TIME)
2372 perf_output_put(&handle, time);
2374 if (record_type & PERF_RECORD_ADDR)
2375 perf_output_put(&handle, addr);
2377 if (record_type & PERF_RECORD_CONFIG)
2378 perf_output_put(&handle, counter->hw_event.config);
2380 if (record_type & PERF_RECORD_CPU)
2381 perf_output_put(&handle, cpu_entry);
2384 * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2386 if (record_type & PERF_RECORD_GROUP) {
2387 struct perf_counter *leader, *sub;
2388 u64 nr = counter->nr_siblings;
2390 perf_output_put(&handle, nr);
2392 leader = counter->group_leader;
2393 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2395 sub->pmu->read(sub);
2397 group_entry.event = sub->hw_event.config;
2398 group_entry.counter = atomic64_read(&sub->count);
2400 perf_output_put(&handle, group_entry);
2405 perf_output_copy(&handle, callchain, callchain_size);
2407 perf_output_end(&handle);
2414 struct perf_comm_event {
2415 struct task_struct *task;
2420 struct perf_event_header header;
2427 static void perf_counter_comm_output(struct perf_counter *counter,
2428 struct perf_comm_event *comm_event)
2430 struct perf_output_handle handle;
2431 int size = comm_event->event.header.size;
2432 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2437 perf_output_put(&handle, comm_event->event);
2438 perf_output_copy(&handle, comm_event->comm,
2439 comm_event->comm_size);
2440 perf_output_end(&handle);
2443 static int perf_counter_comm_match(struct perf_counter *counter,
2444 struct perf_comm_event *comm_event)
2446 if (counter->hw_event.comm &&
2447 comm_event->event.header.type == PERF_EVENT_COMM)
2453 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2454 struct perf_comm_event *comm_event)
2456 struct perf_counter *counter;
2458 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2462 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2463 if (perf_counter_comm_match(counter, comm_event))
2464 perf_counter_comm_output(counter, comm_event);
2469 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2471 struct perf_cpu_context *cpuctx;
2472 struct perf_counter_context *ctx;
2474 char *comm = comm_event->task->comm;
2476 size = ALIGN(strlen(comm)+1, sizeof(u64));
2478 comm_event->comm = comm;
2479 comm_event->comm_size = size;
2481 comm_event->event.header.size = sizeof(comm_event->event) + size;
2483 cpuctx = &get_cpu_var(perf_cpu_context);
2484 perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2485 put_cpu_var(perf_cpu_context);
2489 * doesn't really matter which of the child contexts the
2490 * events ends up in.
2492 ctx = rcu_dereference(current->perf_counter_ctxp);
2494 perf_counter_comm_ctx(ctx, comm_event);
2498 void perf_counter_comm(struct task_struct *task)
2500 struct perf_comm_event comm_event;
2502 if (!atomic_read(&nr_comm_tracking))
2505 comm_event = (struct perf_comm_event){
2508 .header = { .type = PERF_EVENT_COMM, },
2509 .pid = task->group_leader->pid,
2514 perf_counter_comm_event(&comm_event);
2521 struct perf_mmap_event {
2527 struct perf_event_header header;
2537 static void perf_counter_mmap_output(struct perf_counter *counter,
2538 struct perf_mmap_event *mmap_event)
2540 struct perf_output_handle handle;
2541 int size = mmap_event->event.header.size;
2542 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2547 perf_output_put(&handle, mmap_event->event);
2548 perf_output_copy(&handle, mmap_event->file_name,
2549 mmap_event->file_size);
2550 perf_output_end(&handle);
2553 static int perf_counter_mmap_match(struct perf_counter *counter,
2554 struct perf_mmap_event *mmap_event)
2556 if (counter->hw_event.mmap &&
2557 mmap_event->event.header.type == PERF_EVENT_MMAP)
2560 if (counter->hw_event.munmap &&
2561 mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2567 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2568 struct perf_mmap_event *mmap_event)
2570 struct perf_counter *counter;
2572 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2576 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2577 if (perf_counter_mmap_match(counter, mmap_event))
2578 perf_counter_mmap_output(counter, mmap_event);
2583 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2585 struct perf_cpu_context *cpuctx;
2586 struct perf_counter_context *ctx;
2587 struct file *file = mmap_event->file;
2594 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2596 name = strncpy(tmp, "//enomem", sizeof(tmp));
2599 name = d_path(&file->f_path, buf, PATH_MAX);
2601 name = strncpy(tmp, "//toolong", sizeof(tmp));
2605 name = strncpy(tmp, "//anon", sizeof(tmp));
2610 size = ALIGN(strlen(name)+1, sizeof(u64));
2612 mmap_event->file_name = name;
2613 mmap_event->file_size = size;
2615 mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2617 cpuctx = &get_cpu_var(perf_cpu_context);
2618 perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2619 put_cpu_var(perf_cpu_context);
2623 * doesn't really matter which of the child contexts the
2624 * events ends up in.
2626 ctx = rcu_dereference(current->perf_counter_ctxp);
2628 perf_counter_mmap_ctx(ctx, mmap_event);
2634 void perf_counter_mmap(unsigned long addr, unsigned long len,
2635 unsigned long pgoff, struct file *file)
2637 struct perf_mmap_event mmap_event;
2639 if (!atomic_read(&nr_mmap_tracking))
2642 mmap_event = (struct perf_mmap_event){
2645 .header = { .type = PERF_EVENT_MMAP, },
2646 .pid = current->group_leader->pid,
2647 .tid = current->pid,
2654 perf_counter_mmap_event(&mmap_event);
2657 void perf_counter_munmap(unsigned long addr, unsigned long len,
2658 unsigned long pgoff, struct file *file)
2660 struct perf_mmap_event mmap_event;
2662 if (!atomic_read(&nr_munmap_tracking))
2665 mmap_event = (struct perf_mmap_event){
2668 .header = { .type = PERF_EVENT_MUNMAP, },
2669 .pid = current->group_leader->pid,
2670 .tid = current->pid,
2677 perf_counter_mmap_event(&mmap_event);
2681 * Log irq_period changes so that analyzing tools can re-normalize the
2685 static void perf_log_period(struct perf_counter *counter, u64 period)
2687 struct perf_output_handle handle;
2691 struct perf_event_header header;
2696 .type = PERF_EVENT_PERIOD,
2698 .size = sizeof(freq_event),
2700 .time = sched_clock(),
2704 if (counter->hw.irq_period == period)
2707 ret = perf_output_begin(&handle, counter, sizeof(freq_event), 0, 0);
2711 perf_output_put(&handle, freq_event);
2712 perf_output_end(&handle);
2716 * IRQ throttle logging
2719 static void perf_log_throttle(struct perf_counter *counter, int enable)
2721 struct perf_output_handle handle;
2725 struct perf_event_header header;
2727 } throttle_event = {
2729 .type = PERF_EVENT_THROTTLE + 1,
2731 .size = sizeof(throttle_event),
2733 .time = sched_clock(),
2736 ret = perf_output_begin(&handle, counter, sizeof(throttle_event), 1, 0);
2740 perf_output_put(&handle, throttle_event);
2741 perf_output_end(&handle);
2745 * Generic counter overflow handling.
2748 int perf_counter_overflow(struct perf_counter *counter,
2749 int nmi, struct pt_regs *regs, u64 addr)
2751 int events = atomic_read(&counter->event_limit);
2752 int throttle = counter->pmu->unthrottle != NULL;
2756 counter->hw.interrupts++;
2757 } else if (counter->hw.interrupts != MAX_INTERRUPTS) {
2758 counter->hw.interrupts++;
2759 if (HZ*counter->hw.interrupts > (u64)sysctl_perf_counter_limit) {
2760 counter->hw.interrupts = MAX_INTERRUPTS;
2761 perf_log_throttle(counter, 0);
2767 * XXX event_limit might not quite work as expected on inherited
2771 counter->pending_kill = POLL_IN;
2772 if (events && atomic_dec_and_test(&counter->event_limit)) {
2774 counter->pending_kill = POLL_HUP;
2776 counter->pending_disable = 1;
2777 perf_pending_queue(&counter->pending,
2778 perf_pending_counter);
2780 perf_counter_disable(counter);
2783 perf_counter_output(counter, nmi, regs, addr);
2788 * Generic software counter infrastructure
2791 static void perf_swcounter_update(struct perf_counter *counter)
2793 struct hw_perf_counter *hwc = &counter->hw;
2798 prev = atomic64_read(&hwc->prev_count);
2799 now = atomic64_read(&hwc->count);
2800 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2805 atomic64_add(delta, &counter->count);
2806 atomic64_sub(delta, &hwc->period_left);
2809 static void perf_swcounter_set_period(struct perf_counter *counter)
2811 struct hw_perf_counter *hwc = &counter->hw;
2812 s64 left = atomic64_read(&hwc->period_left);
2813 s64 period = hwc->irq_period;
2815 if (unlikely(left <= -period)) {
2817 atomic64_set(&hwc->period_left, left);
2820 if (unlikely(left <= 0)) {
2822 atomic64_add(period, &hwc->period_left);
2825 atomic64_set(&hwc->prev_count, -left);
2826 atomic64_set(&hwc->count, -left);
2829 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2831 enum hrtimer_restart ret = HRTIMER_RESTART;
2832 struct perf_counter *counter;
2833 struct pt_regs *regs;
2836 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2837 counter->pmu->read(counter);
2839 regs = get_irq_regs();
2841 * In case we exclude kernel IPs or are somehow not in interrupt
2842 * context, provide the next best thing, the user IP.
2844 if ((counter->hw_event.exclude_kernel || !regs) &&
2845 !counter->hw_event.exclude_user)
2846 regs = task_pt_regs(current);
2849 if (perf_counter_overflow(counter, 0, regs, 0))
2850 ret = HRTIMER_NORESTART;
2853 period = max_t(u64, 10000, counter->hw.irq_period);
2854 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
2859 static void perf_swcounter_overflow(struct perf_counter *counter,
2860 int nmi, struct pt_regs *regs, u64 addr)
2862 perf_swcounter_update(counter);
2863 perf_swcounter_set_period(counter);
2864 if (perf_counter_overflow(counter, nmi, regs, addr))
2865 /* soft-disable the counter */
2870 static int perf_swcounter_match(struct perf_counter *counter,
2871 enum perf_event_types type,
2872 u32 event, struct pt_regs *regs)
2874 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2877 if (perf_event_raw(&counter->hw_event))
2880 if (perf_event_type(&counter->hw_event) != type)
2883 if (perf_event_id(&counter->hw_event) != event)
2886 if (counter->hw_event.exclude_user && user_mode(regs))
2889 if (counter->hw_event.exclude_kernel && !user_mode(regs))
2895 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2896 int nmi, struct pt_regs *regs, u64 addr)
2898 int neg = atomic64_add_negative(nr, &counter->hw.count);
2899 if (counter->hw.irq_period && !neg)
2900 perf_swcounter_overflow(counter, nmi, regs, addr);
2903 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2904 enum perf_event_types type, u32 event,
2905 u64 nr, int nmi, struct pt_regs *regs,
2908 struct perf_counter *counter;
2910 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2914 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2915 if (perf_swcounter_match(counter, type, event, regs))
2916 perf_swcounter_add(counter, nr, nmi, regs, addr);
2921 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2924 return &cpuctx->recursion[3];
2927 return &cpuctx->recursion[2];
2930 return &cpuctx->recursion[1];
2932 return &cpuctx->recursion[0];
2935 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2936 u64 nr, int nmi, struct pt_regs *regs,
2939 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2940 int *recursion = perf_swcounter_recursion_context(cpuctx);
2941 struct perf_counter_context *ctx;
2949 perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2950 nr, nmi, regs, addr);
2953 * doesn't really matter which of the child contexts the
2954 * events ends up in.
2956 ctx = rcu_dereference(current->perf_counter_ctxp);
2958 perf_swcounter_ctx_event(ctx, type, event, nr, nmi, regs, addr);
2965 put_cpu_var(perf_cpu_context);
2969 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2971 __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2974 static void perf_swcounter_read(struct perf_counter *counter)
2976 perf_swcounter_update(counter);
2979 static int perf_swcounter_enable(struct perf_counter *counter)
2981 perf_swcounter_set_period(counter);
2985 static void perf_swcounter_disable(struct perf_counter *counter)
2987 perf_swcounter_update(counter);
2990 static const struct pmu perf_ops_generic = {
2991 .enable = perf_swcounter_enable,
2992 .disable = perf_swcounter_disable,
2993 .read = perf_swcounter_read,
2997 * Software counter: cpu wall time clock
3000 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
3002 int cpu = raw_smp_processor_id();
3006 now = cpu_clock(cpu);
3007 prev = atomic64_read(&counter->hw.prev_count);
3008 atomic64_set(&counter->hw.prev_count, now);
3009 atomic64_add(now - prev, &counter->count);
3012 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
3014 struct hw_perf_counter *hwc = &counter->hw;
3015 int cpu = raw_smp_processor_id();
3017 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
3018 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3019 hwc->hrtimer.function = perf_swcounter_hrtimer;
3020 if (hwc->irq_period) {
3021 u64 period = max_t(u64, 10000, hwc->irq_period);
3022 __hrtimer_start_range_ns(&hwc->hrtimer,
3023 ns_to_ktime(period), 0,
3024 HRTIMER_MODE_REL, 0);
3030 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
3032 if (counter->hw.irq_period)
3033 hrtimer_cancel(&counter->hw.hrtimer);
3034 cpu_clock_perf_counter_update(counter);
3037 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
3039 cpu_clock_perf_counter_update(counter);
3042 static const struct pmu perf_ops_cpu_clock = {
3043 .enable = cpu_clock_perf_counter_enable,
3044 .disable = cpu_clock_perf_counter_disable,
3045 .read = cpu_clock_perf_counter_read,
3049 * Software counter: task time clock
3052 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
3057 prev = atomic64_xchg(&counter->hw.prev_count, now);
3059 atomic64_add(delta, &counter->count);
3062 static int task_clock_perf_counter_enable(struct perf_counter *counter)
3064 struct hw_perf_counter *hwc = &counter->hw;
3067 now = counter->ctx->time;
3069 atomic64_set(&hwc->prev_count, now);
3070 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3071 hwc->hrtimer.function = perf_swcounter_hrtimer;
3072 if (hwc->irq_period) {
3073 u64 period = max_t(u64, 10000, hwc->irq_period);
3074 __hrtimer_start_range_ns(&hwc->hrtimer,
3075 ns_to_ktime(period), 0,
3076 HRTIMER_MODE_REL, 0);
3082 static void task_clock_perf_counter_disable(struct perf_counter *counter)
3084 if (counter->hw.irq_period)
3085 hrtimer_cancel(&counter->hw.hrtimer);
3086 task_clock_perf_counter_update(counter, counter->ctx->time);
3090 static void task_clock_perf_counter_read(struct perf_counter *counter)
3095 update_context_time(counter->ctx);
3096 time = counter->ctx->time;
3098 u64 now = perf_clock();
3099 u64 delta = now - counter->ctx->timestamp;
3100 time = counter->ctx->time + delta;
3103 task_clock_perf_counter_update(counter, time);
3106 static const struct pmu perf_ops_task_clock = {
3107 .enable = task_clock_perf_counter_enable,
3108 .disable = task_clock_perf_counter_disable,
3109 .read = task_clock_perf_counter_read,
3113 * Software counter: cpu migrations
3116 static inline u64 get_cpu_migrations(struct perf_counter *counter)
3118 struct task_struct *curr = counter->ctx->task;
3121 return curr->se.nr_migrations;
3122 return cpu_nr_migrations(smp_processor_id());
3125 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
3130 prev = atomic64_read(&counter->hw.prev_count);
3131 now = get_cpu_migrations(counter);
3133 atomic64_set(&counter->hw.prev_count, now);
3137 atomic64_add(delta, &counter->count);
3140 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
3142 cpu_migrations_perf_counter_update(counter);
3145 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
3147 if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
3148 atomic64_set(&counter->hw.prev_count,
3149 get_cpu_migrations(counter));
3153 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
3155 cpu_migrations_perf_counter_update(counter);
3158 static const struct pmu perf_ops_cpu_migrations = {
3159 .enable = cpu_migrations_perf_counter_enable,
3160 .disable = cpu_migrations_perf_counter_disable,
3161 .read = cpu_migrations_perf_counter_read,
3164 #ifdef CONFIG_EVENT_PROFILE
3165 void perf_tpcounter_event(int event_id)
3167 struct pt_regs *regs = get_irq_regs();
3170 regs = task_pt_regs(current);
3172 __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
3174 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
3176 extern int ftrace_profile_enable(int);
3177 extern void ftrace_profile_disable(int);
3179 static void tp_perf_counter_destroy(struct perf_counter *counter)
3181 ftrace_profile_disable(perf_event_id(&counter->hw_event));
3184 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3186 int event_id = perf_event_id(&counter->hw_event);
3189 ret = ftrace_profile_enable(event_id);
3193 counter->destroy = tp_perf_counter_destroy;
3194 counter->hw.irq_period = counter->hw_event.irq_period;
3196 return &perf_ops_generic;
3199 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3205 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
3207 const struct pmu *pmu = NULL;
3210 * Software counters (currently) can't in general distinguish
3211 * between user, kernel and hypervisor events.
3212 * However, context switches and cpu migrations are considered
3213 * to be kernel events, and page faults are never hypervisor
3216 switch (perf_event_id(&counter->hw_event)) {
3217 case PERF_COUNT_CPU_CLOCK:
3218 pmu = &perf_ops_cpu_clock;
3221 case PERF_COUNT_TASK_CLOCK:
3223 * If the user instantiates this as a per-cpu counter,
3224 * use the cpu_clock counter instead.
3226 if (counter->ctx->task)
3227 pmu = &perf_ops_task_clock;
3229 pmu = &perf_ops_cpu_clock;
3232 case PERF_COUNT_PAGE_FAULTS:
3233 case PERF_COUNT_PAGE_FAULTS_MIN:
3234 case PERF_COUNT_PAGE_FAULTS_MAJ:
3235 case PERF_COUNT_CONTEXT_SWITCHES:
3236 pmu = &perf_ops_generic;
3238 case PERF_COUNT_CPU_MIGRATIONS:
3239 if (!counter->hw_event.exclude_kernel)
3240 pmu = &perf_ops_cpu_migrations;
3248 * Allocate and initialize a counter structure
3250 static struct perf_counter *
3251 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
3253 struct perf_counter_context *ctx,
3254 struct perf_counter *group_leader,
3257 const struct pmu *pmu;
3258 struct perf_counter *counter;
3259 struct hw_perf_counter *hwc;
3262 counter = kzalloc(sizeof(*counter), gfpflags);
3264 return ERR_PTR(-ENOMEM);
3267 * Single counters are their own group leaders, with an
3268 * empty sibling list:
3271 group_leader = counter;
3273 mutex_init(&counter->child_mutex);
3274 INIT_LIST_HEAD(&counter->child_list);
3276 INIT_LIST_HEAD(&counter->list_entry);
3277 INIT_LIST_HEAD(&counter->event_entry);
3278 INIT_LIST_HEAD(&counter->sibling_list);
3279 init_waitqueue_head(&counter->waitq);
3281 mutex_init(&counter->mmap_mutex);
3284 counter->hw_event = *hw_event;
3285 counter->group_leader = group_leader;
3286 counter->pmu = NULL;
3288 counter->oncpu = -1;
3290 counter->state = PERF_COUNTER_STATE_INACTIVE;
3291 if (hw_event->disabled)
3292 counter->state = PERF_COUNTER_STATE_OFF;
3297 if (hw_event->freq && hw_event->irq_freq)
3298 hwc->irq_period = div64_u64(TICK_NSEC, hw_event->irq_freq);
3300 hwc->irq_period = hw_event->irq_period;
3303 * we currently do not support PERF_RECORD_GROUP on inherited counters
3305 if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
3308 if (perf_event_raw(hw_event)) {
3309 pmu = hw_perf_counter_init(counter);
3313 switch (perf_event_type(hw_event)) {
3314 case PERF_TYPE_HARDWARE:
3315 pmu = hw_perf_counter_init(counter);
3318 case PERF_TYPE_SOFTWARE:
3319 pmu = sw_perf_counter_init(counter);
3322 case PERF_TYPE_TRACEPOINT:
3323 pmu = tp_perf_counter_init(counter);
3330 else if (IS_ERR(pmu))
3335 return ERR_PTR(err);
3340 atomic_inc(&nr_counters);
3341 if (counter->hw_event.mmap)
3342 atomic_inc(&nr_mmap_tracking);
3343 if (counter->hw_event.munmap)
3344 atomic_inc(&nr_munmap_tracking);
3345 if (counter->hw_event.comm)
3346 atomic_inc(&nr_comm_tracking);
3352 * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
3354 * @hw_event_uptr: event type attributes for monitoring/sampling
3357 * @group_fd: group leader counter fd
3359 SYSCALL_DEFINE5(perf_counter_open,
3360 const struct perf_counter_hw_event __user *, hw_event_uptr,
3361 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
3363 struct perf_counter *counter, *group_leader;
3364 struct perf_counter_hw_event hw_event;
3365 struct perf_counter_context *ctx;
3366 struct file *counter_file = NULL;
3367 struct file *group_file = NULL;
3368 int fput_needed = 0;
3369 int fput_needed2 = 0;
3372 /* for future expandability... */
3376 if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
3380 * Get the target context (task or percpu):
3382 ctx = find_get_context(pid, cpu);
3384 return PTR_ERR(ctx);
3387 * Look up the group leader (we will attach this counter to it):
3389 group_leader = NULL;
3390 if (group_fd != -1) {
3392 group_file = fget_light(group_fd, &fput_needed);
3394 goto err_put_context;
3395 if (group_file->f_op != &perf_fops)
3396 goto err_put_context;
3398 group_leader = group_file->private_data;
3400 * Do not allow a recursive hierarchy (this new sibling
3401 * becoming part of another group-sibling):
3403 if (group_leader->group_leader != group_leader)
3404 goto err_put_context;
3406 * Do not allow to attach to a group in a different
3407 * task or CPU context:
3409 if (group_leader->ctx != ctx)
3410 goto err_put_context;
3412 * Only a group leader can be exclusive or pinned
3414 if (hw_event.exclusive || hw_event.pinned)
3415 goto err_put_context;
3418 counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
3420 ret = PTR_ERR(counter);
3421 if (IS_ERR(counter))
3422 goto err_put_context;
3424 ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3426 goto err_free_put_context;
3428 counter_file = fget_light(ret, &fput_needed2);
3430 goto err_free_put_context;
3432 counter->filp = counter_file;
3433 WARN_ON_ONCE(ctx->parent_ctx);
3434 mutex_lock(&ctx->mutex);
3435 perf_install_in_context(ctx, counter, cpu);
3437 mutex_unlock(&ctx->mutex);
3439 counter->owner = current;
3440 get_task_struct(current);
3441 mutex_lock(¤t->perf_counter_mutex);
3442 list_add_tail(&counter->owner_entry, ¤t->perf_counter_list);
3443 mutex_unlock(¤t->perf_counter_mutex);
3445 fput_light(counter_file, fput_needed2);
3448 fput_light(group_file, fput_needed);
3452 err_free_put_context:
3462 * inherit a counter from parent task to child task:
3464 static struct perf_counter *
3465 inherit_counter(struct perf_counter *parent_counter,
3466 struct task_struct *parent,
3467 struct perf_counter_context *parent_ctx,
3468 struct task_struct *child,
3469 struct perf_counter *group_leader,
3470 struct perf_counter_context *child_ctx)
3472 struct perf_counter *child_counter;
3475 * Instead of creating recursive hierarchies of counters,
3476 * we link inherited counters back to the original parent,
3477 * which has a filp for sure, which we use as the reference
3480 if (parent_counter->parent)
3481 parent_counter = parent_counter->parent;
3483 child_counter = perf_counter_alloc(&parent_counter->hw_event,
3484 parent_counter->cpu, child_ctx,
3485 group_leader, GFP_KERNEL);
3486 if (IS_ERR(child_counter))
3487 return child_counter;
3491 * Make the child state follow the state of the parent counter,
3492 * not its hw_event.disabled bit. We hold the parent's mutex,
3493 * so we won't race with perf_counter_{en,dis}able_family.
3495 if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3496 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3498 child_counter->state = PERF_COUNTER_STATE_OFF;
3501 * Link it up in the child's context:
3503 add_counter_to_ctx(child_counter, child_ctx);
3505 child_counter->parent = parent_counter;
3507 * inherit into child's child as well:
3509 child_counter->hw_event.inherit = 1;
3512 * Get a reference to the parent filp - we will fput it
3513 * when the child counter exits. This is safe to do because
3514 * we are in the parent and we know that the filp still
3515 * exists and has a nonzero count:
3517 atomic_long_inc(&parent_counter->filp->f_count);
3520 * Link this into the parent counter's child list
3522 WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3523 mutex_lock(&parent_counter->child_mutex);
3524 list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3525 mutex_unlock(&parent_counter->child_mutex);
3527 return child_counter;
3530 static int inherit_group(struct perf_counter *parent_counter,
3531 struct task_struct *parent,
3532 struct perf_counter_context *parent_ctx,
3533 struct task_struct *child,
3534 struct perf_counter_context *child_ctx)
3536 struct perf_counter *leader;
3537 struct perf_counter *sub;
3538 struct perf_counter *child_ctr;
3540 leader = inherit_counter(parent_counter, parent, parent_ctx,
3541 child, NULL, child_ctx);
3543 return PTR_ERR(leader);
3544 list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3545 child_ctr = inherit_counter(sub, parent, parent_ctx,
3546 child, leader, child_ctx);
3547 if (IS_ERR(child_ctr))
3548 return PTR_ERR(child_ctr);
3553 static void sync_child_counter(struct perf_counter *child_counter,
3554 struct perf_counter *parent_counter)
3558 child_val = atomic64_read(&child_counter->count);
3561 * Add back the child's count to the parent's count:
3563 atomic64_add(child_val, &parent_counter->count);
3564 atomic64_add(child_counter->total_time_enabled,
3565 &parent_counter->child_total_time_enabled);
3566 atomic64_add(child_counter->total_time_running,
3567 &parent_counter->child_total_time_running);
3570 * Remove this counter from the parent's list
3572 WARN_ON_ONCE(parent_counter->ctx->parent_ctx);
3573 mutex_lock(&parent_counter->child_mutex);
3574 list_del_init(&child_counter->child_list);
3575 mutex_unlock(&parent_counter->child_mutex);
3578 * Release the parent counter, if this was the last
3581 fput(parent_counter->filp);
3585 __perf_counter_exit_task(struct perf_counter *child_counter,
3586 struct perf_counter_context *child_ctx)
3588 struct perf_counter *parent_counter;
3590 update_counter_times(child_counter);
3591 perf_counter_remove_from_context(child_counter);
3593 parent_counter = child_counter->parent;
3595 * It can happen that parent exits first, and has counters
3596 * that are still around due to the child reference. These
3597 * counters need to be zapped - but otherwise linger.
3599 if (parent_counter) {
3600 sync_child_counter(child_counter, parent_counter);
3601 free_counter(child_counter);
3606 * When a child task exits, feed back counter values to parent counters.
3608 void perf_counter_exit_task(struct task_struct *child)
3610 struct perf_counter *child_counter, *tmp;
3611 struct perf_counter_context *child_ctx;
3612 unsigned long flags;
3614 if (likely(!child->perf_counter_ctxp))
3617 local_irq_save(flags);
3619 * We can't reschedule here because interrupts are disabled,
3620 * and either child is current or it is a task that can't be
3621 * scheduled, so we are now safe from rescheduling changing
3624 child_ctx = child->perf_counter_ctxp;
3625 __perf_counter_task_sched_out(child_ctx);
3628 * Take the context lock here so that if find_get_context is
3629 * reading child->perf_counter_ctxp, we wait until it has
3630 * incremented the context's refcount before we do put_ctx below.
3632 spin_lock(&child_ctx->lock);
3633 child->perf_counter_ctxp = NULL;
3634 if (child_ctx->parent_ctx) {
3636 * This context is a clone; unclone it so it can't get
3637 * swapped to another process while we're removing all
3638 * the counters from it.
3640 put_ctx(child_ctx->parent_ctx);
3641 child_ctx->parent_ctx = NULL;
3643 spin_unlock(&child_ctx->lock);
3644 local_irq_restore(flags);
3646 mutex_lock(&child_ctx->mutex);
3649 list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3651 __perf_counter_exit_task(child_counter, child_ctx);
3654 * If the last counter was a group counter, it will have appended all
3655 * its siblings to the list, but we obtained 'tmp' before that which
3656 * will still point to the list head terminating the iteration.
3658 if (!list_empty(&child_ctx->counter_list))
3661 mutex_unlock(&child_ctx->mutex);
3667 * free an unexposed, unused context as created by inheritance by
3668 * init_task below, used by fork() in case of fail.
3670 void perf_counter_free_task(struct task_struct *task)
3672 struct perf_counter_context *ctx = task->perf_counter_ctxp;
3673 struct perf_counter *counter, *tmp;
3678 mutex_lock(&ctx->mutex);
3680 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry) {
3681 struct perf_counter *parent = counter->parent;
3683 if (WARN_ON_ONCE(!parent))
3686 mutex_lock(&parent->child_mutex);
3687 list_del_init(&counter->child_list);
3688 mutex_unlock(&parent->child_mutex);
3692 list_del_counter(counter, ctx);
3693 free_counter(counter);
3696 if (!list_empty(&ctx->counter_list))
3699 mutex_unlock(&ctx->mutex);
3705 * Initialize the perf_counter context in task_struct
3707 int perf_counter_init_task(struct task_struct *child)
3709 struct perf_counter_context *child_ctx, *parent_ctx;
3710 struct perf_counter_context *cloned_ctx;
3711 struct perf_counter *counter;
3712 struct task_struct *parent = current;
3713 int inherited_all = 1;
3716 child->perf_counter_ctxp = NULL;
3718 mutex_init(&child->perf_counter_mutex);
3719 INIT_LIST_HEAD(&child->perf_counter_list);
3721 if (likely(!parent->perf_counter_ctxp))
3725 * This is executed from the parent task context, so inherit
3726 * counters that have been marked for cloning.
3727 * First allocate and initialize a context for the child.
3730 child_ctx = kmalloc(sizeof(struct perf_counter_context), GFP_KERNEL);
3734 __perf_counter_init_context(child_ctx, child);
3735 child->perf_counter_ctxp = child_ctx;
3736 get_task_struct(child);
3739 * If the parent's context is a clone, pin it so it won't get
3742 parent_ctx = perf_pin_task_context(parent);
3745 * No need to check if parent_ctx != NULL here; since we saw
3746 * it non-NULL earlier, the only reason for it to become NULL
3747 * is if we exit, and since we're currently in the middle of
3748 * a fork we can't be exiting at the same time.
3752 * Lock the parent list. No need to lock the child - not PID
3753 * hashed yet and not running, so nobody can access it.
3755 mutex_lock(&parent_ctx->mutex);
3758 * We dont have to disable NMIs - we are only looking at
3759 * the list, not manipulating it:
3761 list_for_each_entry_rcu(counter, &parent_ctx->event_list, event_entry) {
3762 if (counter != counter->group_leader)
3765 if (!counter->hw_event.inherit) {
3770 ret = inherit_group(counter, parent, parent_ctx,
3778 if (inherited_all) {
3780 * Mark the child context as a clone of the parent
3781 * context, or of whatever the parent is a clone of.
3782 * Note that if the parent is a clone, it could get
3783 * uncloned at any point, but that doesn't matter
3784 * because the list of counters and the generation
3785 * count can't have changed since we took the mutex.
3787 cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
3789 child_ctx->parent_ctx = cloned_ctx;
3790 child_ctx->parent_gen = parent_ctx->parent_gen;
3792 child_ctx->parent_ctx = parent_ctx;
3793 child_ctx->parent_gen = parent_ctx->generation;
3795 get_ctx(child_ctx->parent_ctx);
3798 mutex_unlock(&parent_ctx->mutex);
3800 perf_unpin_context(parent_ctx);
3805 static void __cpuinit perf_counter_init_cpu(int cpu)
3807 struct perf_cpu_context *cpuctx;
3809 cpuctx = &per_cpu(perf_cpu_context, cpu);
3810 __perf_counter_init_context(&cpuctx->ctx, NULL);
3812 spin_lock(&perf_resource_lock);
3813 cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3814 spin_unlock(&perf_resource_lock);
3816 hw_perf_counter_setup(cpu);
3819 #ifdef CONFIG_HOTPLUG_CPU
3820 static void __perf_counter_exit_cpu(void *info)
3822 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3823 struct perf_counter_context *ctx = &cpuctx->ctx;
3824 struct perf_counter *counter, *tmp;
3826 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3827 __perf_counter_remove_from_context(counter);
3829 static void perf_counter_exit_cpu(int cpu)
3831 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3832 struct perf_counter_context *ctx = &cpuctx->ctx;
3834 mutex_lock(&ctx->mutex);
3835 smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3836 mutex_unlock(&ctx->mutex);
3839 static inline void perf_counter_exit_cpu(int cpu) { }
3842 static int __cpuinit
3843 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3845 unsigned int cpu = (long)hcpu;
3849 case CPU_UP_PREPARE:
3850 case CPU_UP_PREPARE_FROZEN:
3851 perf_counter_init_cpu(cpu);
3854 case CPU_DOWN_PREPARE:
3855 case CPU_DOWN_PREPARE_FROZEN:
3856 perf_counter_exit_cpu(cpu);
3866 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3867 .notifier_call = perf_cpu_notify,
3870 void __init perf_counter_init(void)
3872 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3873 (void *)(long)smp_processor_id());
3874 register_cpu_notifier(&perf_cpu_nb);
3877 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3879 return sprintf(buf, "%d\n", perf_reserved_percpu);
3883 perf_set_reserve_percpu(struct sysdev_class *class,
3887 struct perf_cpu_context *cpuctx;
3891 err = strict_strtoul(buf, 10, &val);
3894 if (val > perf_max_counters)
3897 spin_lock(&perf_resource_lock);
3898 perf_reserved_percpu = val;
3899 for_each_online_cpu(cpu) {
3900 cpuctx = &per_cpu(perf_cpu_context, cpu);
3901 spin_lock_irq(&cpuctx->ctx.lock);
3902 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3903 perf_max_counters - perf_reserved_percpu);
3904 cpuctx->max_pertask = mpt;
3905 spin_unlock_irq(&cpuctx->ctx.lock);
3907 spin_unlock(&perf_resource_lock);
3912 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3914 return sprintf(buf, "%d\n", perf_overcommit);
3918 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3923 err = strict_strtoul(buf, 10, &val);
3929 spin_lock(&perf_resource_lock);
3930 perf_overcommit = val;
3931 spin_unlock(&perf_resource_lock);
3936 static SYSDEV_CLASS_ATTR(
3939 perf_show_reserve_percpu,
3940 perf_set_reserve_percpu
3943 static SYSDEV_CLASS_ATTR(
3946 perf_show_overcommit,
3950 static struct attribute *perfclass_attrs[] = {
3951 &attr_reserve_percpu.attr,
3952 &attr_overcommit.attr,
3956 static struct attribute_group perfclass_attr_group = {
3957 .attrs = perfclass_attrs,
3958 .name = "perf_counters",
3961 static int __init perf_counter_sysfs_init(void)
3963 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3964 &perfclass_attr_group);
3966 device_initcall(perf_counter_sysfs_init);