arch/s390/oprofile/hwsampler.c

   1 /**
   2  * arch/s390/oprofile/hwsampler.c
   3  *
   4  * Copyright IBM Corp. 2010
   5  * Author: Heinz Graalfs <graalfs@de.ibm.com>
   6  */
   7
   8 #include <linux/kernel_stat.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/smp.h>
  12 #include <linux/errno.h>
  13 #include <linux/workqueue.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/notifier.h>
  16 #include <linux/cpu.h>
  17 #include <linux/semaphore.h>
  18 #include <linux/oom.h>
  19 #include <linux/oprofile.h>
  20
  21 #include <asm/lowcore.h>
  22 #include <asm/irq.h>
  23
  24 #include "hwsampler.h"
  25 #include "op_counter.h"
  26
  27 #define MAX_NUM_SDB 511
  28 #define MIN_NUM_SDB 1
  29
  30 #define ALERT_REQ_MASK   0x4000000000000000ul
  31 #define BUFFER_FULL_MASK 0x8000000000000000ul
  32
  33 #define EI_IEA      (1 << 31)   /* invalid entry address              */
  34 #define EI_ISE      (1 << 30)   /* incorrect SDBT entry               */
  35 #define EI_PRA      (1 << 29)   /* program request alert              */
  36 #define EI_SACA     (1 << 23)   /* sampler authorization change alert */
  37 #define EI_LSDA     (1 << 22)   /* loss of sample data alert          */
  38
  39 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  40
  41 struct hws_execute_parms {
  42         void *buffer;
  43         signed int rc;
  44 };
  45
  46 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  47 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
  48
  49 static DEFINE_MUTEX(hws_sem);
  50 static DEFINE_MUTEX(hws_sem_oom);
  51
  52 static unsigned char hws_flush_all;
  53 static unsigned int hws_oom;
  54 static struct workqueue_struct *hws_wq;
  55
  56 static unsigned int hws_state;
  57 enum {
  58         HWS_INIT = 1,
  59         HWS_DEALLOCATED,
  60         HWS_STOPPED,
  61         HWS_STARTED,
  62         HWS_STOPPING };
  63
  64 /* set to 1 if called by kernel during memory allocation */
  65 static unsigned char oom_killer_was_active;
  66 /* size of SDBT and SDB as of allocate API */
  67 static unsigned long num_sdbt = 100;
  68 static unsigned long num_sdb = 511;
  69 /* sampling interval (machine cycles) */
  70 static unsigned long interval;
  71
  72 static unsigned long min_sampler_rate;
  73 static unsigned long max_sampler_rate;
  74
  75 static int ssctl(void *buffer)
  76 {
  77         int cc;
  78
  79         /* set in order to detect a program check */
  80         cc = 1;
  81
  82         asm volatile(
  83                 "0: .insn s,0xB2870000,0(%1)\n"
  84                 "1: ipm %0\n"
  85                 "   srl %0,28\n"
  86                 "2:\n"
  87                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  88                 : "+d" (cc), "+a" (buffer)
  89                 : "m" (*((struct hws_ssctl_request_block *)buffer))
  90                 : "cc", "memory");
  91
  92         return cc ? -EINVAL : 0 ;
  93 }
  94
  95 static int qsi(void *buffer)
  96 {
  97         int cc;
  98         cc = 1;
  99
 100         asm volatile(
 101                 "0: .insn s,0xB2860000,0(%1)\n"
 102                 "1: lhi %0,0\n"
 103                 "2:\n"
 104                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
 105                 : "=d" (cc), "+a" (buffer)
 106                 : "m" (*((struct hws_qsi_info_block *)buffer))
 107                 : "cc", "memory");
 108
 109         return cc ? -EINVAL : 0;
 110 }
 111
 112 static void execute_qsi(void *parms)
 113 {
 114         struct hws_execute_parms *ep = parms;
 115
 116         ep->rc = qsi(ep->buffer);
 117 }
 118
 119 static void execute_ssctl(void *parms)
 120 {
 121         struct hws_execute_parms *ep = parms;
 122
 123         ep->rc = ssctl(ep->buffer);
 124 }
 125
 126 static int smp_ctl_ssctl_stop(int cpu)
 127 {
 128         int rc;
 129         struct hws_execute_parms ep;
 130         struct hws_cpu_buffer *cb;
 131
 132         cb = &per_cpu(sampler_cpu_buffer, cpu);
 133
 134         cb->ssctl.es = 0;
 135         cb->ssctl.cs = 0;
 136
 137         ep.buffer = &cb->ssctl;
 138         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 139         rc = ep.rc;
 140         if (rc) {
 141                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 142                 dump_stack();
 143         }
 144
 145         ep.buffer = &cb->qsi;
 146         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 147
 148         if (cb->qsi.es || cb->qsi.cs) {
 149                 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
 150                 dump_stack();
 151         }
 152
 153         return rc;
 154 }
 155
 156 static int smp_ctl_ssctl_deactivate(int cpu)
 157 {
 158         int rc;
 159         struct hws_execute_parms ep;
 160         struct hws_cpu_buffer *cb;
 161
 162         cb = &per_cpu(sampler_cpu_buffer, cpu);
 163
 164         cb->ssctl.es = 1;
 165         cb->ssctl.cs = 0;
 166
 167         ep.buffer = &cb->ssctl;
 168         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 169         rc = ep.rc;
 170         if (rc)
 171                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 172
 173         ep.buffer = &cb->qsi;
 174         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 175
 176         if (cb->qsi.cs)
 177                 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
 178
 179         return rc;
 180 }
 181
 182 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
 183 {
 184         int rc;
 185         struct hws_execute_parms ep;
 186         struct hws_cpu_buffer *cb;
 187
 188         cb = &per_cpu(sampler_cpu_buffer, cpu);
 189
 190         cb->ssctl.h = 1;
 191         cb->ssctl.tear = cb->first_sdbt;
 192         cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
 193         cb->ssctl.interval = interval;
 194         cb->ssctl.es = 1;
 195         cb->ssctl.cs = 1;
 196
 197         ep.buffer = &cb->ssctl;
 198         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 199         rc = ep.rc;
 200         if (rc)
 201                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 202
 203         ep.buffer = &cb->qsi;
 204         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 205         if (ep.rc)
 206                 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
 207
 208         return rc;
 209 }
 210
 211 static int smp_ctl_qsi(int cpu)
 212 {
 213         struct hws_execute_parms ep;
 214         struct hws_cpu_buffer *cb;
 215
 216         cb = &per_cpu(sampler_cpu_buffer, cpu);
 217
 218         ep.buffer = &cb->qsi;
 219         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 220
 221         return ep.rc;
 222 }
 223
 224 static inline unsigned long *trailer_entry_ptr(unsigned long v)
 225 {
 226         void *ret;
 227
 228         ret = (void *)v;
 229         ret += PAGE_SIZE;
 230         ret -= sizeof(struct hws_trailer_entry);
 231
 232         return (unsigned long *) ret;
 233 }
 234
 235 /* prototypes for external interrupt handler and worker */
 236 static void hws_ext_handler(unsigned int ext_int_code,
 237                                 unsigned int param32, unsigned long param64);
 238
 239 static void worker(struct work_struct *work);
 240
 241 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
 242                                 unsigned long *dear);
 243
 244 static void init_all_cpu_buffers(void)
 245 {
 246         int cpu;
 247         struct hws_cpu_buffer *cb;
 248
 249         for_each_online_cpu(cpu) {
 250                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 251                 memset(cb, 0, sizeof(struct hws_cpu_buffer));
 252         }
 253 }
 254
 255 static int is_link_entry(unsigned long *s)
 256 {
 257         return *s & 0x1ul ? 1 : 0;
 258 }
 259
 260 static unsigned long *get_next_sdbt(unsigned long *s)
 261 {
 262         return (unsigned long *) (*s & ~0x1ul);
 263 }
 264
 265 static int prepare_cpu_buffers(void)
 266 {
 267         int cpu;
 268         int rc;
 269         struct hws_cpu_buffer *cb;
 270
 271         rc = 0;
 272         for_each_online_cpu(cpu) {
 273                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 274                 atomic_set(&cb->ext_params, 0);
 275                 cb->worker_entry = 0;
 276                 cb->sample_overflow = 0;
 277                 cb->req_alert = 0;
 278                 cb->incorrect_sdbt_entry = 0;
 279                 cb->invalid_entry_address = 0;
 280                 cb->loss_of_sample_data = 0;
 281                 cb->sample_auth_change_alert = 0;
 282                 cb->finish = 0;
 283                 cb->oom = 0;
 284                 cb->stop_mode = 0;
 285         }
 286
 287         return rc;
 288 }
 289
 290 /*
 291  * allocate_sdbt() - allocate sampler memory
 292  * @cpu: the cpu for which sampler memory is allocated
 293  *
 294  * A 4K page is allocated for each requested SDBT.
 295  * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
 296  * Set ALERT_REQ mask in each SDBs trailer.
 297  * Returns zero if successful, <0 otherwise.
 298  */
 299 static int allocate_sdbt(int cpu)
 300 {
 301         int j, k, rc;
 302         unsigned long *sdbt;
 303         unsigned long  sdb;
 304         unsigned long *tail;
 305         unsigned long *trailer;
 306         struct hws_cpu_buffer *cb;
 307
 308         cb = &per_cpu(sampler_cpu_buffer, cpu);
 309
 310         if (cb->first_sdbt)
 311                 return -EINVAL;
 312
 313         sdbt = NULL;
 314         tail = sdbt;
 315
 316         for (j = 0; j < num_sdbt; j++) {
 317                 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 318
 319                 mutex_lock(&hws_sem_oom);
 320                 /* OOM killer might have been activated */
 321                 barrier();
 322                 if (oom_killer_was_active || !sdbt) {
 323                         if (sdbt)
 324                                 free_page((unsigned long)sdbt);
 325
 326                         goto allocate_sdbt_error;
 327                 }
 328                 if (cb->first_sdbt == 0)
 329                         cb->first_sdbt = (unsigned long)sdbt;
 330
 331                 /* link current page to tail of chain */
 332                 if (tail)
 333                         *tail = (unsigned long)(void *)sdbt + 1;
 334
 335                 mutex_unlock(&hws_sem_oom);
 336
 337                 for (k = 0; k < num_sdb; k++) {
 338                         /* get and set SDB page */
 339                         sdb = get_zeroed_page(GFP_KERNEL);
 340
 341                         mutex_lock(&hws_sem_oom);
 342                         /* OOM killer might have been activated */
 343                         barrier();
 344                         if (oom_killer_was_active || !sdb) {
 345                                 if (sdb)
 346                                         free_page(sdb);
 347
 348                                 goto allocate_sdbt_error;
 349                         }
 350                         *sdbt = sdb;
 351                         trailer = trailer_entry_ptr(*sdbt);
 352                         *trailer = ALERT_REQ_MASK;
 353                         sdbt++;
 354                         mutex_unlock(&hws_sem_oom);
 355                 }
 356                 tail = sdbt;
 357         }
 358         mutex_lock(&hws_sem_oom);
 359         if (oom_killer_was_active)
 360                 goto allocate_sdbt_error;
 361
 362         rc = 0;
 363         if (tail)
 364                 *tail = (unsigned long)
 365                         ((void *)cb->first_sdbt) + 1;
 366
 367 allocate_sdbt_exit:
 368         mutex_unlock(&hws_sem_oom);
 369         return rc;
 370
 371 allocate_sdbt_error:
 372         rc = -ENOMEM;
 373         goto allocate_sdbt_exit;
 374 }
 375
 376 /*
 377  * deallocate_sdbt() - deallocate all sampler memory
 378  *
 379  * For each online CPU all SDBT trees are deallocated.
 380  * Returns the number of freed pages.
 381  */
 382 static int deallocate_sdbt(void)
 383 {
 384         int cpu;
 385         int counter;
 386
 387         counter = 0;
 388
 389         for_each_online_cpu(cpu) {
 390                 unsigned long start;
 391                 unsigned long sdbt;
 392                 unsigned long *curr;
 393                 struct hws_cpu_buffer *cb;
 394
 395                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 396
 397                 if (!cb->first_sdbt)
 398                         continue;
 399
 400                 sdbt = cb->first_sdbt;
 401                 curr = (unsigned long *) sdbt;
 402                 start = sdbt;
 403
 404                 /* we'll free the SDBT after all SDBs are processed... */
 405                 while (1) {
 406                         if (!*curr || !sdbt)
 407                                 break;
 408
 409                         /* watch for link entry reset if found */
 410                         if (is_link_entry(curr)) {
 411                                 curr = get_next_sdbt(curr);
 412                                 if (sdbt)
 413                                         free_page(sdbt);
 414
 415                                 /* we are done if we reach the start */
 416                                 if ((unsigned long) curr == start)
 417                                         break;
 418                                 else
 419                                         sdbt = (unsigned long) curr;
 420                         } else {
 421                                 /* process SDB pointer */
 422                                 if (*curr) {
 423                                         free_page(*curr);
 424                                         curr++;
 425                                 }
 426                         }
 427                         counter++;
 428                 }
 429                 cb->first_sdbt = 0;
 430         }
 431         return counter;
 432 }
 433
 434 static int start_sampling(int cpu)
 435 {
 436         int rc;
 437         struct hws_cpu_buffer *cb;
 438
 439         cb = &per_cpu(sampler_cpu_buffer, cpu);
 440         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 441         if (rc) {
 442                 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
 443                 goto start_exit;
 444         }
 445
 446         rc = -EINVAL;
 447         if (!cb->qsi.es) {
 448                 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
 449                 goto start_exit;
 450         }
 451
 452         if (!cb->qsi.cs) {
 453                 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
 454                 goto start_exit;
 455         }
 456
 457         printk(KERN_INFO
 458                 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
 459                 cpu, interval);
 460
 461         rc = 0;
 462
 463 start_exit:
 464         return rc;
 465 }
 466
 467 static int stop_sampling(int cpu)
 468 {
 469         unsigned long v;
 470         int rc;
 471         struct hws_cpu_buffer *cb;
 472
 473         rc = smp_ctl_qsi(cpu);
 474         WARN_ON(rc);
 475
 476         cb = &per_cpu(sampler_cpu_buffer, cpu);
 477         if (!rc && !cb->qsi.es)
 478                 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
 479
 480         rc = smp_ctl_ssctl_stop(cpu);
 481         if (rc) {
 482                 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
 483                                 cpu, rc);
 484                 goto stop_exit;
 485         }
 486
 487         printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
 488
 489 stop_exit:
 490         v = cb->req_alert;
 491         if (v)
 492                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
 493                                 " count=%lu.\n", cpu, v);
 494
 495         v = cb->loss_of_sample_data;
 496         if (v)
 497                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
 498                                 " count=%lu.\n", cpu, v);
 499
 500         v = cb->invalid_entry_address;
 501         if (v)
 502                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
 503                                 " count=%lu.\n", cpu, v);
 504
 505         v = cb->incorrect_sdbt_entry;
 506         if (v)
 507                 printk(KERN_ERR
 508                                 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
 509                                 " count=%lu.\n", cpu, v);
 510
 511         v = cb->sample_auth_change_alert;
 512         if (v)
 513                 printk(KERN_ERR
 514                                 "hwsampler: CPU %d CPUMF Sample authorization change,"
 515                                 " count=%lu.\n", cpu, v);
 516
 517         return rc;
 518 }
 519
 520 static int check_hardware_prerequisites(void)
 521 {
 522         if (!test_facility(68))
 523                 return -EOPNOTSUPP;
 524         return 0;
 525 }
 526 /*
 527  * hws_oom_callback() - the OOM callback function
 528  *
 529  * In case the callback is invoked during memory allocation for the
 530  *  hw sampler, all obtained memory is deallocated and a flag is set
 531  *  so main sampler memory allocation can exit with a failure code.
 532  * In case the callback is invoked during sampling the hw sampler
 533  *  is deactivated for all CPUs.
 534  */
 535 static int hws_oom_callback(struct notifier_block *nfb,
 536         unsigned long dummy, void *parm)
 537 {
 538         unsigned long *freed;
 539         int cpu;
 540         struct hws_cpu_buffer *cb;
 541
 542         freed = parm;
 543
 544         mutex_lock(&hws_sem_oom);
 545
 546         if (hws_state == HWS_DEALLOCATED) {
 547                 /* during memory allocation */
 548                 if (oom_killer_was_active == 0) {
 549                         oom_killer_was_active = 1;
 550                         *freed += deallocate_sdbt();
 551                 }
 552         } else {
 553                 int i;
 554                 cpu = get_cpu();
 555                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 556
 557                 if (!cb->oom) {
 558                         for_each_online_cpu(i) {
 559                                 smp_ctl_ssctl_deactivate(i);
 560                                 cb->oom = 1;
 561                         }
 562                         cb->finish = 1;
 563
 564                         printk(KERN_INFO
 565                                 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
 566                                 cpu);
 567                 }
 568         }
 569
 570         mutex_unlock(&hws_sem_oom);
 571
 572         return NOTIFY_OK;
 573 }
 574
 575 static struct notifier_block hws_oom_notifier = {
 576         .notifier_call = hws_oom_callback
 577 };
 578
 579 static int hws_cpu_callback(struct notifier_block *nfb,
 580         unsigned long action, void *hcpu)
 581 {
 582         /* We do not have sampler space available for all possible CPUs.
 583            All CPUs should be online when hw sampling is activated. */
 584         return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
 585 }
 586
 587 static struct notifier_block hws_cpu_notifier = {
 588         .notifier_call = hws_cpu_callback
 589 };
 590
 591 /**
 592  * hwsampler_deactivate() - set hardware sampling temporarily inactive
 593  * @cpu:  specifies the CPU to be set inactive.
 594  *
 595  * Returns 0 on success, !0 on failure.
 596  */
 597 int hwsampler_deactivate(unsigned int cpu)
 598 {
 599         /*
 600          * Deactivate hw sampling temporarily and flush the buffer
 601          * by pushing all the pending samples to oprofile buffer.
 602          *
 603          * This function can be called under one of the following conditions:
 604          *     Memory unmap, task is exiting.
 605          */
 606         int rc;
 607         struct hws_cpu_buffer *cb;
 608
 609         rc = 0;
 610         mutex_lock(&hws_sem);
 611
 612         cb = &per_cpu(sampler_cpu_buffer, cpu);
 613         if (hws_state == HWS_STARTED) {
 614                 rc = smp_ctl_qsi(cpu);
 615                 WARN_ON(rc);
 616                 if (cb->qsi.cs) {
 617                         rc = smp_ctl_ssctl_deactivate(cpu);
 618                         if (rc) {
 619                                 printk(KERN_INFO
 620                                 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
 621                                 cb->finish = 1;
 622                                 hws_state = HWS_STOPPING;
 623                         } else  {
 624                                 hws_flush_all = 1;
 625                                 /* Add work to queue to read pending samples.*/
 626                                 queue_work_on(cpu, hws_wq, &cb->worker);
 627                         }
 628                 }
 629         }
 630         mutex_unlock(&hws_sem);
 631
 632         if (hws_wq)
 633                 flush_workqueue(hws_wq);
 634
 635         return rc;
 636 }
 637
 638 /**
 639  * hwsampler_activate() - activate/resume hardware sampling which was deactivated
 640  * @cpu:  specifies the CPU to be set active.
 641  *
 642  * Returns 0 on success, !0 on failure.
 643  */
 644 int hwsampler_activate(unsigned int cpu)
 645 {
 646         /*
 647          * Re-activate hw sampling. This should be called in pair with
 648          * hwsampler_deactivate().
 649          */
 650         int rc;
 651         struct hws_cpu_buffer *cb;
 652
 653         rc = 0;
 654         mutex_lock(&hws_sem);
 655
 656         cb = &per_cpu(sampler_cpu_buffer, cpu);
 657         if (hws_state == HWS_STARTED) {
 658                 rc = smp_ctl_qsi(cpu);
 659                 WARN_ON(rc);
 660                 if (!cb->qsi.cs) {
 661                         hws_flush_all = 0;
 662                         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 663                         if (rc) {
 664                                 printk(KERN_ERR
 665                                 "CPU %d, CPUMF activate sampling failed.\n",
 666                                          cpu);
 667                         }
 668                 }
 669         }
 670
 671         mutex_unlock(&hws_sem);
 672
 673         return rc;
 674 }
 675
 676 static void hws_ext_handler(unsigned int ext_int_code,
 677                             unsigned int param32, unsigned long param64)
 678 {
 679         struct hws_cpu_buffer *cb;
 680
 681         kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++;
 682         cb = &__get_cpu_var(sampler_cpu_buffer);
 683         atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
 684         if (hws_wq)
 685                 queue_work(hws_wq, &cb->worker);
 686 }
 687
 688 static int check_qsi_on_setup(void)
 689 {
 690         int rc;
 691         unsigned int cpu;
 692         struct hws_cpu_buffer *cb;
 693
 694         for_each_online_cpu(cpu) {
 695                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 696                 rc = smp_ctl_qsi(cpu);
 697                 WARN_ON(rc);
 698                 if (rc)
 699                         return -EOPNOTSUPP;
 700
 701                 if (!cb->qsi.as) {
 702                         printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
 703                         return -EINVAL;
 704                 }
 705
 706                 if (cb->qsi.es) {
 707                         printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
 708                         rc = smp_ctl_ssctl_stop(cpu);
 709                         if (rc)
 710                                 return -EINVAL;
 711
 712                         printk(KERN_INFO
 713                                 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
 714                 }
 715         }
 716         return 0;
 717 }
 718
 719 static int check_qsi_on_start(void)
 720 {
 721         unsigned int cpu;
 722         int rc;
 723         struct hws_cpu_buffer *cb;
 724
 725         for_each_online_cpu(cpu) {
 726                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 727                 rc = smp_ctl_qsi(cpu);
 728                 WARN_ON(rc);
 729
 730                 if (!cb->qsi.as)
 731                         return -EINVAL;
 732
 733                 if (cb->qsi.es)
 734                         return -EINVAL;
 735
 736                 if (cb->qsi.cs)
 737                         return -EINVAL;
 738         }
 739         return 0;
 740 }
 741
 742 static void worker_on_start(unsigned int cpu)
 743 {
 744         struct hws_cpu_buffer *cb;
 745
 746         cb = &per_cpu(sampler_cpu_buffer, cpu);
 747         cb->worker_entry = cb->first_sdbt;
 748 }
 749
 750 static int worker_check_error(unsigned int cpu, int ext_params)
 751 {
 752         int rc;
 753         unsigned long *sdbt;
 754         struct hws_cpu_buffer *cb;
 755
 756         rc = 0;
 757         cb = &per_cpu(sampler_cpu_buffer, cpu);
 758         sdbt = (unsigned long *) cb->worker_entry;
 759
 760         if (!sdbt || !*sdbt)
 761                 return -EINVAL;
 762
 763         if (ext_params & EI_PRA)
 764                 cb->req_alert++;
 765
 766         if (ext_params & EI_LSDA)
 767                 cb->loss_of_sample_data++;
 768
 769         if (ext_params & EI_IEA) {
 770                 cb->invalid_entry_address++;
 771                 rc = -EINVAL;
 772         }
 773
 774         if (ext_params & EI_ISE) {
 775                 cb->incorrect_sdbt_entry++;
 776                 rc = -EINVAL;
 777         }
 778
 779         if (ext_params & EI_SACA) {
 780                 cb->sample_auth_change_alert++;
 781                 rc = -EINVAL;
 782         }
 783
 784         return rc;
 785 }
 786
 787 static void worker_on_finish(unsigned int cpu)
 788 {
 789         int rc, i;
 790         struct hws_cpu_buffer *cb;
 791
 792         cb = &per_cpu(sampler_cpu_buffer, cpu);
 793
 794         if (cb->finish) {
 795                 rc = smp_ctl_qsi(cpu);
 796                 WARN_ON(rc);
 797                 if (cb->qsi.es) {
 798                         printk(KERN_INFO
 799                                 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
 800                                 cpu);
 801                         rc = smp_ctl_ssctl_stop(cpu);
 802                         if (rc)
 803                                 printk(KERN_INFO
 804                                         "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
 805                                         cpu);
 806
 807                         for_each_online_cpu(i) {
 808                                 if (i == cpu)
 809                                         continue;
 810                                 if (!cb->finish) {
 811                                         cb->finish = 1;
 812                                         queue_work_on(i, hws_wq,
 813                                                 &cb->worker);
 814                                 }
 815                         }
 816                 }
 817         }
 818 }
 819
 820 static void worker_on_interrupt(unsigned int cpu)
 821 {
 822         unsigned long *sdbt;
 823         unsigned char done;
 824         struct hws_cpu_buffer *cb;
 825
 826         cb = &per_cpu(sampler_cpu_buffer, cpu);
 827
 828         sdbt = (unsigned long *) cb->worker_entry;
 829
 830         done = 0;
 831         /* do not proceed if stop was entered,
 832          * forget the buffers not yet processed */
 833         while (!done && !cb->stop_mode) {
 834                 unsigned long *trailer;
 835                 struct hws_trailer_entry *te;
 836                 unsigned long *dear = 0;
 837
 838                 trailer = trailer_entry_ptr(*sdbt);
 839                 /* leave loop if no more work to do */
 840                 if (!(*trailer & BUFFER_FULL_MASK)) {
 841                         done = 1;
 842                         if (!hws_flush_all)
 843                                 continue;
 844                 }
 845
 846                 te = (struct hws_trailer_entry *)trailer;
 847                 cb->sample_overflow += te->overflow;
 848
 849                 add_samples_to_oprofile(cpu, sdbt, dear);
 850
 851                 /* reset trailer */
 852                 xchg((unsigned char *) te, 0x40);
 853
 854                 /* advance to next sdb slot in current sdbt */
 855                 sdbt++;
 856                 /* in case link bit is set use address w/o link bit */
 857                 if (is_link_entry(sdbt))
 858                         sdbt = get_next_sdbt(sdbt);
 859
 860                 cb->worker_entry = (unsigned long)sdbt;
 861         }
 862 }
 863
 864 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
 865                 unsigned long *dear)
 866 {
 867         struct hws_data_entry *sample_data_ptr;
 868         unsigned long *trailer;
 869
 870         trailer = trailer_entry_ptr(*sdbt);
 871         if (dear) {
 872                 if (dear > trailer)
 873                         return;
 874                 trailer = dear;
 875         }
 876
 877         sample_data_ptr = (struct hws_data_entry *)(*sdbt);
 878
 879         while ((unsigned long *)sample_data_ptr < trailer) {
 880                 struct pt_regs *regs = NULL;
 881                 struct task_struct *tsk = NULL;
 882
 883                 /*
 884                  * Check sampling mode, 1 indicates basic (=customer) sampling
 885                  * mode.
 886                  */
 887                 if (sample_data_ptr->def != 1) {
 888                         /* sample slot is not yet written */
 889                         break;
 890                 } else {
 891                         /* make sure we don't use it twice,
 892                          * the next time the sampler will set it again */
 893                         sample_data_ptr->def = 0;
 894                 }
 895
 896                 /* Get pt_regs. */
 897                 if (sample_data_ptr->P == 1) {
 898                         /* userspace sample */
 899                         unsigned int pid = sample_data_ptr->prim_asn;
 900                         if (!counter_config.user)
 901                                 goto skip_sample;
 902                         rcu_read_lock();
 903                         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 904                         if (tsk)
 905                                 regs = task_pt_regs(tsk);
 906                         rcu_read_unlock();
 907                 } else {
 908                         /* kernelspace sample */
 909                         if (!counter_config.kernel)
 910                                 goto skip_sample;
 911                         regs = task_pt_regs(current);
 912                 }
 913
 914                 mutex_lock(&hws_sem);
 915                 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
 916                                 !sample_data_ptr->P, tsk);
 917                 mutex_unlock(&hws_sem);
 918         skip_sample:
 919                 sample_data_ptr++;
 920         }
 921 }
 922
 923 static void worker(struct work_struct *work)
 924 {
 925         unsigned int cpu;
 926         int ext_params;
 927         struct hws_cpu_buffer *cb;
 928
 929         cb = container_of(work, struct hws_cpu_buffer, worker);
 930         cpu = smp_processor_id();
 931         ext_params = atomic_xchg(&cb->ext_params, 0);
 932
 933         if (!cb->worker_entry)
 934                 worker_on_start(cpu);
 935
 936         if (worker_check_error(cpu, ext_params))
 937                 return;
 938
 939         if (!cb->finish)
 940                 worker_on_interrupt(cpu);
 941
 942         if (cb->finish)
 943                 worker_on_finish(cpu);
 944 }
 945
 946 /**
 947  * hwsampler_allocate() - allocate memory for the hardware sampler
 948  * @sdbt:  number of SDBTs per online CPU (must be > 0)
 949  * @sdb:   number of SDBs per SDBT (minimum 1, maximum 511)
 950  *
 951  * Returns 0 on success, !0 on failure.
 952  */
 953 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
 954 {
 955         int cpu, rc;
 956         mutex_lock(&hws_sem);
 957
 958         rc = -EINVAL;
 959         if (hws_state != HWS_DEALLOCATED)
 960                 goto allocate_exit;
 961
 962         if (sdbt < 1)
 963                 goto allocate_exit;
 964
 965         if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
 966                 goto allocate_exit;
 967
 968         num_sdbt = sdbt;
 969         num_sdb = sdb;
 970
 971         oom_killer_was_active = 0;
 972         register_oom_notifier(&hws_oom_notifier);
 973
 974         for_each_online_cpu(cpu) {
 975                 if (allocate_sdbt(cpu)) {
 976                         unregister_oom_notifier(&hws_oom_notifier);
 977                         goto allocate_error;
 978                 }
 979         }
 980         unregister_oom_notifier(&hws_oom_notifier);
 981         if (oom_killer_was_active)
 982                 goto allocate_error;
 983
 984         hws_state = HWS_STOPPED;
 985         rc = 0;
 986
 987 allocate_exit:
 988         mutex_unlock(&hws_sem);
 989         return rc;
 990
 991 allocate_error:
 992         rc = -ENOMEM;
 993         printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
 994         goto allocate_exit;
 995 }
 996
 997 /**
 998  * hwsampler_deallocate() - deallocate hardware sampler memory
 999  *
1000  * Returns 0 on success, !0 on failure.
1001  */
1002 int hwsampler_deallocate(void)
1003 {
1004         int rc;
1005
1006         mutex_lock(&hws_sem);
1007
1008         rc = -EINVAL;
1009         if (hws_state != HWS_STOPPED)
1010                 goto deallocate_exit;
1011
1012         ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1013         deallocate_sdbt();
1014
1015         hws_state = HWS_DEALLOCATED;
1016         rc = 0;
1017
1018 deallocate_exit:
1019         mutex_unlock(&hws_sem);
1020
1021         return rc;
1022 }
1023
1024 unsigned long hwsampler_query_min_interval(void)
1025 {
1026         return min_sampler_rate;
1027 }
1028
1029 unsigned long hwsampler_query_max_interval(void)
1030 {
1031         return max_sampler_rate;
1032 }
1033
1034 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1035 {
1036         struct hws_cpu_buffer *cb;
1037
1038         cb = &per_cpu(sampler_cpu_buffer, cpu);
1039
1040         return cb->sample_overflow;
1041 }
1042
1043 int hwsampler_setup(void)
1044 {
1045         int rc;
1046         int cpu;
1047         struct hws_cpu_buffer *cb;
1048
1049         mutex_lock(&hws_sem);
1050
1051         rc = -EINVAL;
1052         if (hws_state)
1053                 goto setup_exit;
1054
1055         hws_state = HWS_INIT;
1056
1057         init_all_cpu_buffers();
1058
1059         rc = check_hardware_prerequisites();
1060         if (rc)
1061                 goto setup_exit;
1062
1063         rc = check_qsi_on_setup();
1064         if (rc)
1065                 goto setup_exit;
1066
1067         rc = -EINVAL;
1068         hws_wq = create_workqueue("hwsampler");
1069         if (!hws_wq)
1070                 goto setup_exit;
1071
1072         register_cpu_notifier(&hws_cpu_notifier);
1073
1074         for_each_online_cpu(cpu) {
1075                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1076                 INIT_WORK(&cb->worker, worker);
1077                 rc = smp_ctl_qsi(cpu);
1078                 WARN_ON(rc);
1079                 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1080                         if (min_sampler_rate) {
1081                                 printk(KERN_WARNING
1082                                         "hwsampler: different min sampler rate values.\n");
1083                                 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1084                                         min_sampler_rate =
1085                                                 cb->qsi.min_sampl_rate;
1086                         } else
1087                                 min_sampler_rate = cb->qsi.min_sampl_rate;
1088                 }
1089                 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1090                         if (max_sampler_rate) {
1091                                 printk(KERN_WARNING
1092                                         "hwsampler: different max sampler rate values.\n");
1093                                 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1094                                         max_sampler_rate =
1095                                                 cb->qsi.max_sampl_rate;
1096                         } else
1097                                 max_sampler_rate = cb->qsi.max_sampl_rate;
1098                 }
1099         }
1100         register_external_interrupt(0x1407, hws_ext_handler);
1101
1102         hws_state = HWS_DEALLOCATED;
1103         rc = 0;
1104
1105 setup_exit:
1106         mutex_unlock(&hws_sem);
1107         return rc;
1108 }
1109
1110 int hwsampler_shutdown(void)
1111 {
1112         int rc;
1113
1114         mutex_lock(&hws_sem);
1115
1116         rc = -EINVAL;
1117         if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1118                 mutex_unlock(&hws_sem);
1119
1120                 if (hws_wq)
1121                         flush_workqueue(hws_wq);
1122
1123                 mutex_lock(&hws_sem);
1124
1125                 if (hws_state == HWS_STOPPED) {
1126                         ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1127                         deallocate_sdbt();
1128                 }
1129                 if (hws_wq) {
1130                         destroy_workqueue(hws_wq);
1131                         hws_wq = NULL;
1132                 }
1133
1134                 unregister_external_interrupt(0x1407, hws_ext_handler);
1135                 hws_state = HWS_INIT;
1136                 rc = 0;
1137         }
1138         mutex_unlock(&hws_sem);
1139
1140         unregister_cpu_notifier(&hws_cpu_notifier);
1141
1142         return rc;
1143 }
1144
1145 /**
1146  * hwsampler_start_all() - start hardware sampling on all online CPUs
1147  * @rate:  specifies the used interval when samples are taken
1148  *
1149  * Returns 0 on success, !0 on failure.
1150  */
1151 int hwsampler_start_all(unsigned long rate)
1152 {
1153         int rc, cpu;
1154
1155         mutex_lock(&hws_sem);
1156
1157         hws_oom = 0;
1158
1159         rc = -EINVAL;
1160         if (hws_state != HWS_STOPPED)
1161                 goto start_all_exit;
1162
1163         interval = rate;
1164
1165         /* fail if rate is not valid */
1166         if (interval < min_sampler_rate || interval > max_sampler_rate)
1167                 goto start_all_exit;
1168
1169         rc = check_qsi_on_start();
1170         if (rc)
1171                 goto start_all_exit;
1172
1173         rc = prepare_cpu_buffers();
1174         if (rc)
1175                 goto start_all_exit;
1176
1177         for_each_online_cpu(cpu) {
1178                 rc = start_sampling(cpu);
1179                 if (rc)
1180                         break;
1181         }
1182         if (rc) {
1183                 for_each_online_cpu(cpu) {
1184                         stop_sampling(cpu);
1185                 }
1186                 goto start_all_exit;
1187         }
1188         hws_state = HWS_STARTED;
1189         rc = 0;
1190
1191 start_all_exit:
1192         mutex_unlock(&hws_sem);
1193
1194         if (rc)
1195                 return rc;
1196
1197         register_oom_notifier(&hws_oom_notifier);
1198         hws_oom = 1;
1199         hws_flush_all = 0;
1200         /* now let them in, 1407 CPUMF external interrupts */
1201         ctl_set_bit(0, 5); /* set CR0 bit 58 */
1202
1203         return 0;
1204 }
1205
1206 /**
1207  * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1208  *
1209  * Returns 0 on success, !0 on failure.
1210  */
1211 int hwsampler_stop_all(void)
1212 {
1213         int tmp_rc, rc, cpu;
1214         struct hws_cpu_buffer *cb;
1215
1216         mutex_lock(&hws_sem);
1217
1218         rc = 0;
1219         if (hws_state == HWS_INIT) {
1220                 mutex_unlock(&hws_sem);
1221                 return rc;
1222         }
1223         hws_state = HWS_STOPPING;
1224         mutex_unlock(&hws_sem);
1225
1226         for_each_online_cpu(cpu) {
1227                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1228                 cb->stop_mode = 1;
1229                 tmp_rc = stop_sampling(cpu);
1230                 if (tmp_rc)
1231                         rc = tmp_rc;
1232         }
1233
1234         if (hws_wq)
1235                 flush_workqueue(hws_wq);
1236
1237         mutex_lock(&hws_sem);
1238         if (hws_oom) {
1239                 unregister_oom_notifier(&hws_oom_notifier);
1240                 hws_oom = 0;
1241         }
1242         hws_state = HWS_STOPPED;
1243         mutex_unlock(&hws_sem);
1244
1245         return rc;
1246 }