drivers/cpufreq/intel_pstate.c

   1 /*
   2  * intel_pstate.c: Native P state management for Intel processors
   3  *
   4  * (C) Copyright 2012 Intel Corporation
   5  * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; version 2
  10  * of the License.
  11  */
  12
  13 #include <linux/kernel.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/module.h>
  16 #include <linux/ktime.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/tick.h>
  19 #include <linux/slab.h>
  20 #include <linux/sched.h>
  21 #include <linux/list.h>
  22 #include <linux/cpu.h>
  23 #include <linux/cpufreq.h>
  24 #include <linux/sysfs.h>
  25 #include <linux/types.h>
  26 #include <linux/fs.h>
  27 #include <linux/debugfs.h>
  28 #include <linux/acpi.h>
  29 #include <trace/events/power.h>
  30
  31 #include <asm/div64.h>
  32 #include <asm/msr.h>
  33 #include <asm/cpu_device_id.h>
  34
  35 #define BYT_RATIOS              0x66a
  36 #define BYT_VIDS                0x66b
  37 #define BYT_TURBO_RATIOS        0x66c
  38 #define BYT_TURBO_VIDS          0x66d
  39
  40 #define FRAC_BITS 8
  41 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  42 #define fp_toint(X) ((X) >> FRAC_BITS)
  43
  44
  45 static inline int32_t mul_fp(int32_t x, int32_t y)
  46 {
  47         return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  48 }
  49
  50 static inline int32_t div_fp(int32_t x, int32_t y)
  51 {
  52         return div_s64((int64_t)x << FRAC_BITS, y);
  53 }
  54
  55 struct sample {
  56         int32_t core_pct_busy;
  57         u64 aperf;
  58         u64 mperf;
  59         int freq;
  60         ktime_t time;
  61 };
  62
  63 struct pstate_data {
  64         int     current_pstate;
  65         int     min_pstate;
  66         int     max_pstate;
  67         int     turbo_pstate;
  68 };
  69
  70 struct vid_data {
  71         int min;
  72         int max;
  73         int turbo;
  74         int32_t ratio;
  75 };
  76
  77 struct _pid {
  78         int setpoint;
  79         int32_t integral;
  80         int32_t p_gain;
  81         int32_t i_gain;
  82         int32_t d_gain;
  83         int deadband;
  84         int32_t last_err;
  85 };
  86
  87 struct cpudata {
  88         int cpu;
  89
  90         struct timer_list timer;
  91
  92         struct pstate_data pstate;
  93         struct vid_data vid;
  94         struct _pid pid;
  95
  96         ktime_t last_sample_time;
  97         u64     prev_aperf;
  98         u64     prev_mperf;
  99         struct sample sample;
 100 };
 101
 102 static struct cpudata **all_cpu_data;
 103 struct pstate_adjust_policy {
 104         int sample_rate_ms;
 105         int deadband;
 106         int setpoint;
 107         int p_gain_pct;
 108         int d_gain_pct;
 109         int i_gain_pct;
 110 };
 111
 112 struct pstate_funcs {
 113         int (*get_max)(void);
 114         int (*get_min)(void);
 115         int (*get_turbo)(void);
 116         void (*set)(struct cpudata*, int pstate);
 117         void (*get_vid)(struct cpudata *);
 118 };
 119
 120 struct cpu_defaults {
 121         struct pstate_adjust_policy pid_policy;
 122         struct pstate_funcs funcs;
 123 };
 124
 125 static struct pstate_adjust_policy pid_params;
 126 static struct pstate_funcs pstate_funcs;
 127
 128 struct perf_limits {
 129         int no_turbo;
 130         int turbo_disabled;
 131         int max_perf_pct;
 132         int min_perf_pct;
 133         int32_t max_perf;
 134         int32_t min_perf;
 135         int max_policy_pct;
 136         int max_sysfs_pct;
 137 };
 138
 139 static struct perf_limits limits = {
 140         .no_turbo = 0,
 141         .turbo_disabled = 0,
 142         .max_perf_pct = 100,
 143         .max_perf = int_tofp(1),
 144         .min_perf_pct = 0,
 145         .min_perf = 0,
 146         .max_policy_pct = 100,
 147         .max_sysfs_pct = 100,
 148 };
 149
 150 static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
 151                              int deadband, int integral) {
 152         pid->setpoint = setpoint;
 153         pid->deadband  = deadband;
 154         pid->integral  = int_tofp(integral);
 155         pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
 156 }
 157
 158 static inline void pid_p_gain_set(struct _pid *pid, int percent)
 159 {
 160         pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
 161 }
 162
 163 static inline void pid_i_gain_set(struct _pid *pid, int percent)
 164 {
 165         pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
 166 }
 167
 168 static inline void pid_d_gain_set(struct _pid *pid, int percent)
 169 {
 170         pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
 171 }
 172
 173 static signed int pid_calc(struct _pid *pid, int32_t busy)
 174 {
 175         signed int result;
 176         int32_t pterm, dterm, fp_error;
 177         int32_t integral_limit;
 178
 179         fp_error = int_tofp(pid->setpoint) - busy;
 180
 181         if (abs(fp_error) <= int_tofp(pid->deadband))
 182                 return 0;
 183
 184         pterm = mul_fp(pid->p_gain, fp_error);
 185
 186         pid->integral += fp_error;
 187
 188         /* limit the integral term */
 189         integral_limit = int_tofp(30);
 190         if (pid->integral > integral_limit)
 191                 pid->integral = integral_limit;
 192         if (pid->integral < -integral_limit)
 193                 pid->integral = -integral_limit;
 194
 195         dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
 196         pid->last_err = fp_error;
 197
 198         result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
 199         result = result + (1 << (FRAC_BITS-1));
 200         return (signed int)fp_toint(result);
 201 }
 202
 203 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
 204 {
 205         pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
 206         pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
 207         pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
 208
 209         pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
 210 }
 211
 212 static inline void intel_pstate_reset_all_pid(void)
 213 {
 214         unsigned int cpu;
 215
 216         for_each_online_cpu(cpu) {
 217                 if (all_cpu_data[cpu])
 218                         intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
 219         }
 220 }
 221
 222 static inline void update_turbo_state(void)
 223 {
 224         u64 misc_en;
 225         struct cpudata *cpu;
 226
 227         cpu = all_cpu_data[0];
 228         rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
 229         limits.turbo_disabled =
 230                 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
 231                  cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
 232 }
 233
 234 /************************** debugfs begin ************************/
 235 static int pid_param_set(void *data, u64 val)
 236 {
 237         *(u32 *)data = val;
 238         intel_pstate_reset_all_pid();
 239         return 0;
 240 }
 241
 242 static int pid_param_get(void *data, u64 *val)
 243 {
 244         *val = *(u32 *)data;
 245         return 0;
 246 }
 247 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
 248
 249 struct pid_param {
 250         char *name;
 251         void *value;
 252 };
 253
 254 static struct pid_param pid_files[] = {
 255         {"sample_rate_ms", &pid_params.sample_rate_ms},
 256         {"d_gain_pct", &pid_params.d_gain_pct},
 257         {"i_gain_pct", &pid_params.i_gain_pct},
 258         {"deadband", &pid_params.deadband},
 259         {"setpoint", &pid_params.setpoint},
 260         {"p_gain_pct", &pid_params.p_gain_pct},
 261         {NULL, NULL}
 262 };
 263
 264 static void __init intel_pstate_debug_expose_params(void)
 265 {
 266         struct dentry *debugfs_parent;
 267         int i = 0;
 268
 269         debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
 270         if (IS_ERR_OR_NULL(debugfs_parent))
 271                 return;
 272         while (pid_files[i].name) {
 273                 debugfs_create_file(pid_files[i].name, 0660,
 274                                     debugfs_parent, pid_files[i].value,
 275                                     &fops_pid_param);
 276                 i++;
 277         }
 278 }
 279
 280 /************************** debugfs end ************************/
 281
 282 /************************** sysfs begin ************************/
 283 #define show_one(file_name, object)                                     \
 284         static ssize_t show_##file_name                                 \
 285         (struct kobject *kobj, struct attribute *attr, char *buf)       \
 286         {                                                               \
 287                 return sprintf(buf, "%u\n", limits.object);             \
 288         }
 289
 290 static ssize_t show_no_turbo(struct kobject *kobj,
 291                              struct attribute *attr, char *buf)
 292 {
 293         ssize_t ret;
 294
 295         update_turbo_state();
 296         if (limits.turbo_disabled)
 297                 ret = sprintf(buf, "%u\n", limits.turbo_disabled);
 298         else
 299                 ret = sprintf(buf, "%u\n", limits.no_turbo);
 300
 301         return ret;
 302 }
 303
 304 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 305                               const char *buf, size_t count)
 306 {
 307         unsigned int input;
 308         int ret;
 309
 310         ret = sscanf(buf, "%u", &input);
 311         if (ret != 1)
 312                 return -EINVAL;
 313
 314         update_turbo_state();
 315         if (limits.turbo_disabled) {
 316                 pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
 317                 return -EPERM;
 318         }
 319         limits.no_turbo = clamp_t(int, input, 0, 1);
 320
 321         return count;
 322 }
 323
 324 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
 325                                   const char *buf, size_t count)
 326 {
 327         unsigned int input;
 328         int ret;
 329
 330         ret = sscanf(buf, "%u", &input);
 331         if (ret != 1)
 332                 return -EINVAL;
 333
 334         limits.max_sysfs_pct = clamp_t(int, input, 0 , 100);
 335         limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
 336         limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
 337
 338         return count;
 339 }
 340
 341 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
 342                                   const char *buf, size_t count)
 343 {
 344         unsigned int input;
 345         int ret;
 346
 347         ret = sscanf(buf, "%u", &input);
 348         if (ret != 1)
 349                 return -EINVAL;
 350         limits.min_perf_pct = clamp_t(int, input, 0 , 100);
 351         limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
 352
 353         return count;
 354 }
 355
 356 show_one(max_perf_pct, max_perf_pct);
 357 show_one(min_perf_pct, min_perf_pct);
 358
 359 define_one_global_rw(no_turbo);
 360 define_one_global_rw(max_perf_pct);
 361 define_one_global_rw(min_perf_pct);
 362
 363 static struct attribute *intel_pstate_attributes[] = {
 364         &no_turbo.attr,
 365         &max_perf_pct.attr,
 366         &min_perf_pct.attr,
 367         NULL
 368 };
 369
 370 static struct attribute_group intel_pstate_attr_group = {
 371         .attrs = intel_pstate_attributes,
 372 };
 373
 374 static void __init intel_pstate_sysfs_expose_params(void)
 375 {
 376         struct kobject *intel_pstate_kobject;
 377         int rc;
 378
 379         intel_pstate_kobject = kobject_create_and_add("intel_pstate",
 380                                                 &cpu_subsys.dev_root->kobj);
 381         BUG_ON(!intel_pstate_kobject);
 382         rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
 383         BUG_ON(rc);
 384 }
 385
 386 /************************** sysfs end ************************/
 387 static int byt_get_min_pstate(void)
 388 {
 389         u64 value;
 390
 391         rdmsrl(BYT_RATIOS, value);
 392         return (value >> 8) & 0x7F;
 393 }
 394
 395 static int byt_get_max_pstate(void)
 396 {
 397         u64 value;
 398
 399         rdmsrl(BYT_RATIOS, value);
 400         return (value >> 16) & 0x7F;
 401 }
 402
 403 static int byt_get_turbo_pstate(void)
 404 {
 405         u64 value;
 406
 407         rdmsrl(BYT_TURBO_RATIOS, value);
 408         return value & 0x7F;
 409 }
 410
 411 static void byt_set_pstate(struct cpudata *cpudata, int pstate)
 412 {
 413         u64 val;
 414         int32_t vid_fp;
 415         u32 vid;
 416
 417         val = pstate << 8;
 418         if (limits.no_turbo && !limits.turbo_disabled)
 419                 val |= (u64)1 << 32;
 420
 421         vid_fp = cpudata->vid.min + mul_fp(
 422                 int_tofp(pstate - cpudata->pstate.min_pstate),
 423                 cpudata->vid.ratio);
 424
 425         vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
 426         vid = fp_toint(vid_fp);
 427
 428         if (pstate > cpudata->pstate.max_pstate)
 429                 vid = cpudata->vid.turbo;
 430
 431         val |= vid;
 432
 433         wrmsrl(MSR_IA32_PERF_CTL, val);
 434 }
 435
 436 static void byt_get_vid(struct cpudata *cpudata)
 437 {
 438         u64 value;
 439
 440         rdmsrl(BYT_VIDS, value);
 441         cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
 442         cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
 443         cpudata->vid.ratio = div_fp(
 444                 cpudata->vid.max - cpudata->vid.min,
 445                 int_tofp(cpudata->pstate.max_pstate -
 446                         cpudata->pstate.min_pstate));
 447
 448         rdmsrl(BYT_TURBO_VIDS, value);
 449         cpudata->vid.turbo = value & 0x7f;
 450 }
 451
 452 static int core_get_min_pstate(void)
 453 {
 454         u64 value;
 455
 456         rdmsrl(MSR_PLATFORM_INFO, value);
 457         return (value >> 40) & 0xFF;
 458 }
 459
 460 static int core_get_max_pstate(void)
 461 {
 462         u64 value;
 463
 464         rdmsrl(MSR_PLATFORM_INFO, value);
 465         return (value >> 8) & 0xFF;
 466 }
 467
 468 static int core_get_turbo_pstate(void)
 469 {
 470         u64 value;
 471         int nont, ret;
 472
 473         rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
 474         nont = core_get_max_pstate();
 475         ret = (value) & 255;
 476         if (ret <= nont)
 477                 ret = nont;
 478         return ret;
 479 }
 480
 481 static void core_set_pstate(struct cpudata *cpudata, int pstate)
 482 {
 483         u64 val;
 484
 485         val = pstate << 8;
 486         if (limits.no_turbo && !limits.turbo_disabled)
 487                 val |= (u64)1 << 32;
 488
 489         wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
 490 }
 491
 492 static struct cpu_defaults core_params = {
 493         .pid_policy = {
 494                 .sample_rate_ms = 10,
 495                 .deadband = 0,
 496                 .setpoint = 97,
 497                 .p_gain_pct = 20,
 498                 .d_gain_pct = 0,
 499                 .i_gain_pct = 0,
 500         },
 501         .funcs = {
 502                 .get_max = core_get_max_pstate,
 503                 .get_min = core_get_min_pstate,
 504                 .get_turbo = core_get_turbo_pstate,
 505                 .set = core_set_pstate,
 506         },
 507 };
 508
 509 static struct cpu_defaults byt_params = {
 510         .pid_policy = {
 511                 .sample_rate_ms = 10,
 512                 .deadband = 0,
 513                 .setpoint = 97,
 514                 .p_gain_pct = 14,
 515                 .d_gain_pct = 0,
 516                 .i_gain_pct = 4,
 517         },
 518         .funcs = {
 519                 .get_max = byt_get_max_pstate,
 520                 .get_min = byt_get_min_pstate,
 521                 .get_turbo = byt_get_turbo_pstate,
 522                 .set = byt_set_pstate,
 523                 .get_vid = byt_get_vid,
 524         },
 525 };
 526
 527 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
 528 {
 529         int max_perf = cpu->pstate.turbo_pstate;
 530         int max_perf_adj;
 531         int min_perf;
 532
 533         if (limits.no_turbo || limits.turbo_disabled)
 534                 max_perf = cpu->pstate.max_pstate;
 535
 536         max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf), limits.max_perf));
 537         *max = clamp_t(int, max_perf_adj,
 538                         cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
 539
 540         min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.min_perf));
 541         *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
 542 }
 543
 544 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
 545 {
 546         int max_perf, min_perf;
 547
 548         update_turbo_state();
 549
 550         intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
 551
 552         pstate = clamp_t(int, pstate, min_perf, max_perf);
 553
 554         if (pstate == cpu->pstate.current_pstate)
 555                 return;
 556
 557         trace_cpu_frequency(pstate * 100000, cpu->cpu);
 558
 559         cpu->pstate.current_pstate = pstate;
 560
 561         pstate_funcs.set(cpu, pstate);
 562 }
 563
 564 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 565 {
 566         cpu->pstate.min_pstate = pstate_funcs.get_min();
 567         cpu->pstate.max_pstate = pstate_funcs.get_max();
 568         cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
 569
 570         if (pstate_funcs.get_vid)
 571                 pstate_funcs.get_vid(cpu);
 572         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
 573 }
 574
 575 static inline void intel_pstate_calc_busy(struct cpudata *cpu)
 576 {
 577         struct sample *sample = &cpu->sample;
 578         int64_t core_pct;
 579
 580         core_pct = int_tofp(sample->aperf) * int_tofp(100);
 581         core_pct = div64_u64(core_pct, int_tofp(sample->mperf));
 582
 583         sample->freq = fp_toint(
 584                 mul_fp(int_tofp(cpu->pstate.max_pstate * 1000), core_pct));
 585
 586         sample->core_pct_busy = (int32_t)core_pct;
 587 }
 588
 589 static inline void intel_pstate_sample(struct cpudata *cpu)
 590 {
 591         u64 aperf, mperf;
 592         unsigned long flags;
 593
 594         local_irq_save(flags);
 595         rdmsrl(MSR_IA32_APERF, aperf);
 596         rdmsrl(MSR_IA32_MPERF, mperf);
 597         local_irq_restore(flags);
 598
 599         cpu->last_sample_time = cpu->sample.time;
 600         cpu->sample.time = ktime_get();
 601         cpu->sample.aperf = aperf;
 602         cpu->sample.mperf = mperf;
 603         cpu->sample.aperf -= cpu->prev_aperf;
 604         cpu->sample.mperf -= cpu->prev_mperf;
 605
 606         intel_pstate_calc_busy(cpu);
 607
 608         cpu->prev_aperf = aperf;
 609         cpu->prev_mperf = mperf;
 610 }
 611
 612 static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
 613 {
 614         int delay;
 615
 616         delay = msecs_to_jiffies(pid_params.sample_rate_ms);
 617         mod_timer_pinned(&cpu->timer, jiffies + delay);
 618 }
 619
 620 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
 621 {
 622         int32_t core_busy, max_pstate, current_pstate, sample_ratio;
 623         u32 duration_us;
 624         u32 sample_time;
 625
 626         core_busy = cpu->sample.core_pct_busy;
 627         max_pstate = int_tofp(cpu->pstate.max_pstate);
 628         current_pstate = int_tofp(cpu->pstate.current_pstate);
 629         core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
 630
 631         sample_time = pid_params.sample_rate_ms  * USEC_PER_MSEC;
 632         duration_us = (u32) ktime_us_delta(cpu->sample.time,
 633                                            cpu->last_sample_time);
 634         if (duration_us > sample_time * 3) {
 635                 sample_ratio = div_fp(int_tofp(sample_time),
 636                                       int_tofp(duration_us));
 637                 core_busy = mul_fp(core_busy, sample_ratio);
 638         }
 639
 640         return core_busy;
 641 }
 642
 643 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 644 {
 645         int32_t busy_scaled;
 646         struct _pid *pid;
 647         signed int ctl;
 648
 649         pid = &cpu->pid;
 650         busy_scaled = intel_pstate_get_scaled_busy(cpu);
 651
 652         ctl = pid_calc(pid, busy_scaled);
 653
 654         /* Negative values of ctl increase the pstate and vice versa */
 655         intel_pstate_set_pstate(cpu, cpu->pstate.current_pstate - ctl);
 656 }
 657
 658 static void intel_pstate_timer_func(unsigned long __data)
 659 {
 660         struct cpudata *cpu = (struct cpudata *) __data;
 661         struct sample *sample;
 662
 663         intel_pstate_sample(cpu);
 664
 665         sample = &cpu->sample;
 666
 667         intel_pstate_adjust_busy_pstate(cpu);
 668
 669         trace_pstate_sample(fp_toint(sample->core_pct_busy),
 670                         fp_toint(intel_pstate_get_scaled_busy(cpu)),
 671                         cpu->pstate.current_pstate,
 672                         sample->mperf,
 673                         sample->aperf,
 674                         sample->freq);
 675
 676         intel_pstate_set_sample_time(cpu);
 677 }
 678
 679 #define ICPU(model, policy) \
 680         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
 681                         (unsigned long)&policy }
 682
 683 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
 684         ICPU(0x2a, core_params),
 685         ICPU(0x2d, core_params),
 686         ICPU(0x37, byt_params),
 687         ICPU(0x3a, core_params),
 688         ICPU(0x3c, core_params),
 689         ICPU(0x3d, core_params),
 690         ICPU(0x3e, core_params),
 691         ICPU(0x3f, core_params),
 692         ICPU(0x45, core_params),
 693         ICPU(0x46, core_params),
 694         ICPU(0x4c, byt_params),
 695         ICPU(0x4f, core_params),
 696         ICPU(0x56, core_params),
 697         {}
 698 };
 699 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
 700
 701 static int intel_pstate_init_cpu(unsigned int cpunum)
 702 {
 703         struct cpudata *cpu;
 704
 705         all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), GFP_KERNEL);
 706         if (!all_cpu_data[cpunum])
 707                 return -ENOMEM;
 708
 709         cpu = all_cpu_data[cpunum];
 710
 711         cpu->cpu = cpunum;
 712         intel_pstate_get_cpu_pstates(cpu);
 713
 714         init_timer_deferrable(&cpu->timer);
 715         cpu->timer.function = intel_pstate_timer_func;
 716         cpu->timer.data = (unsigned long)cpu;
 717         cpu->timer.expires = jiffies + HZ/100;
 718         intel_pstate_busy_pid_reset(cpu);
 719         intel_pstate_sample(cpu);
 720
 721         add_timer_on(&cpu->timer, cpunum);
 722
 723         pr_debug("Intel pstate controlling: cpu %d\n", cpunum);
 724
 725         return 0;
 726 }
 727
 728 static unsigned int intel_pstate_get(unsigned int cpu_num)
 729 {
 730         struct sample *sample;
 731         struct cpudata *cpu;
 732
 733         cpu = all_cpu_data[cpu_num];
 734         if (!cpu)
 735                 return 0;
 736         sample = &cpu->sample;
 737         return sample->freq;
 738 }
 739
 740 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 741 {
 742         if (!policy->cpuinfo.max_freq)
 743                 return -ENODEV;
 744
 745         if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
 746                 limits.min_perf_pct = 100;
 747                 limits.min_perf = int_tofp(1);
 748                 limits.max_policy_pct = 100;
 749                 limits.max_perf_pct = 100;
 750                 limits.max_perf = int_tofp(1);
 751                 limits.no_turbo = 0;
 752                 return 0;
 753         }
 754         limits.min_perf_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
 755         limits.min_perf_pct = clamp_t(int, limits.min_perf_pct, 0 , 100);
 756         limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
 757
 758         limits.max_policy_pct = (policy->max * 100) / policy->cpuinfo.max_freq;
 759         limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100);
 760         limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
 761         limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
 762
 763         return 0;
 764 }
 765
 766 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
 767 {
 768         cpufreq_verify_within_cpu_limits(policy);
 769
 770         if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
 771             policy->policy != CPUFREQ_POLICY_PERFORMANCE)
 772                 return -EINVAL;
 773
 774         return 0;
 775 }
 776
 777 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
 778 {
 779         int cpu_num = policy->cpu;
 780         struct cpudata *cpu = all_cpu_data[cpu_num];
 781
 782         pr_info("intel_pstate CPU %d exiting\n", cpu_num);
 783
 784         del_timer_sync(&all_cpu_data[cpu_num]->timer);
 785         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
 786         kfree(all_cpu_data[cpu_num]);
 787         all_cpu_data[cpu_num] = NULL;
 788 }
 789
 790 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 791 {
 792         struct cpudata *cpu;
 793         int rc;
 794
 795         rc = intel_pstate_init_cpu(policy->cpu);
 796         if (rc)
 797                 return rc;
 798
 799         cpu = all_cpu_data[policy->cpu];
 800
 801         if (limits.min_perf_pct == 100 && limits.max_perf_pct == 100)
 802                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 803         else
 804                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
 805
 806         policy->min = cpu->pstate.min_pstate * 100000;
 807         policy->max = cpu->pstate.turbo_pstate * 100000;
 808
 809         /* cpuinfo and default policy values */
 810         policy->cpuinfo.min_freq = cpu->pstate.min_pstate * 100000;
 811         policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate * 100000;
 812         policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 813         cpumask_set_cpu(policy->cpu, policy->cpus);
 814
 815         return 0;
 816 }
 817
 818 static struct cpufreq_driver intel_pstate_driver = {
 819         .flags          = CPUFREQ_CONST_LOOPS,
 820         .verify         = intel_pstate_verify_policy,
 821         .setpolicy      = intel_pstate_set_policy,
 822         .get            = intel_pstate_get,
 823         .init           = intel_pstate_cpu_init,
 824         .stop_cpu       = intel_pstate_stop_cpu,
 825         .name           = "intel_pstate",
 826 };
 827
 828 static int __initdata no_load;
 829
 830 static int intel_pstate_msrs_not_valid(void)
 831 {
 832         /* Check that all the msr's we are using are valid. */
 833         u64 aperf, mperf, tmp;
 834
 835         rdmsrl(MSR_IA32_APERF, aperf);
 836         rdmsrl(MSR_IA32_MPERF, mperf);
 837
 838         if (!pstate_funcs.get_max() ||
 839             !pstate_funcs.get_min() ||
 840             !pstate_funcs.get_turbo())
 841                 return -ENODEV;
 842
 843         rdmsrl(MSR_IA32_APERF, tmp);
 844         if (!(tmp - aperf))
 845                 return -ENODEV;
 846
 847         rdmsrl(MSR_IA32_MPERF, tmp);
 848         if (!(tmp - mperf))
 849                 return -ENODEV;
 850
 851         return 0;
 852 }
 853
 854 static void copy_pid_params(struct pstate_adjust_policy *policy)
 855 {
 856         pid_params.sample_rate_ms = policy->sample_rate_ms;
 857         pid_params.p_gain_pct = policy->p_gain_pct;
 858         pid_params.i_gain_pct = policy->i_gain_pct;
 859         pid_params.d_gain_pct = policy->d_gain_pct;
 860         pid_params.deadband = policy->deadband;
 861         pid_params.setpoint = policy->setpoint;
 862 }
 863
 864 static void copy_cpu_funcs(struct pstate_funcs *funcs)
 865 {
 866         pstate_funcs.get_max   = funcs->get_max;
 867         pstate_funcs.get_min   = funcs->get_min;
 868         pstate_funcs.get_turbo = funcs->get_turbo;
 869         pstate_funcs.set       = funcs->set;
 870         pstate_funcs.get_vid   = funcs->get_vid;
 871 }
 872
 873 #if IS_ENABLED(CONFIG_ACPI)
 874 #include <acpi/processor.h>
 875
 876 static bool intel_pstate_no_acpi_pss(void)
 877 {
 878         int i;
 879
 880         for_each_possible_cpu(i) {
 881                 acpi_status status;
 882                 union acpi_object *pss;
 883                 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 884                 struct acpi_processor *pr = per_cpu(processors, i);
 885
 886                 if (!pr)
 887                         continue;
 888
 889                 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
 890                 if (ACPI_FAILURE(status))
 891                         continue;
 892
 893                 pss = buffer.pointer;
 894                 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
 895                         kfree(pss);
 896                         return false;
 897                 }
 898
 899                 kfree(pss);
 900         }
 901
 902         return true;
 903 }
 904
 905 struct hw_vendor_info {
 906         u16  valid;
 907         char oem_id[ACPI_OEM_ID_SIZE];
 908         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
 909 };
 910
 911 /* Hardware vendor-specific info that has its own power management modes */
 912 static struct hw_vendor_info vendor_info[] = {
 913         {1, "HP    ", "ProLiant"},
 914         {0, "", ""},
 915 };
 916
 917 static bool intel_pstate_platform_pwr_mgmt_exists(void)
 918 {
 919         struct acpi_table_header hdr;
 920         struct hw_vendor_info *v_info;
 921
 922         if (acpi_disabled ||
 923             ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
 924                 return false;
 925
 926         for (v_info = vendor_info; v_info->valid; v_info++) {
 927                 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
 928                     !strncmp(hdr.oem_table_id, v_info->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 929                     intel_pstate_no_acpi_pss())
 930                         return true;
 931         }
 932
 933         return false;
 934 }
 935 #else /* CONFIG_ACPI not enabled */
 936 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
 937 #endif /* CONFIG_ACPI */
 938
 939 static int __init intel_pstate_init(void)
 940 {
 941         int cpu, rc = 0;
 942         const struct x86_cpu_id *id;
 943         struct cpu_defaults *cpu_info;
 944
 945         if (no_load)
 946                 return -ENODEV;
 947
 948         id = x86_match_cpu(intel_pstate_cpu_ids);
 949         if (!id)
 950                 return -ENODEV;
 951
 952         /*
 953          * The Intel pstate driver will be ignored if the platform
 954          * firmware has its own power management modes.
 955          */
 956         if (intel_pstate_platform_pwr_mgmt_exists())
 957                 return -ENODEV;
 958
 959         cpu_info = (struct cpu_defaults *)id->driver_data;
 960
 961         copy_pid_params(&cpu_info->pid_policy);
 962         copy_cpu_funcs(&cpu_info->funcs);
 963
 964         if (intel_pstate_msrs_not_valid())
 965                 return -ENODEV;
 966
 967         pr_info("Intel P-state driver initializing.\n");
 968
 969         all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
 970         if (!all_cpu_data)
 971                 return -ENOMEM;
 972
 973         rc = cpufreq_register_driver(&intel_pstate_driver);
 974         if (rc)
 975                 goto out;
 976
 977         intel_pstate_debug_expose_params();
 978         intel_pstate_sysfs_expose_params();
 979
 980         return rc;
 981 out:
 982         get_online_cpus();
 983         for_each_online_cpu(cpu) {
 984                 if (all_cpu_data[cpu]) {
 985                         del_timer_sync(&all_cpu_data[cpu]->timer);
 986                         kfree(all_cpu_data[cpu]);
 987                 }
 988         }
 989
 990         put_online_cpus();
 991         vfree(all_cpu_data);
 992         return -ENODEV;
 993 }
 994 device_initcall(intel_pstate_init);
 995
 996 static int __init intel_pstate_setup(char *str)
 997 {
 998         if (!str)
 999                 return -EINVAL;
1000
1001         if (!strcmp(str, "disable"))
1002                 no_load = 1;
1003         return 0;
1004 }
1005 early_param("intel_pstate", intel_pstate_setup);
1006
1007 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1008 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1009 MODULE_LICENSE("GPL");