drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/of_irq.h>
  20 #include <linux/io.h>
  21
  22 #include <asm/arch_timer.h>
  23 #include <asm/virt.h>
  24
  25 #include <clocksource/arm_arch_timer.h>
  26
  27 static u32 arch_timer_rate;
  28
  29 enum ppi_nr {
  30         PHYS_SECURE_PPI,
  31         PHYS_NONSECURE_PPI,
  32         VIRT_PPI,
  33         HYP_PPI,
  34         MAX_TIMER_PPI
  35 };
  36
  37 static int arch_timer_ppi[MAX_TIMER_PPI];
  38
  39 static struct clock_event_device __percpu *arch_timer_evt;
  40
  41 static bool arch_timer_use_virtual = true;
  42
  43 /*
  44  * Architected system timer support.
  45  */
  46
  47 static inline irqreturn_t timer_handler(const int access,
  48                                         struct clock_event_device *evt)
  49 {
  50         unsigned long ctrl;
  51         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
  52         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
  53                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
  54                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
  55                 evt->event_handler(evt);
  56                 return IRQ_HANDLED;
  57         }
  58
  59         return IRQ_NONE;
  60 }
  61
  62 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
  63 {
  64         struct clock_event_device *evt = dev_id;
  65
  66         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
  67 }
  68
  69 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
  70 {
  71         struct clock_event_device *evt = dev_id;
  72
  73         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
  74 }
  75
  76 static inline void timer_set_mode(const int access, int mode)
  77 {
  78         unsigned long ctrl;
  79         switch (mode) {
  80         case CLOCK_EVT_MODE_UNUSED:
  81         case CLOCK_EVT_MODE_SHUTDOWN:
  82                 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
  83                 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
  84                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
  85                 break;
  86         default:
  87                 break;
  88         }
  89 }
  90
  91 static void arch_timer_set_mode_virt(enum clock_event_mode mode,
  92                                      struct clock_event_device *clk)
  93 {
  94         timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode);
  95 }
  96
  97 static void arch_timer_set_mode_phys(enum clock_event_mode mode,
  98                                      struct clock_event_device *clk)
  99 {
 100         timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode);
 101 }
 102
 103 static inline void set_next_event(const int access, unsigned long evt)
 104 {
 105         unsigned long ctrl;
 106         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
 107         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 108         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 109         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt);
 110         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
 111 }
 112
 113 static int arch_timer_set_next_event_virt(unsigned long evt,
 114                                           struct clock_event_device *unused)
 115 {
 116         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt);
 117         return 0;
 118 }
 119
 120 static int arch_timer_set_next_event_phys(unsigned long evt,
 121                                           struct clock_event_device *unused)
 122 {
 123         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt);
 124         return 0;
 125 }
 126
 127 static void arch_timer_configure_evtstream(void)
 128 {
 129         int evt_stream_div, pos;
 130
 131         /* Find the closest power of two to the divisor */
 132         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 133         pos = fls(evt_stream_div);
 134         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 135                 pos--;
 136         /* enable event stream */
 137         arch_timer_evtstrm_enable(min(pos, 15));
 138 }
 139
 140 static int __cpuinit arch_timer_setup(struct clock_event_device *clk)
 141 {
 142         clk->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP;
 143         clk->name = "arch_sys_timer";
 144         clk->rating = 450;
 145         if (arch_timer_use_virtual) {
 146                 clk->irq = arch_timer_ppi[VIRT_PPI];
 147                 clk->set_mode = arch_timer_set_mode_virt;
 148                 clk->set_next_event = arch_timer_set_next_event_virt;
 149         } else {
 150                 clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
 151                 clk->set_mode = arch_timer_set_mode_phys;
 152                 clk->set_next_event = arch_timer_set_next_event_phys;
 153         }
 154
 155         clk->cpumask = cpumask_of(smp_processor_id());
 156
 157         clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, NULL);
 158
 159         clockevents_config_and_register(clk, arch_timer_rate,
 160                                         0xf, 0x7fffffff);
 161
 162         if (arch_timer_use_virtual)
 163                 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
 164         else {
 165                 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
 166                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 167                         enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
 168         }
 169
 170         arch_counter_set_user_access();
 171         if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
 172                 arch_timer_configure_evtstream();
 173
 174         return 0;
 175 }
 176
 177 static int arch_timer_available(void)
 178 {
 179         u32 freq;
 180
 181         if (arch_timer_rate == 0) {
 182                 freq = arch_timer_get_cntfrq();
 183
 184                 /* Check the timer frequency. */
 185                 if (freq == 0) {
 186                         pr_warn("Architected timer frequency not available\n");
 187                         return -EINVAL;
 188                 }
 189
 190                 arch_timer_rate = freq;
 191         }
 192
 193         pr_info_once("Architected local timer running at %lu.%02luMHz (%s).\n",
 194                      (unsigned long)arch_timer_rate / 1000000,
 195                      (unsigned long)(arch_timer_rate / 10000) % 100,
 196                      arch_timer_use_virtual ? "virt" : "phys");
 197         return 0;
 198 }
 199
 200 u32 arch_timer_get_rate(void)
 201 {
 202         return arch_timer_rate;
 203 }
 204
 205 u64 arch_timer_read_counter(void)
 206 {
 207         return arch_counter_get_cntvct();
 208 }
 209
 210 static cycle_t arch_counter_read(struct clocksource *cs)
 211 {
 212         return arch_counter_get_cntvct();
 213 }
 214
 215 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 216 {
 217         return arch_counter_get_cntvct();
 218 }
 219
 220 static struct clocksource clocksource_counter = {
 221         .name   = "arch_sys_counter",
 222         .rating = 400,
 223         .read   = arch_counter_read,
 224         .mask   = CLOCKSOURCE_MASK(56),
 225         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 226 };
 227
 228 static struct cyclecounter cyclecounter = {
 229         .read   = arch_counter_read_cc,
 230         .mask   = CLOCKSOURCE_MASK(56),
 231 };
 232
 233 static struct timecounter timecounter;
 234
 235 struct timecounter *arch_timer_get_timecounter(void)
 236 {
 237         return &timecounter;
 238 }
 239
 240 static void __cpuinit arch_timer_stop(struct clock_event_device *clk)
 241 {
 242         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 243                  clk->irq, smp_processor_id());
 244
 245         if (arch_timer_use_virtual)
 246                 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
 247         else {
 248                 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
 249                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 250                         disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 251         }
 252
 253         clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
 254 }
 255
 256 static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self,
 257                                            unsigned long action, void *hcpu)
 258 {
 259         /*
 260          * Grab cpu pointer in each case to avoid spurious
 261          * preemptible warnings
 262          */
 263         switch (action & ~CPU_TASKS_FROZEN) {
 264         case CPU_STARTING:
 265                 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 266                 break;
 267         case CPU_DYING:
 268                 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
 269                 break;
 270         }
 271
 272         return NOTIFY_OK;
 273 }
 274
 275 static struct notifier_block arch_timer_cpu_nb __cpuinitdata = {
 276         .notifier_call = arch_timer_cpu_notify,
 277 };
 278
 279 #ifdef CONFIG_CPU_PM
 280 static unsigned int saved_cntkctl;
 281 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 282                                     unsigned long action, void *hcpu)
 283 {
 284         if (action == CPU_PM_ENTER)
 285                 saved_cntkctl = arch_timer_get_cntkctl();
 286         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 287                 arch_timer_set_cntkctl(saved_cntkctl);
 288         return NOTIFY_OK;
 289 }
 290
 291 static struct notifier_block arch_timer_cpu_pm_notifier = {
 292         .notifier_call = arch_timer_cpu_pm_notify,
 293 };
 294
 295 static int __init arch_timer_cpu_pm_init(void)
 296 {
 297         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 298 }
 299 #else
 300 static int __init arch_timer_cpu_pm_init(void)
 301 {
 302         return 0;
 303 }
 304 #endif
 305
 306 static int __init arch_timer_register(void)
 307 {
 308         int err;
 309         int ppi;
 310
 311         err = arch_timer_available();
 312         if (err)
 313                 goto out;
 314
 315         arch_timer_evt = alloc_percpu(struct clock_event_device);
 316         if (!arch_timer_evt) {
 317                 err = -ENOMEM;
 318                 goto out;
 319         }
 320
 321         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 322         cyclecounter.mult = clocksource_counter.mult;
 323         cyclecounter.shift = clocksource_counter.shift;
 324         timecounter_init(&timecounter, &cyclecounter,
 325                          arch_counter_get_cntvct());
 326
 327         if (arch_timer_use_virtual) {
 328                 ppi = arch_timer_ppi[VIRT_PPI];
 329                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 330                                          "arch_timer", arch_timer_evt);
 331         } else {
 332                 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
 333                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 334                                          "arch_timer", arch_timer_evt);
 335                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 336                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 337                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 338                                                  "arch_timer", arch_timer_evt);
 339                         if (err)
 340                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 341                                                 arch_timer_evt);
 342                 }
 343         }
 344
 345         if (err) {
 346                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 347                        ppi, err);
 348                 goto out_free;
 349         }
 350
 351         err = register_cpu_notifier(&arch_timer_cpu_nb);
 352         if (err)
 353                 goto out_free_irq;
 354
 355         err = arch_timer_cpu_pm_init();
 356         if (err)
 357                 goto out_unreg_notify;
 358
 359         /* Immediately configure the timer on the boot CPU */
 360         arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 361
 362         return 0;
 363
 364 out_unreg_notify:
 365         unregister_cpu_notifier(&arch_timer_cpu_nb);
 366 out_free_irq:
 367         if (arch_timer_use_virtual)
 368                 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
 369         else {
 370                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 371                                 arch_timer_evt);
 372                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 373                         free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 374                                         arch_timer_evt);
 375         }
 376
 377 out_free:
 378         free_percpu(arch_timer_evt);
 379 out:
 380         return err;
 381 }
 382
 383 static void __init arch_timer_init(struct device_node *np)
 384 {
 385         u32 freq;
 386         int i;
 387
 388         if (arch_timer_get_rate()) {
 389                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 390                 return;
 391         }
 392
 393         /* Try to determine the frequency from the device tree or CNTFRQ */
 394         if (!of_property_read_u32(np, "clock-frequency", &freq))
 395                 arch_timer_rate = freq;
 396
 397         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 398                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 399
 400         of_node_put(np);
 401
 402         /*
 403          * If HYP mode is available, we know that the physical timer
 404          * has been configured to be accessible from PL1. Use it, so
 405          * that a guest can use the virtual timer instead.
 406          *
 407          * If no interrupt provided for virtual timer, we'll have to
 408          * stick to the physical timer. It'd better be accessible...
 409          */
 410         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 411                 arch_timer_use_virtual = false;
 412
 413                 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
 414                     !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 415                         pr_warn("arch_timer: No interrupt available, giving up\n");
 416                         return;
 417                 }
 418         }
 419
 420         arch_timer_register();
 421         arch_timer_arch_init();
 422 }
 423 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_init);
 424 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_init);