kernel/time/tick-broadcast.c

   1 /*
   2  * linux/kernel/time/tick-broadcast.c
   3  *
   4  * This file contains functions which emulate a local clock-event
   5  * device via a broadcast event source.
   6  *
   7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10  *
  11  * This code is licenced under the GPL version 2. For details see
  12  * kernel-base/COPYING.
  13  */
  14 #include <linux/cpu.h>
  15 #include <linux/err.h>
  16 #include <linux/hrtimer.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/percpu.h>
  19 #include <linux/profile.h>
  20 #include <linux/sched.h>
  21 #include <linux/smp.h>
  22 #include <linux/module.h>
  23
  24 #include "tick-internal.h"
  25
  26 /*
  27  * Broadcast support for broken x86 hardware, where the local apic
  28  * timer stops in C3 state.
  29  */
  30
  31 static struct tick_device tick_broadcast_device;
  32 static cpumask_var_t tick_broadcast_mask;
  33 static cpumask_var_t tick_broadcast_on;
  34 static cpumask_var_t tmpmask;
  35 static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  36 static int tick_broadcast_force;
  37
  38 #ifdef CONFIG_TICK_ONESHOT
  39 static void tick_broadcast_clear_oneshot(int cpu);
  40 #else
  41 static inline void tick_broadcast_clear_oneshot(int cpu) { }
  42 #endif
  43
  44 /*
  45  * Debugging: see timer_list.c
  46  */
  47 struct tick_device *tick_get_broadcast_device(void)
  48 {
  49         return &tick_broadcast_device;
  50 }
  51
  52 struct cpumask *tick_get_broadcast_mask(void)
  53 {
  54         return tick_broadcast_mask;
  55 }
  56
  57 /*
  58  * Start the device in periodic mode
  59  */
  60 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  61 {
  62         if (bc)
  63                 tick_setup_periodic(bc, 1);
  64 }
  65
  66 /*
  67  * Check, if the device can be utilized as broadcast device:
  68  */
  69 static bool tick_check_broadcast_device(struct clock_event_device *curdev,
  70                                         struct clock_event_device *newdev)
  71 {
  72         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
  73             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
  74                 return false;
  75
  76         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
  77             !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
  78                 return false;
  79
  80         return !curdev || newdev->rating > curdev->rating;
  81 }
  82
  83 /*
  84  * Conditionally install/replace broadcast device
  85  */
  86 void tick_install_broadcast_device(struct clock_event_device *dev)
  87 {
  88         struct clock_event_device *cur = tick_broadcast_device.evtdev;
  89
  90         if (!tick_check_broadcast_device(cur, dev))
  91                 return;
  92
  93         if (!try_module_get(dev->owner))
  94                 return;
  95
  96         clockevents_exchange_device(cur, dev);
  97         if (cur)
  98                 cur->event_handler = clockevents_handle_noop;
  99         tick_broadcast_device.evtdev = dev;
 100         if (!cpumask_empty(tick_broadcast_mask))
 101                 tick_broadcast_start_periodic(dev);
 102         /*
 103          * Inform all cpus about this. We might be in a situation
 104          * where we did not switch to oneshot mode because the per cpu
 105          * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
 106          * of a oneshot capable broadcast device. Without that
 107          * notification the systems stays stuck in periodic mode
 108          * forever.
 109          */
 110         if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
 111                 tick_clock_notify();
 112 }
 113
 114 /*
 115  * Check, if the device is the broadcast device
 116  */
 117 int tick_is_broadcast_device(struct clock_event_device *dev)
 118 {
 119         return (dev && tick_broadcast_device.evtdev == dev);
 120 }
 121
 122 static void err_broadcast(const struct cpumask *mask)
 123 {
 124         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
 125 }
 126
 127 static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
 128 {
 129         if (!dev->broadcast)
 130                 dev->broadcast = tick_broadcast;
 131         if (!dev->broadcast) {
 132                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
 133                              dev->name);
 134                 dev->broadcast = err_broadcast;
 135         }
 136 }
 137
 138 /*
 139  * Check, if the device is disfunctional and a place holder, which
 140  * needs to be handled by the broadcast device.
 141  */
 142 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
 143 {
 144         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 145         unsigned long flags;
 146         int ret;
 147
 148         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 149
 150         /*
 151          * Devices might be registered with both periodic and oneshot
 152          * mode disabled. This signals, that the device needs to be
 153          * operated from the broadcast device and is a placeholder for
 154          * the cpu local device.
 155          */
 156         if (!tick_device_is_functional(dev)) {
 157                 dev->event_handler = tick_handle_periodic;
 158                 tick_device_setup_broadcast_func(dev);
 159                 cpumask_set_cpu(cpu, tick_broadcast_mask);
 160                 tick_broadcast_start_periodic(bc);
 161                 ret = 1;
 162         } else {
 163                 /*
 164                  * Clear the broadcast bit for this cpu if the
 165                  * device is not power state affected.
 166                  */
 167                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 168                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 169                 else
 170                         tick_device_setup_broadcast_func(dev);
 171
 172                 /*
 173                  * Clear the broadcast bit if the CPU is not in
 174                  * periodic broadcast on state.
 175                  */
 176                 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
 177                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 178
 179                 switch (tick_broadcast_device.mode) {
 180                 case TICKDEV_MODE_ONESHOT:
 181                         /*
 182                          * If the system is in oneshot mode we can
 183                          * unconditionally clear the oneshot mask bit,
 184                          * because the CPU is running and therefore
 185                          * not in an idle state which causes the power
 186                          * state affected device to stop. Let the
 187                          * caller initialize the device.
 188                          */
 189                         tick_broadcast_clear_oneshot(cpu);
 190                         ret = 0;
 191                         break;
 192
 193                 case TICKDEV_MODE_PERIODIC:
 194                         /*
 195                          * If the system is in periodic mode, check
 196                          * whether the broadcast device can be
 197                          * switched off now.
 198                          */
 199                         if (cpumask_empty(tick_broadcast_mask) && bc)
 200                                 clockevents_shutdown(bc);
 201                         /*
 202                          * If we kept the cpu in the broadcast mask,
 203                          * tell the caller to leave the per cpu device
 204                          * in shutdown state. The periodic interrupt
 205                          * is delivered by the broadcast device.
 206                          */
 207                         ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
 208                         break;
 209                 default:
 210                         /* Nothing to do */
 211                         ret = 0;
 212                         break;
 213                 }
 214         }
 215         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 216         return ret;
 217 }
 218
 219 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 220 int tick_receive_broadcast(void)
 221 {
 222         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 223         struct clock_event_device *evt = td->evtdev;
 224
 225         if (!evt)
 226                 return -ENODEV;
 227
 228         if (!evt->event_handler)
 229                 return -EINVAL;
 230
 231         evt->event_handler(evt);
 232         return 0;
 233 }
 234 #endif
 235
 236 /*
 237  * Broadcast the event to the cpus, which are set in the mask (mangled).
 238  */
 239 static void tick_do_broadcast(struct cpumask *mask)
 240 {
 241         int cpu = smp_processor_id();
 242         struct tick_device *td;
 243
 244         /*
 245          * Check, if the current cpu is in the mask
 246          */
 247         if (cpumask_test_cpu(cpu, mask)) {
 248                 cpumask_clear_cpu(cpu, mask);
 249                 td = &per_cpu(tick_cpu_device, cpu);
 250                 td->evtdev->event_handler(td->evtdev);
 251         }
 252
 253         if (!cpumask_empty(mask)) {
 254                 /*
 255                  * It might be necessary to actually check whether the devices
 256                  * have different broadcast functions. For now, just use the
 257                  * one of the first device. This works as long as we have this
 258                  * misfeature only on x86 (lapic)
 259                  */
 260                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
 261                 td->evtdev->broadcast(mask);
 262         }
 263 }
 264
 265 /*
 266  * Periodic broadcast:
 267  * - invoke the broadcast handlers
 268  */
 269 static void tick_do_periodic_broadcast(void)
 270 {
 271         raw_spin_lock(&tick_broadcast_lock);
 272
 273         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
 274         tick_do_broadcast(tmpmask);
 275
 276         raw_spin_unlock(&tick_broadcast_lock);
 277 }
 278
 279 /*
 280  * Event handler for periodic broadcast ticks
 281  */
 282 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 283 {
 284         ktime_t next;
 285
 286         tick_do_periodic_broadcast();
 287
 288         /*
 289          * The device is in periodic mode. No reprogramming necessary:
 290          */
 291         if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
 292                 return;
 293
 294         /*
 295          * Setup the next period for devices, which do not have
 296          * periodic mode. We read dev->next_event first and add to it
 297          * when the event already expired. clockevents_program_event()
 298          * sets dev->next_event only when the event is really
 299          * programmed to the device.
 300          */
 301         for (next = dev->next_event; ;) {
 302                 next = ktime_add(next, tick_period);
 303
 304                 if (!clockevents_program_event(dev, next, false))
 305                         return;
 306                 tick_do_periodic_broadcast();
 307         }
 308 }
 309
 310 /*
 311  * Powerstate information: The system enters/leaves a state, where
 312  * affected devices might stop
 313  */
 314 static void tick_do_broadcast_on_off(unsigned long *reason)
 315 {
 316         struct clock_event_device *bc, *dev;
 317         struct tick_device *td;
 318         unsigned long flags;
 319         int cpu, bc_stopped;
 320
 321         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 322
 323         cpu = smp_processor_id();
 324         td = &per_cpu(tick_cpu_device, cpu);
 325         dev = td->evtdev;
 326         bc = tick_broadcast_device.evtdev;
 327
 328         /*
 329          * Is the device not affected by the powerstate ?
 330          */
 331         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 332                 goto out;
 333
 334         if (!tick_device_is_functional(dev))
 335                 goto out;
 336
 337         bc_stopped = cpumask_empty(tick_broadcast_mask);
 338
 339         switch (*reason) {
 340         case CLOCK_EVT_NOTIFY_BROADCAST_ON:
 341         case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
 342                 cpumask_set_cpu(cpu, tick_broadcast_on);
 343                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
 344                         if (tick_broadcast_device.mode ==
 345                             TICKDEV_MODE_PERIODIC)
 346                                 clockevents_shutdown(dev);
 347                 }
 348                 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
 349                         tick_broadcast_force = 1;
 350                 break;
 351         case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
 352                 if (tick_broadcast_force)
 353                         break;
 354                 cpumask_clear_cpu(cpu, tick_broadcast_on);
 355                 if (!tick_device_is_functional(dev))
 356                         break;
 357                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
 358                         if (tick_broadcast_device.mode ==
 359                             TICKDEV_MODE_PERIODIC)
 360                                 tick_setup_periodic(dev, 0);
 361                 }
 362                 break;
 363         }
 364
 365         if (cpumask_empty(tick_broadcast_mask)) {
 366                 if (!bc_stopped)
 367                         clockevents_shutdown(bc);
 368         } else if (bc_stopped) {
 369                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 370                         tick_broadcast_start_periodic(bc);
 371                 else
 372                         tick_broadcast_setup_oneshot(bc);
 373         }
 374 out:
 375         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 376 }
 377
 378 /*
 379  * Powerstate information: The system enters/leaves a state, where
 380  * affected devices might stop.
 381  */
 382 void tick_broadcast_on_off(unsigned long reason, int *oncpu)
 383 {
 384         if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
 385                 printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
 386                        "offline CPU #%d\n", *oncpu);
 387         else
 388                 tick_do_broadcast_on_off(&reason);
 389 }
 390
 391 /*
 392  * Set the periodic handler depending on broadcast on/off
 393  */
 394 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 395 {
 396         if (!broadcast)
 397                 dev->event_handler = tick_handle_periodic;
 398         else
 399                 dev->event_handler = tick_handle_periodic_broadcast;
 400 }
 401
 402 /*
 403  * Remove a CPU from broadcasting
 404  */
 405 void tick_shutdown_broadcast(unsigned int *cpup)
 406 {
 407         struct clock_event_device *bc;
 408         unsigned long flags;
 409         unsigned int cpu = *cpup;
 410
 411         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 412
 413         bc = tick_broadcast_device.evtdev;
 414         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 415         cpumask_clear_cpu(cpu, tick_broadcast_on);
 416
 417         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 418                 if (bc && cpumask_empty(tick_broadcast_mask))
 419                         clockevents_shutdown(bc);
 420         }
 421
 422         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 423 }
 424
 425 void tick_suspend_broadcast(void)
 426 {
 427         struct clock_event_device *bc;
 428         unsigned long flags;
 429
 430         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 431
 432         bc = tick_broadcast_device.evtdev;
 433         if (bc)
 434                 clockevents_shutdown(bc);
 435
 436         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 437 }
 438
 439 int tick_resume_broadcast(void)
 440 {
 441         struct clock_event_device *bc;
 442         unsigned long flags;
 443         int broadcast = 0;
 444
 445         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 446
 447         bc = tick_broadcast_device.evtdev;
 448
 449         if (bc) {
 450                 clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
 451
 452                 switch (tick_broadcast_device.mode) {
 453                 case TICKDEV_MODE_PERIODIC:
 454                         if (!cpumask_empty(tick_broadcast_mask))
 455                                 tick_broadcast_start_periodic(bc);
 456                         broadcast = cpumask_test_cpu(smp_processor_id(),
 457                                                      tick_broadcast_mask);
 458                         break;
 459                 case TICKDEV_MODE_ONESHOT:
 460                         if (!cpumask_empty(tick_broadcast_mask))
 461                                 broadcast = tick_resume_broadcast_oneshot(bc);
 462                         break;
 463                 }
 464         }
 465         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 466
 467         return broadcast;
 468 }
 469
 470
 471 #ifdef CONFIG_TICK_ONESHOT
 472
 473 static cpumask_var_t tick_broadcast_oneshot_mask;
 474 static cpumask_var_t tick_broadcast_pending_mask;
 475 static cpumask_var_t tick_broadcast_force_mask;
 476
 477 /*
 478  * Exposed for debugging: see timer_list.c
 479  */
 480 struct cpumask *tick_get_broadcast_oneshot_mask(void)
 481 {
 482         return tick_broadcast_oneshot_mask;
 483 }
 484
 485 /*
 486  * Called before going idle with interrupts disabled. Checks whether a
 487  * broadcast event from the other core is about to happen. We detected
 488  * that in tick_broadcast_oneshot_control(). The callsite can use this
 489  * to avoid a deep idle transition as we are about to get the
 490  * broadcast IPI right away.
 491  */
 492 int tick_check_broadcast_expired(void)
 493 {
 494         return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
 495 }
 496
 497 /*
 498  * Set broadcast interrupt affinity
 499  */
 500 static void tick_broadcast_set_affinity(struct clock_event_device *bc,
 501                                         const struct cpumask *cpumask)
 502 {
 503         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
 504                 return;
 505
 506         if (cpumask_equal(bc->cpumask, cpumask))
 507                 return;
 508
 509         bc->cpumask = cpumask;
 510         irq_set_affinity(bc->irq, bc->cpumask);
 511 }
 512
 513 static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
 514                                     ktime_t expires, int force)
 515 {
 516         int ret;
 517
 518         if (bc->mode != CLOCK_EVT_MODE_ONESHOT)
 519                 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 520
 521         ret = clockevents_program_event(bc, expires, force);
 522         if (!ret)
 523                 tick_broadcast_set_affinity(bc, cpumask_of(cpu));
 524         return ret;
 525 }
 526
 527 int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 528 {
 529         clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 530         return 0;
 531 }
 532
 533 /*
 534  * Called from irq_enter() when idle was interrupted to reenable the
 535  * per cpu device.
 536  */
 537 void tick_check_oneshot_broadcast(int cpu)
 538 {
 539         if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) {
 540                 struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
 541
 542                 /*
 543                  * We might be in the middle of switching over from
 544                  * periodic to oneshot. If the CPU has not yet
 545                  * switched over, leave the device alone.
 546                  */
 547                 if (td->mode == TICKDEV_MODE_ONESHOT) {
 548                         clockevents_set_mode(td->evtdev,
 549                                              CLOCK_EVT_MODE_ONESHOT);
 550                 }
 551         }
 552 }
 553
 554 /*
 555  * Handle oneshot mode broadcasting
 556  */
 557 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 558 {
 559         struct tick_device *td;
 560         ktime_t now, next_event;
 561         int cpu, next_cpu = 0;
 562
 563         raw_spin_lock(&tick_broadcast_lock);
 564 again:
 565         dev->next_event.tv64 = KTIME_MAX;
 566         next_event.tv64 = KTIME_MAX;
 567         cpumask_clear(tmpmask);
 568         now = ktime_get();
 569         /* Find all expired events */
 570         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
 571                 td = &per_cpu(tick_cpu_device, cpu);
 572                 if (td->evtdev->next_event.tv64 <= now.tv64) {
 573                         cpumask_set_cpu(cpu, tmpmask);
 574                         /*
 575                          * Mark the remote cpu in the pending mask, so
 576                          * it can avoid reprogramming the cpu local
 577                          * timer in tick_broadcast_oneshot_control().
 578                          */
 579                         cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
 580                 } else if (td->evtdev->next_event.tv64 < next_event.tv64) {
 581                         next_event.tv64 = td->evtdev->next_event.tv64;
 582                         next_cpu = cpu;
 583                 }
 584         }
 585
 586         /*
 587          * Remove the current cpu from the pending mask. The event is
 588          * delivered immediately in tick_do_broadcast() !
 589          */
 590         cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
 591
 592         /* Take care of enforced broadcast requests */
 593         cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
 594         cpumask_clear(tick_broadcast_force_mask);
 595
 596         /*
 597          * Sanity check. Catch the case where we try to broadcast to
 598          * offline cpus.
 599          */
 600         if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
 601                 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
 602
 603         /*
 604          * Wakeup the cpus which have an expired event.
 605          */
 606         tick_do_broadcast(tmpmask);
 607
 608         /*
 609          * Two reasons for reprogram:
 610          *
 611          * - The global event did not expire any CPU local
 612          * events. This happens in dyntick mode, as the maximum PIT
 613          * delta is quite small.
 614          *
 615          * - There are pending events on sleeping CPUs which were not
 616          * in the event mask
 617          */
 618         if (next_event.tv64 != KTIME_MAX) {
 619                 /*
 620                  * Rearm the broadcast device. If event expired,
 621                  * repeat the above
 622                  */
 623                 if (tick_broadcast_set_event(dev, next_cpu, next_event, 0))
 624                         goto again;
 625         }
 626         raw_spin_unlock(&tick_broadcast_lock);
 627 }
 628
 629 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
 630 {
 631         if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 632                 return 0;
 633         if (bc->next_event.tv64 == KTIME_MAX)
 634                 return 0;
 635         return bc->bound_on == cpu ? -EBUSY : 0;
 636 }
 637
 638 static void broadcast_shutdown_local(struct clock_event_device *bc,
 639                                      struct clock_event_device *dev)
 640 {
 641         /*
 642          * For hrtimer based broadcasting we cannot shutdown the cpu
 643          * local device if our own event is the first one to expire or
 644          * if we own the broadcast timer.
 645          */
 646         if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
 647                 if (broadcast_needs_cpu(bc, smp_processor_id()))
 648                         return;
 649                 if (dev->next_event.tv64 < bc->next_event.tv64)
 650                         return;
 651         }
 652         clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
 653 }
 654
 655 static void broadcast_move_bc(int deadcpu)
 656 {
 657         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 658
 659         if (!bc || !broadcast_needs_cpu(bc, deadcpu))
 660                 return;
 661         /* This moves the broadcast assignment to this cpu */
 662         clockevents_program_event(bc, bc->next_event, 1);
 663 }
 664
 665 /*
 666  * Powerstate information: The system enters/leaves a state, where
 667  * affected devices might stop
 668  * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups.
 669  */
 670 int tick_broadcast_oneshot_control(unsigned long reason)
 671 {
 672         struct clock_event_device *bc, *dev;
 673         struct tick_device *td;
 674         unsigned long flags;
 675         ktime_t now;
 676         int cpu, ret = 0;
 677
 678         /*
 679          * Periodic mode does not care about the enter/exit of power
 680          * states
 681          */
 682         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 683                 return 0;
 684
 685         /*
 686          * We are called with preemtion disabled from the depth of the
 687          * idle code, so we can't be moved away.
 688          */
 689         cpu = smp_processor_id();
 690         td = &per_cpu(tick_cpu_device, cpu);
 691         dev = td->evtdev;
 692
 693         if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 694                 return 0;
 695
 696         bc = tick_broadcast_device.evtdev;
 697
 698         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 699         if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
 700                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
 701                         WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
 702                         broadcast_shutdown_local(bc, dev);
 703                         /*
 704                          * We only reprogram the broadcast timer if we
 705                          * did not mark ourself in the force mask and
 706                          * if the cpu local event is earlier than the
 707                          * broadcast event. If the current CPU is in
 708                          * the force mask, then we are going to be
 709                          * woken by the IPI right away.
 710                          */
 711                         if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) &&
 712                             dev->next_event.tv64 < bc->next_event.tv64)
 713                                 tick_broadcast_set_event(bc, cpu, dev->next_event, 1);
 714                 }
 715                 /*
 716                  * If the current CPU owns the hrtimer broadcast
 717                  * mechanism, it cannot go deep idle and we remove the
 718                  * CPU from the broadcast mask. We don't have to go
 719                  * through the EXIT path as the local timer is not
 720                  * shutdown.
 721                  */
 722                 ret = broadcast_needs_cpu(bc, cpu);
 723                 if (ret)
 724                         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 725         } else {
 726                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
 727                         clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
 728                         /*
 729                          * The cpu which was handling the broadcast
 730                          * timer marked this cpu in the broadcast
 731                          * pending mask and fired the broadcast
 732                          * IPI. So we are going to handle the expired
 733                          * event anyway via the broadcast IPI
 734                          * handler. No need to reprogram the timer
 735                          * with an already expired event.
 736                          */
 737                         if (cpumask_test_and_clear_cpu(cpu,
 738                                        tick_broadcast_pending_mask))
 739                                 goto out;
 740
 741                         /*
 742                          * Bail out if there is no next event.
 743                          */
 744                         if (dev->next_event.tv64 == KTIME_MAX)
 745                                 goto out;
 746                         /*
 747                          * If the pending bit is not set, then we are
 748                          * either the CPU handling the broadcast
 749                          * interrupt or we got woken by something else.
 750                          *
 751                          * We are not longer in the broadcast mask, so
 752                          * if the cpu local expiry time is already
 753                          * reached, we would reprogram the cpu local
 754                          * timer with an already expired event.
 755                          *
 756                          * This can lead to a ping-pong when we return
 757                          * to idle and therefor rearm the broadcast
 758                          * timer before the cpu local timer was able
 759                          * to fire. This happens because the forced
 760                          * reprogramming makes sure that the event
 761                          * will happen in the future and depending on
 762                          * the min_delta setting this might be far
 763                          * enough out that the ping-pong starts.
 764                          *
 765                          * If the cpu local next_event has expired
 766                          * then we know that the broadcast timer
 767                          * next_event has expired as well and
 768                          * broadcast is about to be handled. So we
 769                          * avoid reprogramming and enforce that the
 770                          * broadcast handler, which did not run yet,
 771                          * will invoke the cpu local handler.
 772                          *
 773                          * We cannot call the handler directly from
 774                          * here, because we might be in a NOHZ phase
 775                          * and we did not go through the irq_enter()
 776                          * nohz fixups.
 777                          */
 778                         now = ktime_get();
 779                         if (dev->next_event.tv64 <= now.tv64) {
 780                                 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
 781                                 goto out;
 782                         }
 783                         /*
 784                          * We got woken by something else. Reprogram
 785                          * the cpu local timer device.
 786                          */
 787                         tick_program_event(dev->next_event, 1);
 788                 }
 789         }
 790 out:
 791         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 792         return ret;
 793 }
 794
 795 /*
 796  * Reset the one shot broadcast for a cpu
 797  *
 798  * Called with tick_broadcast_lock held
 799  */
 800 static void tick_broadcast_clear_oneshot(int cpu)
 801 {
 802         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 803         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 804 }
 805
 806 static void tick_broadcast_init_next_event(struct cpumask *mask,
 807                                            ktime_t expires)
 808 {
 809         struct tick_device *td;
 810         int cpu;
 811
 812         for_each_cpu(cpu, mask) {
 813                 td = &per_cpu(tick_cpu_device, cpu);
 814                 if (td->evtdev)
 815                         td->evtdev->next_event = expires;
 816         }
 817 }
 818
 819 /**
 820  * tick_broadcast_setup_oneshot - setup the broadcast device
 821  */
 822 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 823 {
 824         int cpu = smp_processor_id();
 825
 826         /* Set it up only once ! */
 827         if (bc->event_handler != tick_handle_oneshot_broadcast) {
 828                 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
 829
 830                 bc->event_handler = tick_handle_oneshot_broadcast;
 831
 832                 /*
 833                  * We must be careful here. There might be other CPUs
 834                  * waiting for periodic broadcast. We need to set the
 835                  * oneshot_mask bits for those and program the
 836                  * broadcast device to fire.
 837                  */
 838                 cpumask_copy(tmpmask, tick_broadcast_mask);
 839                 cpumask_clear_cpu(cpu, tmpmask);
 840                 cpumask_or(tick_broadcast_oneshot_mask,
 841                            tick_broadcast_oneshot_mask, tmpmask);
 842
 843                 if (was_periodic && !cpumask_empty(tmpmask)) {
 844                         clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 845                         tick_broadcast_init_next_event(tmpmask,
 846                                                        tick_next_period);
 847                         tick_broadcast_set_event(bc, cpu, tick_next_period, 1);
 848                 } else
 849                         bc->next_event.tv64 = KTIME_MAX;
 850         } else {
 851                 /*
 852                  * The first cpu which switches to oneshot mode sets
 853                  * the bit for all other cpus which are in the general
 854                  * (periodic) broadcast mask. So the bit is set and
 855                  * would prevent the first broadcast enter after this
 856                  * to program the bc device.
 857                  */
 858                 tick_broadcast_clear_oneshot(cpu);
 859         }
 860 }
 861
 862 /*
 863  * Select oneshot operating mode for the broadcast device
 864  */
 865 void tick_broadcast_switch_to_oneshot(void)
 866 {
 867         struct clock_event_device *bc;
 868         unsigned long flags;
 869
 870         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 871
 872         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 873         bc = tick_broadcast_device.evtdev;
 874         if (bc)
 875                 tick_broadcast_setup_oneshot(bc);
 876
 877         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 878 }
 879
 880
 881 /*
 882  * Remove a dead CPU from broadcasting
 883  */
 884 void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
 885 {
 886         unsigned long flags;
 887         unsigned int cpu = *cpup;
 888
 889         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 890
 891         /*
 892          * Clear the broadcast masks for the dead cpu, but do not stop
 893          * the broadcast device!
 894          */
 895         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 896         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 897         cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
 898
 899         broadcast_move_bc(cpu);
 900
 901         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 902 }
 903
 904 /*
 905  * Check, whether the broadcast device is in one shot mode
 906  */
 907 int tick_broadcast_oneshot_active(void)
 908 {
 909         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
 910 }
 911
 912 /*
 913  * Check whether the broadcast device supports oneshot.
 914  */
 915 bool tick_broadcast_oneshot_available(void)
 916 {
 917         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 918
 919         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
 920 }
 921
 922 #endif
 923
 924 void __init tick_broadcast_init(void)
 925 {
 926         zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
 927         zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
 928         zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
 929 #ifdef CONFIG_TICK_ONESHOT
 930         zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
 931         zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
 932         zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
 933 #endif
 934 }