drivers/irqchip/irq-gic-v3.c

   1 /*
   2  * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/of.h>
  23 #include <linux/of_address.h>
  24 #include <linux/of_irq.h>
  25 #include <linux/percpu.h>
  26 #include <linux/slab.h>
  27
  28 #include <linux/irqchip.h>
  29 #include <linux/irqchip/arm-gic-v3.h>
  30 #include <linux/irqchip/irq-partition-percpu.h>
  31
  32 #include <asm/cputype.h>
  33 #include <asm/exception.h>
  34 #include <asm/smp_plat.h>
  35 #include <asm/virt.h>
  36
  37 #include "irq-gic-common.h"
  38
  39 struct redist_region {
  40         void __iomem            *redist_base;
  41         phys_addr_t             phys_base;
  42 };
  43
  44 struct gic_chip_data {
  45         struct fwnode_handle    *fwnode;
  46         void __iomem            *dist_base;
  47         struct redist_region    *redist_regions;
  48         struct rdists           rdists;
  49         struct irq_domain       *domain;
  50         u64                     redist_stride;
  51         u32                     nr_redist_regions;
  52         unsigned int            irq_nr;
  53         struct partition_desc   *ppi_descs[16];
  54 };
  55
  56 static struct gic_chip_data gic_data __read_mostly;
  57 static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
  58
  59 #define gic_data_rdist()                (this_cpu_ptr(gic_data.rdists.rdist))
  60 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
  61 #define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
  62
  63 /* Our default, arbitrary priority value. Linux only uses one anyway. */
  64 #define DEFAULT_PMR_VALUE       0xf0
  65
  66 static inline unsigned int gic_irq(struct irq_data *d)
  67 {
  68         return d->hwirq;
  69 }
  70
  71 static inline int gic_irq_in_rdist(struct irq_data *d)
  72 {
  73         return gic_irq(d) < 32;
  74 }
  75
  76 static inline void __iomem *gic_dist_base(struct irq_data *d)
  77 {
  78         if (gic_irq_in_rdist(d))        /* SGI+PPI -> SGI_base for this CPU */
  79                 return gic_data_rdist_sgi_base();
  80
  81         if (d->hwirq <= 1023)           /* SPI -> dist_base */
  82                 return gic_data.dist_base;
  83
  84         return NULL;
  85 }
  86
  87 static void gic_do_wait_for_rwp(void __iomem *base)
  88 {
  89         u32 count = 1000000;    /* 1s! */
  90
  91         while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
  92                 count--;
  93                 if (!count) {
  94                         pr_err_ratelimited("RWP timeout, gone fishing\n");
  95                         return;
  96                 }
  97                 cpu_relax();
  98                 udelay(1);
  99         };
 100 }
 101
 102 /* Wait for completion of a distributor change */
 103 static void gic_dist_wait_for_rwp(void)
 104 {
 105         gic_do_wait_for_rwp(gic_data.dist_base);
 106 }
 107
 108 /* Wait for completion of a redistributor change */
 109 static void gic_redist_wait_for_rwp(void)
 110 {
 111         gic_do_wait_for_rwp(gic_data_rdist_rd_base());
 112 }
 113
 114 #ifdef CONFIG_ARM64
 115 static DEFINE_STATIC_KEY_FALSE(is_cavium_thunderx);
 116
 117 static u64 __maybe_unused gic_read_iar(void)
 118 {
 119         if (static_branch_unlikely(&is_cavium_thunderx))
 120                 return gic_read_iar_cavium_thunderx();
 121         else
 122                 return gic_read_iar_common();
 123 }
 124 #endif
 125
 126 static void gic_enable_redist(bool enable)
 127 {
 128         void __iomem *rbase;
 129         u32 count = 1000000;    /* 1s! */
 130         u32 val;
 131
 132         rbase = gic_data_rdist_rd_base();
 133
 134         val = readl_relaxed(rbase + GICR_WAKER);
 135         if (enable)
 136                 /* Wake up this CPU redistributor */
 137                 val &= ~GICR_WAKER_ProcessorSleep;
 138         else
 139                 val |= GICR_WAKER_ProcessorSleep;
 140         writel_relaxed(val, rbase + GICR_WAKER);
 141
 142         if (!enable) {          /* Check that GICR_WAKER is writeable */
 143                 val = readl_relaxed(rbase + GICR_WAKER);
 144                 if (!(val & GICR_WAKER_ProcessorSleep))
 145                         return; /* No PM support in this redistributor */
 146         }
 147
 148         while (--count) {
 149                 val = readl_relaxed(rbase + GICR_WAKER);
 150                 if (enable ^ (val & GICR_WAKER_ChildrenAsleep))
 151                         break;
 152                 cpu_relax();
 153                 udelay(1);
 154         };
 155         if (!count)
 156                 pr_err_ratelimited("redistributor failed to %s...\n",
 157                                    enable ? "wakeup" : "sleep");
 158 }
 159
 160 /*
 161  * Routines to disable, enable, EOI and route interrupts
 162  */
 163 static int gic_peek_irq(struct irq_data *d, u32 offset)
 164 {
 165         u32 mask = 1 << (gic_irq(d) % 32);
 166         void __iomem *base;
 167
 168         if (gic_irq_in_rdist(d))
 169                 base = gic_data_rdist_sgi_base();
 170         else
 171                 base = gic_data.dist_base;
 172
 173         return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
 174 }
 175
 176 static void gic_poke_irq(struct irq_data *d, u32 offset)
 177 {
 178         u32 mask = 1 << (gic_irq(d) % 32);
 179         void (*rwp_wait)(void);
 180         void __iomem *base;
 181
 182         if (gic_irq_in_rdist(d)) {
 183                 base = gic_data_rdist_sgi_base();
 184                 rwp_wait = gic_redist_wait_for_rwp;
 185         } else {
 186                 base = gic_data.dist_base;
 187                 rwp_wait = gic_dist_wait_for_rwp;
 188         }
 189
 190         writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
 191         rwp_wait();
 192 }
 193
 194 static void gic_mask_irq(struct irq_data *d)
 195 {
 196         gic_poke_irq(d, GICD_ICENABLER);
 197 }
 198
 199 static void gic_eoimode1_mask_irq(struct irq_data *d)
 200 {
 201         gic_mask_irq(d);
 202         /*
 203          * When masking a forwarded interrupt, make sure it is
 204          * deactivated as well.
 205          *
 206          * This ensures that an interrupt that is getting
 207          * disabled/masked will not get "stuck", because there is
 208          * noone to deactivate it (guest is being terminated).
 209          */
 210         if (irqd_is_forwarded_to_vcpu(d))
 211                 gic_poke_irq(d, GICD_ICACTIVER);
 212 }
 213
 214 static void gic_unmask_irq(struct irq_data *d)
 215 {
 216         gic_poke_irq(d, GICD_ISENABLER);
 217 }
 218
 219 static int gic_irq_set_irqchip_state(struct irq_data *d,
 220                                      enum irqchip_irq_state which, bool val)
 221 {
 222         u32 reg;
 223
 224         if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
 225                 return -EINVAL;
 226
 227         switch (which) {
 228         case IRQCHIP_STATE_PENDING:
 229                 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
 230                 break;
 231
 232         case IRQCHIP_STATE_ACTIVE:
 233                 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
 234                 break;
 235
 236         case IRQCHIP_STATE_MASKED:
 237                 reg = val ? GICD_ICENABLER : GICD_ISENABLER;
 238                 break;
 239
 240         default:
 241                 return -EINVAL;
 242         }
 243
 244         gic_poke_irq(d, reg);
 245         return 0;
 246 }
 247
 248 static int gic_irq_get_irqchip_state(struct irq_data *d,
 249                                      enum irqchip_irq_state which, bool *val)
 250 {
 251         if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
 252                 return -EINVAL;
 253
 254         switch (which) {
 255         case IRQCHIP_STATE_PENDING:
 256                 *val = gic_peek_irq(d, GICD_ISPENDR);
 257                 break;
 258
 259         case IRQCHIP_STATE_ACTIVE:
 260                 *val = gic_peek_irq(d, GICD_ISACTIVER);
 261                 break;
 262
 263         case IRQCHIP_STATE_MASKED:
 264                 *val = !gic_peek_irq(d, GICD_ISENABLER);
 265                 break;
 266
 267         default:
 268                 return -EINVAL;
 269         }
 270
 271         return 0;
 272 }
 273
 274 static void gic_eoi_irq(struct irq_data *d)
 275 {
 276         gic_write_eoir(gic_irq(d));
 277 }
 278
 279 static void gic_eoimode1_eoi_irq(struct irq_data *d)
 280 {
 281         /*
 282          * No need to deactivate an LPI, or an interrupt that
 283          * is is getting forwarded to a vcpu.
 284          */
 285         if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
 286                 return;
 287         gic_write_dir(gic_irq(d));
 288 }
 289
 290 static int gic_set_type(struct irq_data *d, unsigned int type)
 291 {
 292         unsigned int irq = gic_irq(d);
 293         void (*rwp_wait)(void);
 294         void __iomem *base;
 295
 296         /* Interrupt configuration for SGIs can't be changed */
 297         if (irq < 16)
 298                 return -EINVAL;
 299
 300         /* SPIs have restrictions on the supported types */
 301         if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
 302                          type != IRQ_TYPE_EDGE_RISING)
 303                 return -EINVAL;
 304
 305         if (gic_irq_in_rdist(d)) {
 306                 base = gic_data_rdist_sgi_base();
 307                 rwp_wait = gic_redist_wait_for_rwp;
 308         } else {
 309                 base = gic_data.dist_base;
 310                 rwp_wait = gic_dist_wait_for_rwp;
 311         }
 312
 313         return gic_configure_irq(irq, type, base, rwp_wait);
 314 }
 315
 316 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 317 {
 318         if (vcpu)
 319                 irqd_set_forwarded_to_vcpu(d);
 320         else
 321                 irqd_clr_forwarded_to_vcpu(d);
 322         return 0;
 323 }
 324
 325 static u64 gic_mpidr_to_affinity(unsigned long mpidr)
 326 {
 327         u64 aff;
 328
 329         aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
 330                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 331                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
 332                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 333
 334         return aff;
 335 }
 336
 337 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 338 {
 339         u32 irqnr;
 340
 341         do {
 342                 irqnr = gic_read_iar();
 343
 344                 if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
 345                         int err;
 346
 347                         if (static_key_true(&supports_deactivate))
 348                                 gic_write_eoir(irqnr);
 349
 350                         err = handle_domain_irq(gic_data.domain, irqnr, regs);
 351                         if (err) {
 352                                 WARN_ONCE(true, "Unexpected interrupt received!\n");
 353                                 if (static_key_true(&supports_deactivate)) {
 354                                         if (irqnr < 8192)
 355                                                 gic_write_dir(irqnr);
 356                                 } else {
 357                                         gic_write_eoir(irqnr);
 358                                 }
 359                         }
 360                         continue;
 361                 }
 362                 if (irqnr < 16) {
 363                         gic_write_eoir(irqnr);
 364                         if (static_key_true(&supports_deactivate))
 365                                 gic_write_dir(irqnr);
 366 #ifdef CONFIG_SMP
 367                         /*
 368                          * Unlike GICv2, we don't need an smp_rmb() here.
 369                          * The control dependency from gic_read_iar to
 370                          * the ISB in gic_write_eoir is enough to ensure
 371                          * that any shared data read by handle_IPI will
 372                          * be read after the ACK.
 373                          */
 374                         handle_IPI(irqnr, regs);
 375 #else
 376                         WARN_ONCE(true, "Unexpected SGI received!\n");
 377 #endif
 378                         continue;
 379                 }
 380         } while (irqnr != ICC_IAR1_EL1_SPURIOUS);
 381 }
 382
 383 static void __init gic_dist_init(void)
 384 {
 385         unsigned int i;
 386         u64 affinity;
 387         void __iomem *base = gic_data.dist_base;
 388
 389         /* Disable the distributor */
 390         writel_relaxed(0, base + GICD_CTLR);
 391         gic_dist_wait_for_rwp();
 392
 393         /*
 394          * Configure SPIs as non-secure Group-1. This will only matter
 395          * if the GIC only has a single security state. This will not
 396          * do the right thing if the kernel is running in secure mode,
 397          * but that's not the intended use case anyway.
 398          */
 399         for (i = 32; i < gic_data.irq_nr; i += 32)
 400                 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
 401
 402         gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
 403
 404         /* Enable distributor with ARE, Group1 */
 405         writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
 406                        base + GICD_CTLR);
 407
 408         /*
 409          * Set all global interrupts to the boot CPU only. ARE must be
 410          * enabled.
 411          */
 412         affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
 413         for (i = 32; i < gic_data.irq_nr; i++)
 414                 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
 415 }
 416
 417 static int gic_populate_rdist(void)
 418 {
 419         unsigned long mpidr = cpu_logical_map(smp_processor_id());
 420         u64 typer;
 421         u32 aff;
 422         int i;
 423
 424         /*
 425          * Convert affinity to a 32bit value that can be matched to
 426          * GICR_TYPER bits [63:32].
 427          */
 428         aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
 429                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 430                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
 431                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 432
 433         for (i = 0; i < gic_data.nr_redist_regions; i++) {
 434                 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
 435                 u32 reg;
 436
 437                 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
 438                 if (reg != GIC_PIDR2_ARCH_GICv3 &&
 439                     reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
 440                         pr_warn("No redistributor present @%p\n", ptr);
 441                         break;
 442                 }
 443
 444                 do {
 445                         typer = gic_read_typer(ptr + GICR_TYPER);
 446                         if ((typer >> 32) == aff) {
 447                                 u64 offset = ptr - gic_data.redist_regions[i].redist_base;
 448                                 gic_data_rdist_rd_base() = ptr;
 449                                 gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset;
 450                                 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
 451                                         smp_processor_id(), mpidr, i,
 452                                         &gic_data_rdist()->phys_base);
 453                                 return 0;
 454                         }
 455
 456                         if (gic_data.redist_stride) {
 457                                 ptr += gic_data.redist_stride;
 458                         } else {
 459                                 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
 460                                 if (typer & GICR_TYPER_VLPIS)
 461                                         ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
 462                         }
 463                 } while (!(typer & GICR_TYPER_LAST));
 464         }
 465
 466         /* We couldn't even deal with ourselves... */
 467         WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
 468              smp_processor_id(), mpidr);
 469         return -ENODEV;
 470 }
 471
 472 static void gic_cpu_sys_reg_init(void)
 473 {
 474         /*
 475          * Need to check that the SRE bit has actually been set. If
 476          * not, it means that SRE is disabled at EL2. We're going to
 477          * die painfully, and there is nothing we can do about it.
 478          *
 479          * Kindly inform the luser.
 480          */
 481         if (!gic_enable_sre())
 482                 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
 483
 484         /* Set priority mask register */
 485         gic_write_pmr(DEFAULT_PMR_VALUE);
 486
 487         if (static_key_true(&supports_deactivate)) {
 488                 /* EOI drops priority only (mode 1) */
 489                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
 490         } else {
 491                 /* EOI deactivates interrupt too (mode 0) */
 492                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
 493         }
 494
 495         /* ... and let's hit the road... */
 496         gic_write_grpen1(1);
 497 }
 498
 499 static int gic_dist_supports_lpis(void)
 500 {
 501         return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS);
 502 }
 503
 504 static void gic_cpu_init(void)
 505 {
 506         void __iomem *rbase;
 507
 508         /* Register ourselves with the rest of the world */
 509         if (gic_populate_rdist())
 510                 return;
 511
 512         gic_enable_redist(true);
 513
 514         rbase = gic_data_rdist_sgi_base();
 515
 516         /* Configure SGIs/PPIs as non-secure Group-1 */
 517         writel_relaxed(~0, rbase + GICR_IGROUPR0);
 518
 519         gic_cpu_config(rbase, gic_redist_wait_for_rwp);
 520
 521         /* Give LPIs a spin */
 522         if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
 523                 its_cpu_init();
 524
 525         /* initialise system registers */
 526         gic_cpu_sys_reg_init();
 527 }
 528
 529 #ifdef CONFIG_SMP
 530 static int gic_secondary_init(struct notifier_block *nfb,
 531                               unsigned long action, void *hcpu)
 532 {
 533         if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
 534                 gic_cpu_init();
 535         return NOTIFY_OK;
 536 }
 537
 538 /*
 539  * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
 540  * priority because the GIC needs to be up before the ARM generic timers.
 541  */
 542 static struct notifier_block gic_cpu_notifier = {
 543         .notifier_call = gic_secondary_init,
 544         .priority = 100,
 545 };
 546
 547 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
 548                                    unsigned long cluster_id)
 549 {
 550         int next_cpu, cpu = *base_cpu;
 551         unsigned long mpidr = cpu_logical_map(cpu);
 552         u16 tlist = 0;
 553
 554         while (cpu < nr_cpu_ids) {
 555                 /*
 556                  * If we ever get a cluster of more than 16 CPUs, just
 557                  * scream and skip that CPU.
 558                  */
 559                 if (WARN_ON((mpidr & 0xff) >= 16))
 560                         goto out;
 561
 562                 tlist |= 1 << (mpidr & 0xf);
 563
 564                 next_cpu = cpumask_next(cpu, mask);
 565                 if (next_cpu >= nr_cpu_ids)
 566                         goto out;
 567                 cpu = next_cpu;
 568
 569                 mpidr = cpu_logical_map(cpu);
 570
 571                 if (cluster_id != (mpidr & ~0xffUL)) {
 572                         cpu--;
 573                         goto out;
 574                 }
 575         }
 576 out:
 577         *base_cpu = cpu;
 578         return tlist;
 579 }
 580
 581 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
 582         (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
 583                 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
 584
 585 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
 586 {
 587         u64 val;
 588
 589         val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)     |
 590                MPIDR_TO_SGI_AFFINITY(cluster_id, 2)     |
 591                irq << ICC_SGI1R_SGI_ID_SHIFT            |
 592                MPIDR_TO_SGI_AFFINITY(cluster_id, 1)     |
 593                tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
 594
 595         pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
 596         gic_write_sgi1r(val);
 597 }
 598
 599 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 600 {
 601         int cpu;
 602
 603         if (WARN_ON(irq >= 16))
 604                 return;
 605
 606         /*
 607          * Ensure that stores to Normal memory are visible to the
 608          * other CPUs before issuing the IPI.
 609          */
 610         smp_wmb();
 611
 612         for_each_cpu(cpu, mask) {
 613                 unsigned long cluster_id = cpu_logical_map(cpu) & ~0xffUL;
 614                 u16 tlist;
 615
 616                 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
 617                 gic_send_sgi(cluster_id, tlist, irq);
 618         }
 619
 620         /* Force the above writes to ICC_SGI1R_EL1 to be executed */
 621         isb();
 622 }
 623
 624 static void gic_smp_init(void)
 625 {
 626         set_smp_cross_call(gic_raise_softirq);
 627         register_cpu_notifier(&gic_cpu_notifier);
 628 }
 629
 630 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 631                             bool force)
 632 {
 633         unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
 634         void __iomem *reg;
 635         int enabled;
 636         u64 val;
 637
 638         if (gic_irq_in_rdist(d))
 639                 return -EINVAL;
 640
 641         /* If interrupt was enabled, disable it first */
 642         enabled = gic_peek_irq(d, GICD_ISENABLER);
 643         if (enabled)
 644                 gic_mask_irq(d);
 645
 646         reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
 647         val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
 648
 649         gic_write_irouter(val, reg);
 650
 651         /*
 652          * If the interrupt was enabled, enabled it again. Otherwise,
 653          * just wait for the distributor to have digested our changes.
 654          */
 655         if (enabled)
 656                 gic_unmask_irq(d);
 657         else
 658                 gic_dist_wait_for_rwp();
 659
 660         return IRQ_SET_MASK_OK;
 661 }
 662 #else
 663 #define gic_set_affinity        NULL
 664 #define gic_smp_init()          do { } while(0)
 665 #endif
 666
 667 #ifdef CONFIG_CPU_PM
 668 static int gic_cpu_pm_notifier(struct notifier_block *self,
 669                                unsigned long cmd, void *v)
 670 {
 671         if (cmd == CPU_PM_EXIT) {
 672                 gic_enable_redist(true);
 673                 gic_cpu_sys_reg_init();
 674         } else if (cmd == CPU_PM_ENTER) {
 675                 gic_write_grpen1(0);
 676                 gic_enable_redist(false);
 677         }
 678         return NOTIFY_OK;
 679 }
 680
 681 static struct notifier_block gic_cpu_pm_notifier_block = {
 682         .notifier_call = gic_cpu_pm_notifier,
 683 };
 684
 685 static void gic_cpu_pm_init(void)
 686 {
 687         cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
 688 }
 689
 690 #else
 691 static inline void gic_cpu_pm_init(void) { }
 692 #endif /* CONFIG_CPU_PM */
 693
 694 static struct irq_chip gic_chip = {
 695         .name                   = "GICv3",
 696         .irq_mask               = gic_mask_irq,
 697         .irq_unmask             = gic_unmask_irq,
 698         .irq_eoi                = gic_eoi_irq,
 699         .irq_set_type           = gic_set_type,
 700         .irq_set_affinity       = gic_set_affinity,
 701         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
 702         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
 703         .flags                  = IRQCHIP_SET_TYPE_MASKED,
 704 };
 705
 706 static struct irq_chip gic_eoimode1_chip = {
 707         .name                   = "GICv3",
 708         .irq_mask               = gic_eoimode1_mask_irq,
 709         .irq_unmask             = gic_unmask_irq,
 710         .irq_eoi                = gic_eoimode1_eoi_irq,
 711         .irq_set_type           = gic_set_type,
 712         .irq_set_affinity       = gic_set_affinity,
 713         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
 714         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
 715         .irq_set_vcpu_affinity  = gic_irq_set_vcpu_affinity,
 716         .flags                  = IRQCHIP_SET_TYPE_MASKED,
 717 };
 718
 719 #define GIC_ID_NR               (1U << gic_data.rdists.id_bits)
 720
 721 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
 722                               irq_hw_number_t hw)
 723 {
 724         struct irq_chip *chip = &gic_chip;
 725
 726         if (static_key_true(&supports_deactivate))
 727                 chip = &gic_eoimode1_chip;
 728
 729         /* SGIs are private to the core kernel */
 730         if (hw < 16)
 731                 return -EPERM;
 732         /* Nothing here */
 733         if (hw >= gic_data.irq_nr && hw < 8192)
 734                 return -EPERM;
 735         /* Off limits */
 736         if (hw >= GIC_ID_NR)
 737                 return -EPERM;
 738
 739         /* PPIs */
 740         if (hw < 32) {
 741                 irq_set_percpu_devid(irq);
 742                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
 743                                     handle_percpu_devid_irq, NULL, NULL);
 744                 irq_set_status_flags(irq, IRQ_NOAUTOEN);
 745         }
 746         /* SPIs */
 747         if (hw >= 32 && hw < gic_data.irq_nr) {
 748                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
 749                                     handle_fasteoi_irq, NULL, NULL);
 750                 irq_set_probe(irq);
 751         }
 752         /* LPIs */
 753         if (hw >= 8192 && hw < GIC_ID_NR) {
 754                 if (!gic_dist_supports_lpis())
 755                         return -EPERM;
 756                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
 757                                     handle_fasteoi_irq, NULL, NULL);
 758         }
 759
 760         return 0;
 761 }
 762
 763 static int gic_irq_domain_translate(struct irq_domain *d,
 764                                     struct irq_fwspec *fwspec,
 765                                     unsigned long *hwirq,
 766                                     unsigned int *type)
 767 {
 768         if (is_of_node(fwspec->fwnode)) {
 769                 if (fwspec->param_count < 3)
 770                         return -EINVAL;
 771
 772                 switch (fwspec->param[0]) {
 773                 case 0:                 /* SPI */
 774                         *hwirq = fwspec->param[1] + 32;
 775                         break;
 776                 case 1:                 /* PPI */
 777                         *hwirq = fwspec->param[1] + 16;
 778                         break;
 779                 case GIC_IRQ_TYPE_LPI:  /* LPI */
 780                         *hwirq = fwspec->param[1];
 781                         break;
 782                 default:
 783                         return -EINVAL;
 784                 }
 785
 786                 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
 787                 return 0;
 788         }
 789
 790         return -EINVAL;
 791 }
 792
 793 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 794                                 unsigned int nr_irqs, void *arg)
 795 {
 796         int i, ret;
 797         irq_hw_number_t hwirq;
 798         unsigned int type = IRQ_TYPE_NONE;
 799         struct irq_fwspec *fwspec = arg;
 800
 801         ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
 802         if (ret)
 803                 return ret;
 804
 805         for (i = 0; i < nr_irqs; i++)
 806                 gic_irq_domain_map(domain, virq + i, hwirq + i);
 807
 808         return 0;
 809 }
 810
 811 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
 812                                 unsigned int nr_irqs)
 813 {
 814         int i;
 815
 816         for (i = 0; i < nr_irqs; i++) {
 817                 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
 818                 irq_set_handler(virq + i, NULL);
 819                 irq_domain_reset_irq_data(d);
 820         }
 821 }
 822
 823 static int gic_irq_domain_select(struct irq_domain *d,
 824                                  struct irq_fwspec *fwspec,
 825                                  enum irq_domain_bus_token bus_token)
 826 {
 827         /* Not for us */
 828         if (fwspec->fwnode != d->fwnode)
 829                 return 0;
 830
 831         /* If this is not DT, then we have a single domain */
 832         if (!is_of_node(fwspec->fwnode))
 833                 return 1;
 834
 835         /*
 836          * If this is a PPI and we have a 4th (non-null) parameter,
 837          * then we need to match the partition domain.
 838          */
 839         if (fwspec->param_count >= 4 &&
 840             fwspec->param[0] == 1 && fwspec->param[3] != 0)
 841                 return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
 842
 843         return d == gic_data.domain;
 844 }
 845
 846 static const struct irq_domain_ops gic_irq_domain_ops = {
 847         .translate = gic_irq_domain_translate,
 848         .alloc = gic_irq_domain_alloc,
 849         .free = gic_irq_domain_free,
 850         .select = gic_irq_domain_select,
 851 };
 852
 853 static int partition_domain_translate(struct irq_domain *d,
 854                                       struct irq_fwspec *fwspec,
 855                                       unsigned long *hwirq,
 856                                       unsigned int *type)
 857 {
 858         struct device_node *np;
 859         int ret;
 860
 861         np = of_find_node_by_phandle(fwspec->param[3]);
 862         if (WARN_ON(!np))
 863                 return -EINVAL;
 864
 865         ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
 866                                      of_node_to_fwnode(np));
 867         if (ret < 0)
 868                 return ret;
 869
 870         *hwirq = ret;
 871         *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
 872
 873         return 0;
 874 }
 875
 876 static const struct irq_domain_ops partition_domain_ops = {
 877         .translate = partition_domain_translate,
 878         .select = gic_irq_domain_select,
 879 };
 880
 881 static void gicv3_enable_quirks(void)
 882 {
 883 #ifdef CONFIG_ARM64
 884         if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154))
 885                 static_branch_enable(&is_cavium_thunderx);
 886 #endif
 887 }
 888
 889 static int __init gic_init_bases(void __iomem *dist_base,
 890                                  struct redist_region *rdist_regs,
 891                                  u32 nr_redist_regions,
 892                                  u64 redist_stride,
 893                                  struct fwnode_handle *handle)
 894 {
 895         struct device_node *node;
 896         u32 typer;
 897         int gic_irqs;
 898         int err;
 899
 900         if (!is_hyp_mode_available())
 901                 static_key_slow_dec(&supports_deactivate);
 902
 903         if (static_key_true(&supports_deactivate))
 904                 pr_info("GIC: Using split EOI/Deactivate mode\n");
 905
 906         gic_data.fwnode = handle;
 907         gic_data.dist_base = dist_base;
 908         gic_data.redist_regions = rdist_regs;
 909         gic_data.nr_redist_regions = nr_redist_regions;
 910         gic_data.redist_stride = redist_stride;
 911
 912         gicv3_enable_quirks();
 913
 914         /*
 915          * Find out how many interrupts are supported.
 916          * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
 917          */
 918         typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
 919         gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer);
 920         gic_irqs = GICD_TYPER_IRQS(typer);
 921         if (gic_irqs > 1020)
 922                 gic_irqs = 1020;
 923         gic_data.irq_nr = gic_irqs;
 924
 925         gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
 926                                                  &gic_data);
 927         gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
 928
 929         if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
 930                 err = -ENOMEM;
 931                 goto out_free;
 932         }
 933
 934         set_handle_irq(gic_handle_irq);
 935
 936         node = to_of_node(handle);
 937         if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis() &&
 938             node) /* Temp hack to prevent ITS init for ACPI */
 939                 its_init(node, &gic_data.rdists, gic_data.domain);
 940
 941         gic_smp_init();
 942         gic_dist_init();
 943         gic_cpu_init();
 944         gic_cpu_pm_init();
 945
 946         return 0;
 947
 948 out_free:
 949         if (gic_data.domain)
 950                 irq_domain_remove(gic_data.domain);
 951         free_percpu(gic_data.rdists.rdist);
 952         return err;
 953 }
 954
 955 static int __init gic_validate_dist_version(void __iomem *dist_base)
 956 {
 957         u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
 958
 959         if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
 960                 return -ENODEV;
 961
 962         return 0;
 963 }
 964
 965 static int get_cpu_number(struct device_node *dn)
 966 {
 967         const __be32 *cell;
 968         u64 hwid;
 969         int i;
 970
 971         cell = of_get_property(dn, "reg", NULL);
 972         if (!cell)
 973                 return -1;
 974
 975         hwid = of_read_number(cell, of_n_addr_cells(dn));
 976
 977         /*
 978          * Non affinity bits must be set to 0 in the DT
 979          */
 980         if (hwid & ~MPIDR_HWID_BITMASK)
 981                 return -1;
 982
 983         for (i = 0; i < num_possible_cpus(); i++)
 984                 if (cpu_logical_map(i) == hwid)
 985                         return i;
 986
 987         return -1;
 988 }
 989
 990 /* Create all possible partitions at boot time */
 991 static void gic_populate_ppi_partitions(struct device_node *gic_node)
 992 {
 993         struct device_node *parts_node, *child_part;
 994         int part_idx = 0, i;
 995         int nr_parts;
 996         struct partition_affinity *parts;
 997
 998         parts_node = of_find_node_by_name(gic_node, "ppi-partitions");
 999         if (!parts_node)
1000                 return;
1001
1002         nr_parts = of_get_child_count(parts_node);
1003
1004         if (!nr_parts)
1005                 return;
1006
1007         parts = kzalloc(sizeof(*parts) * nr_parts, GFP_KERNEL);
1008         if (WARN_ON(!parts))
1009                 return;
1010
1011         for_each_child_of_node(parts_node, child_part) {
1012                 struct partition_affinity *part;
1013                 int n;
1014
1015                 part = &parts[part_idx];
1016
1017                 part->partition_id = of_node_to_fwnode(child_part);
1018
1019                 pr_info("GIC: PPI partition %s[%d] { ",
1020                         child_part->name, part_idx);
1021
1022                 n = of_property_count_elems_of_size(child_part, "affinity",
1023                                                     sizeof(u32));
1024                 WARN_ON(n <= 0);
1025
1026                 for (i = 0; i < n; i++) {
1027                         int err, cpu;
1028                         u32 cpu_phandle;
1029                         struct device_node *cpu_node;
1030
1031                         err = of_property_read_u32_index(child_part, "affinity",
1032                                                          i, &cpu_phandle);
1033                         if (WARN_ON(err))
1034                                 continue;
1035
1036                         cpu_node = of_find_node_by_phandle(cpu_phandle);
1037                         if (WARN_ON(!cpu_node))
1038                                 continue;
1039
1040                         cpu = get_cpu_number(cpu_node);
1041                         if (WARN_ON(cpu == -1))
1042                                 continue;
1043
1044                         pr_cont("%s[%d] ", cpu_node->full_name, cpu);
1045
1046                         cpumask_set_cpu(cpu, &part->mask);
1047                 }
1048
1049                 pr_cont("}\n");
1050                 part_idx++;
1051         }
1052
1053         for (i = 0; i < 16; i++) {
1054                 unsigned int irq;
1055                 struct partition_desc *desc;
1056                 struct irq_fwspec ppi_fwspec = {
1057                         .fwnode         = gic_data.fwnode,
1058                         .param_count    = 3,
1059                         .param          = {
1060                                 [0]     = 1,
1061                                 [1]     = i,
1062                                 [2]     = IRQ_TYPE_NONE,
1063                         },
1064                 };
1065
1066                 irq = irq_create_fwspec_mapping(&ppi_fwspec);
1067                 if (WARN_ON(!irq))
1068                         continue;
1069                 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
1070                                              irq, &partition_domain_ops);
1071                 if (WARN_ON(!desc))
1072                         continue;
1073
1074                 gic_data.ppi_descs[i] = desc;
1075         }
1076 }
1077
1078 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
1079 {
1080         void __iomem *dist_base;
1081         struct redist_region *rdist_regs;
1082         u64 redist_stride;
1083         u32 nr_redist_regions;
1084         int err, i;
1085
1086         dist_base = of_iomap(node, 0);
1087         if (!dist_base) {
1088                 pr_err("%s: unable to map gic dist registers\n",
1089                         node->full_name);
1090                 return -ENXIO;
1091         }
1092
1093         err = gic_validate_dist_version(dist_base);
1094         if (err) {
1095                 pr_err("%s: no distributor detected, giving up\n",
1096                         node->full_name);
1097                 goto out_unmap_dist;
1098         }
1099
1100         if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
1101                 nr_redist_regions = 1;
1102
1103         rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL);
1104         if (!rdist_regs) {
1105                 err = -ENOMEM;
1106                 goto out_unmap_dist;
1107         }
1108
1109         for (i = 0; i < nr_redist_regions; i++) {
1110                 struct resource res;
1111                 int ret;
1112
1113                 ret = of_address_to_resource(node, 1 + i, &res);
1114                 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
1115                 if (ret || !rdist_regs[i].redist_base) {
1116                         pr_err("%s: couldn't map region %d\n",
1117                                node->full_name, i);
1118                         err = -ENODEV;
1119                         goto out_unmap_rdist;
1120                 }
1121                 rdist_regs[i].phys_base = res.start;
1122         }
1123
1124         if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
1125                 redist_stride = 0;
1126
1127         err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
1128                              redist_stride, &node->fwnode);
1129         if (err)
1130                 goto out_unmap_rdist;
1131
1132         gic_populate_ppi_partitions(node);
1133         return 0;
1134
1135 out_unmap_rdist:
1136         for (i = 0; i < nr_redist_regions; i++)
1137                 if (rdist_regs[i].redist_base)
1138                         iounmap(rdist_regs[i].redist_base);
1139         kfree(rdist_regs);
1140 out_unmap_dist:
1141         iounmap(dist_base);
1142         return err;
1143 }
1144
1145 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);