drivers/irqchip/irq-gic-v3.c

   1 /*
   2  * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/of.h>
  23 #include <linux/of_address.h>
  24 #include <linux/of_irq.h>
  25 #include <linux/percpu.h>
  26 #include <linux/slab.h>
  27
  28 #include <linux/irqchip.h>
  29 #include <linux/irqchip/arm-gic-v3.h>
  30 #include <linux/irqchip/irq-partition-percpu.h>
  31
  32 #include <asm/cputype.h>
  33 #include <asm/exception.h>
  34 #include <asm/smp_plat.h>
  35 #include <asm/virt.h>
  36
  37 #include "irq-gic-common.h"
  38
  39 struct redist_region {
  40         void __iomem            *redist_base;
  41         phys_addr_t             phys_base;
  42 };
  43
  44 struct gic_chip_data {
  45         struct fwnode_handle    *fwnode;
  46         void __iomem            *dist_base;
  47         struct redist_region    *redist_regions;
  48         struct rdists           rdists;
  49         struct irq_domain       *domain;
  50         u64                     redist_stride;
  51         u32                     nr_redist_regions;
  52         unsigned int            irq_nr;
  53         struct partition_desc   *ppi_descs[16];
  54 };
  55
  56 static struct gic_chip_data gic_data __read_mostly;
  57 static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
  58
  59 #define gic_data_rdist()                (this_cpu_ptr(gic_data.rdists.rdist))
  60 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
  61 #define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
  62
  63 /* Our default, arbitrary priority value. Linux only uses one anyway. */
  64 #define DEFAULT_PMR_VALUE       0xf0
  65
  66 static inline unsigned int gic_irq(struct irq_data *d)
  67 {
  68         return d->hwirq;
  69 }
  70
  71 static inline int gic_irq_in_rdist(struct irq_data *d)
  72 {
  73         return gic_irq(d) < 32;
  74 }
  75
  76 static inline void __iomem *gic_dist_base(struct irq_data *d)
  77 {
  78         if (gic_irq_in_rdist(d))        /* SGI+PPI -> SGI_base for this CPU */
  79                 return gic_data_rdist_sgi_base();
  80
  81         if (d->hwirq <= 1023)           /* SPI -> dist_base */
  82                 return gic_data.dist_base;
  83
  84         return NULL;
  85 }
  86
  87 static void gic_do_wait_for_rwp(void __iomem *base)
  88 {
  89         u32 count = 1000000;    /* 1s! */
  90
  91         while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
  92                 count--;
  93                 if (!count) {
  94                         pr_err_ratelimited("RWP timeout, gone fishing\n");
  95                         return;
  96                 }
  97                 cpu_relax();
  98                 udelay(1);
  99         };
 100 }
 101
 102 /* Wait for completion of a distributor change */
 103 static void gic_dist_wait_for_rwp(void)
 104 {
 105         gic_do_wait_for_rwp(gic_data.dist_base);
 106 }
 107
 108 /* Wait for completion of a redistributor change */
 109 static void gic_redist_wait_for_rwp(void)
 110 {
 111         gic_do_wait_for_rwp(gic_data_rdist_rd_base());
 112 }
 113
 114 #ifdef CONFIG_ARM64
 115 static DEFINE_STATIC_KEY_FALSE(is_cavium_thunderx);
 116
 117 static u64 __maybe_unused gic_read_iar(void)
 118 {
 119         if (static_branch_unlikely(&is_cavium_thunderx))
 120                 return gic_read_iar_cavium_thunderx();
 121         else
 122                 return gic_read_iar_common();
 123 }
 124 #endif
 125
 126 static void gic_enable_redist(bool enable)
 127 {
 128         void __iomem *rbase;
 129         u32 count = 1000000;    /* 1s! */
 130         u32 val;
 131
 132         rbase = gic_data_rdist_rd_base();
 133
 134         val = readl_relaxed(rbase + GICR_WAKER);
 135         if (enable)
 136                 /* Wake up this CPU redistributor */
 137                 val &= ~GICR_WAKER_ProcessorSleep;
 138         else
 139                 val |= GICR_WAKER_ProcessorSleep;
 140         writel_relaxed(val, rbase + GICR_WAKER);
 141
 142         if (!enable) {          /* Check that GICR_WAKER is writeable */
 143                 val = readl_relaxed(rbase + GICR_WAKER);
 144                 if (!(val & GICR_WAKER_ProcessorSleep))
 145                         return; /* No PM support in this redistributor */
 146         }
 147
 148         while (count--) {
 149                 val = readl_relaxed(rbase + GICR_WAKER);
 150                 if (enable ^ (val & GICR_WAKER_ChildrenAsleep))
 151                         break;
 152                 cpu_relax();
 153                 udelay(1);
 154         };
 155         if (!count)
 156                 pr_err_ratelimited("redistributor failed to %s...\n",
 157                                    enable ? "wakeup" : "sleep");
 158 }
 159
 160 /*
 161  * Routines to disable, enable, EOI and route interrupts
 162  */
 163 static int gic_peek_irq(struct irq_data *d, u32 offset)
 164 {
 165         u32 mask = 1 << (gic_irq(d) % 32);
 166         void __iomem *base;
 167
 168         if (gic_irq_in_rdist(d))
 169                 base = gic_data_rdist_sgi_base();
 170         else
 171                 base = gic_data.dist_base;
 172
 173         return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
 174 }
 175
 176 static void gic_poke_irq(struct irq_data *d, u32 offset)
 177 {
 178         u32 mask = 1 << (gic_irq(d) % 32);
 179         void (*rwp_wait)(void);
 180         void __iomem *base;
 181
 182         if (gic_irq_in_rdist(d)) {
 183                 base = gic_data_rdist_sgi_base();
 184                 rwp_wait = gic_redist_wait_for_rwp;
 185         } else {
 186                 base = gic_data.dist_base;
 187                 rwp_wait = gic_dist_wait_for_rwp;
 188         }
 189
 190         writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
 191         rwp_wait();
 192 }
 193
 194 static void gic_mask_irq(struct irq_data *d)
 195 {
 196         gic_poke_irq(d, GICD_ICENABLER);
 197 }
 198
 199 static void gic_eoimode1_mask_irq(struct irq_data *d)
 200 {
 201         gic_mask_irq(d);
 202         /*
 203          * When masking a forwarded interrupt, make sure it is
 204          * deactivated as well.
 205          *
 206          * This ensures that an interrupt that is getting
 207          * disabled/masked will not get "stuck", because there is
 208          * noone to deactivate it (guest is being terminated).
 209          */
 210         if (irqd_is_forwarded_to_vcpu(d))
 211                 gic_poke_irq(d, GICD_ICACTIVER);
 212 }
 213
 214 static void gic_unmask_irq(struct irq_data *d)
 215 {
 216         gic_poke_irq(d, GICD_ISENABLER);
 217 }
 218
 219 static int gic_irq_set_irqchip_state(struct irq_data *d,
 220                                      enum irqchip_irq_state which, bool val)
 221 {
 222         u32 reg;
 223
 224         if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
 225                 return -EINVAL;
 226
 227         switch (which) {
 228         case IRQCHIP_STATE_PENDING:
 229                 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
 230                 break;
 231
 232         case IRQCHIP_STATE_ACTIVE:
 233                 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
 234                 break;
 235
 236         case IRQCHIP_STATE_MASKED:
 237                 reg = val ? GICD_ICENABLER : GICD_ISENABLER;
 238                 break;
 239
 240         default:
 241                 return -EINVAL;
 242         }
 243
 244         gic_poke_irq(d, reg);
 245         return 0;
 246 }
 247
 248 static int gic_irq_get_irqchip_state(struct irq_data *d,
 249                                      enum irqchip_irq_state which, bool *val)
 250 {
 251         if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
 252                 return -EINVAL;
 253
 254         switch (which) {
 255         case IRQCHIP_STATE_PENDING:
 256                 *val = gic_peek_irq(d, GICD_ISPENDR);
 257                 break;
 258
 259         case IRQCHIP_STATE_ACTIVE:
 260                 *val = gic_peek_irq(d, GICD_ISACTIVER);
 261                 break;
 262
 263         case IRQCHIP_STATE_MASKED:
 264                 *val = !gic_peek_irq(d, GICD_ISENABLER);
 265                 break;
 266
 267         default:
 268                 return -EINVAL;
 269         }
 270
 271         return 0;
 272 }
 273
 274 static void gic_eoi_irq(struct irq_data *d)
 275 {
 276         gic_write_eoir(gic_irq(d));
 277 }
 278
 279 static void gic_eoimode1_eoi_irq(struct irq_data *d)
 280 {
 281         /*
 282          * No need to deactivate an LPI, or an interrupt that
 283          * is is getting forwarded to a vcpu.
 284          */
 285         if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
 286                 return;
 287         gic_write_dir(gic_irq(d));
 288 }
 289
 290 static int gic_set_type(struct irq_data *d, unsigned int type)
 291 {
 292         unsigned int irq = gic_irq(d);
 293         void (*rwp_wait)(void);
 294         void __iomem *base;
 295
 296         /* Interrupt configuration for SGIs can't be changed */
 297         if (irq < 16)
 298                 return -EINVAL;
 299
 300         /* SPIs have restrictions on the supported types */
 301         if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
 302                          type != IRQ_TYPE_EDGE_RISING)
 303                 return -EINVAL;
 304
 305         if (gic_irq_in_rdist(d)) {
 306                 base = gic_data_rdist_sgi_base();
 307                 rwp_wait = gic_redist_wait_for_rwp;
 308         } else {
 309                 base = gic_data.dist_base;
 310                 rwp_wait = gic_dist_wait_for_rwp;
 311         }
 312
 313         return gic_configure_irq(irq, type, base, rwp_wait);
 314 }
 315
 316 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 317 {
 318         if (vcpu)
 319                 irqd_set_forwarded_to_vcpu(d);
 320         else
 321                 irqd_clr_forwarded_to_vcpu(d);
 322         return 0;
 323 }
 324
 325 static u64 gic_mpidr_to_affinity(unsigned long mpidr)
 326 {
 327         u64 aff;
 328
 329         aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
 330                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 331                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
 332                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 333
 334         return aff;
 335 }
 336
 337 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 338 {
 339         u32 irqnr;
 340
 341         do {
 342                 irqnr = gic_read_iar();
 343
 344                 if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
 345                         int err;
 346
 347                         if (static_key_true(&supports_deactivate))
 348                                 gic_write_eoir(irqnr);
 349
 350                         err = handle_domain_irq(gic_data.domain, irqnr, regs);
 351                         if (err) {
 352                                 WARN_ONCE(true, "Unexpected interrupt received!\n");
 353                                 if (static_key_true(&supports_deactivate)) {
 354                                         if (irqnr < 8192)
 355                                                 gic_write_dir(irqnr);
 356                                 } else {
 357                                         gic_write_eoir(irqnr);
 358                                 }
 359                         }
 360                         continue;
 361                 }
 362                 if (irqnr < 16) {
 363                         gic_write_eoir(irqnr);
 364                         if (static_key_true(&supports_deactivate))
 365                                 gic_write_dir(irqnr);
 366 #ifdef CONFIG_SMP
 367                         /*
 368                          * Unlike GICv2, we don't need an smp_rmb() here.
 369                          * The control dependency from gic_read_iar to
 370                          * the ISB in gic_write_eoir is enough to ensure
 371                          * that any shared data read by handle_IPI will
 372                          * be read after the ACK.
 373                          */
 374                         handle_IPI(irqnr, regs);
 375 #else
 376                         WARN_ONCE(true, "Unexpected SGI received!\n");
 377 #endif
 378                         continue;
 379                 }
 380         } while (irqnr != ICC_IAR1_EL1_SPURIOUS);
 381 }
 382
 383 static void __init gic_dist_init(void)
 384 {
 385         unsigned int i;
 386         u64 affinity;
 387         void __iomem *base = gic_data.dist_base;
 388
 389         /* Disable the distributor */
 390         writel_relaxed(0, base + GICD_CTLR);
 391         gic_dist_wait_for_rwp();
 392
 393         /*
 394          * Configure SPIs as non-secure Group-1. This will only matter
 395          * if the GIC only has a single security state. This will not
 396          * do the right thing if the kernel is running in secure mode,
 397          * but that's not the intended use case anyway.
 398          */
 399         for (i = 32; i < gic_data.irq_nr; i += 32)
 400                 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
 401
 402         gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
 403
 404         /* Enable distributor with ARE, Group1 */
 405         writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
 406                        base + GICD_CTLR);
 407
 408         /*
 409          * Set all global interrupts to the boot CPU only. ARE must be
 410          * enabled.
 411          */
 412         affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
 413         for (i = 32; i < gic_data.irq_nr; i++)
 414                 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
 415 }
 416
 417 static int gic_populate_rdist(void)
 418 {
 419         unsigned long mpidr = cpu_logical_map(smp_processor_id());
 420         u64 typer;
 421         u32 aff;
 422         int i;
 423
 424         /*
 425          * Convert affinity to a 32bit value that can be matched to
 426          * GICR_TYPER bits [63:32].
 427          */
 428         aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
 429                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 430                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
 431                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 432
 433         for (i = 0; i < gic_data.nr_redist_regions; i++) {
 434                 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
 435                 u32 reg;
 436
 437                 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
 438                 if (reg != GIC_PIDR2_ARCH_GICv3 &&
 439                     reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
 440                         pr_warn("No redistributor present @%p\n", ptr);
 441                         break;
 442                 }
 443
 444                 do {
 445                         typer = gic_read_typer(ptr + GICR_TYPER);
 446                         if ((typer >> 32) == aff) {
 447                                 u64 offset = ptr - gic_data.redist_regions[i].redist_base;
 448                                 gic_data_rdist_rd_base() = ptr;
 449                                 gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset;
 450                                 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
 451                                         smp_processor_id(), mpidr, i,
 452                                         &gic_data_rdist()->phys_base);
 453                                 return 0;
 454                         }
 455
 456                         if (gic_data.redist_stride) {
 457                                 ptr += gic_data.redist_stride;
 458                         } else {
 459                                 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
 460                                 if (typer & GICR_TYPER_VLPIS)
 461                                         ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
 462                         }
 463                 } while (!(typer & GICR_TYPER_LAST));
 464         }
 465
 466         /* We couldn't even deal with ourselves... */
 467         WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
 468              smp_processor_id(), mpidr);
 469         return -ENODEV;
 470 }
 471
 472 static void gic_cpu_sys_reg_init(void)
 473 {
 474         /*
 475          * Need to check that the SRE bit has actually been set. If
 476          * not, it means that SRE is disabled at EL2. We're going to
 477          * die painfully, and there is nothing we can do about it.
 478          *
 479          * Kindly inform the luser.
 480          */
 481         if (!gic_enable_sre())
 482                 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
 483
 484         /* Set priority mask register */
 485         gic_write_pmr(DEFAULT_PMR_VALUE);
 486
 487         if (static_key_true(&supports_deactivate)) {
 488                 /* EOI drops priority only (mode 1) */
 489                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
 490         } else {
 491                 /* EOI deactivates interrupt too (mode 0) */
 492                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
 493         }
 494
 495         /* ... and let's hit the road... */
 496         gic_write_grpen1(1);
 497 }
 498
 499 static int gic_dist_supports_lpis(void)
 500 {
 501         return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS);
 502 }
 503
 504 static void gic_cpu_init(void)
 505 {
 506         void __iomem *rbase;
 507
 508         /* Register ourselves with the rest of the world */
 509         if (gic_populate_rdist())
 510                 return;
 511
 512         gic_enable_redist(true);
 513
 514         rbase = gic_data_rdist_sgi_base();
 515
 516         /* Configure SGIs/PPIs as non-secure Group-1 */
 517         writel_relaxed(~0, rbase + GICR_IGROUPR0);
 518
 519         gic_cpu_config(rbase, gic_redist_wait_for_rwp);
 520
 521         /* Give LPIs a spin */
 522         if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
 523                 its_cpu_init();
 524
 525         /* initialise system registers */
 526         gic_cpu_sys_reg_init();
 527 }
 528
 529 #ifdef CONFIG_SMP
 530 static int gic_secondary_init(struct notifier_block *nfb,
 531                               unsigned long action, void *hcpu)
 532 {
 533         if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
 534                 gic_cpu_init();
 535         return NOTIFY_OK;
 536 }
 537
 538 /*
 539  * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
 540  * priority because the GIC needs to be up before the ARM generic timers.
 541  */
 542 static struct notifier_block gic_cpu_notifier = {
 543         .notifier_call = gic_secondary_init,
 544         .priority = 100,
 545 };
 546
 547 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
 548                                    unsigned long cluster_id)
 549 {
 550         int cpu = *base_cpu;
 551         unsigned long mpidr = cpu_logical_map(cpu);
 552         u16 tlist = 0;
 553
 554         while (cpu < nr_cpu_ids) {
 555                 /*
 556                  * If we ever get a cluster of more than 16 CPUs, just
 557                  * scream and skip that CPU.
 558                  */
 559                 if (WARN_ON((mpidr & 0xff) >= 16))
 560                         goto out;
 561
 562                 tlist |= 1 << (mpidr & 0xf);
 563
 564                 cpu = cpumask_next(cpu, mask);
 565                 if (cpu >= nr_cpu_ids)
 566                         goto out;
 567
 568                 mpidr = cpu_logical_map(cpu);
 569
 570                 if (cluster_id != (mpidr & ~0xffUL)) {
 571                         cpu--;
 572                         goto out;
 573                 }
 574         }
 575 out:
 576         *base_cpu = cpu;
 577         return tlist;
 578 }
 579
 580 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
 581         (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
 582                 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
 583
 584 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
 585 {
 586         u64 val;
 587
 588         val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)     |
 589                MPIDR_TO_SGI_AFFINITY(cluster_id, 2)     |
 590                irq << ICC_SGI1R_SGI_ID_SHIFT            |
 591                MPIDR_TO_SGI_AFFINITY(cluster_id, 1)     |
 592                tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
 593
 594         pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
 595         gic_write_sgi1r(val);
 596 }
 597
 598 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 599 {
 600         int cpu;
 601
 602         if (WARN_ON(irq >= 16))
 603                 return;
 604
 605         /*
 606          * Ensure that stores to Normal memory are visible to the
 607          * other CPUs before issuing the IPI.
 608          */
 609         smp_wmb();
 610
 611         for_each_cpu(cpu, mask) {
 612                 unsigned long cluster_id = cpu_logical_map(cpu) & ~0xffUL;
 613                 u16 tlist;
 614
 615                 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
 616                 gic_send_sgi(cluster_id, tlist, irq);
 617         }
 618
 619         /* Force the above writes to ICC_SGI1R_EL1 to be executed */
 620         isb();
 621 }
 622
 623 static void gic_smp_init(void)
 624 {
 625         set_smp_cross_call(gic_raise_softirq);
 626         register_cpu_notifier(&gic_cpu_notifier);
 627 }
 628
 629 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 630                             bool force)
 631 {
 632         unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
 633         void __iomem *reg;
 634         int enabled;
 635         u64 val;
 636
 637         if (gic_irq_in_rdist(d))
 638                 return -EINVAL;
 639
 640         /* If interrupt was enabled, disable it first */
 641         enabled = gic_peek_irq(d, GICD_ISENABLER);
 642         if (enabled)
 643                 gic_mask_irq(d);
 644
 645         reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
 646         val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
 647
 648         gic_write_irouter(val, reg);
 649
 650         /*
 651          * If the interrupt was enabled, enabled it again. Otherwise,
 652          * just wait for the distributor to have digested our changes.
 653          */
 654         if (enabled)
 655                 gic_unmask_irq(d);
 656         else
 657                 gic_dist_wait_for_rwp();
 658
 659         return IRQ_SET_MASK_OK;
 660 }
 661 #else
 662 #define gic_set_affinity        NULL
 663 #define gic_smp_init()          do { } while(0)
 664 #endif
 665
 666 #ifdef CONFIG_CPU_PM
 667 static int gic_cpu_pm_notifier(struct notifier_block *self,
 668                                unsigned long cmd, void *v)
 669 {
 670         if (cmd == CPU_PM_EXIT) {
 671                 gic_enable_redist(true);
 672                 gic_cpu_sys_reg_init();
 673         } else if (cmd == CPU_PM_ENTER) {
 674                 gic_write_grpen1(0);
 675                 gic_enable_redist(false);
 676         }
 677         return NOTIFY_OK;
 678 }
 679
 680 static struct notifier_block gic_cpu_pm_notifier_block = {
 681         .notifier_call = gic_cpu_pm_notifier,
 682 };
 683
 684 static void gic_cpu_pm_init(void)
 685 {
 686         cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
 687 }
 688
 689 #else
 690 static inline void gic_cpu_pm_init(void) { }
 691 #endif /* CONFIG_CPU_PM */
 692
 693 static struct irq_chip gic_chip = {
 694         .name                   = "GICv3",
 695         .irq_mask               = gic_mask_irq,
 696         .irq_unmask             = gic_unmask_irq,
 697         .irq_eoi                = gic_eoi_irq,
 698         .irq_set_type           = gic_set_type,
 699         .irq_set_affinity       = gic_set_affinity,
 700         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
 701         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
 702         .flags                  = IRQCHIP_SET_TYPE_MASKED,
 703 };
 704
 705 static struct irq_chip gic_eoimode1_chip = {
 706         .name                   = "GICv3",
 707         .irq_mask               = gic_eoimode1_mask_irq,
 708         .irq_unmask             = gic_unmask_irq,
 709         .irq_eoi                = gic_eoimode1_eoi_irq,
 710         .irq_set_type           = gic_set_type,
 711         .irq_set_affinity       = gic_set_affinity,
 712         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
 713         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
 714         .irq_set_vcpu_affinity  = gic_irq_set_vcpu_affinity,
 715         .flags                  = IRQCHIP_SET_TYPE_MASKED,
 716 };
 717
 718 #define GIC_ID_NR               (1U << gic_data.rdists.id_bits)
 719
 720 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
 721                               irq_hw_number_t hw)
 722 {
 723         struct irq_chip *chip = &gic_chip;
 724
 725         if (static_key_true(&supports_deactivate))
 726                 chip = &gic_eoimode1_chip;
 727
 728         /* SGIs are private to the core kernel */
 729         if (hw < 16)
 730                 return -EPERM;
 731         /* Nothing here */
 732         if (hw >= gic_data.irq_nr && hw < 8192)
 733                 return -EPERM;
 734         /* Off limits */
 735         if (hw >= GIC_ID_NR)
 736                 return -EPERM;
 737
 738         /* PPIs */
 739         if (hw < 32) {
 740                 irq_set_percpu_devid(irq);
 741                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
 742                                     handle_percpu_devid_irq, NULL, NULL);
 743                 irq_set_status_flags(irq, IRQ_NOAUTOEN);
 744         }
 745         /* SPIs */
 746         if (hw >= 32 && hw < gic_data.irq_nr) {
 747                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
 748                                     handle_fasteoi_irq, NULL, NULL);
 749                 irq_set_probe(irq);
 750         }
 751         /* LPIs */
 752         if (hw >= 8192 && hw < GIC_ID_NR) {
 753                 if (!gic_dist_supports_lpis())
 754                         return -EPERM;
 755                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
 756                                     handle_fasteoi_irq, NULL, NULL);
 757         }
 758
 759         return 0;
 760 }
 761
 762 static int gic_irq_domain_translate(struct irq_domain *d,
 763                                     struct irq_fwspec *fwspec,
 764                                     unsigned long *hwirq,
 765                                     unsigned int *type)
 766 {
 767         if (is_of_node(fwspec->fwnode)) {
 768                 if (fwspec->param_count < 3)
 769                         return -EINVAL;
 770
 771                 switch (fwspec->param[0]) {
 772                 case 0:                 /* SPI */
 773                         *hwirq = fwspec->param[1] + 32;
 774                         break;
 775                 case 1:                 /* PPI */
 776                         *hwirq = fwspec->param[1] + 16;
 777                         break;
 778                 case GIC_IRQ_TYPE_LPI:  /* LPI */
 779                         *hwirq = fwspec->param[1];
 780                         break;
 781                 default:
 782                         return -EINVAL;
 783                 }
 784
 785                 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
 786                 return 0;
 787         }
 788
 789         return -EINVAL;
 790 }
 791
 792 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 793                                 unsigned int nr_irqs, void *arg)
 794 {
 795         int i, ret;
 796         irq_hw_number_t hwirq;
 797         unsigned int type = IRQ_TYPE_NONE;
 798         struct irq_fwspec *fwspec = arg;
 799
 800         ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
 801         if (ret)
 802                 return ret;
 803
 804         for (i = 0; i < nr_irqs; i++)
 805                 gic_irq_domain_map(domain, virq + i, hwirq + i);
 806
 807         return 0;
 808 }
 809
 810 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
 811                                 unsigned int nr_irqs)
 812 {
 813         int i;
 814
 815         for (i = 0; i < nr_irqs; i++) {
 816                 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
 817                 irq_set_handler(virq + i, NULL);
 818                 irq_domain_reset_irq_data(d);
 819         }
 820 }
 821
 822 static int gic_irq_domain_select(struct irq_domain *d,
 823                                  struct irq_fwspec *fwspec,
 824                                  enum irq_domain_bus_token bus_token)
 825 {
 826         /* Not for us */
 827         if (fwspec->fwnode != d->fwnode)
 828                 return 0;
 829
 830         /* If this is not DT, then we have a single domain */
 831         if (!is_of_node(fwspec->fwnode))
 832                 return 1;
 833
 834         /*
 835          * If this is a PPI and we have a 4th (non-null) parameter,
 836          * then we need to match the partition domain.
 837          */
 838         if (fwspec->param_count >= 4 &&
 839             fwspec->param[0] == 1 && fwspec->param[3] != 0)
 840                 return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
 841
 842         return d == gic_data.domain;
 843 }
 844
 845 static const struct irq_domain_ops gic_irq_domain_ops = {
 846         .translate = gic_irq_domain_translate,
 847         .alloc = gic_irq_domain_alloc,
 848         .free = gic_irq_domain_free,
 849         .select = gic_irq_domain_select,
 850 };
 851
 852 static int partition_domain_translate(struct irq_domain *d,
 853                                       struct irq_fwspec *fwspec,
 854                                       unsigned long *hwirq,
 855                                       unsigned int *type)
 856 {
 857         struct device_node *np;
 858         int ret;
 859
 860         np = of_find_node_by_phandle(fwspec->param[3]);
 861         if (WARN_ON(!np))
 862                 return -EINVAL;
 863
 864         ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
 865                                      of_node_to_fwnode(np));
 866         if (ret < 0)
 867                 return ret;
 868
 869         *hwirq = ret;
 870         *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
 871
 872         return 0;
 873 }
 874
 875 static const struct irq_domain_ops partition_domain_ops = {
 876         .translate = partition_domain_translate,
 877         .select = gic_irq_domain_select,
 878 };
 879
 880 static void gicv3_enable_quirks(void)
 881 {
 882 #ifdef CONFIG_ARM64
 883         if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154))
 884                 static_branch_enable(&is_cavium_thunderx);
 885 #endif
 886 }
 887
 888 static int __init gic_init_bases(void __iomem *dist_base,
 889                                  struct redist_region *rdist_regs,
 890                                  u32 nr_redist_regions,
 891                                  u64 redist_stride,
 892                                  struct fwnode_handle *handle)
 893 {
 894         struct device_node *node;
 895         u32 typer;
 896         int gic_irqs;
 897         int err;
 898
 899         if (!is_hyp_mode_available())
 900                 static_key_slow_dec(&supports_deactivate);
 901
 902         if (static_key_true(&supports_deactivate))
 903                 pr_info("GIC: Using split EOI/Deactivate mode\n");
 904
 905         gic_data.fwnode = handle;
 906         gic_data.dist_base = dist_base;
 907         gic_data.redist_regions = rdist_regs;
 908         gic_data.nr_redist_regions = nr_redist_regions;
 909         gic_data.redist_stride = redist_stride;
 910
 911         gicv3_enable_quirks();
 912
 913         /*
 914          * Find out how many interrupts are supported.
 915          * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
 916          */
 917         typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
 918         gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer);
 919         gic_irqs = GICD_TYPER_IRQS(typer);
 920         if (gic_irqs > 1020)
 921                 gic_irqs = 1020;
 922         gic_data.irq_nr = gic_irqs;
 923
 924         gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
 925                                                  &gic_data);
 926         gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
 927
 928         if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
 929                 err = -ENOMEM;
 930                 goto out_free;
 931         }
 932
 933         set_handle_irq(gic_handle_irq);
 934
 935         node = to_of_node(handle);
 936         if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis() &&
 937             node) /* Temp hack to prevent ITS init for ACPI */
 938                 its_init(node, &gic_data.rdists, gic_data.domain);
 939
 940         gic_smp_init();
 941         gic_dist_init();
 942         gic_cpu_init();
 943         gic_cpu_pm_init();
 944
 945         return 0;
 946
 947 out_free:
 948         if (gic_data.domain)
 949                 irq_domain_remove(gic_data.domain);
 950         free_percpu(gic_data.rdists.rdist);
 951         return err;
 952 }
 953
 954 static int __init gic_validate_dist_version(void __iomem *dist_base)
 955 {
 956         u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
 957
 958         if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
 959                 return -ENODEV;
 960
 961         return 0;
 962 }
 963
 964 static int get_cpu_number(struct device_node *dn)
 965 {
 966         const __be32 *cell;
 967         u64 hwid;
 968         int i;
 969
 970         cell = of_get_property(dn, "reg", NULL);
 971         if (!cell)
 972                 return -1;
 973
 974         hwid = of_read_number(cell, of_n_addr_cells(dn));
 975
 976         /*
 977          * Non affinity bits must be set to 0 in the DT
 978          */
 979         if (hwid & ~MPIDR_HWID_BITMASK)
 980                 return -1;
 981
 982         for (i = 0; i < num_possible_cpus(); i++)
 983                 if (cpu_logical_map(i) == hwid)
 984                         return i;
 985
 986         return -1;
 987 }
 988
 989 /* Create all possible partitions at boot time */
 990 static void gic_populate_ppi_partitions(struct device_node *gic_node)
 991 {
 992         struct device_node *parts_node, *child_part;
 993         int part_idx = 0, i;
 994         int nr_parts;
 995         struct partition_affinity *parts;
 996
 997         parts_node = of_find_node_by_name(gic_node, "ppi-partitions");
 998         if (!parts_node)
 999                 return;
1000
1001         nr_parts = of_get_child_count(parts_node);
1002
1003         if (!nr_parts)
1004                 return;
1005
1006         parts = kzalloc(sizeof(*parts) * nr_parts, GFP_KERNEL);
1007         if (WARN_ON(!parts))
1008                 return;
1009
1010         for_each_child_of_node(parts_node, child_part) {
1011                 struct partition_affinity *part;
1012                 int n;
1013
1014                 part = &parts[part_idx];
1015
1016                 part->partition_id = of_node_to_fwnode(child_part);
1017
1018                 pr_info("GIC: PPI partition %s[%d] { ",
1019                         child_part->name, part_idx);
1020
1021                 n = of_property_count_elems_of_size(child_part, "affinity",
1022                                                     sizeof(u32));
1023                 WARN_ON(n <= 0);
1024
1025                 for (i = 0; i < n; i++) {
1026                         int err, cpu;
1027                         u32 cpu_phandle;
1028                         struct device_node *cpu_node;
1029
1030                         err = of_property_read_u32_index(child_part, "affinity",
1031                                                          i, &cpu_phandle);
1032                         if (WARN_ON(err))
1033                                 continue;
1034
1035                         cpu_node = of_find_node_by_phandle(cpu_phandle);
1036                         if (WARN_ON(!cpu_node))
1037                                 continue;
1038
1039                         cpu = get_cpu_number(cpu_node);
1040                         if (WARN_ON(cpu == -1))
1041                                 continue;
1042
1043                         pr_cont("%s[%d] ", cpu_node->full_name, cpu);
1044
1045                         cpumask_set_cpu(cpu, &part->mask);
1046                 }
1047
1048                 pr_cont("}\n");
1049                 part_idx++;
1050         }
1051
1052         for (i = 0; i < 16; i++) {
1053                 unsigned int irq;
1054                 struct partition_desc *desc;
1055                 struct irq_fwspec ppi_fwspec = {
1056                         .fwnode         = gic_data.fwnode,
1057                         .param_count    = 3,
1058                         .param          = {
1059                                 [0]     = 1,
1060                                 [1]     = i,
1061                                 [2]     = IRQ_TYPE_NONE,
1062                         },
1063                 };
1064
1065                 irq = irq_create_fwspec_mapping(&ppi_fwspec);
1066                 if (WARN_ON(!irq))
1067                         continue;
1068                 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
1069                                              irq, &partition_domain_ops);
1070                 if (WARN_ON(!desc))
1071                         continue;
1072
1073                 gic_data.ppi_descs[i] = desc;
1074         }
1075 }
1076
1077 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
1078 {
1079         void __iomem *dist_base;
1080         struct redist_region *rdist_regs;
1081         u64 redist_stride;
1082         u32 nr_redist_regions;
1083         int err, i;
1084
1085         dist_base = of_iomap(node, 0);
1086         if (!dist_base) {
1087                 pr_err("%s: unable to map gic dist registers\n",
1088                         node->full_name);
1089                 return -ENXIO;
1090         }
1091
1092         err = gic_validate_dist_version(dist_base);
1093         if (err) {
1094                 pr_err("%s: no distributor detected, giving up\n",
1095                         node->full_name);
1096                 goto out_unmap_dist;
1097         }
1098
1099         if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
1100                 nr_redist_regions = 1;
1101
1102         rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL);
1103         if (!rdist_regs) {
1104                 err = -ENOMEM;
1105                 goto out_unmap_dist;
1106         }
1107
1108         for (i = 0; i < nr_redist_regions; i++) {
1109                 struct resource res;
1110                 int ret;
1111
1112                 ret = of_address_to_resource(node, 1 + i, &res);
1113                 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
1114                 if (ret || !rdist_regs[i].redist_base) {
1115                         pr_err("%s: couldn't map region %d\n",
1116                                node->full_name, i);
1117                         err = -ENODEV;
1118                         goto out_unmap_rdist;
1119                 }
1120                 rdist_regs[i].phys_base = res.start;
1121         }
1122
1123         if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
1124                 redist_stride = 0;
1125
1126         err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
1127                              redist_stride, &node->fwnode);
1128         if (err)
1129                 goto out_unmap_rdist;
1130
1131         gic_populate_ppi_partitions(node);
1132         return 0;
1133
1134 out_unmap_rdist:
1135         for (i = 0; i < nr_redist_regions; i++)
1136                 if (rdist_regs[i].redist_base)
1137                         iounmap(rdist_regs[i].redist_base);
1138         kfree(rdist_regs);
1139 out_unmap_dist:
1140         iounmap(dist_base);
1141         return err;
1142 }
1143
1144 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);