db55d18691ed1357f3971c39f7a7f6733bd67157
[firefly-linux-kernel-4.4.55.git] / arch / arm / mm / cache-l2x0.c
1 /*
2  * arch/arm/mm/cache-l2x0.c - L210/L220 cache controller support
3  *
4  * Copyright (C) 2007 ARM Limited
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18  */
19 #include <linux/err.h>
20 #include <linux/init.h>
21 #include <linux/spinlock.h>
22 #include <linux/io.h>
23 #include <linux/of.h>
24 #include <linux/of_address.h>
25
26 #include <asm/cacheflush.h>
27 #include <asm/hardware/cache-l2x0.h>
28
29 #define CACHE_LINE_SIZE         32
30
31 static void __iomem *l2x0_base;
32 static DEFINE_RAW_SPINLOCK(l2x0_lock);
33 static u32 l2x0_way_mask;       /* Bitmask of active ways */
34 static u32 l2x0_size;
35 static unsigned long sync_reg_offset = L2X0_CACHE_SYNC;
36
37 struct l2x0_regs l2x0_saved_regs;
38
39 struct l2x0_of_data {
40         void (*setup)(const struct device_node *, u32 *, u32 *);
41         void (*save)(void);
42         struct outer_cache_fns outer_cache;
43 };
44
45 static bool of_init = false;
46
47 static inline void cache_wait_way(void __iomem *reg, unsigned long mask)
48 {
49         /* wait for cache operation by line or way to complete */
50         while (readl_relaxed(reg) & mask)
51                 cpu_relax();
52 }
53
54 #ifdef CONFIG_CACHE_PL310
55 static inline void cache_wait(void __iomem *reg, unsigned long mask)
56 {
57         /* cache operations by line are atomic on PL310 */
58 }
59 #else
60 #define cache_wait      cache_wait_way
61 #endif
62
63 static inline void cache_sync(void)
64 {
65         void __iomem *base = l2x0_base;
66
67         writel_relaxed(0, base + sync_reg_offset);
68         cache_wait(base + L2X0_CACHE_SYNC, 1);
69 }
70
71 static inline void l2x0_clean_line(unsigned long addr)
72 {
73         void __iomem *base = l2x0_base;
74         cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
75         writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
76 }
77
78 static inline void l2x0_inv_line(unsigned long addr)
79 {
80         void __iomem *base = l2x0_base;
81         cache_wait(base + L2X0_INV_LINE_PA, 1);
82         writel_relaxed(addr, base + L2X0_INV_LINE_PA);
83 }
84
85 #if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915)
86 static inline void debug_writel(unsigned long val)
87 {
88         if (outer_cache.set_debug)
89                 outer_cache.set_debug(val);
90 }
91
92 static void pl310_set_debug(unsigned long val)
93 {
94         writel_relaxed(val, l2x0_base + L2X0_DEBUG_CTRL);
95 }
96 #else
97 /* Optimised out for non-errata case */
98 static inline void debug_writel(unsigned long val)
99 {
100 }
101
102 #define pl310_set_debug NULL
103 #endif
104
105 #ifdef CONFIG_PL310_ERRATA_588369
106 static inline void l2x0_flush_line(unsigned long addr)
107 {
108         void __iomem *base = l2x0_base;
109
110         /* Clean by PA followed by Invalidate by PA */
111         cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
112         writel_relaxed(addr, base + L2X0_CLEAN_LINE_PA);
113         cache_wait(base + L2X0_INV_LINE_PA, 1);
114         writel_relaxed(addr, base + L2X0_INV_LINE_PA);
115 }
116 #else
117
118 static inline void l2x0_flush_line(unsigned long addr)
119 {
120         void __iomem *base = l2x0_base;
121         cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
122         writel_relaxed(addr, base + L2X0_CLEAN_INV_LINE_PA);
123 }
124 #endif
125
126 static void l2x0_cache_sync(void)
127 {
128         unsigned long flags;
129
130         raw_spin_lock_irqsave(&l2x0_lock, flags);
131         cache_sync();
132         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
133 }
134
135 static void __l2x0_flush_all(void)
136 {
137         debug_writel(0x03);
138         writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_INV_WAY);
139         cache_wait_way(l2x0_base + L2X0_CLEAN_INV_WAY, l2x0_way_mask);
140         cache_sync();
141         debug_writel(0x00);
142 }
143
144 static void l2x0_flush_all(void)
145 {
146         unsigned long flags;
147
148         /* clean all ways */
149         raw_spin_lock_irqsave(&l2x0_lock, flags);
150         __l2x0_flush_all();
151         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
152 }
153
154 static void l2x0_clean_all(void)
155 {
156         unsigned long flags;
157
158         /* clean all ways */
159         raw_spin_lock_irqsave(&l2x0_lock, flags);
160         writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_WAY);
161         cache_wait_way(l2x0_base + L2X0_CLEAN_WAY, l2x0_way_mask);
162         cache_sync();
163         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
164 }
165
166 static void l2x0_inv_all(void)
167 {
168         unsigned long flags;
169
170         /* invalidate all ways */
171         raw_spin_lock_irqsave(&l2x0_lock, flags);
172         /* Invalidating when L2 is enabled is a nono */
173         BUG_ON(readl(l2x0_base + L2X0_CTRL) & 1);
174         writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY);
175         cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask);
176         cache_sync();
177         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
178 }
179
180 static void l2x0_inv_range(unsigned long start, unsigned long end)
181 {
182         void __iomem *base = l2x0_base;
183         unsigned long flags;
184
185         raw_spin_lock_irqsave(&l2x0_lock, flags);
186         if (start & (CACHE_LINE_SIZE - 1)) {
187                 start &= ~(CACHE_LINE_SIZE - 1);
188                 debug_writel(0x03);
189                 l2x0_flush_line(start);
190                 debug_writel(0x00);
191                 start += CACHE_LINE_SIZE;
192         }
193
194         if (end & (CACHE_LINE_SIZE - 1)) {
195                 end &= ~(CACHE_LINE_SIZE - 1);
196                 debug_writel(0x03);
197                 l2x0_flush_line(end);
198                 debug_writel(0x00);
199         }
200
201         while (start < end) {
202                 unsigned long blk_end = start + min(end - start, 4096UL);
203
204                 while (start < blk_end) {
205                         l2x0_inv_line(start);
206                         start += CACHE_LINE_SIZE;
207                 }
208
209                 if (blk_end < end) {
210                         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
211                         raw_spin_lock_irqsave(&l2x0_lock, flags);
212                 }
213         }
214         cache_wait(base + L2X0_INV_LINE_PA, 1);
215         cache_sync();
216         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
217 }
218
219 static void l2x0_clean_range(unsigned long start, unsigned long end)
220 {
221         void __iomem *base = l2x0_base;
222         unsigned long flags;
223
224         if ((end - start) >= l2x0_size) {
225                 l2x0_clean_all();
226                 return;
227         }
228
229         raw_spin_lock_irqsave(&l2x0_lock, flags);
230         start &= ~(CACHE_LINE_SIZE - 1);
231         while (start < end) {
232                 unsigned long blk_end = start + min(end - start, 4096UL);
233
234                 while (start < blk_end) {
235                         l2x0_clean_line(start);
236                         start += CACHE_LINE_SIZE;
237                 }
238
239                 if (blk_end < end) {
240                         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
241                         raw_spin_lock_irqsave(&l2x0_lock, flags);
242                 }
243         }
244         cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
245         cache_sync();
246         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
247 }
248
249 static void l2x0_flush_range(unsigned long start, unsigned long end)
250 {
251         void __iomem *base = l2x0_base;
252         unsigned long flags;
253
254         if ((end - start) >= l2x0_size) {
255                 l2x0_flush_all();
256                 return;
257         }
258
259         raw_spin_lock_irqsave(&l2x0_lock, flags);
260         start &= ~(CACHE_LINE_SIZE - 1);
261         while (start < end) {
262                 unsigned long blk_end = start + min(end - start, 4096UL);
263
264                 debug_writel(0x03);
265                 while (start < blk_end) {
266                         l2x0_flush_line(start);
267                         start += CACHE_LINE_SIZE;
268                 }
269                 debug_writel(0x00);
270
271                 if (blk_end < end) {
272                         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
273                         raw_spin_lock_irqsave(&l2x0_lock, flags);
274                 }
275         }
276         cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
277         cache_sync();
278         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
279 }
280
281 static void l2x0_disable(void)
282 {
283         unsigned long flags;
284
285         raw_spin_lock_irqsave(&l2x0_lock, flags);
286         __l2x0_flush_all();
287         writel_relaxed(0, l2x0_base + L2X0_CTRL);
288         dsb();
289         raw_spin_unlock_irqrestore(&l2x0_lock, flags);
290 }
291
292 static void l2x0_unlock(u32 cache_id)
293 {
294         int lockregs;
295         int i;
296
297         if (cache_id == L2X0_CACHE_ID_PART_L310)
298                 lockregs = 8;
299         else
300                 /* L210 and unknown types */
301                 lockregs = 1;
302
303         for (i = 0; i < lockregs; i++) {
304                 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_D_BASE +
305                                i * L2X0_LOCKDOWN_STRIDE);
306                 writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_I_BASE +
307                                i * L2X0_LOCKDOWN_STRIDE);
308         }
309 }
310
311 void __init l2x0_init(void __iomem *base, u32 aux_val, u32 aux_mask)
312 {
313         u32 aux;
314         u32 cache_id;
315         u32 way_size = 0;
316         int ways;
317         const char *type;
318
319         l2x0_base = base;
320
321         cache_id = readl_relaxed(l2x0_base + L2X0_CACHE_ID);
322         aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
323
324         aux &= aux_mask;
325         aux |= aux_val;
326
327         /* Determine the number of ways */
328         switch (cache_id & L2X0_CACHE_ID_PART_MASK) {
329         case L2X0_CACHE_ID_PART_L310:
330                 if (aux & (1 << 16))
331                         ways = 16;
332                 else
333                         ways = 8;
334                 type = "L310";
335 #ifdef CONFIG_PL310_ERRATA_753970
336                 /* Unmapped register. */
337                 sync_reg_offset = L2X0_DUMMY_REG;
338 #endif
339                 outer_cache.set_debug = pl310_set_debug;
340                 break;
341         case L2X0_CACHE_ID_PART_L210:
342                 ways = (aux >> 13) & 0xf;
343                 type = "L210";
344                 break;
345         default:
346                 /* Assume unknown chips have 8 ways */
347                 ways = 8;
348                 type = "L2x0 series";
349                 break;
350         }
351
352         l2x0_way_mask = (1 << ways) - 1;
353
354         /*
355          * L2 cache Size =  Way size * Number of ways
356          */
357         way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17;
358         way_size = 1 << (way_size + 3);
359         l2x0_size = ways * way_size * SZ_1K;
360
361         /*
362          * Check if l2x0 controller is already enabled.
363          * If you are booting from non-secure mode
364          * accessing the below registers will fault.
365          */
366         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) {
367                 /* Make sure that I&D is not locked down when starting */
368                 l2x0_unlock(cache_id);
369
370                 /* l2x0 controller is disabled */
371                 writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL);
372
373                 l2x0_inv_all();
374
375                 /* enable L2X0 */
376                 writel_relaxed(1, l2x0_base + L2X0_CTRL);
377         }
378
379         /* Re-read it in case some bits are reserved. */
380         aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
381
382         /* Save the value for resuming. */
383         l2x0_saved_regs.aux_ctrl = aux;
384
385         if (!of_init) {
386                 outer_cache.inv_range = l2x0_inv_range;
387                 outer_cache.clean_range = l2x0_clean_range;
388                 outer_cache.flush_range = l2x0_flush_range;
389                 outer_cache.sync = l2x0_cache_sync;
390                 outer_cache.flush_all = l2x0_flush_all;
391                 outer_cache.inv_all = l2x0_inv_all;
392                 outer_cache.disable = l2x0_disable;
393         }
394
395         printk(KERN_INFO "%s cache controller enabled\n", type);
396         printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d B\n",
397                         ways, cache_id, aux, l2x0_size);
398 }
399
400 #ifdef CONFIG_OF
401 static void __init l2x0_of_setup(const struct device_node *np,
402                                  u32 *aux_val, u32 *aux_mask)
403 {
404         u32 data[2] = { 0, 0 };
405         u32 tag = 0;
406         u32 dirty = 0;
407         u32 val = 0, mask = 0;
408
409         of_property_read_u32(np, "arm,tag-latency", &tag);
410         if (tag) {
411                 mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK;
412                 val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT;
413         }
414
415         of_property_read_u32_array(np, "arm,data-latency",
416                                    data, ARRAY_SIZE(data));
417         if (data[0] && data[1]) {
418                 mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK |
419                         L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK;
420                 val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |
421                        ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);
422         }
423
424         of_property_read_u32(np, "arm,dirty-latency", &dirty);
425         if (dirty) {
426                 mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK;
427                 val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;
428         }
429
430         *aux_val &= ~mask;
431         *aux_val |= val;
432         *aux_mask &= ~mask;
433 }
434
435 static void __init pl310_of_setup(const struct device_node *np,
436                                   u32 *aux_val, u32 *aux_mask)
437 {
438         u32 data[3] = { 0, 0, 0 };
439         u32 tag[3] = { 0, 0, 0 };
440         u32 filter[2] = { 0, 0 };
441
442         of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));
443         if (tag[0] && tag[1] && tag[2])
444                 writel_relaxed(
445                         ((tag[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) |
446                         ((tag[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) |
447                         ((tag[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT),
448                         l2x0_base + L2X0_TAG_LATENCY_CTRL);
449
450         of_property_read_u32_array(np, "arm,data-latency",
451                                    data, ARRAY_SIZE(data));
452         if (data[0] && data[1] && data[2])
453                 writel_relaxed(
454                         ((data[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) |
455                         ((data[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) |
456                         ((data[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT),
457                         l2x0_base + L2X0_DATA_LATENCY_CTRL);
458
459         of_property_read_u32_array(np, "arm,filter-ranges",
460                                    filter, ARRAY_SIZE(filter));
461         if (filter[1]) {
462                 writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M),
463                                l2x0_base + L2X0_ADDR_FILTER_END);
464                 writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L2X0_ADDR_FILTER_EN,
465                                l2x0_base + L2X0_ADDR_FILTER_START);
466         }
467 }
468
469 static void __init pl310_save(void)
470 {
471         u32 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) &
472                 L2X0_CACHE_ID_RTL_MASK;
473
474         l2x0_saved_regs.tag_latency = readl_relaxed(l2x0_base +
475                 L2X0_TAG_LATENCY_CTRL);
476         l2x0_saved_regs.data_latency = readl_relaxed(l2x0_base +
477                 L2X0_DATA_LATENCY_CTRL);
478         l2x0_saved_regs.filter_end = readl_relaxed(l2x0_base +
479                 L2X0_ADDR_FILTER_END);
480         l2x0_saved_regs.filter_start = readl_relaxed(l2x0_base +
481                 L2X0_ADDR_FILTER_START);
482
483         if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) {
484                 /*
485                  * From r2p0, there is Prefetch offset/control register
486                  */
487                 l2x0_saved_regs.prefetch_ctrl = readl_relaxed(l2x0_base +
488                         L2X0_PREFETCH_CTRL);
489                 /*
490                  * From r3p0, there is Power control register
491                  */
492                 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0)
493                         l2x0_saved_regs.pwr_ctrl = readl_relaxed(l2x0_base +
494                                 L2X0_POWER_CTRL);
495         }
496 }
497
498 static void l2x0_resume(void)
499 {
500         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) {
501                 /* restore aux ctrl and enable l2 */
502                 l2x0_unlock(readl_relaxed(l2x0_base + L2X0_CACHE_ID));
503
504                 writel_relaxed(l2x0_saved_regs.aux_ctrl, l2x0_base +
505                         L2X0_AUX_CTRL);
506
507                 l2x0_inv_all();
508
509                 writel_relaxed(1, l2x0_base + L2X0_CTRL);
510         }
511 }
512
513 static void pl310_resume(void)
514 {
515         u32 l2x0_revision;
516
517         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) {
518                 /* restore pl310 setup */
519                 writel_relaxed(l2x0_saved_regs.tag_latency,
520                         l2x0_base + L2X0_TAG_LATENCY_CTRL);
521                 writel_relaxed(l2x0_saved_regs.data_latency,
522                         l2x0_base + L2X0_DATA_LATENCY_CTRL);
523                 writel_relaxed(l2x0_saved_regs.filter_end,
524                         l2x0_base + L2X0_ADDR_FILTER_END);
525                 writel_relaxed(l2x0_saved_regs.filter_start,
526                         l2x0_base + L2X0_ADDR_FILTER_START);
527
528                 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) &
529                         L2X0_CACHE_ID_RTL_MASK;
530
531                 if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) {
532                         writel_relaxed(l2x0_saved_regs.prefetch_ctrl,
533                                 l2x0_base + L2X0_PREFETCH_CTRL);
534                         if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0)
535                                 writel_relaxed(l2x0_saved_regs.pwr_ctrl,
536                                         l2x0_base + L2X0_POWER_CTRL);
537                 }
538         }
539
540         l2x0_resume();
541 }
542
543 static const struct l2x0_of_data pl310_data = {
544         .setup = pl310_of_setup,
545         .save  = pl310_save,
546         .outer_cache = {
547                 .resume      = pl310_resume,
548                 .inv_range   = l2x0_inv_range,
549                 .clean_range = l2x0_clean_range,
550                 .flush_range = l2x0_flush_range,
551                 .sync        = l2x0_cache_sync,
552                 .flush_all   = l2x0_flush_all,
553                 .inv_all     = l2x0_inv_all,
554                 .disable     = l2x0_disable,
555                 .set_debug   = pl310_set_debug,
556         },
557 };
558
559 static const struct l2x0_of_data l2x0_data = {
560         .setup = l2x0_of_setup,
561         .save  = NULL,
562         .outer_cache = {
563                 .resume      = l2x0_resume,
564                 .inv_range   = l2x0_inv_range,
565                 .clean_range = l2x0_clean_range,
566                 .flush_range = l2x0_flush_range,
567                 .sync        = l2x0_cache_sync,
568                 .flush_all   = l2x0_flush_all,
569                 .inv_all     = l2x0_inv_all,
570                 .disable     = l2x0_disable,
571         },
572 };
573
574 static const struct of_device_id l2x0_ids[] __initconst = {
575         { .compatible = "arm,pl310-cache", .data = (void *)&pl310_data },
576         { .compatible = "arm,l220-cache", .data = (void *)&l2x0_data },
577         { .compatible = "arm,l210-cache", .data = (void *)&l2x0_data },
578         {}
579 };
580
581 int __init l2x0_of_init(u32 aux_val, u32 aux_mask)
582 {
583         struct device_node *np;
584         const struct l2x0_of_data *data;
585         struct resource res;
586
587         np = of_find_matching_node(NULL, l2x0_ids);
588         if (!np)
589                 return -ENODEV;
590
591         if (of_address_to_resource(np, 0, &res))
592                 return -ENODEV;
593
594         l2x0_base = ioremap(res.start, resource_size(&res));
595         if (!l2x0_base)
596                 return -ENOMEM;
597
598         l2x0_saved_regs.phy_base = res.start;
599
600         data = of_match_node(l2x0_ids, np)->data;
601
602         /* L2 configuration can only be changed if the cache is disabled */
603         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) {
604                 if (data->setup)
605                         data->setup(np, &aux_val, &aux_mask);
606         }
607
608         if (data->save)
609                 data->save();
610
611         of_init = true;
612         l2x0_init(l2x0_base, aux_val, aux_mask);
613
614         memcpy(&outer_cache, &data->outer_cache, sizeof(outer_cache));
615
616         return 0;
617 }
618 #endif