2 * Copyright (C) 2013 Imagination Technologies
3 * Author: Paul Burton <paul.burton@mips.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
11 #include <linux/errno.h>
12 #include <linux/percpu.h>
13 #include <linux/spinlock.h>
15 #include <asm/mips-cps.h>
16 #include <asm/mipsregs.h>
18 void __iomem
*mips_gcr_base
;
19 void __iomem
*mips_cm_l2sync_base
;
22 static char *cm2_tr
[8] = {
23 "mem", "gcr", "gic", "mmio",
24 "0x04", "cpc", "0x06", "0x07"
27 /* CM3 Tag ECC transaction type */
28 static char *cm3_tr
[16] = {
35 [0x6] = "IReqNoRespDat",
36 [0x7] = "IReqWRespDat",
38 [0x9] = "RespDataFol",
40 [0xb] = "RespDataOnly",
41 [0xc] = "IRespNoData",
42 [0xd] = "IRespDataFol",
44 [0xf] = "IRespDataOnly"
47 static char *cm2_cmd
[32] = {
49 [0x01] = "Legacy Write",
50 [0x02] = "Legacy Read",
56 [0x08] = "Coherent Read Own",
57 [0x09] = "Coherent Read Share",
58 [0x0a] = "Coherent Read Discard",
59 [0x0b] = "Coherent Ready Share Always",
60 [0x0c] = "Coherent Upgrade",
61 [0x0d] = "Coherent Writeback",
64 [0x10] = "Coherent Copyback",
65 [0x11] = "Coherent Copyback Invalidate",
66 [0x12] = "Coherent Invalidate",
67 [0x13] = "Coherent Write Invalidate",
68 [0x14] = "Coherent Completion Sync",
82 /* CM3 Tag ECC command type */
83 static char *cm3_cmd
[16] = {
84 [0x0] = "Legacy Read",
85 [0x1] = "Legacy Write",
86 [0x2] = "Coherent Read Own",
87 [0x3] = "Coherent Read Share",
88 [0x4] = "Coherent Read Discard",
89 [0x5] = "Coherent Evicted",
90 [0x6] = "Coherent Upgrade",
91 [0x7] = "Coherent Upgrade for Store Conditional",
92 [0x8] = "Coherent Writeback",
93 [0x9] = "Coherent Write Invalidate",
102 /* CM3 Tag ECC command group */
103 static char *cm3_cmd_group
[8] = {
114 static char *cm2_core
[8] = {
115 "Invalid/OK", "Invalid/Data",
116 "Shared/OK", "Shared/Data",
117 "Modified/OK", "Modified/Data",
118 "Exclusive/OK", "Exclusive/Data"
121 static char *cm2_causes
[32] = {
122 "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR",
123 "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07",
124 "0x08", "0x09", "0x0a", "0x0b",
125 "0x0c", "0x0d", "0x0e", "0x0f",
126 "0x10", "0x11", "0x12", "0x13",
127 "0x14", "0x15", "0x16", "INTVN_WR_ERR",
128 "INTVN_RD_ERR", "0x19", "0x1a", "0x1b",
129 "0x1c", "0x1d", "0x1e", "0x1f"
132 static char *cm3_causes
[32] = {
133 "0x0", "MP_CORRECTABLE_ECC_ERR", "MP_REQUEST_DECODE_ERR",
134 "MP_UNCORRECTABLE_ECC_ERR", "MP_PARITY_ERR", "MP_COHERENCE_ERR",
135 "CMBIU_REQUEST_DECODE_ERR", "CMBIU_PARITY_ERR", "CMBIU_AXI_RESP_ERR",
136 "0x9", "RBI_BUS_ERR", "0xb", "0xc", "0xd", "0xe", "0xf", "0x10",
137 "0x11", "0x12", "0x13", "0x14", "0x15", "0x16", "0x17", "0x18",
138 "0x19", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f"
141 static DEFINE_PER_CPU_ALIGNED(spinlock_t
, cm_core_lock
);
142 static DEFINE_PER_CPU_ALIGNED(unsigned long, cm_core_lock_flags
);
144 phys_addr_t
__mips_cm_phys_base(void)
146 u32 config3
= read_c0_config3();
149 /* Check the CMGCRBase register is implemented */
150 if (!(config3
& MIPS_CONF3_CMGCR
))
153 /* Read the address from CMGCRBase */
154 cmgcr
= read_c0_cmgcrbase();
155 return (cmgcr
& MIPS_CMGCRF_BASE
) << (36 - 32);
158 phys_addr_t
mips_cm_phys_base(void)
159 __attribute__((weak
, alias("__mips_cm_phys_base")));
161 phys_addr_t
__mips_cm_l2sync_phys_base(void)
166 * If the L2-only sync region is already enabled then leave it at it's
169 base_reg
= read_gcr_l2_only_sync_base();
170 if (base_reg
& CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN
)
171 return base_reg
& CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE
;
173 /* Default to following the CM */
174 return mips_cm_phys_base() + MIPS_CM_GCR_SIZE
;
177 phys_addr_t
mips_cm_l2sync_phys_base(void)
178 __attribute__((weak
, alias("__mips_cm_l2sync_phys_base")));
180 static void mips_cm_probe_l2sync(void)
185 /* L2-only sync was introduced with CM major revision 6 */
186 major_rev
= (read_gcr_rev() & CM_GCR_REV_MAJOR
) >>
187 __ffs(CM_GCR_REV_MAJOR
);
191 /* Find a location for the L2 sync region */
192 addr
= mips_cm_l2sync_phys_base();
193 BUG_ON((addr
& CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE
) != addr
);
197 /* Set the region base address & enable it */
198 write_gcr_l2_only_sync_base(addr
| CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN
);
201 mips_cm_l2sync_base
= ioremap_nocache(addr
, MIPS_CM_L2SYNC_SIZE
);
204 int mips_cm_probe(void)
211 * No need to probe again if we have already been
217 addr
= mips_cm_phys_base();
218 BUG_ON((addr
& CM_GCR_BASE_GCRBASE
) != addr
);
222 mips_gcr_base
= ioremap_nocache(addr
, MIPS_CM_GCR_SIZE
);
226 /* sanity check that we're looking at a CM */
227 base_reg
= read_gcr_base();
228 if ((base_reg
& CM_GCR_BASE_GCRBASE
) != addr
) {
229 pr_err("GCRs appear to have been moved (expected them at 0x%08lx)!\n",
230 (unsigned long)addr
);
231 mips_gcr_base
= NULL
;
235 /* set default target to memory */
236 change_gcr_base(CM_GCR_BASE_CMDEFTGT
, CM_GCR_BASE_CMDEFTGT_MEM
);
238 /* disable CM regions */
239 write_gcr_reg0_base(CM_GCR_REGn_BASE_BASEADDR
);
240 write_gcr_reg0_mask(CM_GCR_REGn_MASK_ADDRMASK
);
241 write_gcr_reg1_base(CM_GCR_REGn_BASE_BASEADDR
);
242 write_gcr_reg1_mask(CM_GCR_REGn_MASK_ADDRMASK
);
243 write_gcr_reg2_base(CM_GCR_REGn_BASE_BASEADDR
);
244 write_gcr_reg2_mask(CM_GCR_REGn_MASK_ADDRMASK
);
245 write_gcr_reg3_base(CM_GCR_REGn_BASE_BASEADDR
);
246 write_gcr_reg3_mask(CM_GCR_REGn_MASK_ADDRMASK
);
248 /* probe for an L2-only sync region */
249 mips_cm_probe_l2sync();
251 /* determine register width for this CM */
252 mips_cm_is64
= IS_ENABLED(CONFIG_64BIT
) && (mips_cm_revision() >= CM_REV_CM3
);
254 for_each_possible_cpu(cpu
)
255 spin_lock_init(&per_cpu(cm_core_lock
, cpu
));
260 void mips_cm_lock_other(unsigned int cluster
, unsigned int core
,
261 unsigned int vp
, unsigned int block
)
263 unsigned int curr_core
, cm_rev
;
266 cm_rev
= mips_cm_revision();
269 if (cm_rev
>= CM_REV_CM3
) {
270 val
= core
<< __ffs(CM3_GCR_Cx_OTHER_CORE
);
271 val
|= vp
<< __ffs(CM3_GCR_Cx_OTHER_VP
);
273 if (cm_rev
>= CM_REV_CM3_5
) {
274 val
|= CM_GCR_Cx_OTHER_CLUSTER_EN
;
275 val
|= cluster
<< __ffs(CM_GCR_Cx_OTHER_CLUSTER
);
276 val
|= block
<< __ffs(CM_GCR_Cx_OTHER_BLOCK
);
278 WARN_ON(cluster
!= 0);
279 WARN_ON(block
!= CM_GCR_Cx_OTHER_BLOCK_LOCAL
);
283 * We need to disable interrupts in SMP systems in order to
284 * ensure that we don't interrupt the caller with code which
285 * may modify the redirect register. We do so here in a
286 * slightly obscure way by using a spin lock, since this has
287 * the neat property of also catching any nested uses of
288 * mips_cm_lock_other() leading to a deadlock or a nice warning
289 * with lockdep enabled.
291 spin_lock_irqsave(this_cpu_ptr(&cm_core_lock
),
292 *this_cpu_ptr(&cm_core_lock_flags
));
294 WARN_ON(cluster
!= 0);
295 WARN_ON(block
!= CM_GCR_Cx_OTHER_BLOCK_LOCAL
);
298 * We only have a GCR_CL_OTHER per core in systems with
299 * CM 2.5 & older, so have to ensure other VP(E)s don't
302 curr_core
= cpu_core(¤t_cpu_data
);
303 spin_lock_irqsave(&per_cpu(cm_core_lock
, curr_core
),
304 per_cpu(cm_core_lock_flags
, curr_core
));
306 val
= core
<< __ffs(CM_GCR_Cx_OTHER_CORENUM
);
309 write_gcr_cl_other(val
);
312 * Ensure the core-other region reflects the appropriate core &
313 * VP before any accesses to it occur.
318 void mips_cm_unlock_other(void)
320 unsigned int curr_core
;
322 if (mips_cm_revision() < CM_REV_CM3
) {
323 curr_core
= cpu_core(¤t_cpu_data
);
324 spin_unlock_irqrestore(&per_cpu(cm_core_lock
, curr_core
),
325 per_cpu(cm_core_lock_flags
, curr_core
));
327 spin_unlock_irqrestore(this_cpu_ptr(&cm_core_lock
),
328 *this_cpu_ptr(&cm_core_lock_flags
));
334 void mips_cm_error_report(void)
336 u64 cm_error
, cm_addr
, cm_other
;
337 unsigned long revision
;
341 if (!mips_cm_present())
344 revision
= mips_cm_revision();
345 cm_error
= read_gcr_error_cause();
346 cm_addr
= read_gcr_error_addr();
347 cm_other
= read_gcr_error_mult();
349 if (revision
< CM_REV_CM3
) { /* CM2 */
350 cause
= cm_error
>> __ffs(CM_GCR_ERROR_CAUSE_ERRTYPE
);
351 ocause
= cm_other
>> __ffs(CM_GCR_ERROR_MULT_ERR2ND
);
357 unsigned long cca_bits
= (cm_error
>> 15) & 7;
358 unsigned long tr_bits
= (cm_error
>> 12) & 7;
359 unsigned long cmd_bits
= (cm_error
>> 7) & 0x1f;
360 unsigned long stag_bits
= (cm_error
>> 3) & 15;
361 unsigned long sport_bits
= (cm_error
>> 0) & 7;
363 snprintf(buf
, sizeof(buf
),
364 "CCA=%lu TR=%s MCmd=%s STag=%lu "
365 "SPort=%lu\n", cca_bits
, cm2_tr
[tr_bits
],
366 cm2_cmd
[cmd_bits
], stag_bits
, sport_bits
);
368 /* glob state & sresp together */
369 unsigned long c3_bits
= (cm_error
>> 18) & 7;
370 unsigned long c2_bits
= (cm_error
>> 15) & 7;
371 unsigned long c1_bits
= (cm_error
>> 12) & 7;
372 unsigned long c0_bits
= (cm_error
>> 9) & 7;
373 unsigned long sc_bit
= (cm_error
>> 8) & 1;
374 unsigned long cmd_bits
= (cm_error
>> 3) & 0x1f;
375 unsigned long sport_bits
= (cm_error
>> 0) & 7;
377 snprintf(buf
, sizeof(buf
),
378 "C3=%s C2=%s C1=%s C0=%s SC=%s "
379 "MCmd=%s SPort=%lu\n",
380 cm2_core
[c3_bits
], cm2_core
[c2_bits
],
381 cm2_core
[c1_bits
], cm2_core
[c0_bits
],
382 sc_bit
? "True" : "False",
383 cm2_cmd
[cmd_bits
], sport_bits
);
385 pr_err("CM_ERROR=%08llx %s <%s>\n", cm_error
,
386 cm2_causes
[cause
], buf
);
387 pr_err("CM_ADDR =%08llx\n", cm_addr
);
388 pr_err("CM_OTHER=%08llx %s\n", cm_other
, cm2_causes
[ocause
]);
390 ulong core_id_bits
, vp_id_bits
, cmd_bits
, cmd_group_bits
;
391 ulong cm3_cca_bits
, mcp_bits
, cm3_tr_bits
, sched_bit
;
393 cause
= cm_error
>> __ffs64(CM3_GCR_ERROR_CAUSE_ERRTYPE
);
394 ocause
= cm_other
>> __ffs(CM_GCR_ERROR_MULT_ERR2ND
);
399 /* Used by cause == {1,2,3} */
400 core_id_bits
= (cm_error
>> 22) & 0xf;
401 vp_id_bits
= (cm_error
>> 18) & 0xf;
402 cmd_bits
= (cm_error
>> 14) & 0xf;
403 cmd_group_bits
= (cm_error
>> 11) & 0xf;
404 cm3_cca_bits
= (cm_error
>> 8) & 7;
405 mcp_bits
= (cm_error
>> 5) & 0xf;
406 cm3_tr_bits
= (cm_error
>> 1) & 0xf;
407 sched_bit
= cm_error
& 0x1;
409 if (cause
== 1 || cause
== 3) { /* Tag ECC */
410 unsigned long tag_ecc
= (cm_error
>> 57) & 0x1;
411 unsigned long tag_way_bits
= (cm_error
>> 29) & 0xffff;
412 unsigned long dword_bits
= (cm_error
>> 49) & 0xff;
413 unsigned long data_way_bits
= (cm_error
>> 45) & 0xf;
414 unsigned long data_sets_bits
= (cm_error
>> 29) & 0xfff;
415 unsigned long bank_bit
= (cm_error
>> 28) & 0x1;
416 snprintf(buf
, sizeof(buf
),
417 "%s ECC Error: Way=%lu (DWORD=%lu, Sets=%lu)"
418 "Bank=%lu CoreID=%lu VPID=%lu Command=%s"
419 "Command Group=%s CCA=%lu MCP=%d"
420 "Transaction type=%s Scheduler=%lu\n",
421 tag_ecc
? "TAG" : "DATA",
422 tag_ecc
? (unsigned long)ffs(tag_way_bits
) - 1 :
423 data_way_bits
, bank_bit
, dword_bits
,
425 core_id_bits
, vp_id_bits
,
427 cm3_cmd_group
[cmd_group_bits
],
428 cm3_cca_bits
, 1 << mcp_bits
,
429 cm3_tr
[cm3_tr_bits
], sched_bit
);
430 } else if (cause
== 2) {
431 unsigned long data_error_type
= (cm_error
>> 41) & 0xfff;
432 unsigned long data_decode_cmd
= (cm_error
>> 37) & 0xf;
433 unsigned long data_decode_group
= (cm_error
>> 34) & 0x7;
434 unsigned long data_decode_destination_id
= (cm_error
>> 28) & 0x3f;
436 snprintf(buf
, sizeof(buf
),
437 "Decode Request Error: Type=%lu, Command=%lu"
438 "Command Group=%lu Destination ID=%lu"
439 "CoreID=%lu VPID=%lu Command=%s"
440 "Command Group=%s CCA=%lu MCP=%d"
441 "Transaction type=%s Scheduler=%lu\n",
442 data_error_type
, data_decode_cmd
,
443 data_decode_group
, data_decode_destination_id
,
444 core_id_bits
, vp_id_bits
,
446 cm3_cmd_group
[cmd_group_bits
],
447 cm3_cca_bits
, 1 << mcp_bits
,
448 cm3_tr
[cm3_tr_bits
], sched_bit
);
453 pr_err("CM_ERROR=%llx %s <%s>\n", cm_error
,
454 cm3_causes
[cause
], buf
);
455 pr_err("CM_ADDR =%llx\n", cm_addr
);
456 pr_err("CM_OTHER=%llx %s\n", cm_other
, cm3_causes
[ocause
]);
459 /* reprime cause register */
460 write_gcr_error_cause(cm_error
);