1 // SPDX-License-Identifier: GPL-2.0
3 * EDAC driver for Intel(R) Xeon(R) Skylake processors
4 * Copyright (c) 2016, Intel Corporation.
7 #include <linux/kernel.h>
8 #include <linux/processor.h>
9 #include <asm/cpu_device_id.h>
10 #include <asm/intel-family.h>
13 #include "edac_module.h"
14 #include "skx_common.h"
16 #define EDAC_MOD_STR "skx_edac"
21 #define skx_printk(level, fmt, arg...) \
22 edac_printk(level, "skx", fmt, ##arg)
24 #define skx_mc_printk(mci, level, fmt, arg...) \
25 edac_mc_chipset_printk(mci, level, "skx", fmt, ##arg)
27 static struct list_head
*skx_edac_list
;
29 static u64 skx_tolm
, skx_tohm
;
30 static int skx_num_sockets
;
31 static unsigned int nvdimm_count
;
33 #define MASK26 0x3FFFFFF /* Mask for 2^26 */
34 #define MASK29 0x1FFFFFFF /* Mask for 2^29 */
36 static struct skx_dev
*get_skx_dev(struct pci_bus
*bus
, u8 idx
)
40 list_for_each_entry(d
, skx_edac_list
, list
) {
41 if (d
->seg
== pci_domain_nr(bus
) && d
->bus
[idx
] == bus
->number
)
49 CHAN0
, CHAN1
, CHAN2
, SAD_ALL
, UTIL_ALL
, SAD
54 u16 devfn
[SKX_NUM_IMC
];
61 * List of PCI device ids that we need together with some device
62 * number and function numbers to tell which memory controller the
65 static const struct munit skx_all_munits
[] = {
66 { 0x2054, { }, 1, 1, SAD_ALL
},
67 { 0x2055, { }, 1, 1, UTIL_ALL
},
68 { 0x2040, { PCI_DEVFN(10, 0), PCI_DEVFN(12, 0) }, 2, 2, CHAN0
},
69 { 0x2044, { PCI_DEVFN(10, 4), PCI_DEVFN(12, 4) }, 2, 2, CHAN1
},
70 { 0x2048, { PCI_DEVFN(11, 0), PCI_DEVFN(13, 0) }, 2, 2, CHAN2
},
71 { 0x208e, { }, 1, 0, SAD
},
75 static int get_all_munits(const struct munit
*m
)
77 struct pci_dev
*pdev
, *prev
;
84 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, m
->did
, prev
);
88 if (m
->per_socket
== SKX_NUM_IMC
) {
89 for (i
= 0; i
< SKX_NUM_IMC
; i
++)
90 if (m
->devfn
[i
] == pdev
->devfn
)
95 d
= get_skx_dev(pdev
->bus
, m
->busidx
);
99 /* Be sure that the device is enabled */
100 if (unlikely(pci_enable_device(pdev
) < 0)) {
101 skx_printk(KERN_ERR
, "Couldn't enable device %04x:%04x\n",
102 PCI_VENDOR_ID_INTEL
, m
->did
);
107 case CHAN0
: case CHAN1
: case CHAN2
:
109 d
->imc
[i
].chan
[m
->mtype
].cdev
= pdev
;
121 * one of these devices per core, including cores
122 * that don't exist on this SKU. Ignore any that
123 * read a route table of zero, make sure all the
124 * non-zero values match.
126 pci_read_config_dword(pdev
, 0xB4, ®
);
128 if (d
->mcroute
== 0) {
130 } else if (d
->mcroute
!= reg
) {
131 skx_printk(KERN_ERR
, "mcroute mismatch\n");
148 static const struct x86_cpu_id skx_cpuids
[] = {
149 { X86_VENDOR_INTEL
, 6, INTEL_FAM6_SKYLAKE_X
, 0, 0 },
152 MODULE_DEVICE_TABLE(x86cpu
, skx_cpuids
);
154 #define SKX_GET_MTMTR(dev, reg) \
155 pci_read_config_dword((dev), 0x87c, &(reg))
157 static bool skx_check_ecc(struct pci_dev
*pdev
)
161 SKX_GET_MTMTR(pdev
, mtmtr
);
163 return !!GET_BITFIELD(mtmtr
, 2, 2);
166 static int skx_get_dimm_config(struct mem_ctl_info
*mci
)
168 struct skx_pvt
*pvt
= mci
->pvt_info
;
169 struct skx_imc
*imc
= pvt
->imc
;
170 u32 mtr
, amap
, mcddrtcfg
;
171 struct dimm_info
*dimm
;
175 for (i
= 0; i
< SKX_NUM_CHANNELS
; i
++) {
177 pci_read_config_dword(imc
->chan
[i
].cdev
, 0x8C, &amap
);
178 pci_read_config_dword(imc
->chan
[i
].cdev
, 0x400, &mcddrtcfg
);
179 for (j
= 0; j
< SKX_NUM_DIMMS
; j
++) {
180 dimm
= EDAC_DIMM_PTR(mci
->layers
, mci
->dimms
,
181 mci
->n_layers
, i
, j
, 0);
182 pci_read_config_dword(imc
->chan
[i
].cdev
,
184 if (IS_DIMM_PRESENT(mtr
)) {
185 ndimms
+= skx_get_dimm_info(mtr
, amap
, dimm
, imc
, i
, j
);
186 } else if (IS_NVDIMM_PRESENT(mcddrtcfg
, j
)) {
187 ndimms
+= skx_get_nvdimm_info(dimm
, imc
, i
, j
,
192 if (ndimms
&& !skx_check_ecc(imc
->chan
[0].cdev
)) {
193 skx_printk(KERN_ERR
, "ECC is disabled on imc %d\n", imc
->mc
);
201 #define SKX_MAX_SAD 24
203 #define SKX_GET_SAD(d, i, reg) \
204 pci_read_config_dword((d)->sad_all, 0x60 + 8 * (i), &(reg))
205 #define SKX_GET_ILV(d, i, reg) \
206 pci_read_config_dword((d)->sad_all, 0x64 + 8 * (i), &(reg))
208 #define SKX_SAD_MOD3MODE(sad) GET_BITFIELD((sad), 30, 31)
209 #define SKX_SAD_MOD3(sad) GET_BITFIELD((sad), 27, 27)
210 #define SKX_SAD_LIMIT(sad) (((u64)GET_BITFIELD((sad), 7, 26) << 26) | MASK26)
211 #define SKX_SAD_MOD3ASMOD2(sad) GET_BITFIELD((sad), 5, 6)
212 #define SKX_SAD_ATTR(sad) GET_BITFIELD((sad), 3, 4)
213 #define SKX_SAD_INTERLEAVE(sad) GET_BITFIELD((sad), 1, 2)
214 #define SKX_SAD_ENABLE(sad) GET_BITFIELD((sad), 0, 0)
216 #define SKX_ILV_REMOTE(tgt) (((tgt) & 8) == 0)
217 #define SKX_ILV_TARGET(tgt) ((tgt) & 7)
219 static bool skx_sad_decode(struct decoded_addr
*res
)
221 struct skx_dev
*d
= list_first_entry(skx_edac_list
, typeof(*d
), list
);
222 u64 addr
= res
->addr
;
223 int i
, idx
, tgt
, lchan
, shift
;
225 u64 limit
, prev_limit
;
228 /* Simple sanity check for I/O space or out of range */
229 if (addr
>= skx_tohm
|| (addr
>= skx_tolm
&& addr
< BIT_ULL(32))) {
230 edac_dbg(0, "Address 0x%llx out of range\n", addr
);
236 for (i
= 0; i
< SKX_MAX_SAD
; i
++) {
237 SKX_GET_SAD(d
, i
, sad
);
238 limit
= SKX_SAD_LIMIT(sad
);
239 if (SKX_SAD_ENABLE(sad
)) {
240 if (addr
>= prev_limit
&& addr
<= limit
)
243 prev_limit
= limit
+ 1;
245 edac_dbg(0, "No SAD entry for 0x%llx\n", addr
);
249 SKX_GET_ILV(d
, i
, ilv
);
251 switch (SKX_SAD_INTERLEAVE(sad
)) {
253 idx
= GET_BITFIELD(addr
, 6, 8);
256 idx
= GET_BITFIELD(addr
, 8, 10);
259 idx
= GET_BITFIELD(addr
, 12, 14);
262 idx
= GET_BITFIELD(addr
, 30, 32);
266 tgt
= GET_BITFIELD(ilv
, 4 * idx
, 4 * idx
+ 3);
268 /* If point to another node, find it and start over */
269 if (SKX_ILV_REMOTE(tgt
)) {
271 edac_dbg(0, "Double remote!\n");
275 list_for_each_entry(d
, skx_edac_list
, list
) {
276 if (d
->imc
[0].src_id
== SKX_ILV_TARGET(tgt
))
279 edac_dbg(0, "Can't find node %d\n", SKX_ILV_TARGET(tgt
));
283 if (SKX_SAD_MOD3(sad
) == 0) {
284 lchan
= SKX_ILV_TARGET(tgt
);
286 switch (SKX_SAD_MOD3MODE(sad
)) {
297 edac_dbg(0, "illegal mod3mode\n");
300 switch (SKX_SAD_MOD3ASMOD2(sad
)) {
302 lchan
= (addr
>> shift
) % 3;
305 lchan
= (addr
>> shift
) % 2;
308 lchan
= (addr
>> shift
) % 2;
309 lchan
= (lchan
<< 1) | !lchan
;
312 lchan
= ((addr
>> shift
) % 2) << 1;
315 lchan
= (lchan
<< 1) | (SKX_ILV_TARGET(tgt
) & 1);
319 res
->socket
= d
->imc
[0].src_id
;
320 res
->imc
= GET_BITFIELD(d
->mcroute
, lchan
* 3, lchan
* 3 + 2);
321 res
->channel
= GET_BITFIELD(d
->mcroute
, lchan
* 2 + 18, lchan
* 2 + 19);
323 edac_dbg(2, "0x%llx: socket=%d imc=%d channel=%d\n",
324 res
->addr
, res
->socket
, res
->imc
, res
->channel
);
328 #define SKX_MAX_TAD 8
330 #define SKX_GET_TADBASE(d, mc, i, reg) \
331 pci_read_config_dword((d)->imc[mc].chan[0].cdev, 0x850 + 4 * (i), &(reg))
332 #define SKX_GET_TADWAYNESS(d, mc, i, reg) \
333 pci_read_config_dword((d)->imc[mc].chan[0].cdev, 0x880 + 4 * (i), &(reg))
334 #define SKX_GET_TADCHNILVOFFSET(d, mc, ch, i, reg) \
335 pci_read_config_dword((d)->imc[mc].chan[ch].cdev, 0x90 + 4 * (i), &(reg))
337 #define SKX_TAD_BASE(b) ((u64)GET_BITFIELD((b), 12, 31) << 26)
338 #define SKX_TAD_SKT_GRAN(b) GET_BITFIELD((b), 4, 5)
339 #define SKX_TAD_CHN_GRAN(b) GET_BITFIELD((b), 6, 7)
340 #define SKX_TAD_LIMIT(b) (((u64)GET_BITFIELD((b), 12, 31) << 26) | MASK26)
341 #define SKX_TAD_OFFSET(b) ((u64)GET_BITFIELD((b), 4, 23) << 26)
342 #define SKX_TAD_SKTWAYS(b) (1 << GET_BITFIELD((b), 10, 11))
343 #define SKX_TAD_CHNWAYS(b) (GET_BITFIELD((b), 8, 9) + 1)
345 /* which bit used for both socket and channel interleave */
346 static int skx_granularity
[] = { 6, 8, 12, 30 };
348 static u64
skx_do_interleave(u64 addr
, int shift
, int ways
, u64 lowbits
)
354 return addr
| (lowbits
& ((1ull << shift
) - 1));
357 static bool skx_tad_decode(struct decoded_addr
*res
)
360 u32 base
, wayness
, chnilvoffset
;
361 int skt_interleave_bit
, chn_interleave_bit
;
364 for (i
= 0; i
< SKX_MAX_TAD
; i
++) {
365 SKX_GET_TADBASE(res
->dev
, res
->imc
, i
, base
);
366 SKX_GET_TADWAYNESS(res
->dev
, res
->imc
, i
, wayness
);
367 if (SKX_TAD_BASE(base
) <= res
->addr
&& res
->addr
<= SKX_TAD_LIMIT(wayness
))
370 edac_dbg(0, "No TAD entry for 0x%llx\n", res
->addr
);
374 res
->sktways
= SKX_TAD_SKTWAYS(wayness
);
375 res
->chanways
= SKX_TAD_CHNWAYS(wayness
);
376 skt_interleave_bit
= skx_granularity
[SKX_TAD_SKT_GRAN(base
)];
377 chn_interleave_bit
= skx_granularity
[SKX_TAD_CHN_GRAN(base
)];
379 SKX_GET_TADCHNILVOFFSET(res
->dev
, res
->imc
, res
->channel
, i
, chnilvoffset
);
380 channel_addr
= res
->addr
- SKX_TAD_OFFSET(chnilvoffset
);
382 if (res
->chanways
== 3 && skt_interleave_bit
> chn_interleave_bit
) {
383 /* Must handle channel first, then socket */
384 channel_addr
= skx_do_interleave(channel_addr
, chn_interleave_bit
,
385 res
->chanways
, channel_addr
);
386 channel_addr
= skx_do_interleave(channel_addr
, skt_interleave_bit
,
387 res
->sktways
, channel_addr
);
389 /* Handle socket then channel. Preserve low bits from original address */
390 channel_addr
= skx_do_interleave(channel_addr
, skt_interleave_bit
,
391 res
->sktways
, res
->addr
);
392 channel_addr
= skx_do_interleave(channel_addr
, chn_interleave_bit
,
393 res
->chanways
, res
->addr
);
396 res
->chan_addr
= channel_addr
;
398 edac_dbg(2, "0x%llx: chan_addr=0x%llx sktways=%d chanways=%d\n",
399 res
->addr
, res
->chan_addr
, res
->sktways
, res
->chanways
);
403 #define SKX_MAX_RIR 4
405 #define SKX_GET_RIRWAYNESS(d, mc, ch, i, reg) \
406 pci_read_config_dword((d)->imc[mc].chan[ch].cdev, \
407 0x108 + 4 * (i), &(reg))
408 #define SKX_GET_RIRILV(d, mc, ch, idx, i, reg) \
409 pci_read_config_dword((d)->imc[mc].chan[ch].cdev, \
410 0x120 + 16 * (idx) + 4 * (i), &(reg))
412 #define SKX_RIR_VALID(b) GET_BITFIELD((b), 31, 31)
413 #define SKX_RIR_LIMIT(b) (((u64)GET_BITFIELD((b), 1, 11) << 29) | MASK29)
414 #define SKX_RIR_WAYS(b) (1 << GET_BITFIELD((b), 28, 29))
415 #define SKX_RIR_CHAN_RANK(b) GET_BITFIELD((b), 16, 19)
416 #define SKX_RIR_OFFSET(b) ((u64)(GET_BITFIELD((b), 2, 15) << 26))
418 static bool skx_rir_decode(struct decoded_addr
*res
)
420 int i
, idx
, chan_rank
;
423 u64 rank_addr
, prev_limit
= 0, limit
;
425 if (res
->dev
->imc
[res
->imc
].chan
[res
->channel
].dimms
[0].close_pg
)
430 for (i
= 0; i
< SKX_MAX_RIR
; i
++) {
431 SKX_GET_RIRWAYNESS(res
->dev
, res
->imc
, res
->channel
, i
, rirway
);
432 limit
= SKX_RIR_LIMIT(rirway
);
433 if (SKX_RIR_VALID(rirway
)) {
434 if (prev_limit
<= res
->chan_addr
&&
435 res
->chan_addr
<= limit
)
440 edac_dbg(0, "No RIR entry for 0x%llx\n", res
->addr
);
444 rank_addr
= res
->chan_addr
>> shift
;
445 rank_addr
/= SKX_RIR_WAYS(rirway
);
447 rank_addr
|= res
->chan_addr
& GENMASK_ULL(shift
- 1, 0);
449 res
->rank_address
= rank_addr
;
450 idx
= (res
->chan_addr
>> shift
) % SKX_RIR_WAYS(rirway
);
452 SKX_GET_RIRILV(res
->dev
, res
->imc
, res
->channel
, idx
, i
, rirlv
);
453 res
->rank_address
= rank_addr
- SKX_RIR_OFFSET(rirlv
);
454 chan_rank
= SKX_RIR_CHAN_RANK(rirlv
);
455 res
->channel_rank
= chan_rank
;
456 res
->dimm
= chan_rank
/ 4;
457 res
->rank
= chan_rank
% 4;
459 edac_dbg(2, "0x%llx: dimm=%d rank=%d chan_rank=%d rank_addr=0x%llx\n",
460 res
->addr
, res
->dimm
, res
->rank
,
461 res
->channel_rank
, res
->rank_address
);
465 static u8 skx_close_row
[] = {
466 15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33
469 static u8 skx_close_column
[] = {
470 3, 4, 5, 14, 19, 23, 24, 25, 26, 27
473 static u8 skx_open_row
[] = {
474 14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33
477 static u8 skx_open_column
[] = {
478 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
481 static u8 skx_open_fine_column
[] = {
482 3, 4, 5, 7, 8, 9, 10, 11, 12, 13
485 static int skx_bits(u64 addr
, int nbits
, u8
*bits
)
489 for (i
= 0; i
< nbits
; i
++)
490 res
|= ((addr
>> bits
[i
]) & 1) << i
;
494 static int skx_bank_bits(u64 addr
, int b0
, int b1
, int do_xor
, int x0
, int x1
)
496 int ret
= GET_BITFIELD(addr
, b0
, b0
) | (GET_BITFIELD(addr
, b1
, b1
) << 1);
499 ret
^= GET_BITFIELD(addr
, x0
, x0
) | (GET_BITFIELD(addr
, x1
, x1
) << 1);
504 static bool skx_mad_decode(struct decoded_addr
*r
)
506 struct skx_dimm
*dimm
= &r
->dev
->imc
[r
->imc
].chan
[r
->channel
].dimms
[r
->dimm
];
507 int bg0
= dimm
->fine_grain_bank
? 6 : 13;
509 if (dimm
->close_pg
) {
510 r
->row
= skx_bits(r
->rank_address
, dimm
->rowbits
, skx_close_row
);
511 r
->column
= skx_bits(r
->rank_address
, dimm
->colbits
, skx_close_column
);
512 r
->column
|= 0x400; /* C10 is autoprecharge, always set */
513 r
->bank_address
= skx_bank_bits(r
->rank_address
, 8, 9, dimm
->bank_xor_enable
, 22, 28);
514 r
->bank_group
= skx_bank_bits(r
->rank_address
, 6, 7, dimm
->bank_xor_enable
, 20, 21);
516 r
->row
= skx_bits(r
->rank_address
, dimm
->rowbits
, skx_open_row
);
517 if (dimm
->fine_grain_bank
)
518 r
->column
= skx_bits(r
->rank_address
, dimm
->colbits
, skx_open_fine_column
);
520 r
->column
= skx_bits(r
->rank_address
, dimm
->colbits
, skx_open_column
);
521 r
->bank_address
= skx_bank_bits(r
->rank_address
, 18, 19, dimm
->bank_xor_enable
, 22, 23);
522 r
->bank_group
= skx_bank_bits(r
->rank_address
, bg0
, 17, dimm
->bank_xor_enable
, 20, 21);
524 r
->row
&= (1u << dimm
->rowbits
) - 1;
526 edac_dbg(2, "0x%llx: row=0x%x col=0x%x bank_addr=%d bank_group=%d\n",
527 r
->addr
, r
->row
, r
->column
, r
->bank_address
,
532 static bool skx_decode(struct decoded_addr
*res
)
534 return skx_sad_decode(res
) && skx_tad_decode(res
) &&
535 skx_rir_decode(res
) && skx_mad_decode(res
);
538 static struct notifier_block skx_mce_dec
= {
539 .notifier_call
= skx_mce_check_error
,
540 .priority
= MCE_PRIO_EDAC
,
543 #ifdef CONFIG_EDAC_DEBUG
546 * Exercise the address decode logic by writing an address to
547 * /sys/kernel/debug/edac/skx_test/addr.
549 static struct dentry
*skx_test
;
551 static int debugfs_u64_set(void *data
, u64 val
)
555 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val
);
557 memset(&m
, 0, sizeof(m
));
558 /* ADDRV + MemRd + Unknown channel */
559 m
.status
= MCI_STATUS_ADDRV
+ 0x90;
560 /* One corrected error */
561 m
.status
|= BIT_ULL(MCI_STATUS_CEC_SHIFT
);
563 skx_mce_check_error(NULL
, 0, &m
);
567 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo
, NULL
, debugfs_u64_set
, "%llu\n");
569 static void setup_skx_debug(void)
571 skx_test
= edac_debugfs_create_dir("skx_test");
575 if (!edac_debugfs_create_file("addr", 0200, skx_test
,
576 NULL
, &fops_u64_wo
)) {
577 debugfs_remove(skx_test
);
582 static void teardown_skx_debug(void)
584 debugfs_remove_recursive(skx_test
);
587 static inline void setup_skx_debug(void) {}
588 static inline void teardown_skx_debug(void) {}
589 #endif /*CONFIG_EDAC_DEBUG*/
593 * make sure we are running on the correct cpu model
594 * search for all the devices we need
595 * check which DIMMs are present.
597 static int __init
skx_init(void)
599 const struct x86_cpu_id
*id
;
600 const struct munit
*m
;
602 int rc
= 0, i
, off
[3] = {0xd0, 0xd4, 0xd8};
603 u8 mc
= 0, src_id
, node_id
;
608 owner
= edac_get_owner();
609 if (owner
&& strncmp(owner
, EDAC_MOD_STR
, sizeof(EDAC_MOD_STR
)))
612 id
= x86_match_cpu(skx_cpuids
);
616 rc
= skx_get_hi_lo(0x2034, off
, &skx_tolm
, &skx_tohm
);
620 rc
= skx_get_all_bus_mappings(0x2016, 0xcc, SKX
, &skx_edac_list
);
624 edac_dbg(2, "No memory controllers found\n");
627 skx_num_sockets
= rc
;
629 for (m
= skx_all_munits
; m
->did
; m
++) {
630 rc
= get_all_munits(m
);
633 if (rc
!= m
->per_socket
* skx_num_sockets
) {
634 edac_dbg(2, "Expected %d, got %d of 0x%x\n",
635 m
->per_socket
* skx_num_sockets
, rc
, m
->did
);
641 list_for_each_entry(d
, skx_edac_list
, list
) {
642 rc
= skx_get_src_id(d
, 0xf0, &src_id
);
645 rc
= skx_get_node_id(d
, &node_id
);
648 edac_dbg(2, "src_id=%d node_id=%d\n", src_id
, node_id
);
649 for (i
= 0; i
< SKX_NUM_IMC
; i
++) {
652 d
->imc
[i
].src_id
= src_id
;
653 d
->imc
[i
].node_id
= node_id
;
654 rc
= skx_register_mci(&d
->imc
[i
], d
->imc
[i
].chan
[0].cdev
,
655 "Skylake Socket", EDAC_MOD_STR
,
656 skx_get_dimm_config
);
662 skx_set_decode(skx_decode
);
664 if (nvdimm_count
&& skx_adxl_get() == -ENODEV
)
665 skx_printk(KERN_NOTICE
, "Only decoding DDR4 address!\n");
667 /* Ensure that the OPSTATE is set correctly for POLL or NMI */
672 mce_register_decode_chain(&skx_mce_dec
);
680 static void __exit
skx_exit(void)
683 mce_unregister_decode_chain(&skx_mce_dec
);
684 teardown_skx_debug();
690 module_init(skx_init
);
691 module_exit(skx_exit
);
693 module_param(edac_op_state
, int, 0444);
694 MODULE_PARM_DESC(edac_op_state
, "EDAC Error Reporting state: 0=Poll,1=NMI");
696 MODULE_LICENSE("GPL v2");
697 MODULE_AUTHOR("Tony Luck");
698 MODULE_DESCRIPTION("MC Driver for Intel Skylake server processors");