2 * edac_mc kernel module
3 * (C) 2005, 2006 Linux Networx (http://lnxi.com)
4 * This file may be distributed under the terms of the
5 * GNU General Public License.
7 * Written by Thayne Harbaugh
8 * Based on work by Dan Hollis <goemon at anime dot net> and others.
9 * http://www.anime.net/~goemon/linux-ecc/
11 * Modified by Dave Peterson and Doug Thompson
15 #include <linux/module.h>
16 #include <linux/proc_fs.h>
17 #include <linux/kernel.h>
18 #include <linux/types.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/sysctl.h>
22 #include <linux/highmem.h>
23 #include <linux/timer.h>
24 #include <linux/slab.h>
25 #include <linux/jiffies.h>
26 #include <linux/spinlock.h>
27 #include <linux/list.h>
28 #include <linux/ctype.h>
29 #include <linux/edac.h>
30 #include <linux/bitops.h>
31 #include <linux/uaccess.h>
34 #include "edac_module.h"
35 #include <ras/ras_event.h>
37 #ifdef CONFIG_EDAC_ATOMIC_SCRUB
40 #define edac_atomic_scrub(va, size) do { } while (0)
43 int edac_op_state
= EDAC_OPSTATE_INVAL
;
44 EXPORT_SYMBOL_GPL(edac_op_state
);
46 static int edac_report
= EDAC_REPORTING_ENABLED
;
48 /* lock to memory controller's control array */
49 static DEFINE_MUTEX(mem_ctls_mutex
);
50 static LIST_HEAD(mc_devices
);
53 * Used to lock EDAC MC to just one module, avoiding two drivers e. g.
54 * apei/ghes and i7core_edac to be used at the same time.
56 static const char *edac_mc_owner
;
58 int edac_get_report_status(void)
62 EXPORT_SYMBOL_GPL(edac_get_report_status
);
64 void edac_set_report_status(int new)
66 if (new == EDAC_REPORTING_ENABLED
||
67 new == EDAC_REPORTING_DISABLED
||
68 new == EDAC_REPORTING_FORCE
)
71 EXPORT_SYMBOL_GPL(edac_set_report_status
);
73 static int edac_report_set(const char *str
, const struct kernel_param
*kp
)
78 if (!strncmp(str
, "on", 2))
79 edac_report
= EDAC_REPORTING_ENABLED
;
80 else if (!strncmp(str
, "off", 3))
81 edac_report
= EDAC_REPORTING_DISABLED
;
82 else if (!strncmp(str
, "force", 5))
83 edac_report
= EDAC_REPORTING_FORCE
;
88 static int edac_report_get(char *buffer
, const struct kernel_param
*kp
)
92 switch (edac_report
) {
93 case EDAC_REPORTING_ENABLED
:
94 ret
= sprintf(buffer
, "on");
96 case EDAC_REPORTING_DISABLED
:
97 ret
= sprintf(buffer
, "off");
99 case EDAC_REPORTING_FORCE
:
100 ret
= sprintf(buffer
, "force");
110 static const struct kernel_param_ops edac_report_ops
= {
111 .set
= edac_report_set
,
112 .get
= edac_report_get
,
115 module_param_cb(edac_report
, &edac_report_ops
, &edac_report
, 0644);
117 unsigned int edac_dimm_info_location(struct dimm_info
*dimm
, char *buf
,
120 struct mem_ctl_info
*mci
= dimm
->mci
;
124 for (i
= 0; i
< mci
->n_layers
; i
++) {
125 n
= snprintf(p
, len
, "%s %d ",
126 edac_layer_name
[mci
->layers
[i
].type
],
138 #ifdef CONFIG_EDAC_DEBUG
140 static void edac_mc_dump_channel(struct rank_info
*chan
)
142 edac_dbg(4, " channel->chan_idx = %d\n", chan
->chan_idx
);
143 edac_dbg(4, " channel = %p\n", chan
);
144 edac_dbg(4, " channel->csrow = %p\n", chan
->csrow
);
145 edac_dbg(4, " channel->dimm = %p\n", chan
->dimm
);
148 static void edac_mc_dump_dimm(struct dimm_info
*dimm
, int number
)
152 edac_dimm_info_location(dimm
, location
, sizeof(location
));
154 edac_dbg(4, "%s%i: %smapped as virtual row %d, chan %d\n",
155 dimm
->mci
->csbased
? "rank" : "dimm",
156 number
, location
, dimm
->csrow
, dimm
->cschannel
);
157 edac_dbg(4, " dimm = %p\n", dimm
);
158 edac_dbg(4, " dimm->label = '%s'\n", dimm
->label
);
159 edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm
->nr_pages
);
160 edac_dbg(4, " dimm->grain = %d\n", dimm
->grain
);
161 edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm
->nr_pages
);
164 static void edac_mc_dump_csrow(struct csrow_info
*csrow
)
166 edac_dbg(4, "csrow->csrow_idx = %d\n", csrow
->csrow_idx
);
167 edac_dbg(4, " csrow = %p\n", csrow
);
168 edac_dbg(4, " csrow->first_page = 0x%lx\n", csrow
->first_page
);
169 edac_dbg(4, " csrow->last_page = 0x%lx\n", csrow
->last_page
);
170 edac_dbg(4, " csrow->page_mask = 0x%lx\n", csrow
->page_mask
);
171 edac_dbg(4, " csrow->nr_channels = %d\n", csrow
->nr_channels
);
172 edac_dbg(4, " csrow->channels = %p\n", csrow
->channels
);
173 edac_dbg(4, " csrow->mci = %p\n", csrow
->mci
);
176 static void edac_mc_dump_mci(struct mem_ctl_info
*mci
)
178 edac_dbg(3, "\tmci = %p\n", mci
);
179 edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci
->mtype_cap
);
180 edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci
->edac_ctl_cap
);
181 edac_dbg(3, "\tmci->edac_cap = %lx\n", mci
->edac_cap
);
182 edac_dbg(4, "\tmci->edac_check = %p\n", mci
->edac_check
);
183 edac_dbg(3, "\tmci->nr_csrows = %d, csrows = %p\n",
184 mci
->nr_csrows
, mci
->csrows
);
185 edac_dbg(3, "\tmci->nr_dimms = %d, dimms = %p\n",
186 mci
->tot_dimms
, mci
->dimms
);
187 edac_dbg(3, "\tdev = %p\n", mci
->pdev
);
188 edac_dbg(3, "\tmod_name:ctl_name = %s:%s\n",
189 mci
->mod_name
, mci
->ctl_name
);
190 edac_dbg(3, "\tpvt_info = %p\n\n", mci
->pvt_info
);
193 #endif /* CONFIG_EDAC_DEBUG */
195 const char * const edac_mem_types
[] = {
196 [MEM_EMPTY
] = "Empty",
197 [MEM_RESERVED
] = "Reserved",
198 [MEM_UNKNOWN
] = "Unknown",
202 [MEM_SDR
] = "Unbuffered-SDR",
203 [MEM_RDR
] = "Registered-SDR",
204 [MEM_DDR
] = "Unbuffered-DDR",
205 [MEM_RDDR
] = "Registered-DDR",
207 [MEM_DDR2
] = "Unbuffered-DDR2",
208 [MEM_FB_DDR2
] = "FullyBuffered-DDR2",
209 [MEM_RDDR2
] = "Registered-DDR2",
211 [MEM_DDR3
] = "Unbuffered-DDR3",
212 [MEM_RDDR3
] = "Registered-DDR3",
213 [MEM_LRDDR3
] = "Load-Reduced-DDR3-RAM",
214 [MEM_DDR4
] = "Unbuffered-DDR4",
215 [MEM_RDDR4
] = "Registered-DDR4",
216 [MEM_LRDDR4
] = "Load-Reduced-DDR4-RAM",
217 [MEM_NVDIMM
] = "Non-volatile-RAM",
219 EXPORT_SYMBOL_GPL(edac_mem_types
);
222 * edac_align_ptr - Prepares the pointer offsets for a single-shot allocation
223 * @p: pointer to a pointer with the memory offset to be used. At
224 * return, this will be incremented to point to the next offset
225 * @size: Size of the data structure to be reserved
226 * @n_elems: Number of elements that should be reserved
228 * If 'size' is a constant, the compiler will optimize this whole function
229 * down to either a no-op or the addition of a constant to the value of '*p'.
231 * The 'p' pointer is absolutely needed to keep the proper advancing
232 * further in memory to the proper offsets when allocating the struct along
233 * with its embedded structs, as edac_device_alloc_ctl_info() does it
234 * above, for example.
236 * At return, the pointer 'p' will be incremented to be used on a next call
239 void *edac_align_ptr(void **p
, unsigned int size
, int n_elems
)
241 unsigned int align
, r
;
244 *p
+= size
* n_elems
;
247 * 'p' can possibly be an unaligned item X such that sizeof(X) is
248 * 'size'. Adjust 'p' so that its alignment is at least as
249 * stringent as what the compiler would provide for X and return
250 * the aligned result.
251 * Here we assume that the alignment of a "long long" is the most
252 * stringent alignment that the compiler will ever provide by default.
253 * As far as I know, this is a reasonable assumption.
255 if (size
> sizeof(long))
256 align
= sizeof(long long);
257 else if (size
> sizeof(int))
258 align
= sizeof(long);
259 else if (size
> sizeof(short))
261 else if (size
> sizeof(char))
262 align
= sizeof(short);
266 r
= (unsigned long)p
% align
;
273 return (void *)(((unsigned long)ptr
) + align
- r
);
276 static void _edac_mc_free(struct mem_ctl_info
*mci
)
278 struct csrow_info
*csr
;
282 for (i
= 0; i
< mci
->tot_dimms
; i
++)
283 kfree(mci
->dimms
[i
]);
288 for (row
= 0; row
< mci
->nr_csrows
; row
++) {
289 csr
= mci
->csrows
[row
];
294 for (chn
= 0; chn
< mci
->num_cschannel
; chn
++)
295 kfree(csr
->channels
[chn
]);
296 kfree(csr
->channels
);
305 struct mem_ctl_info
*edac_mc_alloc(unsigned int mc_num
,
306 unsigned int n_layers
,
307 struct edac_mc_layer
*layers
,
310 struct mem_ctl_info
*mci
;
311 struct edac_mc_layer
*layer
;
312 struct csrow_info
*csr
;
313 struct rank_info
*chan
;
314 struct dimm_info
*dimm
;
315 u32
*ce_per_layer
[EDAC_MAX_LAYERS
], *ue_per_layer
[EDAC_MAX_LAYERS
];
316 unsigned int pos
[EDAC_MAX_LAYERS
];
317 unsigned int size
, tot_dimms
= 1, count
= 1;
318 unsigned int tot_csrows
= 1, tot_channels
= 1, tot_errcount
= 0;
319 void *pvt
, *p
, *ptr
= NULL
;
320 int i
, j
, row
, chn
, n
, len
, off
;
321 bool per_rank
= false;
323 BUG_ON(n_layers
> EDAC_MAX_LAYERS
|| n_layers
== 0);
325 * Calculate the total amount of dimms and csrows/cschannels while
326 * in the old API emulation mode
328 for (i
= 0; i
< n_layers
; i
++) {
329 tot_dimms
*= layers
[i
].size
;
330 if (layers
[i
].is_virt_csrow
)
331 tot_csrows
*= layers
[i
].size
;
333 tot_channels
*= layers
[i
].size
;
335 if (layers
[i
].type
== EDAC_MC_LAYER_CHIP_SELECT
)
339 /* Figure out the offsets of the various items from the start of an mc
340 * structure. We want the alignment of each item to be at least as
341 * stringent as what the compiler would provide if we could simply
342 * hardcode everything into a single struct.
344 mci
= edac_align_ptr(&ptr
, sizeof(*mci
), 1);
345 layer
= edac_align_ptr(&ptr
, sizeof(*layer
), n_layers
);
346 for (i
= 0; i
< n_layers
; i
++) {
347 count
*= layers
[i
].size
;
348 edac_dbg(4, "errcount layer %d size %d\n", i
, count
);
349 ce_per_layer
[i
] = edac_align_ptr(&ptr
, sizeof(u32
), count
);
350 ue_per_layer
[i
] = edac_align_ptr(&ptr
, sizeof(u32
), count
);
351 tot_errcount
+= 2 * count
;
354 edac_dbg(4, "allocating %d error counters\n", tot_errcount
);
355 pvt
= edac_align_ptr(&ptr
, sz_pvt
, 1);
356 size
= ((unsigned long)pvt
) + sz_pvt
;
358 edac_dbg(1, "allocating %u bytes for mci data (%d %s, %d csrows/channels)\n",
361 per_rank
? "ranks" : "dimms",
362 tot_csrows
* tot_channels
);
364 mci
= kzalloc(size
, GFP_KERNEL
);
368 /* Adjust pointers so they point within the memory we just allocated
369 * rather than an imaginary chunk of memory located at address 0.
371 layer
= (struct edac_mc_layer
*)(((char *)mci
) + ((unsigned long)layer
));
372 for (i
= 0; i
< n_layers
; i
++) {
373 mci
->ce_per_layer
[i
] = (u32
*)((char *)mci
+ ((unsigned long)ce_per_layer
[i
]));
374 mci
->ue_per_layer
[i
] = (u32
*)((char *)mci
+ ((unsigned long)ue_per_layer
[i
]));
376 pvt
= sz_pvt
? (((char *)mci
) + ((unsigned long)pvt
)) : NULL
;
378 /* setup index and various internal pointers */
379 mci
->mc_idx
= mc_num
;
380 mci
->tot_dimms
= tot_dimms
;
382 mci
->n_layers
= n_layers
;
384 memcpy(mci
->layers
, layers
, sizeof(*layer
) * n_layers
);
385 mci
->nr_csrows
= tot_csrows
;
386 mci
->num_cschannel
= tot_channels
;
387 mci
->csbased
= per_rank
;
390 * Alocate and fill the csrow/channels structs
392 mci
->csrows
= kcalloc(tot_csrows
, sizeof(*mci
->csrows
), GFP_KERNEL
);
395 for (row
= 0; row
< tot_csrows
; row
++) {
396 csr
= kzalloc(sizeof(**mci
->csrows
), GFP_KERNEL
);
399 mci
->csrows
[row
] = csr
;
400 csr
->csrow_idx
= row
;
402 csr
->nr_channels
= tot_channels
;
403 csr
->channels
= kcalloc(tot_channels
, sizeof(*csr
->channels
),
408 for (chn
= 0; chn
< tot_channels
; chn
++) {
409 chan
= kzalloc(sizeof(**csr
->channels
), GFP_KERNEL
);
412 csr
->channels
[chn
] = chan
;
413 chan
->chan_idx
= chn
;
419 * Allocate and fill the dimm structs
421 mci
->dimms
= kcalloc(tot_dimms
, sizeof(*mci
->dimms
), GFP_KERNEL
);
425 memset(&pos
, 0, sizeof(pos
));
428 for (i
= 0; i
< tot_dimms
; i
++) {
429 chan
= mci
->csrows
[row
]->channels
[chn
];
430 off
= EDAC_DIMM_OFF(layer
, n_layers
, pos
[0], pos
[1], pos
[2]);
431 if (off
< 0 || off
>= tot_dimms
) {
432 edac_mc_printk(mci
, KERN_ERR
, "EDAC core bug: EDAC_DIMM_OFF is trying to do an illegal data access\n");
436 dimm
= kzalloc(sizeof(**mci
->dimms
), GFP_KERNEL
);
439 mci
->dimms
[off
] = dimm
;
443 * Copy DIMM location and initialize it.
445 len
= sizeof(dimm
->label
);
447 n
= snprintf(p
, len
, "mc#%u", mc_num
);
450 for (j
= 0; j
< n_layers
; j
++) {
451 n
= snprintf(p
, len
, "%s#%u",
452 edac_layer_name
[layers
[j
].type
],
456 dimm
->location
[j
] = pos
[j
];
462 /* Link it to the csrows old API data */
465 dimm
->cschannel
= chn
;
467 /* Increment csrow location */
468 if (layers
[0].is_virt_csrow
) {
470 if (chn
== tot_channels
) {
476 if (row
== tot_csrows
) {
482 /* Increment dimm location */
483 for (j
= n_layers
- 1; j
>= 0; j
--) {
485 if (pos
[j
] < layers
[j
].size
)
491 mci
->op_state
= OP_ALLOC
;
500 EXPORT_SYMBOL_GPL(edac_mc_alloc
);
502 void edac_mc_free(struct mem_ctl_info
*mci
)
506 /* If we're not yet registered with sysfs free only what was allocated
507 * in edac_mc_alloc().
509 if (!device_is_registered(&mci
->dev
)) {
514 /* the mci instance is freed here, when the sysfs object is dropped */
515 edac_unregister_sysfs(mci
);
517 EXPORT_SYMBOL_GPL(edac_mc_free
);
519 bool edac_has_mcs(void)
523 mutex_lock(&mem_ctls_mutex
);
525 ret
= list_empty(&mc_devices
);
527 mutex_unlock(&mem_ctls_mutex
);
531 EXPORT_SYMBOL_GPL(edac_has_mcs
);
533 /* Caller must hold mem_ctls_mutex */
534 static struct mem_ctl_info
*__find_mci_by_dev(struct device
*dev
)
536 struct mem_ctl_info
*mci
;
537 struct list_head
*item
;
541 list_for_each(item
, &mc_devices
) {
542 mci
= list_entry(item
, struct mem_ctl_info
, link
);
544 if (mci
->pdev
== dev
)
554 * scan list of controllers looking for the one that manages
556 * @dev: pointer to a struct device related with the MCI
558 struct mem_ctl_info
*find_mci_by_dev(struct device
*dev
)
560 struct mem_ctl_info
*ret
;
562 mutex_lock(&mem_ctls_mutex
);
563 ret
= __find_mci_by_dev(dev
);
564 mutex_unlock(&mem_ctls_mutex
);
568 EXPORT_SYMBOL_GPL(find_mci_by_dev
);
571 * edac_mc_workq_function
572 * performs the operation scheduled by a workq request
574 static void edac_mc_workq_function(struct work_struct
*work_req
)
576 struct delayed_work
*d_work
= to_delayed_work(work_req
);
577 struct mem_ctl_info
*mci
= to_edac_mem_ctl_work(d_work
);
579 mutex_lock(&mem_ctls_mutex
);
581 if (mci
->op_state
!= OP_RUNNING_POLL
) {
582 mutex_unlock(&mem_ctls_mutex
);
586 if (edac_op_state
== EDAC_OPSTATE_POLL
)
587 mci
->edac_check(mci
);
589 mutex_unlock(&mem_ctls_mutex
);
591 /* Queue ourselves again. */
592 edac_queue_work(&mci
->work
, msecs_to_jiffies(edac_mc_get_poll_msec()));
596 * edac_mc_reset_delay_period(unsigned long value)
598 * user space has updated our poll period value, need to
599 * reset our workq delays
601 void edac_mc_reset_delay_period(unsigned long value
)
603 struct mem_ctl_info
*mci
;
604 struct list_head
*item
;
606 mutex_lock(&mem_ctls_mutex
);
608 list_for_each(item
, &mc_devices
) {
609 mci
= list_entry(item
, struct mem_ctl_info
, link
);
611 if (mci
->op_state
== OP_RUNNING_POLL
)
612 edac_mod_work(&mci
->work
, value
);
614 mutex_unlock(&mem_ctls_mutex
);
619 /* Return 0 on success, 1 on failure.
620 * Before calling this function, caller must
621 * assign a unique value to mci->mc_idx.
625 * called with the mem_ctls_mutex lock held
627 static int add_mc_to_global_list(struct mem_ctl_info
*mci
)
629 struct list_head
*item
, *insert_before
;
630 struct mem_ctl_info
*p
;
632 insert_before
= &mc_devices
;
634 p
= __find_mci_by_dev(mci
->pdev
);
635 if (unlikely(p
!= NULL
))
638 list_for_each(item
, &mc_devices
) {
639 p
= list_entry(item
, struct mem_ctl_info
, link
);
641 if (p
->mc_idx
>= mci
->mc_idx
) {
642 if (unlikely(p
->mc_idx
== mci
->mc_idx
))
645 insert_before
= item
;
650 list_add_tail_rcu(&mci
->link
, insert_before
);
654 edac_printk(KERN_WARNING
, EDAC_MC
,
655 "%s (%s) %s %s already assigned %d\n", dev_name(p
->pdev
),
656 edac_dev_name(mci
), p
->mod_name
, p
->ctl_name
, p
->mc_idx
);
660 edac_printk(KERN_WARNING
, EDAC_MC
,
661 "bug in low-level driver: attempt to assign\n"
662 " duplicate mc_idx %d in %s()\n", p
->mc_idx
, __func__
);
666 static int del_mc_from_global_list(struct mem_ctl_info
*mci
)
668 list_del_rcu(&mci
->link
);
670 /* these are for safe removal of devices from global list while
671 * NMI handlers may be traversing list
674 INIT_LIST_HEAD(&mci
->link
);
676 return list_empty(&mc_devices
);
679 struct mem_ctl_info
*edac_mc_find(int idx
)
681 struct mem_ctl_info
*mci
;
682 struct list_head
*item
;
684 mutex_lock(&mem_ctls_mutex
);
686 list_for_each(item
, &mc_devices
) {
687 mci
= list_entry(item
, struct mem_ctl_info
, link
);
688 if (mci
->mc_idx
== idx
)
694 mutex_unlock(&mem_ctls_mutex
);
697 EXPORT_SYMBOL(edac_mc_find
);
699 const char *edac_get_owner(void)
701 return edac_mc_owner
;
703 EXPORT_SYMBOL_GPL(edac_get_owner
);
705 /* FIXME - should a warning be printed if no error detection? correction? */
706 int edac_mc_add_mc_with_groups(struct mem_ctl_info
*mci
,
707 const struct attribute_group
**groups
)
712 #ifdef CONFIG_EDAC_DEBUG
713 if (edac_debug_level
>= 3)
714 edac_mc_dump_mci(mci
);
716 if (edac_debug_level
>= 4) {
719 for (i
= 0; i
< mci
->nr_csrows
; i
++) {
720 struct csrow_info
*csrow
= mci
->csrows
[i
];
724 for (j
= 0; j
< csrow
->nr_channels
; j
++)
725 nr_pages
+= csrow
->channels
[j
]->dimm
->nr_pages
;
728 edac_mc_dump_csrow(csrow
);
729 for (j
= 0; j
< csrow
->nr_channels
; j
++)
730 if (csrow
->channels
[j
]->dimm
->nr_pages
)
731 edac_mc_dump_channel(csrow
->channels
[j
]);
733 for (i
= 0; i
< mci
->tot_dimms
; i
++)
734 if (mci
->dimms
[i
]->nr_pages
)
735 edac_mc_dump_dimm(mci
->dimms
[i
], i
);
738 mutex_lock(&mem_ctls_mutex
);
740 if (edac_mc_owner
&& edac_mc_owner
!= mci
->mod_name
) {
745 if (add_mc_to_global_list(mci
))
748 /* set load time so that error rate can be tracked */
749 mci
->start_time
= jiffies
;
751 mci
->bus
= edac_get_sysfs_subsys();
753 if (edac_create_sysfs_mci_device(mci
, groups
)) {
754 edac_mc_printk(mci
, KERN_WARNING
,
755 "failed to create sysfs device\n");
759 if (mci
->edac_check
) {
760 mci
->op_state
= OP_RUNNING_POLL
;
762 INIT_DELAYED_WORK(&mci
->work
, edac_mc_workq_function
);
763 edac_queue_work(&mci
->work
, msecs_to_jiffies(edac_mc_get_poll_msec()));
766 mci
->op_state
= OP_RUNNING_INTERRUPT
;
769 /* Report action taken */
770 edac_mc_printk(mci
, KERN_INFO
,
771 "Giving out device to module %s controller %s: DEV %s (%s)\n",
772 mci
->mod_name
, mci
->ctl_name
, mci
->dev_name
,
773 edac_op_state_to_string(mci
->op_state
));
775 edac_mc_owner
= mci
->mod_name
;
777 mutex_unlock(&mem_ctls_mutex
);
781 del_mc_from_global_list(mci
);
784 mutex_unlock(&mem_ctls_mutex
);
787 EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups
);
789 struct mem_ctl_info
*edac_mc_del_mc(struct device
*dev
)
791 struct mem_ctl_info
*mci
;
795 mutex_lock(&mem_ctls_mutex
);
797 /* find the requested mci struct in the global list */
798 mci
= __find_mci_by_dev(dev
);
800 mutex_unlock(&mem_ctls_mutex
);
804 /* mark MCI offline: */
805 mci
->op_state
= OP_OFFLINE
;
807 if (del_mc_from_global_list(mci
))
808 edac_mc_owner
= NULL
;
810 mutex_unlock(&mem_ctls_mutex
);
813 edac_stop_work(&mci
->work
);
815 /* remove from sysfs */
816 edac_remove_sysfs_mci_device(mci
);
818 edac_printk(KERN_INFO
, EDAC_MC
,
819 "Removed device %d for %s %s: DEV %s\n", mci
->mc_idx
,
820 mci
->mod_name
, mci
->ctl_name
, edac_dev_name(mci
));
824 EXPORT_SYMBOL_GPL(edac_mc_del_mc
);
826 static void edac_mc_scrub_block(unsigned long page
, unsigned long offset
,
831 unsigned long flags
= 0;
835 /* ECC error page was not in our memory. Ignore it. */
836 if (!pfn_valid(page
))
839 /* Find the actual page structure then map it and fix */
840 pg
= pfn_to_page(page
);
843 local_irq_save(flags
);
845 virt_addr
= kmap_atomic(pg
);
847 /* Perform architecture specific atomic scrub operation */
848 edac_atomic_scrub(virt_addr
+ offset
, size
);
850 /* Unmap and complete */
851 kunmap_atomic(virt_addr
);
854 local_irq_restore(flags
);
857 /* FIXME - should return -1 */
858 int edac_mc_find_csrow_by_page(struct mem_ctl_info
*mci
, unsigned long page
)
860 struct csrow_info
**csrows
= mci
->csrows
;
863 edac_dbg(1, "MC%d: 0x%lx\n", mci
->mc_idx
, page
);
866 for (i
= 0; i
< mci
->nr_csrows
; i
++) {
867 struct csrow_info
*csrow
= csrows
[i
];
869 for (j
= 0; j
< csrow
->nr_channels
; j
++) {
870 struct dimm_info
*dimm
= csrow
->channels
[j
]->dimm
;
876 edac_dbg(3, "MC%d: first(0x%lx) page(0x%lx) last(0x%lx) mask(0x%lx)\n",
878 csrow
->first_page
, page
, csrow
->last_page
,
881 if ((page
>= csrow
->first_page
) &&
882 (page
<= csrow
->last_page
) &&
883 ((page
& csrow
->page_mask
) ==
884 (csrow
->first_page
& csrow
->page_mask
))) {
891 edac_mc_printk(mci
, KERN_ERR
,
892 "could not look up page error address %lx\n",
893 (unsigned long)page
);
897 EXPORT_SYMBOL_GPL(edac_mc_find_csrow_by_page
);
899 const char *edac_layer_name
[] = {
900 [EDAC_MC_LAYER_BRANCH
] = "branch",
901 [EDAC_MC_LAYER_CHANNEL
] = "channel",
902 [EDAC_MC_LAYER_SLOT
] = "slot",
903 [EDAC_MC_LAYER_CHIP_SELECT
] = "csrow",
904 [EDAC_MC_LAYER_ALL_MEM
] = "memory",
906 EXPORT_SYMBOL_GPL(edac_layer_name
);
908 static void edac_inc_ce_error(struct mem_ctl_info
*mci
,
909 bool enable_per_layer_report
,
910 const int pos
[EDAC_MAX_LAYERS
],
917 if (!enable_per_layer_report
) {
918 mci
->ce_noinfo_count
+= count
;
922 for (i
= 0; i
< mci
->n_layers
; i
++) {
926 mci
->ce_per_layer
[i
][index
] += count
;
928 if (i
< mci
->n_layers
- 1)
929 index
*= mci
->layers
[i
+ 1].size
;
933 static void edac_inc_ue_error(struct mem_ctl_info
*mci
,
934 bool enable_per_layer_report
,
935 const int pos
[EDAC_MAX_LAYERS
],
942 if (!enable_per_layer_report
) {
943 mci
->ue_noinfo_count
+= count
;
947 for (i
= 0; i
< mci
->n_layers
; i
++) {
951 mci
->ue_per_layer
[i
][index
] += count
;
953 if (i
< mci
->n_layers
- 1)
954 index
*= mci
->layers
[i
+ 1].size
;
958 static void edac_ce_error(struct mem_ctl_info
*mci
,
959 const u16 error_count
,
960 const int pos
[EDAC_MAX_LAYERS
],
962 const char *location
,
965 const char *other_detail
,
966 const bool enable_per_layer_report
,
967 const unsigned long page_frame_number
,
968 const unsigned long offset_in_page
,
971 unsigned long remapped_page
;
977 if (edac_mc_get_log_ce()) {
978 if (other_detail
&& *other_detail
)
979 edac_mc_printk(mci
, KERN_WARNING
,
980 "%d CE %s%son %s (%s %s - %s)\n",
981 error_count
, msg
, msg_aux
, label
,
982 location
, detail
, other_detail
);
984 edac_mc_printk(mci
, KERN_WARNING
,
985 "%d CE %s%son %s (%s %s)\n",
986 error_count
, msg
, msg_aux
, label
,
989 edac_inc_ce_error(mci
, enable_per_layer_report
, pos
, error_count
);
991 if (mci
->scrub_mode
== SCRUB_SW_SRC
) {
993 * Some memory controllers (called MCs below) can remap
994 * memory so that it is still available at a different
995 * address when PCI devices map into memory.
996 * MC's that can't do this, lose the memory where PCI
997 * devices are mapped. This mapping is MC-dependent
998 * and so we call back into the MC driver for it to
999 * map the MC page to a physical (CPU) page which can
1000 * then be mapped to a virtual page - which can then
1003 remapped_page
= mci
->ctl_page_to_phys
?
1004 mci
->ctl_page_to_phys(mci
, page_frame_number
) :
1007 edac_mc_scrub_block(remapped_page
,
1008 offset_in_page
, grain
);
1012 static void edac_ue_error(struct mem_ctl_info
*mci
,
1013 const u16 error_count
,
1014 const int pos
[EDAC_MAX_LAYERS
],
1016 const char *location
,
1019 const char *other_detail
,
1020 const bool enable_per_layer_report
)
1027 if (edac_mc_get_log_ue()) {
1028 if (other_detail
&& *other_detail
)
1029 edac_mc_printk(mci
, KERN_WARNING
,
1030 "%d UE %s%son %s (%s %s - %s)\n",
1031 error_count
, msg
, msg_aux
, label
,
1032 location
, detail
, other_detail
);
1034 edac_mc_printk(mci
, KERN_WARNING
,
1035 "%d UE %s%son %s (%s %s)\n",
1036 error_count
, msg
, msg_aux
, label
,
1040 if (edac_mc_get_panic_on_ue()) {
1041 if (other_detail
&& *other_detail
)
1042 panic("UE %s%son %s (%s%s - %s)\n",
1043 msg
, msg_aux
, label
, location
, detail
, other_detail
);
1045 panic("UE %s%son %s (%s%s)\n",
1046 msg
, msg_aux
, label
, location
, detail
);
1049 edac_inc_ue_error(mci
, enable_per_layer_report
, pos
, error_count
);
1052 void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type
,
1053 struct mem_ctl_info
*mci
,
1054 struct edac_raw_error_desc
*e
)
1057 int pos
[EDAC_MAX_LAYERS
] = { e
->top_layer
, e
->mid_layer
, e
->low_layer
};
1059 /* Memory type dependent details about the error */
1060 if (type
== HW_EVENT_ERR_CORRECTED
) {
1061 snprintf(detail
, sizeof(detail
),
1062 "page:0x%lx offset:0x%lx grain:%ld syndrome:0x%lx",
1063 e
->page_frame_number
, e
->offset_in_page
,
1064 e
->grain
, e
->syndrome
);
1065 edac_ce_error(mci
, e
->error_count
, pos
, e
->msg
, e
->location
, e
->label
,
1066 detail
, e
->other_detail
, e
->enable_per_layer_report
,
1067 e
->page_frame_number
, e
->offset_in_page
, e
->grain
);
1069 snprintf(detail
, sizeof(detail
),
1070 "page:0x%lx offset:0x%lx grain:%ld",
1071 e
->page_frame_number
, e
->offset_in_page
, e
->grain
);
1073 edac_ue_error(mci
, e
->error_count
, pos
, e
->msg
, e
->location
, e
->label
,
1074 detail
, e
->other_detail
, e
->enable_per_layer_report
);
1079 EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error
);
1081 void edac_mc_handle_error(const enum hw_event_mc_err_type type
,
1082 struct mem_ctl_info
*mci
,
1083 const u16 error_count
,
1084 const unsigned long page_frame_number
,
1085 const unsigned long offset_in_page
,
1086 const unsigned long syndrome
,
1087 const int top_layer
,
1088 const int mid_layer
,
1089 const int low_layer
,
1091 const char *other_detail
)
1094 int row
= -1, chan
= -1;
1095 int pos
[EDAC_MAX_LAYERS
] = { top_layer
, mid_layer
, low_layer
};
1096 int i
, n_labels
= 0;
1098 struct edac_raw_error_desc
*e
= &mci
->error_desc
;
1100 edac_dbg(3, "MC%d\n", mci
->mc_idx
);
1102 /* Fills the error report buffer */
1103 memset(e
, 0, sizeof (*e
));
1104 e
->error_count
= error_count
;
1105 e
->top_layer
= top_layer
;
1106 e
->mid_layer
= mid_layer
;
1107 e
->low_layer
= low_layer
;
1108 e
->page_frame_number
= page_frame_number
;
1109 e
->offset_in_page
= offset_in_page
;
1110 e
->syndrome
= syndrome
;
1112 e
->other_detail
= other_detail
;
1115 * Check if the event report is consistent and if the memory
1116 * location is known. If it is known, enable_per_layer_report will be
1117 * true, the DIMM(s) label info will be filled and the per-layer
1118 * error counters will be incremented.
1120 for (i
= 0; i
< mci
->n_layers
; i
++) {
1121 if (pos
[i
] >= (int)mci
->layers
[i
].size
) {
1123 edac_mc_printk(mci
, KERN_ERR
,
1124 "INTERNAL ERROR: %s value is out of range (%d >= %d)\n",
1125 edac_layer_name
[mci
->layers
[i
].type
],
1126 pos
[i
], mci
->layers
[i
].size
);
1128 * Instead of just returning it, let's use what's
1129 * known about the error. The increment routines and
1130 * the DIMM filter logic will do the right thing by
1131 * pointing the likely damaged DIMMs.
1136 e
->enable_per_layer_report
= true;
1140 * Get the dimm label/grain that applies to the match criteria.
1141 * As the error algorithm may not be able to point to just one memory
1142 * stick, the logic here will get all possible labels that could
1143 * pottentially be affected by the error.
1144 * On FB-DIMM memory controllers, for uncorrected errors, it is common
1145 * to have only the MC channel and the MC dimm (also called "branch")
1146 * but the channel is not known, as the memory is arranged in pairs,
1147 * where each memory belongs to a separate channel within the same
1153 for (i
= 0; i
< mci
->tot_dimms
; i
++) {
1154 struct dimm_info
*dimm
= mci
->dimms
[i
];
1156 if (top_layer
>= 0 && top_layer
!= dimm
->location
[0])
1158 if (mid_layer
>= 0 && mid_layer
!= dimm
->location
[1])
1160 if (low_layer
>= 0 && low_layer
!= dimm
->location
[2])
1163 /* get the max grain, over the error match range */
1164 if (dimm
->grain
> e
->grain
)
1165 e
->grain
= dimm
->grain
;
1168 * If the error is memory-controller wide, there's no need to
1169 * seek for the affected DIMMs because the whole
1170 * channel/memory controller/... may be affected.
1171 * Also, don't show errors for empty DIMM slots.
1173 if (e
->enable_per_layer_report
&& dimm
->nr_pages
) {
1174 if (n_labels
>= EDAC_MAX_LABELS
) {
1175 e
->enable_per_layer_report
= false;
1179 if (p
!= e
->label
) {
1180 strcpy(p
, OTHER_LABEL
);
1181 p
+= strlen(OTHER_LABEL
);
1183 strcpy(p
, dimm
->label
);
1188 * get csrow/channel of the DIMM, in order to allow
1189 * incrementing the compat API counters
1191 edac_dbg(4, "%s csrows map: (%d,%d)\n",
1192 mci
->csbased
? "rank" : "dimm",
1193 dimm
->csrow
, dimm
->cschannel
);
1196 else if (row
>= 0 && row
!= dimm
->csrow
)
1200 chan
= dimm
->cschannel
;
1201 else if (chan
>= 0 && chan
!= dimm
->cschannel
)
1206 if (!e
->enable_per_layer_report
) {
1207 strcpy(e
->label
, "any memory");
1209 edac_dbg(4, "csrow/channel to increment: (%d,%d)\n", row
, chan
);
1211 strcpy(e
->label
, "unknown memory");
1212 if (type
== HW_EVENT_ERR_CORRECTED
) {
1214 mci
->csrows
[row
]->ce_count
+= error_count
;
1216 mci
->csrows
[row
]->channels
[chan
]->ce_count
+= error_count
;
1220 mci
->csrows
[row
]->ue_count
+= error_count
;
1223 /* Fill the RAM location data */
1226 for (i
= 0; i
< mci
->n_layers
; i
++) {
1230 p
+= sprintf(p
, "%s:%d ",
1231 edac_layer_name
[mci
->layers
[i
].type
],
1234 if (p
> e
->location
)
1237 /* Sanity-check driver-supplied grain value. */
1238 if (WARN_ON_ONCE(!e
->grain
))
1241 grain_bits
= fls_long(e
->grain
- 1);
1243 /* Report the error via the trace interface */
1244 if (IS_ENABLED(CONFIG_RAS
))
1245 trace_mc_event(type
, e
->msg
, e
->label
, e
->error_count
,
1246 mci
->mc_idx
, e
->top_layer
, e
->mid_layer
,
1248 (e
->page_frame_number
<< PAGE_SHIFT
) | e
->offset_in_page
,
1249 grain_bits
, e
->syndrome
, e
->other_detail
);
1251 edac_raw_mc_handle_error(type
, mci
, e
);
1253 EXPORT_SYMBOL_GPL(edac_mc_handle_error
);