[mirror_ubuntu-bionic-kernel.git] / drivers / ras / cec.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/kernel.h>
#include <linux/workqueue.h>

#include <asm/mce.h>

#include "debugfs.h"

/*
 * RAS Correctable Errors Collector
 *
 * This is a simple gadget which collects correctable errors and counts their
 * occurrence per physical page address.
 *
 * We've opted for possibly the simplest data structure to collect those - an
 * array of the size of a memory page. It stores 512 u64's with the following
 * structure:
 *
 * [63 ... PFN ... 12 | 11 ... generation ... 10 | 9 ... count ... 0]
 *
 * The generation in the two highest order bits is two bits which are set to 11b
 * on every insertion. During the course of each entry's existence, the
 * generation field gets decremented during spring cleaning to 10b, then 01b and
 * then 00b.
 *
 * This way we're employing the natural numeric ordering to make sure that newly
 * inserted/touched elements have higher 12-bit counts (which we've manufactured)
 * and thus iterating over the array initially won't kick out those elements
 * which were inserted last.
 *
 * Spring cleaning is what we do when we reach a certain number CLEAN_ELEMS of
 * elements entered into the array, during which, we're decaying all elements.
 * If, after decay, an element gets inserted again, its generation is set to 11b
 * to make sure it has higher numerical count than other, older elements and
 * thus emulate an an LRU-like behavior when deleting elements to free up space
 * in the page.
 *
 * When an element reaches it's max count of count_threshold, we try to poison
 * it by assuming that errors triggered count_threshold times in a single page
 * are excessive and that page shouldn't be used anymore. count_threshold is
 * initialized to COUNT_MASK which is the maximum.
 *
 * That error event entry causes cec_add_elem() to return !0 value and thus
 * signal to its callers to log the error.
 *
 * To the question why we've chosen a page and moving elements around with
 * memmove(), it is because it is a very simple structure to handle and max data
 * movement is 4K which on highly optimized modern CPUs is almost unnoticeable.
 * We wanted to avoid the pointer traversal of more complex structures like a
 * linked list or some sort of a balancing search tree.
 *
 * Deleting an element takes O(n) but since it is only a single page, it should
 * be fast enough and it shouldn't happen all too often depending on error
 * patterns.
 */

#undef pr_fmt
#define pr_fmt(fmt) "RAS: " fmt

/*
 * We use DECAY_BITS bits of PAGE_SHIFT bits for counting decay, i.e., how long
 * elements have stayed in the array without having been accessed again.
 */
#define DECAY_BITS		2
#define DECAY_MASK		((1ULL << DECAY_BITS) - 1)
#define MAX_ELEMS		(PAGE_SIZE / sizeof(u64))

/*
 * Threshold amount of inserted elements after which we start spring
 * cleaning.
 */
#define CLEAN_ELEMS		(MAX_ELEMS >> DECAY_BITS)

/* Bits which count the number of errors happened in this 4K page. */
#define COUNT_BITS		(PAGE_SHIFT - DECAY_BITS)
#define COUNT_MASK		((1ULL << COUNT_BITS) - 1)
#define FULL_COUNT_MASK		(PAGE_SIZE - 1)

/*
 * u64: [ 63 ... 12 | DECAY_BITS | COUNT_BITS ]
 */

#define PFN(e)			((e) >> PAGE_SHIFT)
#define DECAY(e)		(((e) >> COUNT_BITS) & DECAY_MASK)
#define COUNT(e)		((unsigned int)(e) & COUNT_MASK)
#define FULL_COUNT(e)		((e) & (PAGE_SIZE - 1))

static struct ce_array {
	u64 *array;			/* container page */
	unsigned int n;			/* number of elements in the array */

	unsigned int decay_count;	/*
					 * number of element insertions/increments
					 * since the last spring cleaning.
					 */

	u64 pfns_poisoned;		/*
					 * number of PFNs which got poisoned.
					 */

	u64 ces_entered;		/*
					 * The number of correctable errors
					 * entered into the collector.
					 */

	u64 decays_done;		/*
					 * Times we did spring cleaning.
					 */

	union {
		struct {
			__u32	disabled : 1,	/* cmdline disabled */
			__resv   : 31;
		};
		__u32 flags;
	};
} ce_arr;

static DEFINE_MUTEX(ce_mutex);
static u64 dfs_pfn;

/* Amount of errors after which we offline */
static unsigned int count_threshold = COUNT_MASK;

/* Each element "decays" each decay_interval which is 24hrs by default. */
#define CEC_DECAY_DEFAULT_INTERVAL	24 * 60 * 60	/* 24 hrs */
#define CEC_DECAY_MIN_INTERVAL		 1 * 60 * 60	/* 1h */
#define CEC_DECAY_MAX_INTERVAL	   30 *	24 * 60 * 60	/* one month */
static struct delayed_work cec_work;
static u64 decay_interval = CEC_DECAY_DEFAULT_INTERVAL;

/*
 * Decrement decay value. We're using DECAY_BITS bits to denote decay of an
 * element in the array. On insertion and any access, it gets reset to max.
 */
static void do_spring_cleaning(struct ce_array *ca)
{
	int i;

	for (i = 0; i < ca->n; i++) {
		u8 decay = DECAY(ca->array[i]);

		if (!decay)
			continue;

		decay--;

		ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);
		ca->array[i] |= (decay << COUNT_BITS);
	}
	ca->decay_count = 0;
	ca->decays_done++;
}

/*
 * @interval in seconds
 */
static void cec_mod_work(unsigned long interval)
{
	unsigned long iv;

	iv = interval * HZ;
	mod_delayed_work(system_wq, &cec_work, round_jiffies(iv));
}

static void cec_work_fn(struct work_struct *work)
{
	mutex_lock(&ce_mutex);
	do_spring_cleaning(&ce_arr);
	mutex_unlock(&ce_mutex);

	cec_mod_work(decay_interval);
}

/*
 * @to: index of the smallest element which is >= then @pfn.
 *
 * Return the index of the pfn if found, otherwise negative value.
 */
static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
{
	int min = 0, max = ca->n - 1;
	u64 this_pfn;

	while (min <= max) {
		int i = (min + max) >> 1;

		this_pfn = PFN(ca->array[i]);

		if (this_pfn < pfn)
			min = i + 1;
		else if (this_pfn > pfn)
			max = i - 1;
		else if (this_pfn == pfn) {
			if (to)
				*to = i;

			return i;
		}
	}

	/*
	 * When the loop terminates without finding @pfn, min has the index of
	 * the element slot where the new @pfn should be inserted. The loop
	 * terminates when min > max, which means the min index points to the
	 * bigger element while the max index to the smaller element, in-between
	 * which the new @pfn belongs to.
	 *
	 * For more details, see exercise 1, Section 6.2.1 in TAOCP, vol. 3.
	 */
	if (to)
		*to = min;

	return -ENOKEY;
}

static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
{
	WARN_ON(!to);

	if (!ca->n) {
		*to = 0;
		return -ENOKEY;
	}
	return __find_elem(ca, pfn, to);
}

static void del_elem(struct ce_array *ca, int idx)
{
	/* Save us a function call when deleting the last element. */
	if (ca->n - (idx + 1))
		memmove((void *)&ca->array[idx],
			(void *)&ca->array[idx + 1],
			(ca->n - (idx + 1)) * sizeof(u64));

	ca->n--;
}

static u64 del_lru_elem_unlocked(struct ce_array *ca)
{
	unsigned int min = FULL_COUNT_MASK;
	int i, min_idx = 0;

	for (i = 0; i < ca->n; i++) {
		unsigned int this = FULL_COUNT(ca->array[i]);

		if (min > this) {
			min = this;
			min_idx = i;
		}
	}

	del_elem(ca, min_idx);

	return PFN(ca->array[min_idx]);
}

/*
 * We return the 0th pfn in the error case under the assumption that it cannot
 * be poisoned and excessive CEs in there are a serious deal anyway.
 */
static u64 __maybe_unused del_lru_elem(void)
{
	struct ce_array *ca = &ce_arr;
	u64 pfn;

	if (!ca->n)
		return 0;

	mutex_lock(&ce_mutex);
	pfn = del_lru_elem_unlocked(ca);
	mutex_unlock(&ce_mutex);

	return pfn;
}


int cec_add_elem(u64 pfn)
{
	struct ce_array *ca = &ce_arr;
	unsigned int to;
	int count, ret = 0;

	/*
	 * We can be called very early on the identify_cpu() path where we are
	 * not initialized yet. We ignore the error for simplicity.
	 */
	if (!ce_arr.array || ce_arr.disabled)
		return -ENODEV;

	ca->ces_entered++;

	mutex_lock(&ce_mutex);

	if (ca->n == MAX_ELEMS)
		WARN_ON(!del_lru_elem_unlocked(ca));

	ret = find_elem(ca, pfn, &to);
	if (ret < 0) {
		/*
		 * Shift range [to-end] to make room for one more element.
		 */
		memmove((void *)&ca->array[to + 1],
			(void *)&ca->array[to],
			(ca->n - to) * sizeof(u64));

		ca->array[to] = (pfn << PAGE_SHIFT) |
				(DECAY_MASK << COUNT_BITS) | 1;

		ca->n++;

		ret = 0;

		goto decay;
	}

	count = COUNT(ca->array[to]);

	if (count < count_threshold) {
		ca->array[to] |= (DECAY_MASK << COUNT_BITS);
		ca->array[to]++;

		ret = 0;
	} else {
		u64 pfn = ca->array[to] >> PAGE_SHIFT;

		if (!pfn_valid(pfn)) {
			pr_warn("CEC: Invalid pfn: 0x%llx\n", pfn);
		} else {
			/* We have reached max count for this page, soft-offline it. */
			pr_err("Soft-offlining pfn: 0x%llx\n", pfn);
			memory_failure_queue(pfn, 0, MF_SOFT_OFFLINE);
			ca->pfns_poisoned++;
		}

		del_elem(ca, to);

		/*
		 * Return a >0 value to denote that we've reached the offlining
		 * threshold.
		 */
		ret = 1;

		goto unlock;
	}

decay:
	ca->decay_count++;

	if (ca->decay_count >= CLEAN_ELEMS)
		do_spring_cleaning(ca);

unlock:
	mutex_unlock(&ce_mutex);

	return ret;
}

static int u64_get(void *data, u64 *val)
{
	*val = *(u64 *)data;

	return 0;
}

static int pfn_set(void *data, u64 val)
{
	*(u64 *)data = val;

	cec_add_elem(val);

	return 0;
}

DEFINE_DEBUGFS_ATTRIBUTE(pfn_ops, u64_get, pfn_set, "0x%llx\n");

static int decay_interval_set(void *data, u64 val)
{
	*(u64 *)data = val;

	if (val < CEC_DECAY_MIN_INTERVAL)
		return -EINVAL;

	if (val > CEC_DECAY_MAX_INTERVAL)
		return -EINVAL;

	decay_interval = val;

	cec_mod_work(decay_interval);
	return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(decay_interval_ops, u64_get, decay_interval_set, "%lld\n");

static int count_threshold_set(void *data, u64 val)
{
	*(u64 *)data = val;

	if (val > COUNT_MASK)
		val = COUNT_MASK;

	count_threshold = val;

	return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(count_threshold_ops, u64_get, count_threshold_set, "%lld\n");

static int array_dump(struct seq_file *m, void *v)
{
	struct ce_array *ca = &ce_arr;
	u64 prev = 0;
	int i;

	mutex_lock(&ce_mutex);

	seq_printf(m, "{ n: %d\n", ca->n);
	for (i = 0; i < ca->n; i++) {
		u64 this = PFN(ca->array[i]);

		seq_printf(m, " %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));

		WARN_ON(prev > this);

		prev = this;
	}

	seq_printf(m, "}\n");

	seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n",
		   ca->ces_entered, ca->pfns_poisoned);

	seq_printf(m, "Flags: 0x%x\n", ca->flags);

	seq_printf(m, "Decay interval: %lld seconds\n", decay_interval);
	seq_printf(m, "Decays: %lld\n", ca->decays_done);

	seq_printf(m, "Action threshold: %d\n", count_threshold);

	mutex_unlock(&ce_mutex);

	return 0;
}

static int array_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, array_dump, NULL);
}

static const struct file_operations array_ops = {
	.owner	 = THIS_MODULE,
	.open	 = array_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
	.release = single_release,
};

static int __init create_debugfs_nodes(void)
{
	struct dentry *d, *pfn, *decay, *count, *array;

	d = debugfs_create_dir("cec", ras_debugfs_dir);
	if (!d) {
		pr_warn("Error creating cec debugfs node!\n");
		return -1;
	}

	pfn = debugfs_create_file("pfn", S_IRUSR | S_IWUSR, d, &dfs_pfn, &pfn_ops);
	if (!pfn) {
		pr_warn("Error creating pfn debugfs node!\n");
		goto err;
	}

	array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_ops);
	if (!array) {
		pr_warn("Error creating array debugfs node!\n");
		goto err;
	}

	decay = debugfs_create_file("decay_interval", S_IRUSR | S_IWUSR, d,
				    &decay_interval, &decay_interval_ops);
	if (!decay) {
		pr_warn("Error creating decay_interval debugfs node!\n");
		goto err;
	}

	count = debugfs_create_file("count_threshold", S_IRUSR | S_IWUSR, d,
				    &count_threshold, &count_threshold_ops);
	if (!count) {
		pr_warn("Error creating count_threshold debugfs node!\n");
		goto err;
	}


	return 0;

err:
	debugfs_remove_recursive(d);

	return 1;
}

void __init cec_init(void)
{
	if (ce_arr.disabled)
		return;

	ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL);
	if (!ce_arr.array) {
		pr_err("Error allocating CE array page!\n");
		return;
	}

	if (create_debugfs_nodes())
		return;

	INIT_DELAYED_WORK(&cec_work, cec_work_fn);
	schedule_delayed_work(&cec_work, CEC_DECAY_DEFAULT_INTERVAL);

	pr_info("Correctable Errors collector initialized.\n");
}

int __init parse_cec_param(char *str)
{
	if (!str)
		return 0;

	if (*str == '=')
		str++;

	if (!strcmp(str, "cec_disable"))
		ce_arr.disabled = 1;
	else
		return 0;

	return 1;
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
011d8261 BP	2	#include <linux/mm.h>
	3	#include <linux/gfp.h>
	4	#include <linux/kernel.h>
2328874d	5	#include <linux/workqueue.h>
011d8261 BP	6
	7	#include <asm/mce.h>
	8
	9	#include "debugfs.h"
	10
	11	/*
	12	* RAS Correctable Errors Collector
	13	*
	14	* This is a simple gadget which collects correctable errors and counts their
	15	* occurrence per physical page address.
	16	*
	17	* We've opted for possibly the simplest data structure to collect those - an
	18	* array of the size of a memory page. It stores 512 u64's with the following
	19	* structure:
	20	*
	21	* [63 ... PFN ... 12 \| 11 ... generation ... 10 \| 9 ... count ... 0]
	22	*
	23	* The generation in the two highest order bits is two bits which are set to 11b
	24	* on every insertion. During the course of each entry's existence, the
	25	* generation field gets decremented during spring cleaning to 10b, then 01b and
	26	* then 00b.
	27	*
	28	* This way we're employing the natural numeric ordering to make sure that newly
	29	* inserted/touched elements have higher 12-bit counts (which we've manufactured)
	30	* and thus iterating over the array initially won't kick out those elements
	31	* which were inserted last.
	32	*
	33	* Spring cleaning is what we do when we reach a certain number CLEAN_ELEMS of
	34	* elements entered into the array, during which, we're decaying all elements.
	35	* If, after decay, an element gets inserted again, its generation is set to 11b
	36	* to make sure it has higher numerical count than other, older elements and
	37	* thus emulate an an LRU-like behavior when deleting elements to free up space
	38	* in the page.
	39	*
	40	* When an element reaches it's max count of count_threshold, we try to poison
	41	* it by assuming that errors triggered count_threshold times in a single page
	42	* are excessive and that page shouldn't be used anymore. count_threshold is
	43	* initialized to COUNT_MASK which is the maximum.
	44	*
	45	* That error event entry causes cec_add_elem() to return !0 value and thus
	46	* signal to its callers to log the error.
	47	*
	48	* To the question why we've chosen a page and moving elements around with
	49	* memmove(), it is because it is a very simple structure to handle and max data
	50	* movement is 4K which on highly optimized modern CPUs is almost unnoticeable.
	51	* We wanted to avoid the pointer traversal of more complex structures like a
	52	* linked list or some sort of a balancing search tree.
	53	*
	54	* Deleting an element takes O(n) but since it is only a single page, it should
	55	* be fast enough and it shouldn't happen all too often depending on error
	56	* patterns.
	57	*/
	58
	59	#undef pr_fmt
	60	#define pr_fmt(fmt) "RAS: " fmt
	61
	62	/*
	63	* We use DECAY_BITS bits of PAGE_SHIFT bits for counting decay, i.e., how long
	64	* elements have stayed in the array without having been accessed again.
	65	*/
	66	#define DECAY_BITS 2
	67	#define DECAY_MASK ((1ULL << DECAY_BITS) - 1)
	68	#define MAX_ELEMS (PAGE_SIZE / sizeof(u64))
	69
70	/*
71	* Threshold amount of inserted elements after which we start spring
72	* cleaning.
73	*/
74	#define CLEAN_ELEMS (MAX_ELEMS >> DECAY_BITS)
75
76	/* Bits which count the number of errors happened in this 4K page. */
77	#define COUNT_BITS (PAGE_SHIFT - DECAY_BITS)
78	#define COUNT_MASK ((1ULL << COUNT_BITS) - 1)
79	#define FULL_COUNT_MASK (PAGE_SIZE - 1)
80
81	/*
82	* u64: [ 63 ... 12 \| DECAY_BITS \| COUNT_BITS ]
83	*/
84
85	#define PFN(e) ((e) >> PAGE_SHIFT)
86	#define DECAY(e) (((e) >> COUNT_BITS) & DECAY_MASK)
87	#define COUNT(e) ((unsigned int)(e) & COUNT_MASK)
88	#define FULL_COUNT(e) ((e) & (PAGE_SIZE - 1))
89
90	static struct ce_array {
91	u64 array; / container page */
92	unsigned int n; /* number of elements in the array */
93
94	unsigned int decay_count; /*
95	* number of element insertions/increments
96	* since the last spring cleaning.
97	*/
98
99	u64 pfns_poisoned; /*
100	* number of PFNs which got poisoned.
101	*/
102
103	u64 ces_entered; /*
104	* The number of correctable errors
105	* entered into the collector.
106	*/
107
108	u64 decays_done; /*
109	* Times we did spring cleaning.
110	*/
111
112	union {
113	struct {
114	__u32 disabled : 1, /* cmdline disabled */
115	__resv : 31;
116	};
117	__u32 flags;
118	};
119	} ce_arr;
120
121	static DEFINE_MUTEX(ce_mutex);
122	static u64 dfs_pfn;
123
124	/* Amount of errors after which we offline */
125	static unsigned int count_threshold = COUNT_MASK;
126
2328874d CW	127	/* Each element "decays" each decay_interval which is 24hrs by default. */
	128	#define CEC_DECAY_DEFAULT_INTERVAL 24 * 60 * 60 /* 24 hrs */
	129	#define CEC_DECAY_MIN_INTERVAL 1 * 60 * 60 /* 1h */
	130	#define CEC_DECAY_MAX_INTERVAL 30 * 24 * 60 * 60 /* one month */
	131	static struct delayed_work cec_work;
	132	static u64 decay_interval = CEC_DECAY_DEFAULT_INTERVAL;
011d8261 BP	133
	134	/*
	135	* Decrement decay value. We're using DECAY_BITS bits to denote decay of an
	136	* element in the array. On insertion and any access, it gets reset to max.
	137	*/
	138	static void do_spring_cleaning(struct ce_array *ca)
	139	{
	140	int i;
	141
	142	for (i = 0; i < ca->n; i++) {
	143	u8 decay = DECAY(ca->array[i]);
	144
	145	if (!decay)
	146	continue;
	147
	148	decay--;
	149
	150	ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);
	151	ca->array[i] \|= (decay << COUNT_BITS);
	152	}
	153	ca->decay_count = 0;
	154	ca->decays_done++;
	155	}
	156
	157	/*
	158	* @interval in seconds
	159	*/
2328874d	160	static void cec_mod_work(unsigned long interval)
011d8261 BP	161	{
	162	unsigned long iv;
	163
2328874d CW	164	iv = interval * HZ;
2328874d CW	165	mod_delayed_work(system_wq, &cec_work, round_jiffies(iv));
011d8261 BP	166	}
011d8261 BP	167
2328874d	168	static void cec_work_fn(struct work_struct *work)
011d8261	169	{
2328874d	170	mutex_lock(&ce_mutex);
254db5bd	171	do_spring_cleaning(&ce_arr);
2328874d	172	mutex_unlock(&ce_mutex);
011d8261	173
2328874d	174	cec_mod_work(decay_interval);
011d8261 BP	175	}
	176
	177	/*
	178	* @to: index of the smallest element which is >= then @pfn.
	179	*
	180	* Return the index of the pfn if found, otherwise negative value.
	181	*/
	182	static int __find_elem(struct ce_array ca, u64 pfn, unsigned int to)
	183	{
61d9d59b	184	int min = 0, max = ca->n - 1;
011d8261	185	u64 this_pfn;
011d8261	186
61d9d59b BP	187	while (min <= max) {
61d9d59b BP	188	int i = (min + max) >> 1;
011d8261	189
61d9d59b	190	this_pfn = PFN(ca->array[i]);
011d8261 BP	191
011d8261 BP	192	if (this_pfn < pfn)
61d9d59b	193	min = i + 1;
011d8261	194	else if (this_pfn > pfn)
61d9d59b BP	195	max = i - 1;
	196	else if (this_pfn == pfn) {
	197	if (to)
	198	*to = i;
	199
	200	return i;
011d8261 BP	201	}
	202	}
	203
61d9d59b BP	204	/*
	205	* When the loop terminates without finding @pfn, min has the index of
	206	* the element slot where the new @pfn should be inserted. The loop
	207	* terminates when min > max, which means the min index points to the
	208	* bigger element while the max index to the smaller element, in-between
	209	* which the new @pfn belongs to.
	210	*
	211	* For more details, see exercise 1, Section 6.2.1 in TAOCP, vol. 3.
	212	*/
011d8261 BP	213	if (to)
	214	*to = min;
	215
011d8261 BP	216	return -ENOKEY;
	217	}
	218
	219	static int find_elem(struct ce_array ca, u64 pfn, unsigned int to)
	220	{
	221	WARN_ON(!to);
	222
	223	if (!ca->n) {
	224	*to = 0;
	225	return -ENOKEY;
	226	}
	227	return __find_elem(ca, pfn, to);
	228	}
	229
	230	static void del_elem(struct ce_array *ca, int idx)
	231	{
	232	/* Save us a function call when deleting the last element. */
	233	if (ca->n - (idx + 1))
	234	memmove((void *)&ca->array[idx],
	235	(void *)&ca->array[idx + 1],
	236	(ca->n - (idx + 1)) * sizeof(u64));
	237
	238	ca->n--;
	239	}
	240
	241	static u64 del_lru_elem_unlocked(struct ce_array *ca)
	242	{
	243	unsigned int min = FULL_COUNT_MASK;
	244	int i, min_idx = 0;
	245
	246	for (i = 0; i < ca->n; i++) {
	247	unsigned int this = FULL_COUNT(ca->array[i]);
	248
	249	if (min > this) {
	250	min = this;
	251	min_idx = i;
	252	}
	253	}
	254
	255	del_elem(ca, min_idx);
	256
	257	return PFN(ca->array[min_idx]);
	258	}
	259
	260	/*
	261	* We return the 0th pfn in the error case under the assumption that it cannot
	262	* be poisoned and excessive CEs in there are a serious deal anyway.
	263	*/
	264	static u64 __maybe_unused del_lru_elem(void)
	265	{
	266	struct ce_array *ca = &ce_arr;
	267	u64 pfn;
	268
	269	if (!ca->n)
	270	return 0;
	271
	272	mutex_lock(&ce_mutex);
	273	pfn = del_lru_elem_unlocked(ca);
	274	mutex_unlock(&ce_mutex);
	275
	276	return pfn;
	277	}
	278
	279
280	int cec_add_elem(u64 pfn)
281	{
282	struct ce_array *ca = &ce_arr;
283	unsigned int to;
284	int count, ret = 0;
285
286	/*
287	* We can be called very early on the identify_cpu() path where we are
288	* not initialized yet. We ignore the error for simplicity.
289	*/
290	if (!ce_arr.array \|\| ce_arr.disabled)
291	return -ENODEV;
292
293	ca->ces_entered++;
294
295	mutex_lock(&ce_mutex);
296
297	if (ca->n == MAX_ELEMS)
298	WARN_ON(!del_lru_elem_unlocked(ca));
299
300	ret = find_elem(ca, pfn, &to);
301	if (ret < 0) {
302	/*
303	* Shift range [to-end] to make room for one more element.
304	*/
305	memmove((void *)&ca->array[to + 1],
306	(void *)&ca->array[to],
307	(ca->n - to) * sizeof(u64));
308
309	ca->array[to] = (pfn << PAGE_SHIFT) \|
310	(DECAY_MASK << COUNT_BITS) \| 1;
311
312	ca->n++;
313
314	ret = 0;
315
316	goto decay;
317	}
318
319	count = COUNT(ca->array[to]);
320
321	if (count < count_threshold) {
322	ca->array[to] \|= (DECAY_MASK << COUNT_BITS);
323	ca->array[to]++;
324
325	ret = 0;
326	} else {
327	u64 pfn = ca->array[to] >> PAGE_SHIFT;
328
329	if (!pfn_valid(pfn)) {
330	pr_warn("CEC: Invalid pfn: 0x%llx\n", pfn);
331	} else {
332	/* We have reached max count for this page, soft-offline it. */
333	pr_err("Soft-offlining pfn: 0x%llx\n", pfn);
334	memory_failure_queue(pfn, 0, MF_SOFT_OFFLINE);
335	ca->pfns_poisoned++;
336	}
337
338	del_elem(ca, to);
339
340	/*
341	* Return a >0 value to denote that we've reached the offlining
342	* threshold.
343	*/
344	ret = 1;
345
346	goto unlock;
347	}
348
349	decay:
350	ca->decay_count++;
351
352	if (ca->decay_count >= CLEAN_ELEMS)
353	do_spring_cleaning(ca);
354
355	unlock:
356	mutex_unlock(&ce_mutex);
357
358	return ret;
359	}
360
361	static int u64_get(void data, u64 val)
362	{
363	val = (u64 *)data;
364
365	return 0;
366	}
367
368	static int pfn_set(void *data, u64 val)
369	{
370	(u64 )data = val;
371
ca5195c7 BP	372	cec_add_elem(val);
	373
	374	return 0;
011d8261 BP	375	}
	376
	377	DEFINE_DEBUGFS_ATTRIBUTE(pfn_ops, u64_get, pfn_set, "0x%llx\n");
	378
	379	static int decay_interval_set(void *data, u64 val)
	380	{
	381	(u64 )data = val;
	382
2328874d	383	if (val < CEC_DECAY_MIN_INTERVAL)
011d8261 BP	384	return -EINVAL;
011d8261 BP	385
2328874d	386	if (val > CEC_DECAY_MAX_INTERVAL)
011d8261 BP	387	return -EINVAL;
011d8261 BP	388
2328874d	389	decay_interval = val;
011d8261	390
2328874d	391	cec_mod_work(decay_interval);
011d8261 BP	392	return 0;
	393	}
	394	DEFINE_DEBUGFS_ATTRIBUTE(decay_interval_ops, u64_get, decay_interval_set, "%lld\n");
	395
	396	static int count_threshold_set(void *data, u64 val)
	397	{
	398	(u64 )data = val;
	399
	400	if (val > COUNT_MASK)
	401	val = COUNT_MASK;
	402
	403	count_threshold = val;
	404
	405	return 0;
	406	}
	407	DEFINE_DEBUGFS_ATTRIBUTE(count_threshold_ops, u64_get, count_threshold_set, "%lld\n");
	408
	409	static int array_dump(struct seq_file m, void v)
	410	{
	411	struct ce_array *ca = &ce_arr;
	412	u64 prev = 0;
	413	int i;
	414
	415	mutex_lock(&ce_mutex);
	416
	417	seq_printf(m, "{ n: %d\n", ca->n);
	418	for (i = 0; i < ca->n; i++) {
	419	u64 this = PFN(ca->array[i]);
	420
	421	seq_printf(m, " %03d: [%016llx\|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));
	422
	423	WARN_ON(prev > this);
	424
	425	prev = this;
	426	}
	427
	428	seq_printf(m, "}\n");
	429
	430	seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n",
	431	ca->ces_entered, ca->pfns_poisoned);
	432
	433	seq_printf(m, "Flags: 0x%x\n", ca->flags);
	434
2328874d	435	seq_printf(m, "Decay interval: %lld seconds\n", decay_interval);
011d8261 BP	436	seq_printf(m, "Decays: %lld\n", ca->decays_done);
	437
	438	seq_printf(m, "Action threshold: %d\n", count_threshold);
	439
	440	mutex_unlock(&ce_mutex);
	441
	442	return 0;
	443	}
	444
	445	static int array_open(struct inode inode, struct file filp)
	446	{
	447	return single_open(filp, array_dump, NULL);
	448	}
	449
	450	static const struct file_operations array_ops = {
	451	.owner = THIS_MODULE,
	452	.open = array_open,
	453	.read = seq_read,
	454	.llseek = seq_lseek,
	455	.release = single_release,
	456	};
	457
	458	static int __init create_debugfs_nodes(void)
	459	{
	460	struct dentry d, pfn, decay, count, *array;
	461
	462	d = debugfs_create_dir("cec", ras_debugfs_dir);
	463	if (!d) {
	464	pr_warn("Error creating cec debugfs node!\n");
	465	return -1;
	466	}
	467
	468	pfn = debugfs_create_file("pfn", S_IRUSR \| S_IWUSR, d, &dfs_pfn, &pfn_ops);
	469	if (!pfn) {
	470	pr_warn("Error creating pfn debugfs node!\n");
	471	goto err;
	472	}
	473
	474	array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_ops);
	475	if (!array) {
	476	pr_warn("Error creating array debugfs node!\n");
	477	goto err;
	478	}
	479
	480	decay = debugfs_create_file("decay_interval", S_IRUSR \| S_IWUSR, d,
2328874d	481	&decay_interval, &decay_interval_ops);
011d8261 BP	482	if (!decay) {
	483	pr_warn("Error creating decay_interval debugfs node!\n");
	484	goto err;
	485	}
	486
	487	count = debugfs_create_file("count_threshold", S_IRUSR \| S_IWUSR, d,
	488	&count_threshold, &count_threshold_ops);
32288daf	489	if (!count) {
011d8261 BP	490	pr_warn("Error creating count_threshold debugfs node!\n");
	491	goto err;
	492	}
	493
	494
	495	return 0;
	496
	497	err:
	498	debugfs_remove_recursive(d);
	499
	500	return 1;
	501	}
	502
	503	void __init cec_init(void)
	504	{
	505	if (ce_arr.disabled)
	506	return;
	507
	508	ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL);
	509	if (!ce_arr.array) {
	510	pr_err("Error allocating CE array page!\n");
	511	return;
	512	}
	513
	514	if (create_debugfs_nodes())
	515	return;
	516
2328874d CW	517	INIT_DELAYED_WORK(&cec_work, cec_work_fn);
2328874d CW	518	schedule_delayed_work(&cec_work, CEC_DECAY_DEFAULT_INTERVAL);
011d8261 BP	519
	520	pr_info("Correctable Errors collector initialized.\n");
	521	}
	522
	523	int __init parse_cec_param(char *str)
	524	{
	525	if (!str)
	526	return 0;
	527
	528	if (*str == '=')
	529	str++;
	530
69a33000	531	if (!strcmp(str, "cec_disable"))
011d8261 BP	532	ce_arr.disabled = 1;
	533	else
	534	return 0;
	535
	536	return 1;
	537	}