]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/acpi/nfit.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nfit, libnvdimm: limited/whitelisted dimm command marshaling mechanism
[mirror_ubuntu-bionic-kernel.git] / drivers / acpi / nfit.c
CommitLineData
b94d5230
DW		 1/*
2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 */
13#include <linux/list_sort.h>
14#include <linux/libnvdimm.h>
15#include <linux/module.h>
047fc8a1 16#include <linux/mutex.h>
62232e45 17#include <linux/ndctl.h>
0caeef63 18#include <linux/delay.h>
b94d5230
DW		 19#include <linux/list.h>
20#include <linux/acpi.h>
eaf96153 21#include <linux/sort.h>
c2ad2954 22#include <linux/pmem.h>
047fc8a1 23#include <linux/io.h>
1cf03c00 24#include <linux/nd.h>
96601adb 25#include <asm/cacheflush.h>
b94d5230
DW		 26#include "nfit.h"
27
047fc8a1
RZ		 28/*
29 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
30 * irrelevant.
31 */
2f8e2c87 32#include <linux/io-64-nonatomic-hi-lo.h>
047fc8a1 33
4d88a97a
DW		 34static bool force_enable_dimms;
35module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
36MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
37
1cf03c00
DW		 38static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT;
39module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR);
40MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds");
41
42/* after three payloads of overflow, it's dead jim */
43static unsigned int scrub_overflow_abort = 3;
44module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR);
45MODULE_PARM_DESC(scrub_overflow_abort,
46 "Number of times we overflow ARS results before abort");
47
7ae0fa43
DW		 48static struct workqueue_struct *nfit_wq;
49
20985164
VV		 50struct nfit_table_prev {
51 struct list_head spas;
52 struct list_head memdevs;
53 struct list_head dcrs;
54 struct list_head bdws;
55 struct list_head idts;
56 struct list_head flushes;
57};
58
b94d5230
DW		 59static u8 nfit_uuid[NFIT_UUID_MAX][16];
60
6bc75619 61const u8 *to_nfit_uuid(enum nfit_uuids id)
b94d5230
DW		 62{
63 return nfit_uuid[id];
64}
6bc75619 65EXPORT_SYMBOL(to_nfit_uuid);
b94d5230 66
62232e45
DW		 67static struct acpi_nfit_desc *to_acpi_nfit_desc(
68 struct nvdimm_bus_descriptor *nd_desc)
69{
70 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
71}
72
73static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
74{
75 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
76
77 /*
78 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
79 * acpi_device.
80 */
81 if (!nd_desc->provider_name
82 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
83 return NULL;
84
85 return to_acpi_device(acpi_desc->dev);
86}
87
aef25338
DW		 88static int xlat_status(void *buf, unsigned int cmd)
89{
d4f32367 90 struct nd_cmd_clear_error *clear_err;
aef25338
DW		 91 struct nd_cmd_ars_status *ars_status;
92 struct nd_cmd_ars_start *ars_start;
93 struct nd_cmd_ars_cap *ars_cap;
94 u16 flags;
95
96 switch (cmd) {
97 case ND_CMD_ARS_CAP:
98 ars_cap = buf;
99 if ((ars_cap->status & 0xffff) == NFIT_ARS_CAP_NONE)
100 return -ENOTTY;
101
102 /* Command failed */
103 if (ars_cap->status & 0xffff)
104 return -EIO;
105
106 /* No supported scan types for this range */
107 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
108 if ((ars_cap->status >> 16 & flags) == 0)
109 return -ENOTTY;
110 break;
111 case ND_CMD_ARS_START:
112 ars_start = buf;
113 /* ARS is in progress */
114 if ((ars_start->status & 0xffff) == NFIT_ARS_START_BUSY)
115 return -EBUSY;
116
117 /* Command failed */
118 if (ars_start->status & 0xffff)
119 return -EIO;
120 break;
121 case ND_CMD_ARS_STATUS:
122 ars_status = buf;
123 /* Command failed */
124 if (ars_status->status & 0xffff)
125 return -EIO;
126 /* Check extended status (Upper two bytes) */
127 if (ars_status->status == NFIT_ARS_STATUS_DONE)
128 return 0;
129
130 /* ARS is in progress */
131 if (ars_status->status == NFIT_ARS_STATUS_BUSY)
132 return -EBUSY;
133
134 /* No ARS performed for the current boot */
135 if (ars_status->status == NFIT_ARS_STATUS_NONE)
136 return -EAGAIN;
137
138 /*
139 * ARS interrupted, either we overflowed or some other
140 * agent wants the scan to stop. If we didn't overflow
141 * then just continue with the returned results.
142 */
143 if (ars_status->status == NFIT_ARS_STATUS_INTR) {
144 if (ars_status->flags & NFIT_ARS_F_OVERFLOW)
145 return -ENOSPC;
146 return 0;
147 }
148
149 /* Unknown status */
150 if (ars_status->status >> 16)
151 return -EIO;
152 break;
d4f32367
DW		 153 case ND_CMD_CLEAR_ERROR:
154 clear_err = buf;
155 if (clear_err->status & 0xffff)
156 return -EIO;
157 if (!clear_err->cleared)
158 return -EIO;
159 if (clear_err->length > clear_err->cleared)
160 return clear_err->cleared;
161 break;
aef25338
DW		 162 default:
163 break;
164 }
165
166 return 0;
167}
168
b94d5230
DW		 169static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
170 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
aef25338 171 unsigned int buf_len, int *cmd_rc)
b94d5230 172{
62232e45 173 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
62232e45 174 union acpi_object in_obj, in_buf, *out_obj;
31eca76b 175 const struct nd_cmd_desc *desc = NULL;
62232e45 176 struct device *dev = acpi_desc->dev;
31eca76b 177 struct nd_cmd_pkg *call_pkg = NULL;
62232e45 178 const char *cmd_name, *dimm_name;
31eca76b 179 unsigned long cmd_mask, dsm_mask;
62232e45 180 acpi_handle handle;
31eca76b 181 unsigned int func;
62232e45
DW		 182 const u8 *uuid;
183 u32 offset;
184 int rc, i;
185
31eca76b
DW		 186 func = cmd;
187 if (cmd == ND_CMD_CALL) {
188 call_pkg = buf;
189 func = call_pkg->nd_command;
190 }
191
62232e45
DW		 192 if (nvdimm) {
193 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
194 struct acpi_device *adev = nfit_mem->adev;
195
196 if (!adev)
197 return -ENOTTY;
31eca76b
DW		 198 if (call_pkg && nfit_mem->family != call_pkg->nd_family)
199 return -ENOTTY;
200
047fc8a1 201 dimm_name = nvdimm_name(nvdimm);
62232e45 202 cmd_name = nvdimm_cmd_name(cmd);
e3654eca 203 cmd_mask = nvdimm_cmd_mask(nvdimm);
31eca76b 204 dsm_mask = nfit_mem->dsm_mask;
62232e45 205 desc = nd_cmd_dimm_desc(cmd);
31eca76b 206 uuid = to_nfit_uuid(nfit_mem->family);
62232e45
DW		 207 handle = adev->handle;
208 } else {
209 struct acpi_device *adev = to_acpi_dev(acpi_desc);
210
211 cmd_name = nvdimm_bus_cmd_name(cmd);
e3654eca 212 cmd_mask = nd_desc->cmd_mask;
31eca76b 213 dsm_mask = cmd_mask;
62232e45
DW		 214 desc = nd_cmd_bus_desc(cmd);
215 uuid = to_nfit_uuid(NFIT_DEV_BUS);
216 handle = adev->handle;
217 dimm_name = "bus";
218 }
219
220 if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
221 return -ENOTTY;
222
31eca76b 223 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask))
62232e45
DW		 224 return -ENOTTY;
225
226 in_obj.type = ACPI_TYPE_PACKAGE;
227 in_obj.package.count = 1;
228 in_obj.package.elements = &in_buf;
229 in_buf.type = ACPI_TYPE_BUFFER;
230 in_buf.buffer.pointer = buf;
231 in_buf.buffer.length = 0;
232
233 /* libnvdimm has already validated the input envelope */
234 for (i = 0; i < desc->in_num; i++)
235 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
236 i, buf);
237
31eca76b
DW		 238 if (call_pkg) {
239 /* skip over package wrapper */
240 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
241 in_buf.buffer.length = call_pkg->nd_size_in;
242 }
243
62232e45 244 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
31eca76b
DW		 245 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n",
246 __func__, dimm_name, cmd, func,
247 in_buf.buffer.length);
248 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4,
249 in_buf.buffer.pointer,
250 min_t(u32, 256, in_buf.buffer.length), true);
62232e45
DW		 251 }
252
31eca76b 253 out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj);
62232e45
DW		 254 if (!out_obj) {
255 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name,
256 cmd_name);
257 return -EINVAL;
258 }
259
31eca76b
DW		 260 if (call_pkg) {
261 call_pkg->nd_fw_size = out_obj->buffer.length;
262 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
263 out_obj->buffer.pointer,
264 min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
265
266 ACPI_FREE(out_obj);
267 /*
268 * Need to support FW function w/o known size in advance.
269 * Caller can determine required size based upon nd_fw_size.
270 * If we return an error (like elsewhere) then caller wouldn't
271 * be able to rely upon data returned to make calculation.
272 */
273 return 0;
274 }
275
62232e45
DW		 276 if (out_obj->package.type != ACPI_TYPE_BUFFER) {
277 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n",
278 __func__, dimm_name, cmd_name, out_obj->type);
279 rc = -EINVAL;
280 goto out;
281 }
282
283 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
284 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
285 dimm_name, cmd_name, out_obj->buffer.length);
286 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
287 4, out_obj->buffer.pointer, min_t(u32, 128,
288 out_obj->buffer.length), true);
289 }
290
291 for (i = 0, offset = 0; i < desc->out_num; i++) {
292 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
293 (u32 *) out_obj->buffer.pointer);
294
295 if (offset + out_size > out_obj->buffer.length) {
296 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n",
297 __func__, dimm_name, cmd_name, i);
298 break;
299 }
300
301 if (in_buf.buffer.length + offset + out_size > buf_len) {
302 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n",
303 __func__, dimm_name, cmd_name, i);
304 rc = -ENXIO;
305 goto out;
306 }
307 memcpy(buf + in_buf.buffer.length + offset,
308 out_obj->buffer.pointer + offset, out_size);
309 offset += out_size;
310 }
311 if (offset + in_buf.buffer.length < buf_len) {
312 if (i >= 1) {
313 /*
314 * status valid, return the number of bytes left
315 * unfilled in the output buffer
316 */
317 rc = buf_len - offset - in_buf.buffer.length;
aef25338
DW		 318 if (cmd_rc)
319 *cmd_rc = xlat_status(buf, cmd);
62232e45
DW		 320 } else {
321 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
322 __func__, dimm_name, cmd_name, buf_len,
323 offset);
324 rc = -ENXIO;
325 }
326 } else
327 rc = 0;
328
329 out:
330 ACPI_FREE(out_obj);
331
332 return rc;
b94d5230
DW		 333}
334
335static const char *spa_type_name(u16 type)
336{
337 static const char *to_name[] = {
338 [NFIT_SPA_VOLATILE] = "volatile",
339 [NFIT_SPA_PM] = "pmem",
340 [NFIT_SPA_DCR] = "dimm-control-region",
341 [NFIT_SPA_BDW] = "block-data-window",
342 [NFIT_SPA_VDISK] = "volatile-disk",
343 [NFIT_SPA_VCD] = "volatile-cd",
344 [NFIT_SPA_PDISK] = "persistent-disk",
345 [NFIT_SPA_PCD] = "persistent-cd",
346
347 };
348
349 if (type > NFIT_SPA_PCD)
350 return "unknown";
351
352 return to_name[type];
353}
354
355static int nfit_spa_type(struct acpi_nfit_system_address *spa)
356{
357 int i;
358
359 for (i = 0; i < NFIT_UUID_MAX; i++)
360 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0)
361 return i;
362 return -1;
363}
364
365static bool add_spa(struct acpi_nfit_desc *acpi_desc,
20985164 366 struct nfit_table_prev *prev,
b94d5230
DW		 367 struct acpi_nfit_system_address *spa)
368{
826c416f 369 size_t length = min_t(size_t, sizeof(*spa), spa->header.length);
b94d5230 370 struct device *dev = acpi_desc->dev;
20985164
VV		 371 struct nfit_spa *nfit_spa;
372
373 list_for_each_entry(nfit_spa, &prev->spas, list) {
826c416f 374 if (memcmp(nfit_spa->spa, spa, length) == 0) {
20985164
VV		 375 list_move_tail(&nfit_spa->list, &acpi_desc->spas);
376 return true;
377 }
378 }
b94d5230 379
20985164 380 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa), GFP_KERNEL);
b94d5230
DW		 381 if (!nfit_spa)
382 return false;
383 INIT_LIST_HEAD(&nfit_spa->list);
384 nfit_spa->spa = spa;
385 list_add_tail(&nfit_spa->list, &acpi_desc->spas);
386 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__,
387 spa->range_index,
388 spa_type_name(nfit_spa_type(spa)));
389 return true;
390}
391
392static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
20985164 393 struct nfit_table_prev *prev,
b94d5230
DW		 394 struct acpi_nfit_memory_map *memdev)
395{
826c416f 396 size_t length = min_t(size_t, sizeof(*memdev), memdev->header.length);
b94d5230 397 struct device *dev = acpi_desc->dev;
20985164 398 struct nfit_memdev *nfit_memdev;
b94d5230 399
20985164 400 list_for_each_entry(nfit_memdev, &prev->memdevs, list)
826c416f 401 if (memcmp(nfit_memdev->memdev, memdev, length) == 0) {
20985164
VV		 402 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
403 return true;
404 }
405
406 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev), GFP_KERNEL);
b94d5230
DW		 407 if (!nfit_memdev)
408 return false;
409 INIT_LIST_HEAD(&nfit_memdev->list);
410 nfit_memdev->memdev = memdev;
411 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
412 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
413 __func__, memdev->device_handle, memdev->range_index,
414 memdev->region_index);
415 return true;
416}
417
418static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
20985164 419 struct nfit_table_prev *prev,
b94d5230
DW		 420 struct acpi_nfit_control_region *dcr)
421{
826c416f 422 size_t length = min_t(size_t, sizeof(*dcr), dcr->header.length);
b94d5230 423 struct device *dev = acpi_desc->dev;
20985164
VV		 424 struct nfit_dcr *nfit_dcr;
425
426 list_for_each_entry(nfit_dcr, &prev->dcrs, list)
826c416f 427 if (memcmp(nfit_dcr->dcr, dcr, length) == 0) {
20985164
VV		 428 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
429 return true;
430 }
b94d5230 431
20985164 432 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr), GFP_KERNEL);
b94d5230
DW		 433 if (!nfit_dcr)
434 return false;
435 INIT_LIST_HEAD(&nfit_dcr->list);
436 nfit_dcr->dcr = dcr;
437 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
438 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__,
439 dcr->region_index, dcr->windows);
440 return true;
441}
442
443static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
20985164 444 struct nfit_table_prev *prev,
b94d5230
DW		 445 struct acpi_nfit_data_region *bdw)
446{
826c416f 447 size_t length = min_t(size_t, sizeof(*bdw), bdw->header.length);
b94d5230 448 struct device *dev = acpi_desc->dev;
20985164
VV		 449 struct nfit_bdw *nfit_bdw;
450
451 list_for_each_entry(nfit_bdw, &prev->bdws, list)
826c416f 452 if (memcmp(nfit_bdw->bdw, bdw, length) == 0) {
20985164
VV		 453 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
454 return true;
455 }
b94d5230 456
20985164 457 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw), GFP_KERNEL);
b94d5230
DW		 458 if (!nfit_bdw)
459 return false;
460 INIT_LIST_HEAD(&nfit_bdw->list);
461 nfit_bdw->bdw = bdw;
462 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
463 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__,
464 bdw->region_index, bdw->windows);
465 return true;
466}
467
047fc8a1 468static bool add_idt(struct acpi_nfit_desc *acpi_desc,
20985164 469 struct nfit_table_prev *prev,
047fc8a1
RZ		 470 struct acpi_nfit_interleave *idt)
471{
826c416f 472 size_t length = min_t(size_t, sizeof(*idt), idt->header.length);
047fc8a1 473 struct device *dev = acpi_desc->dev;
20985164
VV		 474 struct nfit_idt *nfit_idt;
475
476 list_for_each_entry(nfit_idt, &prev->idts, list)
826c416f 477 if (memcmp(nfit_idt->idt, idt, length) == 0) {
20985164
VV		 478 list_move_tail(&nfit_idt->list, &acpi_desc->idts);
479 return true;
480 }
047fc8a1 481
20985164 482 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt), GFP_KERNEL);
047fc8a1
RZ		 483 if (!nfit_idt)
484 return false;
485 INIT_LIST_HEAD(&nfit_idt->list);
486 nfit_idt->idt = idt;
487 list_add_tail(&nfit_idt->list, &acpi_desc->idts);
488 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__,
489 idt->interleave_index, idt->line_count);
490 return true;
491}
492
c2ad2954 493static bool add_flush(struct acpi_nfit_desc *acpi_desc,
20985164 494 struct nfit_table_prev *prev,
c2ad2954
RZ		 495 struct acpi_nfit_flush_address *flush)
496{
826c416f 497 size_t length = min_t(size_t, sizeof(*flush), flush->header.length);
c2ad2954 498 struct device *dev = acpi_desc->dev;
20985164 499 struct nfit_flush *nfit_flush;
c2ad2954 500
20985164 501 list_for_each_entry(nfit_flush, &prev->flushes, list)
826c416f 502 if (memcmp(nfit_flush->flush, flush, length) == 0) {
20985164
VV		 503 list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
504 return true;
505 }
506
507 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush), GFP_KERNEL);
c2ad2954
RZ		 508 if (!nfit_flush)
509 return false;
510 INIT_LIST_HEAD(&nfit_flush->list);
511 nfit_flush->flush = flush;
512 list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
513 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__,
514 flush->device_handle, flush->hint_count);
515 return true;
516}
517
20985164
VV		 518static void *add_table(struct acpi_nfit_desc *acpi_desc,
519 struct nfit_table_prev *prev, void *table, const void *end)
b94d5230
DW		 520{
521 struct device *dev = acpi_desc->dev;
522 struct acpi_nfit_header *hdr;
523 void *err = ERR_PTR(-ENOMEM);
524
525 if (table >= end)
526 return NULL;
527
528 hdr = table;
564d5011
VV		 529 if (!hdr->length) {
530 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
531 hdr->type);
532 return NULL;
533 }
534
b94d5230
DW		 535 switch (hdr->type) {
536 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
20985164 537 if (!add_spa(acpi_desc, prev, table))
b94d5230
DW		 538 return err;
539 break;
540 case ACPI_NFIT_TYPE_MEMORY_MAP:
20985164 541 if (!add_memdev(acpi_desc, prev, table))
b94d5230
DW		 542 return err;
543 break;
544 case ACPI_NFIT_TYPE_CONTROL_REGION:
20985164 545 if (!add_dcr(acpi_desc, prev, table))
b94d5230
DW		 546 return err;
547 break;
548 case ACPI_NFIT_TYPE_DATA_REGION:
20985164 549 if (!add_bdw(acpi_desc, prev, table))
b94d5230
DW		 550 return err;
551 break;
b94d5230 552 case ACPI_NFIT_TYPE_INTERLEAVE:
20985164 553 if (!add_idt(acpi_desc, prev, table))
047fc8a1 554 return err;
b94d5230
DW		 555 break;
556 case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
20985164 557 if (!add_flush(acpi_desc, prev, table))
c2ad2954 558 return err;
b94d5230
DW		 559 break;
560 case ACPI_NFIT_TYPE_SMBIOS:
561 dev_dbg(dev, "%s: smbios\n", __func__);
562 break;
563 default:
564 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
565 break;
566 }
567
568 return table + hdr->length;
569}
570
571static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
572 struct nfit_mem *nfit_mem)
573{
574 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
575 u16 dcr = nfit_mem->dcr->region_index;
576 struct nfit_spa *nfit_spa;
577
578 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
579 u16 range_index = nfit_spa->spa->range_index;
580 int type = nfit_spa_type(nfit_spa->spa);
581 struct nfit_memdev *nfit_memdev;
582
583 if (type != NFIT_SPA_BDW)
584 continue;
585
586 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
587 if (nfit_memdev->memdev->range_index != range_index)
588 continue;
589 if (nfit_memdev->memdev->device_handle != device_handle)
590 continue;
591 if (nfit_memdev->memdev->region_index != dcr)
592 continue;
593
594 nfit_mem->spa_bdw = nfit_spa->spa;
595 return;
596 }
597 }
598
599 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
600 nfit_mem->spa_dcr->range_index);
601 nfit_mem->bdw = NULL;
602}
603
6697b2cf 604static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
b94d5230
DW		 605 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
606{
607 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
047fc8a1 608 struct nfit_memdev *nfit_memdev;
c2ad2954 609 struct nfit_flush *nfit_flush;
b94d5230 610 struct nfit_bdw *nfit_bdw;
047fc8a1
RZ		 611 struct nfit_idt *nfit_idt;
612 u16 idt_idx, range_index;
b94d5230 613
b94d5230
DW		 614 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
615 if (nfit_bdw->bdw->region_index != dcr)
616 continue;
617 nfit_mem->bdw = nfit_bdw->bdw;
618 break;
619 }
620
621 if (!nfit_mem->bdw)
6697b2cf 622 return;
b94d5230
DW		 623
624 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
047fc8a1
RZ		 625
626 if (!nfit_mem->spa_bdw)
6697b2cf 627 return;
047fc8a1
RZ		 628
629 range_index = nfit_mem->spa_bdw->range_index;
630 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
631 if (nfit_memdev->memdev->range_index != range_index ||
632 nfit_memdev->memdev->region_index != dcr)
633 continue;
634 nfit_mem->memdev_bdw = nfit_memdev->memdev;
635 idt_idx = nfit_memdev->memdev->interleave_index;
636 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
637 if (nfit_idt->idt->interleave_index != idt_idx)
638 continue;
639 nfit_mem->idt_bdw = nfit_idt->idt;
640 break;
641 }
c2ad2954
RZ		 642
643 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
644 if (nfit_flush->flush->device_handle !=
645 nfit_memdev->memdev->device_handle)
646 continue;
647 nfit_mem->nfit_flush = nfit_flush;
648 break;
649 }
047fc8a1
RZ		 650 break;
651 }
b94d5230
DW		 652}
653
654static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
655 struct acpi_nfit_system_address *spa)
656{
657 struct nfit_mem *nfit_mem, *found;
658 struct nfit_memdev *nfit_memdev;
659 int type = nfit_spa_type(spa);
b94d5230
DW		 660
661 switch (type) {
662 case NFIT_SPA_DCR:
663 case NFIT_SPA_PM:
664 break;
665 default:
666 return 0;
667 }
668
669 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
6697b2cf
DW		 670 struct nfit_dcr *nfit_dcr;
671 u32 device_handle;
672 u16 dcr;
b94d5230
DW		 673
674 if (nfit_memdev->memdev->range_index != spa->range_index)
675 continue;
676 found = NULL;
677 dcr = nfit_memdev->memdev->region_index;
6697b2cf 678 device_handle = nfit_memdev->memdev->device_handle;
b94d5230 679 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
6697b2cf
DW		 680 if (__to_nfit_memdev(nfit_mem)->device_handle
681 == device_handle) {
b94d5230
DW		 682 found = nfit_mem;
683 break;
684 }
685
686 if (found)
687 nfit_mem = found;
688 else {
689 nfit_mem = devm_kzalloc(acpi_desc->dev,
690 sizeof(*nfit_mem), GFP_KERNEL);
691 if (!nfit_mem)
692 return -ENOMEM;
693 INIT_LIST_HEAD(&nfit_mem->list);
6697b2cf
DW		 694 list_add(&nfit_mem->list, &acpi_desc->dimms);
695 }
696
697 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
698 if (nfit_dcr->dcr->region_index != dcr)
699 continue;
700 /*
701 * Record the control region for the dimm. For
702 * the ACPI 6.1 case, where there are separate
703 * control regions for the pmem vs blk
704 * interfaces, be sure to record the extended
705 * blk details.
706 */
707 if (!nfit_mem->dcr)
708 nfit_mem->dcr = nfit_dcr->dcr;
709 else if (nfit_mem->dcr->windows == 0
710 && nfit_dcr->dcr->windows)
711 nfit_mem->dcr = nfit_dcr->dcr;
712 break;
713 }
714
715 if (dcr && !nfit_mem->dcr) {
716 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
717 spa->range_index, dcr);
718 return -ENODEV;
b94d5230
DW		 719 }
720
721 if (type == NFIT_SPA_DCR) {
047fc8a1
RZ		 722 struct nfit_idt *nfit_idt;
723 u16 idt_idx;
724
b94d5230
DW		 725 /* multiple dimms may share a SPA when interleaved */
726 nfit_mem->spa_dcr = spa;
727 nfit_mem->memdev_dcr = nfit_memdev->memdev;
047fc8a1
RZ		 728 idt_idx = nfit_memdev->memdev->interleave_index;
729 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
730 if (nfit_idt->idt->interleave_index != idt_idx)
731 continue;
732 nfit_mem->idt_dcr = nfit_idt->idt;
733 break;
734 }
6697b2cf 735 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
b94d5230
DW		 736 } else {
737 /*
738 * A single dimm may belong to multiple SPA-PM
739 * ranges, record at least one in addition to
740 * any SPA-DCR range.
741 */
742 nfit_mem->memdev_pmem = nfit_memdev->memdev;
743 }
b94d5230
DW		 744 }
745
746 return 0;
747}
748
749static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
750{
751 struct nfit_mem *a = container_of(_a, typeof(*a), list);
752 struct nfit_mem *b = container_of(_b, typeof(*b), list);
753 u32 handleA, handleB;
754
755 handleA = __to_nfit_memdev(a)->device_handle;
756 handleB = __to_nfit_memdev(b)->device_handle;
757 if (handleA < handleB)
758 return -1;
759 else if (handleA > handleB)
760 return 1;
761 return 0;
762}
763
764static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
765{
766 struct nfit_spa *nfit_spa;
767
768 /*
769 * For each SPA-DCR or SPA-PMEM address range find its
770 * corresponding MEMDEV(s). From each MEMDEV find the
771 * corresponding DCR. Then, if we're operating on a SPA-DCR,
772 * try to find a SPA-BDW and a corresponding BDW that references
773 * the DCR. Throw it all into an nfit_mem object. Note, that
774 * BDWs are optional.
775 */
776 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
777 int rc;
778
779 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
780 if (rc)
781 return rc;
782 }
783
784 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
785
786 return 0;
787}
788
45def22c
DW		 789static ssize_t revision_show(struct device *dev,
790 struct device_attribute *attr, char *buf)
791{
792 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
793 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
794 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
795
6b577c9d 796 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
45def22c
DW		 797}
798static DEVICE_ATTR_RO(revision);
799
800static struct attribute *acpi_nfit_attributes[] = {
801 &dev_attr_revision.attr,
802 NULL,
803};
804
805static struct attribute_group acpi_nfit_attribute_group = {
806 .name = "nfit",
807 .attrs = acpi_nfit_attributes,
808};
809
a61fe6f7 810static const struct attribute_group *acpi_nfit_attribute_groups[] = {
45def22c
DW		 811 &nvdimm_bus_attribute_group,
812 &acpi_nfit_attribute_group,
813 NULL,
814};
815
e6dfb2de
DW		 816static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
817{
818 struct nvdimm *nvdimm = to_nvdimm(dev);
819 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
820
821 return __to_nfit_memdev(nfit_mem);
822}
823
824static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
825{
826 struct nvdimm *nvdimm = to_nvdimm(dev);
827 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
828
829 return nfit_mem->dcr;
830}
831
832static ssize_t handle_show(struct device *dev,
833 struct device_attribute *attr, char *buf)
834{
835 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
836
837 return sprintf(buf, "%#x\n", memdev->device_handle);
838}
839static DEVICE_ATTR_RO(handle);
840
841static ssize_t phys_id_show(struct device *dev,
842 struct device_attribute *attr, char *buf)
843{
844 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
845
846 return sprintf(buf, "%#x\n", memdev->physical_id);
847}
848static DEVICE_ATTR_RO(phys_id);
849
850static ssize_t vendor_show(struct device *dev,
851 struct device_attribute *attr, char *buf)
852{
853 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
854
855 return sprintf(buf, "%#x\n", dcr->vendor_id);
856}
857static DEVICE_ATTR_RO(vendor);
858
859static ssize_t rev_id_show(struct device *dev,
860 struct device_attribute *attr, char *buf)
861{
862 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
863
864 return sprintf(buf, "%#x\n", dcr->revision_id);
865}
866static DEVICE_ATTR_RO(rev_id);
867
868static ssize_t device_show(struct device *dev,
869 struct device_attribute *attr, char *buf)
870{
871 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
872
873 return sprintf(buf, "%#x\n", dcr->device_id);
874}
875static DEVICE_ATTR_RO(device);
876
877static ssize_t format_show(struct device *dev,
878 struct device_attribute *attr, char *buf)
879{
880 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
881
882 return sprintf(buf, "%#x\n", dcr->code);
883}
884static DEVICE_ATTR_RO(format);
885
886static ssize_t serial_show(struct device *dev,
887 struct device_attribute *attr, char *buf)
888{
889 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
890
891 return sprintf(buf, "%#x\n", dcr->serial_number);
892}
893static DEVICE_ATTR_RO(serial);
894
58138820
DW		 895static ssize_t flags_show(struct device *dev,
896 struct device_attribute *attr, char *buf)
897{
898 u16 flags = to_nfit_memdev(dev)->flags;
899
900 return sprintf(buf, "%s%s%s%s%s\n",
402bae59
TK		 901 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
902 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
903 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
ca321d1c 904 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
402bae59 905 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "");
58138820
DW		 906}
907static DEVICE_ATTR_RO(flags);
908
e6dfb2de
DW		 909static struct attribute *acpi_nfit_dimm_attributes[] = {
910 &dev_attr_handle.attr,
911 &dev_attr_phys_id.attr,
912 &dev_attr_vendor.attr,
913 &dev_attr_device.attr,
914 &dev_attr_format.attr,
915 &dev_attr_serial.attr,
916 &dev_attr_rev_id.attr,
58138820 917 &dev_attr_flags.attr,
e6dfb2de
DW		 918 NULL,
919};
920
921static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
922 struct attribute *a, int n)
923{
924 struct device *dev = container_of(kobj, struct device, kobj);
925
926 if (to_nfit_dcr(dev))
927 return a->mode;
928 else
929 return 0;
930}
931
932static struct attribute_group acpi_nfit_dimm_attribute_group = {
933 .name = "nfit",
934 .attrs = acpi_nfit_dimm_attributes,
935 .is_visible = acpi_nfit_dimm_attr_visible,
936};
937
938static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
62232e45 939 &nvdimm_attribute_group,
4d88a97a 940 &nd_device_attribute_group,
e6dfb2de
DW		 941 &acpi_nfit_dimm_attribute_group,
942 NULL,
943};
944
945static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
946 u32 device_handle)
947{
948 struct nfit_mem *nfit_mem;
949
950 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
951 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
952 return nfit_mem->nvdimm;
953
954 return NULL;
955}
956
62232e45
DW		 957static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
958 struct nfit_mem *nfit_mem, u32 device_handle)
959{
960 struct acpi_device *adev, *adev_dimm;
961 struct device *dev = acpi_desc->dev;
31eca76b
DW		 962 unsigned long dsm_mask;
963 const u8 *uuid;
60e95f43 964 int i;
62232e45 965
e3654eca
DW		 966 /* nfit test assumes 1:1 relationship between commands and dsms */
967 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
31eca76b 968 nfit_mem->family = NVDIMM_FAMILY_INTEL;
62232e45
DW		 969 adev = to_acpi_dev(acpi_desc);
970 if (!adev)
971 return 0;
972
973 adev_dimm = acpi_find_child_device(adev, device_handle, false);
974 nfit_mem->adev = adev_dimm;
975 if (!adev_dimm) {
976 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
977 device_handle);
4d88a97a 978 return force_enable_dimms ? 0 : -ENODEV;
62232e45
DW		 979 }
980
31eca76b
DW		 981 /*
982 * Until standardization materializes we need to consider up to 3
983 * different command sets. Note, that checking for function0 (bit0)
984 * tells us if any commands are reachable through this uuid.
985 */
986 for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_HPE2; i++)
987 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
988 break;
989
990 /* limit the supported commands to those that are publicly documented */
991 nfit_mem->family = i;
992 if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
993 dsm_mask = 0x3fe;
994 else if (nfit_mem->family == NVDIMM_FAMILY_HPE1)
995 dsm_mask = 0x1c3c76;
996 else if (nfit_mem->family == NVDIMM_FAMILY_HPE2)
997 dsm_mask = 0x1fe;
998 else {
999 dev_err(dev, "unknown dimm command family\n");
1000 nfit_mem->family = -1;
1001 return force_enable_dimms ? 0 : -ENODEV;
1002 }
1003
1004 uuid = to_nfit_uuid(nfit_mem->family);
1005 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
62232e45
DW		 1006 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i))
1007 set_bit(i, &nfit_mem->dsm_mask);
1008
60e95f43 1009 return 0;
62232e45
DW		 1010}
1011
e6dfb2de
DW		 1012static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
1013{
1014 struct nfit_mem *nfit_mem;
4d88a97a 1015 int dimm_count = 0;
e6dfb2de
DW		 1016
1017 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
31eca76b 1018 unsigned long flags = 0, cmd_mask;
e6dfb2de 1019 struct nvdimm *nvdimm;
e6dfb2de 1020 u32 device_handle;
58138820 1021 u16 mem_flags;
62232e45 1022 int rc;
e6dfb2de
DW		 1023
1024 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
1025 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
1026 if (nvdimm) {
20985164 1027 dimm_count++;
e6dfb2de
DW		 1028 continue;
1029 }
1030
1031 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
1032 flags |= NDD_ALIASING;
1033
58138820 1034 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
ca321d1c 1035 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
58138820
DW		 1036 flags |= NDD_UNARMED;
1037
62232e45
DW		 1038 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
1039 if (rc)
1040 continue;
1041
e3654eca 1042 /*
31eca76b
DW		 1043 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
1044 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
1045 * userspace interface.
e3654eca 1046 */
31eca76b
DW		 1047 cmd_mask = 1UL << ND_CMD_CALL;
1048 if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1049 cmd_mask |= nfit_mem->dsm_mask;
1050
e6dfb2de 1051 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
62232e45 1052 acpi_nfit_dimm_attribute_groups,
31eca76b 1053 flags, cmd_mask);
e6dfb2de
DW		 1054 if (!nvdimm)
1055 return -ENOMEM;
1056
1057 nfit_mem->nvdimm = nvdimm;
4d88a97a 1058 dimm_count++;
58138820
DW		 1059
1060 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
1061 continue;
1062
402bae59 1063 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n",
58138820 1064 nvdimm_name(nvdimm),
402bae59
TK		 1065 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
1066 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
1067 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
ca321d1c 1068 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "");
58138820 1069
e6dfb2de
DW		 1070 }
1071
4d88a97a 1072 return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
e6dfb2de
DW		 1073}
1074
62232e45
DW		 1075static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
1076{
1077 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1078 const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS);
1079 struct acpi_device *adev;
1080 int i;
1081
e3654eca 1082 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
62232e45
DW		 1083 adev = to_acpi_dev(acpi_desc);
1084 if (!adev)
1085 return;
1086
d4f32367 1087 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
62232e45 1088 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i))
e3654eca 1089 set_bit(i, &nd_desc->cmd_mask);
62232e45
DW		 1090}
1091
1f7df6f8
DW		 1092static ssize_t range_index_show(struct device *dev,
1093 struct device_attribute *attr, char *buf)
1094{
1095 struct nd_region *nd_region = to_nd_region(dev);
1096 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
1097
1098 return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
1099}
1100static DEVICE_ATTR_RO(range_index);
1101
1102static struct attribute *acpi_nfit_region_attributes[] = {
1103 &dev_attr_range_index.attr,
1104 NULL,
1105};
1106
1107static struct attribute_group acpi_nfit_region_attribute_group = {
1108 .name = "nfit",
1109 .attrs = acpi_nfit_region_attributes,
1110};
1111
1112static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
1113 &nd_region_attribute_group,
1114 &nd_mapping_attribute_group,
3d88002e 1115 &nd_device_attribute_group,
74ae66c3 1116 &nd_numa_attribute_group,
1f7df6f8
DW		 1117 &acpi_nfit_region_attribute_group,
1118 NULL,
1119};
1120
eaf96153
DW		 1121/* enough info to uniquely specify an interleave set */
1122struct nfit_set_info {
1123 struct nfit_set_info_map {
1124 u64 region_offset;
1125 u32 serial_number;
1126 u32 pad;
1127 } mapping[0];
1128};
1129
1130static size_t sizeof_nfit_set_info(int num_mappings)
1131{
1132 return sizeof(struct nfit_set_info)
1133 + num_mappings * sizeof(struct nfit_set_info_map);
1134}
1135
1136static int cmp_map(const void *m0, const void *m1)
1137{
1138 const struct nfit_set_info_map *map0 = m0;
1139 const struct nfit_set_info_map *map1 = m1;
1140
1141 return memcmp(&map0->region_offset, &map1->region_offset,
1142 sizeof(u64));
1143}
1144
1145/* Retrieve the nth entry referencing this spa */
1146static struct acpi_nfit_memory_map *memdev_from_spa(
1147 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
1148{
1149 struct nfit_memdev *nfit_memdev;
1150
1151 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
1152 if (nfit_memdev->memdev->range_index == range_index)
1153 if (n-- == 0)
1154 return nfit_memdev->memdev;
1155 return NULL;
1156}
1157
1158static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
1159 struct nd_region_desc *ndr_desc,
1160 struct acpi_nfit_system_address *spa)
1161{
1162 int i, spa_type = nfit_spa_type(spa);
1163 struct device *dev = acpi_desc->dev;
1164 struct nd_interleave_set *nd_set;
1165 u16 nr = ndr_desc->num_mappings;
1166 struct nfit_set_info *info;
1167
1168 if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE)
1169 /* pass */;
1170 else
1171 return 0;
1172
1173 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
1174 if (!nd_set)
1175 return -ENOMEM;
1176
1177 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
1178 if (!info)
1179 return -ENOMEM;
1180 for (i = 0; i < nr; i++) {
1181 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
1182 struct nfit_set_info_map *map = &info->mapping[i];
1183 struct nvdimm *nvdimm = nd_mapping->nvdimm;
1184 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1185 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
1186 spa->range_index, i);
1187
1188 if (!memdev || !nfit_mem->dcr) {
1189 dev_err(dev, "%s: failed to find DCR\n", __func__);
1190 return -ENODEV;
1191 }
1192
1193 map->region_offset = memdev->region_offset;
1194 map->serial_number = nfit_mem->dcr->serial_number;
1195 }
1196
1197 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
1198 cmp_map, NULL);
1199 nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
1200 ndr_desc->nd_set = nd_set;
1201 devm_kfree(dev, info);
1202
1203 return 0;
1204}
1205
047fc8a1
RZ		 1206static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
1207{
1208 struct acpi_nfit_interleave *idt = mmio->idt;
1209 u32 sub_line_offset, line_index, line_offset;
1210 u64 line_no, table_skip_count, table_offset;
1211
1212 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
1213 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
1214 line_offset = idt->line_offset[line_index]
1215 * mmio->line_size;
1216 table_offset = table_skip_count * mmio->table_size;
1217
1218 return mmio->base_offset + line_offset + table_offset + sub_line_offset;
1219}
1220
c2ad2954
RZ		 1221static void wmb_blk(struct nfit_blk *nfit_blk)
1222{
1223
1224 if (nfit_blk->nvdimm_flush) {
1225 /*
1226 * The first wmb() is needed to 'sfence' all previous writes
1227 * such that they are architecturally visible for the platform
1228 * buffer flush. Note that we've already arranged for pmem
1229 * writes to avoid the cache via arch_memcpy_to_pmem(). The
1230 * final wmb() ensures ordering for the NVDIMM flush write.
1231 */
1232 wmb();
1233 writeq(1, nfit_blk->nvdimm_flush);
1234 wmb();
1235 } else
1236 wmb_pmem();
1237}
1238
de4a196c 1239static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
047fc8a1
RZ		 1240{
1241 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1242 u64 offset = nfit_blk->stat_offset + mmio->size * bw;
1243
1244 if (mmio->num_lines)
1245 offset = to_interleave_offset(offset, mmio);
1246
12f03ee6 1247 return readl(mmio->addr.base + offset);
047fc8a1
RZ		 1248}
1249
1250static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
1251 resource_size_t dpa, unsigned int len, unsigned int write)
1252{
1253 u64 cmd, offset;
1254 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1255
1256 enum {
1257 BCW_OFFSET_MASK = (1ULL << 48)-1,
1258 BCW_LEN_SHIFT = 48,
1259 BCW_LEN_MASK = (1ULL << 8) - 1,
1260 BCW_CMD_SHIFT = 56,
1261 };
1262
1263 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
1264 len = len >> L1_CACHE_SHIFT;
1265 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
1266 cmd |= ((u64) write) << BCW_CMD_SHIFT;
1267
1268 offset = nfit_blk->cmd_offset + mmio->size * bw;
1269 if (mmio->num_lines)
1270 offset = to_interleave_offset(offset, mmio);
1271
67a3e8fe 1272 writeq(cmd, mmio->addr.base + offset);
c2ad2954 1273 wmb_blk(nfit_blk);
f0f2c072 1274
aef25338 1275 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
67a3e8fe 1276 readq(mmio->addr.base + offset);
047fc8a1
RZ		 1277}
1278
1279static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
1280 resource_size_t dpa, void *iobuf, size_t len, int rw,
1281 unsigned int lane)
1282{
1283 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1284 unsigned int copied = 0;
1285 u64 base_offset;
1286 int rc;
1287
1288 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
1289 + lane * mmio->size;
047fc8a1
RZ		 1290 write_blk_ctl(nfit_blk, lane, dpa, len, rw);
1291 while (len) {
1292 unsigned int c;
1293 u64 offset;
1294
1295 if (mmio->num_lines) {
1296 u32 line_offset;
1297
1298 offset = to_interleave_offset(base_offset + copied,
1299 mmio);
1300 div_u64_rem(offset, mmio->line_size, &line_offset);
1301 c = min_t(size_t, len, mmio->line_size - line_offset);
1302 } else {
1303 offset = base_offset + nfit_blk->bdw_offset;
1304 c = len;
1305 }
1306
1307 if (rw)
67a3e8fe 1308 memcpy_to_pmem(mmio->addr.aperture + offset,
c2ad2954 1309 iobuf + copied, c);
67a3e8fe 1310 else {
aef25338 1311 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
67a3e8fe
RZ		 1312 mmio_flush_range((void __force *)
1313 mmio->addr.aperture + offset, c);
1314
c2ad2954 1315 memcpy_from_pmem(iobuf + copied,
67a3e8fe
RZ		 1316 mmio->addr.aperture + offset, c);
1317 }
047fc8a1
RZ		 1318
1319 copied += c;
1320 len -= c;
1321 }
c2ad2954
RZ		 1322
1323 if (rw)
1324 wmb_blk(nfit_blk);
1325
047fc8a1
RZ		 1326 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
1327 return rc;
1328}
1329
1330static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
1331 resource_size_t dpa, void *iobuf, u64 len, int rw)
1332{
1333 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1334 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1335 struct nd_region *nd_region = nfit_blk->nd_region;
1336 unsigned int lane, copied = 0;
1337 int rc = 0;
1338
1339 lane = nd_region_acquire_lane(nd_region);
1340 while (len) {
1341 u64 c = min(len, mmio->size);
1342
1343 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
1344 iobuf + copied, c, rw, lane);
1345 if (rc)
1346 break;
1347
1348 copied += c;
1349 len -= c;
1350 }
1351 nd_region_release_lane(nd_region, lane);
1352
1353 return rc;
1354}
1355
1356static void nfit_spa_mapping_release(struct kref *kref)
1357{
1358 struct nfit_spa_mapping *spa_map = to_spa_map(kref);
1359 struct acpi_nfit_system_address *spa = spa_map->spa;
1360 struct acpi_nfit_desc *acpi_desc = spa_map->acpi_desc;
1361
1362 WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1363 dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index);
67a3e8fe
RZ		 1364 if (spa_map->type == SPA_MAP_APERTURE)
1365 memunmap((void __force *)spa_map->addr.aperture);
1366 else
1367 iounmap(spa_map->addr.base);
047fc8a1
RZ		 1368 release_mem_region(spa->address, spa->length);
1369 list_del(&spa_map->list);
1370 kfree(spa_map);
1371}
1372
1373static struct nfit_spa_mapping *find_spa_mapping(
1374 struct acpi_nfit_desc *acpi_desc,
1375 struct acpi_nfit_system_address *spa)
1376{
1377 struct nfit_spa_mapping *spa_map;
1378
1379 WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1380 list_for_each_entry(spa_map, &acpi_desc->spa_maps, list)
1381 if (spa_map->spa == spa)
1382 return spa_map;
1383
1384 return NULL;
1385}
1386
1387static void nfit_spa_unmap(struct acpi_nfit_desc *acpi_desc,
1388 struct acpi_nfit_system_address *spa)
1389{
1390 struct nfit_spa_mapping *spa_map;
1391
1392 mutex_lock(&acpi_desc->spa_map_mutex);
1393 spa_map = find_spa_mapping(acpi_desc, spa);
1394
1395 if (spa_map)
1396 kref_put(&spa_map->kref, nfit_spa_mapping_release);
1397 mutex_unlock(&acpi_desc->spa_map_mutex);
1398}
1399
1400static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
c2ad2954 1401 struct acpi_nfit_system_address *spa, enum spa_map_type type)
047fc8a1
RZ		 1402{
1403 resource_size_t start = spa->address;
1404 resource_size_t n = spa->length;
1405 struct nfit_spa_mapping *spa_map;
1406 struct resource *res;
1407
1408 WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1409
1410 spa_map = find_spa_mapping(acpi_desc, spa);
1411 if (spa_map) {
1412 kref_get(&spa_map->kref);
67a3e8fe 1413 return spa_map->addr.base;
047fc8a1
RZ		 1414 }
1415
1416 spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL);
1417 if (!spa_map)
1418 return NULL;
1419
1420 INIT_LIST_HEAD(&spa_map->list);
1421 spa_map->spa = spa;
1422 kref_init(&spa_map->kref);
1423 spa_map->acpi_desc = acpi_desc;
1424
1425 res = request_mem_region(start, n, dev_name(acpi_desc->dev));
1426 if (!res)
1427 goto err_mem;
1428
67a3e8fe
RZ		 1429 spa_map->type = type;
1430 if (type == SPA_MAP_APERTURE)
1431 spa_map->addr.aperture = (void __pmem *)memremap(start, n,
1432 ARCH_MEMREMAP_PMEM);
1433 else
1434 spa_map->addr.base = ioremap_nocache(start, n);
1435
c2ad2954 1436
67a3e8fe 1437 if (!spa_map->addr.base)
047fc8a1
RZ		 1438 goto err_map;
1439
1440 list_add_tail(&spa_map->list, &acpi_desc->spa_maps);
67a3e8fe 1441 return spa_map->addr.base;
047fc8a1
RZ		 1442
1443 err_map:
1444 release_mem_region(start, n);
1445 err_mem:
1446 kfree(spa_map);
1447 return NULL;
1448}
1449
1450/**
1451 * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1452 * @nvdimm_bus: NFIT-bus that provided the spa table entry
1453 * @nfit_spa: spa table to map
c2ad2954 1454 * @type: aperture or control region
047fc8a1
RZ		 1455 *
1456 * In the case where block-data-window apertures and
1457 * dimm-control-regions are interleaved they will end up sharing a
1458 * single request_mem_region() + ioremap() for the address range. In
1459 * the style of devm nfit_spa_map() mappings are automatically dropped
1460 * when all region devices referencing the same mapping are disabled /
1461 * unbound.
1462 */
1463static void __iomem *nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
c2ad2954 1464 struct acpi_nfit_system_address *spa, enum spa_map_type type)
047fc8a1
RZ		 1465{
1466 void __iomem *iomem;
1467
1468 mutex_lock(&acpi_desc->spa_map_mutex);
c2ad2954 1469 iomem = __nfit_spa_map(acpi_desc, spa, type);
047fc8a1
RZ		 1470 mutex_unlock(&acpi_desc->spa_map_mutex);
1471
1472 return iomem;
1473}
1474
1475static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
1476 struct acpi_nfit_interleave *idt, u16 interleave_ways)
1477{
1478 if (idt) {
1479 mmio->num_lines = idt->line_count;
1480 mmio->line_size = idt->line_size;
1481 if (interleave_ways == 0)
1482 return -ENXIO;
1483 mmio->table_size = mmio->num_lines * interleave_ways
1484 * mmio->line_size;
1485 }
1486
1487 return 0;
1488}
1489
f0f2c072
RZ		 1490static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
1491 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
1492{
1493 struct nd_cmd_dimm_flags flags;
1494 int rc;
1495
1496 memset(&flags, 0, sizeof(flags));
1497 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
aef25338 1498 sizeof(flags), NULL);
f0f2c072
RZ		 1499
1500 if (rc >= 0 && flags.status == 0)
1501 nfit_blk->dimm_flags = flags.flags;
1502 else if (rc == -ENOTTY) {
1503 /* fall back to a conservative default */
aef25338 1504 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
f0f2c072
RZ		 1505 rc = 0;
1506 } else
1507 rc = -ENXIO;
1508
1509 return rc;
1510}
1511
047fc8a1
RZ		 1512static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
1513 struct device *dev)
1514{
1515 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1516 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1517 struct nd_blk_region *ndbr = to_nd_blk_region(dev);
c2ad2954 1518 struct nfit_flush *nfit_flush;
047fc8a1
RZ		 1519 struct nfit_blk_mmio *mmio;
1520 struct nfit_blk *nfit_blk;
1521 struct nfit_mem *nfit_mem;
1522 struct nvdimm *nvdimm;
1523 int rc;
1524
1525 nvdimm = nd_blk_region_to_dimm(ndbr);
1526 nfit_mem = nvdimm_provider_data(nvdimm);
1527 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
1528 dev_dbg(dev, "%s: missing%s%s%s\n", __func__,
1529 nfit_mem ? "" : " nfit_mem",
193ccca4
DW		 1530 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
1531 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
047fc8a1
RZ		 1532 return -ENXIO;
1533 }
1534
1535 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
1536 if (!nfit_blk)
1537 return -ENOMEM;
1538 nd_blk_region_set_provider_data(ndbr, nfit_blk);
1539 nfit_blk->nd_region = to_nd_region(dev);
1540
1541 /* map block aperture memory */
1542 nfit_blk->bdw_offset = nfit_mem->bdw->offset;
1543 mmio = &nfit_blk->mmio[BDW];
67a3e8fe 1544 mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw,
c2ad2954 1545 SPA_MAP_APERTURE);
67a3e8fe 1546 if (!mmio->addr.base) {
047fc8a1
RZ		 1547 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__,
1548 nvdimm_name(nvdimm));
1549 return -ENOMEM;
1550 }
1551 mmio->size = nfit_mem->bdw->size;
1552 mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
1553 mmio->idt = nfit_mem->idt_bdw;
1554 mmio->spa = nfit_mem->spa_bdw;
1555 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
1556 nfit_mem->memdev_bdw->interleave_ways);
1557 if (rc) {
1558 dev_dbg(dev, "%s: %s failed to init bdw interleave\n",
1559 __func__, nvdimm_name(nvdimm));
1560 return rc;
1561 }
1562
1563 /* map block control memory */
1564 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
1565 nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
1566 mmio = &nfit_blk->mmio[DCR];
67a3e8fe 1567 mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr,
c2ad2954 1568 SPA_MAP_CONTROL);
67a3e8fe 1569 if (!mmio->addr.base) {
047fc8a1
RZ		 1570 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__,
1571 nvdimm_name(nvdimm));
1572 return -ENOMEM;
1573 }
1574 mmio->size = nfit_mem->dcr->window_size;
1575 mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
1576 mmio->idt = nfit_mem->idt_dcr;
1577 mmio->spa = nfit_mem->spa_dcr;
1578 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
1579 nfit_mem->memdev_dcr->interleave_ways);
1580 if (rc) {
1581 dev_dbg(dev, "%s: %s failed to init dcr interleave\n",
1582 __func__, nvdimm_name(nvdimm));
1583 return rc;
1584 }
1585
f0f2c072
RZ		 1586 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
1587 if (rc < 0) {
1588 dev_dbg(dev, "%s: %s failed get DIMM flags\n",
1589 __func__, nvdimm_name(nvdimm));
1590 return rc;
1591 }
1592
c2ad2954
RZ		 1593 nfit_flush = nfit_mem->nfit_flush;
1594 if (nfit_flush && nfit_flush->flush->hint_count != 0) {
1595 nfit_blk->nvdimm_flush = devm_ioremap_nocache(dev,
1596 nfit_flush->flush->hint_address[0], 8);
1597 if (!nfit_blk->nvdimm_flush)
1598 return -ENOMEM;
1599 }
1600
96601adb 1601 if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush)
c2ad2954
RZ		 1602 dev_warn(dev, "unable to guarantee persistence of writes\n");
1603
047fc8a1
RZ		 1604 if (mmio->line_size == 0)
1605 return 0;
1606
1607 if ((u32) nfit_blk->cmd_offset % mmio->line_size
1608 + 8 > mmio->line_size) {
1609 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
1610 return -ENXIO;
1611 } else if ((u32) nfit_blk->stat_offset % mmio->line_size
1612 + 8 > mmio->line_size) {
1613 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
1614 return -ENXIO;
1615 }
1616
1617 return 0;
1618}
1619
1620static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus,
1621 struct device *dev)
1622{
1623 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1624 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1625 struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1626 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1627 int i;
1628
1629 if (!nfit_blk)
1630 return; /* never enabled */
1631
1632 /* auto-free BLK spa mappings */
1633 for (i = 0; i < 2; i++) {
1634 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i];
1635
67a3e8fe 1636 if (mmio->addr.base)
047fc8a1
RZ		 1637 nfit_spa_unmap(acpi_desc, mmio->spa);
1638 }
1639 nd_blk_region_set_provider_data(ndbr, NULL);
1640 /* devm will free nfit_blk */
1641}
1642
aef25338 1643static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
1cf03c00 1644 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
0caeef63 1645{
aef25338 1646 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1cf03c00 1647 struct acpi_nfit_system_address *spa = nfit_spa->spa;
aef25338
DW		 1648 int cmd_rc, rc;
1649
1cf03c00
DW		 1650 cmd->address = spa->address;
1651 cmd->length = spa->length;
aef25338
DW		 1652 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
1653 sizeof(*cmd), &cmd_rc);
1654 if (rc < 0)
1655 return rc;
1cf03c00 1656 return cmd_rc;
0caeef63
VV		 1657}
1658
1cf03c00 1659static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa)
0caeef63
VV		 1660{
1661 int rc;
1cf03c00
DW		 1662 int cmd_rc;
1663 struct nd_cmd_ars_start ars_start;
1664 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1665 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
0caeef63 1666
1cf03c00
DW		 1667 memset(&ars_start, 0, sizeof(ars_start));
1668 ars_start.address = spa->address;
1669 ars_start.length = spa->length;
1670 if (nfit_spa_type(spa) == NFIT_SPA_PM)
1671 ars_start.type = ND_ARS_PERSISTENT;
1672 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
1673 ars_start.type = ND_ARS_VOLATILE;
1674 else
1675 return -ENOTTY;
aef25338 1676
1cf03c00
DW		 1677 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
1678 sizeof(ars_start), &cmd_rc);
aef25338 1679
1cf03c00
DW		 1680 if (rc < 0)
1681 return rc;
1682 return cmd_rc;
0caeef63
VV		 1683}
1684
1cf03c00 1685static int ars_continue(struct acpi_nfit_desc *acpi_desc)
0caeef63 1686{
aef25338 1687 int rc, cmd_rc;
1cf03c00
DW		 1688 struct nd_cmd_ars_start ars_start;
1689 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1690 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
1691
1692 memset(&ars_start, 0, sizeof(ars_start));
1693 ars_start.address = ars_status->restart_address;
1694 ars_start.length = ars_status->restart_length;
1695 ars_start.type = ars_status->type;
1696 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
1697 sizeof(ars_start), &cmd_rc);
1698 if (rc < 0)
1699 return rc;
1700 return cmd_rc;
1701}
0caeef63 1702
1cf03c00
DW		 1703static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
1704{
1705 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1706 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
1707 int rc, cmd_rc;
aef25338 1708
1cf03c00
DW		 1709 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
1710 acpi_desc->ars_status_size, &cmd_rc);
1711 if (rc < 0)
1712 return rc;
1713 return cmd_rc;
0caeef63
VV		 1714}
1715
1716static int ars_status_process_records(struct nvdimm_bus *nvdimm_bus,
1cf03c00 1717 struct nd_cmd_ars_status *ars_status)
0caeef63
VV		 1718{
1719 int rc;
1720 u32 i;
1721
0caeef63
VV		 1722 for (i = 0; i < ars_status->num_records; i++) {
1723 rc = nvdimm_bus_add_poison(nvdimm_bus,
1724 ars_status->records[i].err_address,
1725 ars_status->records[i].length);
1726 if (rc)
1727 return rc;
1728 }
1729
1730 return 0;
1731}
1732
af1996ef
TK		 1733static void acpi_nfit_remove_resource(void *data)
1734{
1735 struct resource *res = data;
1736
1737 remove_resource(res);
1738}
1739
1740static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
1741 struct nd_region_desc *ndr_desc)
1742{
1743 struct resource *res, *nd_res = ndr_desc->res;
1744 int is_pmem, ret;
1745
1746 /* No operation if the region is already registered as PMEM */
1747 is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
1748 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
1749 if (is_pmem == REGION_INTERSECTS)
1750 return 0;
1751
1752 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
1753 if (!res)
1754 return -ENOMEM;
1755
1756 res->name = "Persistent Memory";
1757 res->start = nd_res->start;
1758 res->end = nd_res->end;
1759 res->flags = IORESOURCE_MEM;
1760 res->desc = IORES_DESC_PERSISTENT_MEMORY;
1761
1762 ret = insert_resource(&iomem_resource, res);
1763 if (ret)
1764 return ret;
1765
1766 ret = devm_add_action(acpi_desc->dev, acpi_nfit_remove_resource, res);
1767 if (ret) {
1768 remove_resource(res);
1769 return ret;
1770 }
1771
1772 return 0;
1773}
1774
1f7df6f8
DW		 1775static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
1776 struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc,
1777 struct acpi_nfit_memory_map *memdev,
1cf03c00 1778 struct nfit_spa *nfit_spa)
1f7df6f8
DW		 1779{
1780 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
1781 memdev->device_handle);
1cf03c00 1782 struct acpi_nfit_system_address *spa = nfit_spa->spa;
047fc8a1 1783 struct nd_blk_region_desc *ndbr_desc;
1f7df6f8
DW		 1784 struct nfit_mem *nfit_mem;
1785 int blk_valid = 0;
1786
1787 if (!nvdimm) {
1788 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
1789 spa->range_index, memdev->device_handle);
1790 return -ENODEV;
1791 }
1792
1793 nd_mapping->nvdimm = nvdimm;
1794 switch (nfit_spa_type(spa)) {
1795 case NFIT_SPA_PM:
1796 case NFIT_SPA_VOLATILE:
1797 nd_mapping->start = memdev->address;
1798 nd_mapping->size = memdev->region_size;
1799 break;
1800 case NFIT_SPA_DCR:
1801 nfit_mem = nvdimm_provider_data(nvdimm);
1802 if (!nfit_mem || !nfit_mem->bdw) {
1803 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
1804 spa->range_index, nvdimm_name(nvdimm));
1805 } else {
1806 nd_mapping->size = nfit_mem->bdw->capacity;
1807 nd_mapping->start = nfit_mem->bdw->start_address;
5212e11f 1808 ndr_desc->num_lanes = nfit_mem->bdw->windows;
1f7df6f8
DW		 1809 blk_valid = 1;
1810 }
1811
1812 ndr_desc->nd_mapping = nd_mapping;
1813 ndr_desc->num_mappings = blk_valid;
047fc8a1
RZ		 1814 ndbr_desc = to_blk_region_desc(ndr_desc);
1815 ndbr_desc->enable = acpi_nfit_blk_region_enable;
1816 ndbr_desc->disable = acpi_nfit_blk_region_disable;
6bc75619 1817 ndbr_desc->do_io = acpi_desc->blk_do_io;
1cf03c00
DW		 1818 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
1819 ndr_desc);
1820 if (!nfit_spa->nd_region)
1f7df6f8
DW		 1821 return -ENOMEM;
1822 break;
1823 }
1824
1825 return 0;
1826}
1827
1828static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
1829 struct nfit_spa *nfit_spa)
1830{
1831 static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS];
1832 struct acpi_nfit_system_address *spa = nfit_spa->spa;
047fc8a1
RZ		 1833 struct nd_blk_region_desc ndbr_desc;
1834 struct nd_region_desc *ndr_desc;
1f7df6f8 1835 struct nfit_memdev *nfit_memdev;
1f7df6f8
DW		 1836 struct nvdimm_bus *nvdimm_bus;
1837 struct resource res;
eaf96153 1838 int count = 0, rc;
1f7df6f8 1839
1cf03c00 1840 if (nfit_spa->nd_region)
20985164
VV		 1841 return 0;
1842
1f7df6f8
DW		 1843 if (spa->range_index == 0) {
1844 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n",
1845 __func__);
1846 return 0;
1847 }
1848
1849 memset(&res, 0, sizeof(res));
1850 memset(&nd_mappings, 0, sizeof(nd_mappings));
047fc8a1 1851 memset(&ndbr_desc, 0, sizeof(ndbr_desc));
1f7df6f8
DW		 1852 res.start = spa->address;
1853 res.end = res.start + spa->length - 1;
047fc8a1
RZ		 1854 ndr_desc = &ndbr_desc.ndr_desc;
1855 ndr_desc->res = &res;
1856 ndr_desc->provider_data = nfit_spa;
1857 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
41d7a6d6
TK		 1858 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
1859 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
1860 spa->proximity_domain);
1861 else
1862 ndr_desc->numa_node = NUMA_NO_NODE;
1863
1f7df6f8
DW		 1864 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1865 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1866 struct nd_mapping *nd_mapping;
1f7df6f8
DW		 1867
1868 if (memdev->range_index != spa->range_index)
1869 continue;
1870 if (count >= ND_MAX_MAPPINGS) {
1871 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
1872 spa->range_index, ND_MAX_MAPPINGS);
1873 return -ENXIO;
1874 }
1875 nd_mapping = &nd_mappings[count++];
047fc8a1 1876 rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc,
1cf03c00 1877 memdev, nfit_spa);
1f7df6f8 1878 if (rc)
1cf03c00 1879 goto out;
1f7df6f8
DW		 1880 }
1881
047fc8a1
RZ		 1882 ndr_desc->nd_mapping = nd_mappings;
1883 ndr_desc->num_mappings = count;
1884 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
eaf96153 1885 if (rc)
1cf03c00 1886 goto out;
eaf96153 1887
1f7df6f8
DW		 1888 nvdimm_bus = acpi_desc->nvdimm_bus;
1889 if (nfit_spa_type(spa) == NFIT_SPA_PM) {
af1996ef 1890 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
48901165 1891 if (rc) {
af1996ef
TK		 1892 dev_warn(acpi_desc->dev,
1893 "failed to insert pmem resource to iomem: %d\n",
1894 rc);
48901165 1895 goto out;
0caeef63 1896 }
48901165 1897
1cf03c00
DW		 1898 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
1899 ndr_desc);
1900 if (!nfit_spa->nd_region)
1901 rc = -ENOMEM;
1f7df6f8 1902 } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) {
1cf03c00
DW		 1903 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
1904 ndr_desc);
1905 if (!nfit_spa->nd_region)
1906 rc = -ENOMEM;
1f7df6f8 1907 }
20985164 1908
1cf03c00
DW		 1909 out:
1910 if (rc)
1911 dev_err(acpi_desc->dev, "failed to register spa range %d\n",
1912 nfit_spa->spa->range_index);
1913 return rc;
1914}
1915
1916static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc,
1917 u32 max_ars)
1918{
1919 struct device *dev = acpi_desc->dev;
1920 struct nd_cmd_ars_status *ars_status;
1921
1922 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) {
1923 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size);
1924 return 0;
1925 }
1926
1927 if (acpi_desc->ars_status)
1928 devm_kfree(dev, acpi_desc->ars_status);
1929 acpi_desc->ars_status = NULL;
1930 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL);
1931 if (!ars_status)
1932 return -ENOMEM;
1933 acpi_desc->ars_status = ars_status;
1934 acpi_desc->ars_status_size = max_ars;
1f7df6f8
DW		 1935 return 0;
1936}
1937
1cf03c00
DW		 1938static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc,
1939 struct nfit_spa *nfit_spa)
1940{
1941 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1942 int rc;
1943
1944 if (!nfit_spa->max_ars) {
1945 struct nd_cmd_ars_cap ars_cap;
1946
1947 memset(&ars_cap, 0, sizeof(ars_cap));
1948 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
1949 if (rc < 0)
1950 return rc;
1951 nfit_spa->max_ars = ars_cap.max_ars_out;
1952 nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
1953 /* check that the supported scrub types match the spa type */
1954 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE &&
1955 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0)
1956 return -ENOTTY;
1957 else if (nfit_spa_type(spa) == NFIT_SPA_PM &&
1958 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0)
1959 return -ENOTTY;
1960 }
1961
1962 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars))
1963 return -ENOMEM;
1964
1965 rc = ars_get_status(acpi_desc);
1966 if (rc < 0 && rc != -ENOSPC)
1967 return rc;
1968
1969 if (ars_status_process_records(acpi_desc->nvdimm_bus,
1970 acpi_desc->ars_status))
1971 return -ENOMEM;
1972
1973 return 0;
1974}
1975
1976static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc,
1977 struct nfit_spa *nfit_spa)
1978{
1979 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1980 unsigned int overflow_retry = scrub_overflow_abort;
1981 u64 init_ars_start = 0, init_ars_len = 0;
1982 struct device *dev = acpi_desc->dev;
1983 unsigned int tmo = scrub_timeout;
1984 int rc;
1985
1986 if (nfit_spa->ars_done || !nfit_spa->nd_region)
1987 return;
1988
1989 rc = ars_start(acpi_desc, nfit_spa);
1990 /*
1991 * If we timed out the initial scan we'll still be busy here,
1992 * and will wait another timeout before giving up permanently.
1993 */
1994 if (rc < 0 && rc != -EBUSY)
1995 return;
1996
1997 do {
1998 u64 ars_start, ars_len;
1999
2000 if (acpi_desc->cancel)
2001 break;
2002 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa);
2003 if (rc == -ENOTTY)
2004 break;
2005 if (rc == -EBUSY && !tmo) {
2006 dev_warn(dev, "range %d ars timeout, aborting\n",
2007 spa->range_index);
2008 break;
2009 }
2010
2011 if (rc == -EBUSY) {
2012 /*
2013 * Note, entries may be appended to the list
2014 * while the lock is dropped, but the workqueue
2015 * being active prevents entries being deleted /
2016 * freed.
2017 */
2018 mutex_unlock(&acpi_desc->init_mutex);
2019 ssleep(1);
2020 tmo--;
2021 mutex_lock(&acpi_desc->init_mutex);
2022 continue;
2023 }
2024
2025 /* we got some results, but there are more pending... */
2026 if (rc == -ENOSPC && overflow_retry--) {
2027 if (!init_ars_len) {
2028 init_ars_len = acpi_desc->ars_status->length;
2029 init_ars_start = acpi_desc->ars_status->address;
2030 }
2031 rc = ars_continue(acpi_desc);
2032 }
2033
2034 if (rc < 0) {
2035 dev_warn(dev, "range %d ars continuation failed\n",
2036 spa->range_index);
2037 break;
2038 }
2039
2040 if (init_ars_len) {
2041 ars_start = init_ars_start;
2042 ars_len = init_ars_len;
2043 } else {
2044 ars_start = acpi_desc->ars_status->address;
2045 ars_len = acpi_desc->ars_status->length;
2046 }
2047 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n",
2048 spa->range_index, ars_start, ars_len);
2049 /* notify the region about new poison entries */
2050 nvdimm_region_notify(nfit_spa->nd_region,
2051 NVDIMM_REVALIDATE_POISON);
2052 break;
2053 } while (1);
2054}
2055
2056static void acpi_nfit_scrub(struct work_struct *work)
1f7df6f8 2057{
1cf03c00
DW		 2058 struct device *dev;
2059 u64 init_scrub_length = 0;
1f7df6f8 2060 struct nfit_spa *nfit_spa;
1cf03c00
DW		 2061 u64 init_scrub_address = 0;
2062 bool init_ars_done = false;
2063 struct acpi_nfit_desc *acpi_desc;
2064 unsigned int tmo = scrub_timeout;
2065 unsigned int overflow_retry = scrub_overflow_abort;
2066
2067 acpi_desc = container_of(work, typeof(*acpi_desc), work);
2068 dev = acpi_desc->dev;
1f7df6f8 2069
1cf03c00
DW		 2070 /*
2071 * We scrub in 2 phases. The first phase waits for any platform
2072 * firmware initiated scrubs to complete and then we go search for the
2073 * affected spa regions to mark them scanned. In the second phase we
2074 * initiate a directed scrub for every range that was not scrubbed in
2075 * phase 1.
2076 */
2077
2078 /* process platform firmware initiated scrubs */
2079 retry:
2080 mutex_lock(&acpi_desc->init_mutex);
1f7df6f8 2081 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1cf03c00
DW		 2082 struct nd_cmd_ars_status *ars_status;
2083 struct acpi_nfit_system_address *spa;
2084 u64 ars_start, ars_len;
2085 int rc;
1f7df6f8 2086
1cf03c00
DW		 2087 if (acpi_desc->cancel)
2088 break;
2089
2090 if (nfit_spa->nd_region)
2091 continue;
2092
2093 if (init_ars_done) {
2094 /*
2095 * No need to re-query, we're now just
2096 * reconciling all the ranges covered by the
2097 * initial scrub
2098 */
2099 rc = 0;
2100 } else
2101 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa);
2102
2103 if (rc == -ENOTTY) {
2104 /* no ars capability, just register spa and move on */
2105 acpi_nfit_register_region(acpi_desc, nfit_spa);
2106 continue;
2107 }
2108
2109 if (rc == -EBUSY && !tmo) {
2110 /* fallthrough to directed scrub in phase 2 */
2111 dev_warn(dev, "timeout awaiting ars results, continuing...\n");
2112 break;
2113 } else if (rc == -EBUSY) {
2114 mutex_unlock(&acpi_desc->init_mutex);
2115 ssleep(1);
2116 tmo--;
2117 goto retry;
2118 }
2119
2120 /* we got some results, but there are more pending... */
2121 if (rc == -ENOSPC && overflow_retry--) {
2122 ars_status = acpi_desc->ars_status;
2123 /*
2124 * Record the original scrub range, so that we
2125 * can recall all the ranges impacted by the
2126 * initial scrub.
2127 */
2128 if (!init_scrub_length) {
2129 init_scrub_length = ars_status->length;
2130 init_scrub_address = ars_status->address;
2131 }
2132 rc = ars_continue(acpi_desc);
2133 if (rc == 0) {
2134 mutex_unlock(&acpi_desc->init_mutex);
2135 goto retry;
2136 }
2137 }
2138
2139 if (rc < 0) {
2140 /*
2141 * Initial scrub failed, we'll give it one more
2142 * try below...
2143 */
2144 break;
2145 }
2146
2147 /* We got some final results, record completed ranges */
2148 ars_status = acpi_desc->ars_status;
2149 if (init_scrub_length) {
2150 ars_start = init_scrub_address;
2151 ars_len = ars_start + init_scrub_length;
2152 } else {
2153 ars_start = ars_status->address;
2154 ars_len = ars_status->length;
2155 }
2156 spa = nfit_spa->spa;
2157
2158 if (!init_ars_done) {
2159 init_ars_done = true;
2160 dev_dbg(dev, "init scrub %#llx + %#llx complete\n",
2161 ars_start, ars_len);
2162 }
2163 if (ars_start <= spa->address && ars_start + ars_len
2164 >= spa->address + spa->length)
2165 acpi_nfit_register_region(acpi_desc, nfit_spa);
1f7df6f8 2166 }
1cf03c00
DW		 2167
2168 /*
2169 * For all the ranges not covered by an initial scrub we still
2170 * want to see if there are errors, but it's ok to discover them
2171 * asynchronously.
2172 */
2173 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2174 /*
2175 * Flag all the ranges that still need scrubbing, but
2176 * register them now to make data available.
2177 */
2178 if (nfit_spa->nd_region)
2179 nfit_spa->ars_done = 1;
2180 else
2181 acpi_nfit_register_region(acpi_desc, nfit_spa);
2182 }
2183
2184 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
2185 acpi_nfit_async_scrub(acpi_desc, nfit_spa);
2186 mutex_unlock(&acpi_desc->init_mutex);
2187}
2188
2189static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
2190{
2191 struct nfit_spa *nfit_spa;
2192 int rc;
2193
2194 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
2195 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) {
2196 /* BLK regions don't need to wait for ars results */
2197 rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
2198 if (rc)
2199 return rc;
2200 }
2201
2202 queue_work(nfit_wq, &acpi_desc->work);
1f7df6f8
DW		 2203 return 0;
2204}
2205
20985164
VV		 2206static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
2207 struct nfit_table_prev *prev)
2208{
2209 struct device *dev = acpi_desc->dev;
2210
2211 if (!list_empty(&prev->spas) ||
2212 !list_empty(&prev->memdevs) ||
2213 !list_empty(&prev->dcrs) ||
2214 !list_empty(&prev->bdws) ||
2215 !list_empty(&prev->idts) ||
2216 !list_empty(&prev->flushes)) {
2217 dev_err(dev, "new nfit deletes entries (unsupported)\n");
2218 return -ENXIO;
2219 }
2220 return 0;
2221}
2222
6bc75619 2223int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, acpi_size sz)
b94d5230
DW		 2224{
2225 struct device *dev = acpi_desc->dev;
20985164 2226 struct nfit_table_prev prev;
b94d5230
DW		 2227 const void *end;
2228 u8 *data;
1f7df6f8 2229 int rc;
b94d5230 2230
20985164
VV		 2231 mutex_lock(&acpi_desc->init_mutex);
2232
2233 INIT_LIST_HEAD(&prev.spas);
2234 INIT_LIST_HEAD(&prev.memdevs);
2235 INIT_LIST_HEAD(&prev.dcrs);
2236 INIT_LIST_HEAD(&prev.bdws);
2237 INIT_LIST_HEAD(&prev.idts);
2238 INIT_LIST_HEAD(&prev.flushes);
2239
2240 list_cut_position(&prev.spas, &acpi_desc->spas,
2241 acpi_desc->spas.prev);
2242 list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
2243 acpi_desc->memdevs.prev);
2244 list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
2245 acpi_desc->dcrs.prev);
2246 list_cut_position(&prev.bdws, &acpi_desc->bdws,
2247 acpi_desc->bdws.prev);
2248 list_cut_position(&prev.idts, &acpi_desc->idts,
2249 acpi_desc->idts.prev);
2250 list_cut_position(&prev.flushes, &acpi_desc->flushes,
2251 acpi_desc->flushes.prev);
b94d5230
DW		 2252
2253 data = (u8 *) acpi_desc->nfit;
2254 end = data + sz;
b94d5230 2255 while (!IS_ERR_OR_NULL(data))
20985164 2256 data = add_table(acpi_desc, &prev, data, end);
b94d5230
DW		 2257
2258 if (IS_ERR(data)) {
2259 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__,
2260 PTR_ERR(data));
20985164
VV		 2261 rc = PTR_ERR(data);
2262 goto out_unlock;
b94d5230
DW		 2263 }
2264
20985164
VV		 2265 rc = acpi_nfit_check_deletions(acpi_desc, &prev);
2266 if (rc)
2267 goto out_unlock;
2268
2269 if (nfit_mem_init(acpi_desc) != 0) {
2270 rc = -ENOMEM;
2271 goto out_unlock;
2272 }
b94d5230 2273
62232e45
DW		 2274 acpi_nfit_init_dsms(acpi_desc);
2275
1f7df6f8
DW		 2276 rc = acpi_nfit_register_dimms(acpi_desc);
2277 if (rc)
20985164
VV		 2278 goto out_unlock;
2279
2280 rc = acpi_nfit_register_regions(acpi_desc);
1f7df6f8 2281
20985164
VV		 2282 out_unlock:
2283 mutex_unlock(&acpi_desc->init_mutex);
2284 return rc;
b94d5230 2285}
6bc75619 2286EXPORT_SYMBOL_GPL(acpi_nfit_init);
b94d5230 2287
7ae0fa43
DW		 2288struct acpi_nfit_flush_work {
2289 struct work_struct work;
2290 struct completion cmp;
2291};
2292
2293static void flush_probe(struct work_struct *work)
2294{
2295 struct acpi_nfit_flush_work *flush;
2296
2297 flush = container_of(work, typeof(*flush), work);
2298 complete(&flush->cmp);
2299}
2300
2301static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
2302{
2303 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
2304 struct device *dev = acpi_desc->dev;
2305 struct acpi_nfit_flush_work flush;
2306
2307 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
2308 device_lock(dev);
2309 device_unlock(dev);
2310
2311 /*
2312 * Scrub work could take 10s of seconds, userspace may give up so we
2313 * need to be interruptible while waiting.
2314 */
2315 INIT_WORK_ONSTACK(&flush.work, flush_probe);
2316 COMPLETION_INITIALIZER_ONSTACK(flush.cmp);
2317 queue_work(nfit_wq, &flush.work);
2318 return wait_for_completion_interruptible(&flush.cmp);
2319}
2320
87bf572e
DW		 2321static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
2322 struct nvdimm *nvdimm, unsigned int cmd)
2323{
2324 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
2325
2326 if (nvdimm)
2327 return 0;
2328 if (cmd != ND_CMD_ARS_START)
2329 return 0;
2330
2331 /*
2332 * The kernel and userspace may race to initiate a scrub, but
2333 * the scrub thread is prepared to lose that initial race. It
2334 * just needs guarantees that any ars it initiates are not
2335 * interrupted by any intervening start reqeusts from userspace.
2336 */
2337 if (work_busy(&acpi_desc->work))
2338 return -EBUSY;
2339
2340 return 0;
2341}
2342
a61fe6f7 2343void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
b94d5230
DW		 2344{
2345 struct nvdimm_bus_descriptor *nd_desc;
b94d5230
DW		 2346
2347 dev_set_drvdata(dev, acpi_desc);
2348 acpi_desc->dev = dev;
6bc75619 2349 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
b94d5230
DW		 2350 nd_desc = &acpi_desc->nd_desc;
2351 nd_desc->provider_name = "ACPI.NFIT";
2352 nd_desc->ndctl = acpi_nfit_ctl;
7ae0fa43 2353 nd_desc->flush_probe = acpi_nfit_flush_probe;
87bf572e 2354 nd_desc->clear_to_send = acpi_nfit_clear_to_send;
45def22c 2355 nd_desc->attr_groups = acpi_nfit_attribute_groups;
b94d5230 2356
20985164
VV		 2357 INIT_LIST_HEAD(&acpi_desc->spa_maps);
2358 INIT_LIST_HEAD(&acpi_desc->spas);
2359 INIT_LIST_HEAD(&acpi_desc->dcrs);
2360 INIT_LIST_HEAD(&acpi_desc->bdws);
2361 INIT_LIST_HEAD(&acpi_desc->idts);
2362 INIT_LIST_HEAD(&acpi_desc->flushes);
2363 INIT_LIST_HEAD(&acpi_desc->memdevs);
2364 INIT_LIST_HEAD(&acpi_desc->dimms);
2365 mutex_init(&acpi_desc->spa_map_mutex);
2366 mutex_init(&acpi_desc->init_mutex);
1cf03c00 2367 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub);
20985164 2368}
a61fe6f7 2369EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
20985164
VV		 2370
2371static int acpi_nfit_add(struct acpi_device *adev)
2372{
2373 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
2374 struct acpi_nfit_desc *acpi_desc;
2375 struct device *dev = &adev->dev;
2376 struct acpi_table_header *tbl;
2377 acpi_status status = AE_OK;
2378 acpi_size sz;
2379 int rc;
2380
2381 status = acpi_get_table_with_size("NFIT", 0, &tbl, &sz);
2382 if (ACPI_FAILURE(status)) {
2383 /* This is ok, we could have an nvdimm hotplugged later */
2384 dev_dbg(dev, "failed to find NFIT at startup\n");
2385 return 0;
2386 }
2387
a61fe6f7
DW		 2388 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
2389 if (!acpi_desc)
2390 return -ENOMEM;
2391 acpi_nfit_desc_init(acpi_desc, &adev->dev);
2392 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, &acpi_desc->nd_desc);
2393 if (!acpi_desc->nvdimm_bus)
2394 return -ENOMEM;
20985164 2395
6b577c9d
LK		 2396 /*
2397 * Save the acpi header for later and then skip it,
2398 * making nfit point to the first nfit table header.
2399 */
2400 acpi_desc->acpi_header = *tbl;
2401 acpi_desc->nfit = (void *) tbl + sizeof(struct acpi_table_nfit);
2402 sz -= sizeof(struct acpi_table_nfit);
20985164
VV		 2403
2404 /* Evaluate _FIT and override with that if present */
2405 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
2406 if (ACPI_SUCCESS(status) && buf.length > 0) {
6b577c9d
LK		 2407 union acpi_object *obj;
2408 /*
2409 * Adjust for the acpi_object header of the _FIT
2410 */
2411 obj = buf.pointer;
2412 if (obj->type == ACPI_TYPE_BUFFER) {
2413 acpi_desc->nfit =
2414 (struct acpi_nfit_header *)obj->buffer.pointer;
2415 sz = obj->buffer.length;
2416 } else
2417 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n",
2418 __func__, (int) obj->type);
20985164 2419 }
b94d5230
DW		 2420
2421 rc = acpi_nfit_init(acpi_desc, sz);
2422 if (rc) {
2423 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
2424 return rc;
2425 }
2426 return 0;
2427}
2428
2429static int acpi_nfit_remove(struct acpi_device *adev)
2430{
2431 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
2432
7ae0fa43
DW		 2433 acpi_desc->cancel = 1;
2434 flush_workqueue(nfit_wq);
b94d5230
DW		 2435 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
2436 return 0;
2437}
2438
20985164
VV		 2439static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
2440{
2441 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
2442 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
6b577c9d
LK		 2443 struct acpi_nfit_header *nfit_saved;
2444 union acpi_object *obj;
20985164
VV		 2445 struct device *dev = &adev->dev;
2446 acpi_status status;
2447 int ret;
2448
2449 dev_dbg(dev, "%s: event: %d\n", __func__, event);
2450
2451 device_lock(dev);
2452 if (!dev->driver) {
2453 /* dev->driver may be null if we're being removed */
2454 dev_dbg(dev, "%s: no driver found for dev\n", __func__);
d91e8928 2455 goto out_unlock;
20985164
VV		 2456 }
2457
2458 if (!acpi_desc) {
a61fe6f7
DW		 2459 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
2460 if (!acpi_desc)
2461 goto out_unlock;
2462 acpi_nfit_desc_init(acpi_desc, &adev->dev);
2463 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, &acpi_desc->nd_desc);
2464 if (!acpi_desc->nvdimm_bus)
20985164 2465 goto out_unlock;
7ae0fa43
DW		 2466 } else {
2467 /*
2468 * Finish previous registration before considering new
2469 * regions.
2470 */
2471 flush_workqueue(nfit_wq);
20985164
VV		 2472 }
2473
2474 /* Evaluate _FIT */
2475 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
2476 if (ACPI_FAILURE(status)) {
2477 dev_err(dev, "failed to evaluate _FIT\n");
2478 goto out_unlock;
2479 }
2480
2481 nfit_saved = acpi_desc->nfit;
6b577c9d
LK		 2482 obj = buf.pointer;
2483 if (obj->type == ACPI_TYPE_BUFFER) {
2484 acpi_desc->nfit =
2485 (struct acpi_nfit_header *)obj->buffer.pointer;
2486 ret = acpi_nfit_init(acpi_desc, obj->buffer.length);
2487 if (ret) {
2488 /* Merge failed, restore old nfit, and exit */
2489 acpi_desc->nfit = nfit_saved;
2490 dev_err(dev, "failed to merge updated NFIT\n");
2491 }
2492 } else {
2493 /* Bad _FIT, restore old nfit */
2494 dev_err(dev, "Invalid _FIT\n");
20985164
VV		 2495 }
2496 kfree(buf.pointer);
2497
2498 out_unlock:
2499 device_unlock(dev);
2500}
2501
b94d5230
DW		 2502static const struct acpi_device_id acpi_nfit_ids[] = {
2503 { "ACPI0012", 0 },
2504 { "", 0 },
2505};
2506MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
2507
2508static struct acpi_driver acpi_nfit_driver = {
2509 .name = KBUILD_MODNAME,
2510 .ids = acpi_nfit_ids,
2511 .ops = {
2512 .add = acpi_nfit_add,
2513 .remove = acpi_nfit_remove,
20985164 2514 .notify = acpi_nfit_notify,
b94d5230
DW		 2515 },
2516};
2517
2518static __init int nfit_init(void)
2519{
2520 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
2521 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
2522 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
2523 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
2524 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
2525 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
2526 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
2527
2528 acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]);
2529 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]);
2530 acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]);
2531 acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]);
2532 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]);
2533 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]);
2534 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]);
2535 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]);
2536 acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]);
2537 acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]);
31eca76b
DW		 2538 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1, nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
2539 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2, nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
b94d5230 2540
7ae0fa43
DW		 2541 nfit_wq = create_singlethread_workqueue("nfit");
2542 if (!nfit_wq)
2543 return -ENOMEM;
2544
b94d5230
DW		 2545 return acpi_bus_register_driver(&acpi_nfit_driver);
2546}
2547
2548static __exit void nfit_exit(void)
2549{
2550 acpi_bus_unregister_driver(&acpi_nfit_driver);
7ae0fa43 2551 destroy_workqueue(nfit_wq);
b94d5230
DW		 2552}
2553
2554module_init(nfit_init);
2555module_exit(nfit_exit);
2556MODULE_LICENSE("GPL v2");
2557MODULE_AUTHOR("Intel Corporation");
ubuntu-bionic-kernel mirror