]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/acpi/nfit.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tools/testing/nvdimm: ND_CMD_CALL support
[mirror_ubuntu-artful-kernel.git] / drivers / acpi / nfit.c
CommitLineData
b94d5230
DW		 1/*
2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 */
13#include <linux/list_sort.h>
14#include <linux/libnvdimm.h>
15#include <linux/module.h>
047fc8a1 16#include <linux/mutex.h>
62232e45 17#include <linux/ndctl.h>
0caeef63 18#include <linux/delay.h>
b94d5230
DW		 19#include <linux/list.h>
20#include <linux/acpi.h>
eaf96153 21#include <linux/sort.h>
c2ad2954 22#include <linux/pmem.h>
047fc8a1 23#include <linux/io.h>
1cf03c00 24#include <linux/nd.h>
96601adb 25#include <asm/cacheflush.h>
b94d5230
DW		 26#include "nfit.h"
27
047fc8a1
RZ		 28/*
29 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
30 * irrelevant.
31 */
2f8e2c87 32#include <linux/io-64-nonatomic-hi-lo.h>
047fc8a1 33
4d88a97a
DW		 34static bool force_enable_dimms;
35module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
36MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
37
1cf03c00
DW		 38static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT;
39module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR);
40MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds");
41
42/* after three payloads of overflow, it's dead jim */
43static unsigned int scrub_overflow_abort = 3;
44module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR);
45MODULE_PARM_DESC(scrub_overflow_abort,
46 "Number of times we overflow ARS results before abort");
47
87554098
DW		 48static bool disable_vendor_specific;
49module_param(disable_vendor_specific, bool, S_IRUGO);
50MODULE_PARM_DESC(disable_vendor_specific,
51 "Limit commands to the publicly specified set\n");
52
7ae0fa43
DW		 53static struct workqueue_struct *nfit_wq;
54
20985164
VV		 55struct nfit_table_prev {
56 struct list_head spas;
57 struct list_head memdevs;
58 struct list_head dcrs;
59 struct list_head bdws;
60 struct list_head idts;
61 struct list_head flushes;
62};
63
b94d5230
DW		 64static u8 nfit_uuid[NFIT_UUID_MAX][16];
65
6bc75619 66const u8 *to_nfit_uuid(enum nfit_uuids id)
b94d5230
DW		 67{
68 return nfit_uuid[id];
69}
6bc75619 70EXPORT_SYMBOL(to_nfit_uuid);
b94d5230 71
62232e45
DW		 72static struct acpi_nfit_desc *to_acpi_nfit_desc(
73 struct nvdimm_bus_descriptor *nd_desc)
74{
75 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
76}
77
78static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
79{
80 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
81
82 /*
83 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
84 * acpi_device.
85 */
86 if (!nd_desc->provider_name
87 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
88 return NULL;
89
90 return to_acpi_device(acpi_desc->dev);
91}
92
aef25338
DW		 93static int xlat_status(void *buf, unsigned int cmd)
94{
d4f32367 95 struct nd_cmd_clear_error *clear_err;
aef25338
DW		 96 struct nd_cmd_ars_status *ars_status;
97 struct nd_cmd_ars_start *ars_start;
98 struct nd_cmd_ars_cap *ars_cap;
99 u16 flags;
100
101 switch (cmd) {
102 case ND_CMD_ARS_CAP:
103 ars_cap = buf;
104 if ((ars_cap->status & 0xffff) == NFIT_ARS_CAP_NONE)
105 return -ENOTTY;
106
107 /* Command failed */
108 if (ars_cap->status & 0xffff)
109 return -EIO;
110
111 /* No supported scan types for this range */
112 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
113 if ((ars_cap->status >> 16 & flags) == 0)
114 return -ENOTTY;
115 break;
116 case ND_CMD_ARS_START:
117 ars_start = buf;
118 /* ARS is in progress */
119 if ((ars_start->status & 0xffff) == NFIT_ARS_START_BUSY)
120 return -EBUSY;
121
122 /* Command failed */
123 if (ars_start->status & 0xffff)
124 return -EIO;
125 break;
126 case ND_CMD_ARS_STATUS:
127 ars_status = buf;
128 /* Command failed */
129 if (ars_status->status & 0xffff)
130 return -EIO;
131 /* Check extended status (Upper two bytes) */
132 if (ars_status->status == NFIT_ARS_STATUS_DONE)
133 return 0;
134
135 /* ARS is in progress */
136 if (ars_status->status == NFIT_ARS_STATUS_BUSY)
137 return -EBUSY;
138
139 /* No ARS performed for the current boot */
140 if (ars_status->status == NFIT_ARS_STATUS_NONE)
141 return -EAGAIN;
142
143 /*
144 * ARS interrupted, either we overflowed or some other
145 * agent wants the scan to stop. If we didn't overflow
146 * then just continue with the returned results.
147 */
148 if (ars_status->status == NFIT_ARS_STATUS_INTR) {
149 if (ars_status->flags & NFIT_ARS_F_OVERFLOW)
150 return -ENOSPC;
151 return 0;
152 }
153
154 /* Unknown status */
155 if (ars_status->status >> 16)
156 return -EIO;
157 break;
d4f32367
DW		 158 case ND_CMD_CLEAR_ERROR:
159 clear_err = buf;
160 if (clear_err->status & 0xffff)
161 return -EIO;
162 if (!clear_err->cleared)
163 return -EIO;
164 if (clear_err->length > clear_err->cleared)
165 return clear_err->cleared;
166 break;
aef25338
DW		 167 default:
168 break;
169 }
170
171 return 0;
172}
173
b94d5230
DW		 174static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
175 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
aef25338 176 unsigned int buf_len, int *cmd_rc)
b94d5230 177{
62232e45 178 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
62232e45 179 union acpi_object in_obj, in_buf, *out_obj;
31eca76b 180 const struct nd_cmd_desc *desc = NULL;
62232e45 181 struct device *dev = acpi_desc->dev;
31eca76b 182 struct nd_cmd_pkg *call_pkg = NULL;
62232e45 183 const char *cmd_name, *dimm_name;
31eca76b 184 unsigned long cmd_mask, dsm_mask;
62232e45 185 acpi_handle handle;
31eca76b 186 unsigned int func;
62232e45
DW		 187 const u8 *uuid;
188 u32 offset;
189 int rc, i;
190
31eca76b
DW		 191 func = cmd;
192 if (cmd == ND_CMD_CALL) {
193 call_pkg = buf;
194 func = call_pkg->nd_command;
195 }
196
62232e45
DW		 197 if (nvdimm) {
198 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
199 struct acpi_device *adev = nfit_mem->adev;
200
201 if (!adev)
202 return -ENOTTY;
31eca76b
DW		 203 if (call_pkg && nfit_mem->family != call_pkg->nd_family)
204 return -ENOTTY;
205
047fc8a1 206 dimm_name = nvdimm_name(nvdimm);
62232e45 207 cmd_name = nvdimm_cmd_name(cmd);
e3654eca 208 cmd_mask = nvdimm_cmd_mask(nvdimm);
31eca76b 209 dsm_mask = nfit_mem->dsm_mask;
62232e45 210 desc = nd_cmd_dimm_desc(cmd);
31eca76b 211 uuid = to_nfit_uuid(nfit_mem->family);
62232e45
DW		 212 handle = adev->handle;
213 } else {
214 struct acpi_device *adev = to_acpi_dev(acpi_desc);
215
216 cmd_name = nvdimm_bus_cmd_name(cmd);
e3654eca 217 cmd_mask = nd_desc->cmd_mask;
31eca76b 218 dsm_mask = cmd_mask;
62232e45
DW		 219 desc = nd_cmd_bus_desc(cmd);
220 uuid = to_nfit_uuid(NFIT_DEV_BUS);
221 handle = adev->handle;
222 dimm_name = "bus";
223 }
224
225 if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
226 return -ENOTTY;
227
31eca76b 228 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask))
62232e45
DW		 229 return -ENOTTY;
230
231 in_obj.type = ACPI_TYPE_PACKAGE;
232 in_obj.package.count = 1;
233 in_obj.package.elements = &in_buf;
234 in_buf.type = ACPI_TYPE_BUFFER;
235 in_buf.buffer.pointer = buf;
236 in_buf.buffer.length = 0;
237
238 /* libnvdimm has already validated the input envelope */
239 for (i = 0; i < desc->in_num; i++)
240 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
241 i, buf);
242
31eca76b
DW		 243 if (call_pkg) {
244 /* skip over package wrapper */
245 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
246 in_buf.buffer.length = call_pkg->nd_size_in;
247 }
248
62232e45 249 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
31eca76b
DW		 250 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n",
251 __func__, dimm_name, cmd, func,
252 in_buf.buffer.length);
253 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4,
254 in_buf.buffer.pointer,
255 min_t(u32, 256, in_buf.buffer.length), true);
62232e45
DW		 256 }
257
31eca76b 258 out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj);
62232e45
DW		 259 if (!out_obj) {
260 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name,
261 cmd_name);
262 return -EINVAL;
263 }
264
31eca76b
DW		 265 if (call_pkg) {
266 call_pkg->nd_fw_size = out_obj->buffer.length;
267 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
268 out_obj->buffer.pointer,
269 min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
270
271 ACPI_FREE(out_obj);
272 /*
273 * Need to support FW function w/o known size in advance.
274 * Caller can determine required size based upon nd_fw_size.
275 * If we return an error (like elsewhere) then caller wouldn't
276 * be able to rely upon data returned to make calculation.
277 */
278 return 0;
279 }
280
62232e45
DW		 281 if (out_obj->package.type != ACPI_TYPE_BUFFER) {
282 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n",
283 __func__, dimm_name, cmd_name, out_obj->type);
284 rc = -EINVAL;
285 goto out;
286 }
287
288 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
289 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
290 dimm_name, cmd_name, out_obj->buffer.length);
291 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
292 4, out_obj->buffer.pointer, min_t(u32, 128,
293 out_obj->buffer.length), true);
294 }
295
296 for (i = 0, offset = 0; i < desc->out_num; i++) {
297 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
298 (u32 *) out_obj->buffer.pointer);
299
300 if (offset + out_size > out_obj->buffer.length) {
301 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n",
302 __func__, dimm_name, cmd_name, i);
303 break;
304 }
305
306 if (in_buf.buffer.length + offset + out_size > buf_len) {
307 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n",
308 __func__, dimm_name, cmd_name, i);
309 rc = -ENXIO;
310 goto out;
311 }
312 memcpy(buf + in_buf.buffer.length + offset,
313 out_obj->buffer.pointer + offset, out_size);
314 offset += out_size;
315 }
316 if (offset + in_buf.buffer.length < buf_len) {
317 if (i >= 1) {
318 /*
319 * status valid, return the number of bytes left
320 * unfilled in the output buffer
321 */
322 rc = buf_len - offset - in_buf.buffer.length;
aef25338
DW		 323 if (cmd_rc)
324 *cmd_rc = xlat_status(buf, cmd);
62232e45
DW		 325 } else {
326 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
327 __func__, dimm_name, cmd_name, buf_len,
328 offset);
329 rc = -ENXIO;
330 }
331 } else
332 rc = 0;
333
334 out:
335 ACPI_FREE(out_obj);
336
337 return rc;
b94d5230
DW		 338}
339
340static const char *spa_type_name(u16 type)
341{
342 static const char *to_name[] = {
343 [NFIT_SPA_VOLATILE] = "volatile",
344 [NFIT_SPA_PM] = "pmem",
345 [NFIT_SPA_DCR] = "dimm-control-region",
346 [NFIT_SPA_BDW] = "block-data-window",
347 [NFIT_SPA_VDISK] = "volatile-disk",
348 [NFIT_SPA_VCD] = "volatile-cd",
349 [NFIT_SPA_PDISK] = "persistent-disk",
350 [NFIT_SPA_PCD] = "persistent-cd",
351
352 };
353
354 if (type > NFIT_SPA_PCD)
355 return "unknown";
356
357 return to_name[type];
358}
359
360static int nfit_spa_type(struct acpi_nfit_system_address *spa)
361{
362 int i;
363
364 for (i = 0; i < NFIT_UUID_MAX; i++)
365 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0)
366 return i;
367 return -1;
368}
369
370static bool add_spa(struct acpi_nfit_desc *acpi_desc,
20985164 371 struct nfit_table_prev *prev,
b94d5230
DW		 372 struct acpi_nfit_system_address *spa)
373{
826c416f 374 size_t length = min_t(size_t, sizeof(*spa), spa->header.length);
b94d5230 375 struct device *dev = acpi_desc->dev;
20985164
VV		 376 struct nfit_spa *nfit_spa;
377
378 list_for_each_entry(nfit_spa, &prev->spas, list) {
826c416f 379 if (memcmp(nfit_spa->spa, spa, length) == 0) {
20985164
VV		 380 list_move_tail(&nfit_spa->list, &acpi_desc->spas);
381 return true;
382 }
383 }
b94d5230 384
20985164 385 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa), GFP_KERNEL);
b94d5230
DW		 386 if (!nfit_spa)
387 return false;
388 INIT_LIST_HEAD(&nfit_spa->list);
389 nfit_spa->spa = spa;
390 list_add_tail(&nfit_spa->list, &acpi_desc->spas);
391 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__,
392 spa->range_index,
393 spa_type_name(nfit_spa_type(spa)));
394 return true;
395}
396
397static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
20985164 398 struct nfit_table_prev *prev,
b94d5230
DW		 399 struct acpi_nfit_memory_map *memdev)
400{
826c416f 401 size_t length = min_t(size_t, sizeof(*memdev), memdev->header.length);
b94d5230 402 struct device *dev = acpi_desc->dev;
20985164 403 struct nfit_memdev *nfit_memdev;
b94d5230 404
20985164 405 list_for_each_entry(nfit_memdev, &prev->memdevs, list)
826c416f 406 if (memcmp(nfit_memdev->memdev, memdev, length) == 0) {
20985164
VV		 407 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
408 return true;
409 }
410
411 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev), GFP_KERNEL);
b94d5230
DW		 412 if (!nfit_memdev)
413 return false;
414 INIT_LIST_HEAD(&nfit_memdev->list);
415 nfit_memdev->memdev = memdev;
416 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
417 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
418 __func__, memdev->device_handle, memdev->range_index,
419 memdev->region_index);
420 return true;
421}
422
423static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
20985164 424 struct nfit_table_prev *prev,
b94d5230
DW		 425 struct acpi_nfit_control_region *dcr)
426{
826c416f 427 size_t length = min_t(size_t, sizeof(*dcr), dcr->header.length);
b94d5230 428 struct device *dev = acpi_desc->dev;
20985164
VV		 429 struct nfit_dcr *nfit_dcr;
430
431 list_for_each_entry(nfit_dcr, &prev->dcrs, list)
826c416f 432 if (memcmp(nfit_dcr->dcr, dcr, length) == 0) {
20985164
VV		 433 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
434 return true;
435 }
b94d5230 436
20985164 437 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr), GFP_KERNEL);
b94d5230
DW		 438 if (!nfit_dcr)
439 return false;
440 INIT_LIST_HEAD(&nfit_dcr->list);
441 nfit_dcr->dcr = dcr;
442 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
443 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__,
444 dcr->region_index, dcr->windows);
445 return true;
446}
447
448static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
20985164 449 struct nfit_table_prev *prev,
b94d5230
DW		 450 struct acpi_nfit_data_region *bdw)
451{
826c416f 452 size_t length = min_t(size_t, sizeof(*bdw), bdw->header.length);
b94d5230 453 struct device *dev = acpi_desc->dev;
20985164
VV		 454 struct nfit_bdw *nfit_bdw;
455
456 list_for_each_entry(nfit_bdw, &prev->bdws, list)
826c416f 457 if (memcmp(nfit_bdw->bdw, bdw, length) == 0) {
20985164
VV		 458 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
459 return true;
460 }
b94d5230 461
20985164 462 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw), GFP_KERNEL);
b94d5230
DW		 463 if (!nfit_bdw)
464 return false;
465 INIT_LIST_HEAD(&nfit_bdw->list);
466 nfit_bdw->bdw = bdw;
467 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
468 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__,
469 bdw->region_index, bdw->windows);
470 return true;
471}
472
047fc8a1 473static bool add_idt(struct acpi_nfit_desc *acpi_desc,
20985164 474 struct nfit_table_prev *prev,
047fc8a1
RZ		 475 struct acpi_nfit_interleave *idt)
476{
826c416f 477 size_t length = min_t(size_t, sizeof(*idt), idt->header.length);
047fc8a1 478 struct device *dev = acpi_desc->dev;
20985164
VV		 479 struct nfit_idt *nfit_idt;
480
481 list_for_each_entry(nfit_idt, &prev->idts, list)
826c416f 482 if (memcmp(nfit_idt->idt, idt, length) == 0) {
20985164
VV		 483 list_move_tail(&nfit_idt->list, &acpi_desc->idts);
484 return true;
485 }
047fc8a1 486
20985164 487 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt), GFP_KERNEL);
047fc8a1
RZ		 488 if (!nfit_idt)
489 return false;
490 INIT_LIST_HEAD(&nfit_idt->list);
491 nfit_idt->idt = idt;
492 list_add_tail(&nfit_idt->list, &acpi_desc->idts);
493 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__,
494 idt->interleave_index, idt->line_count);
495 return true;
496}
497
c2ad2954 498static bool add_flush(struct acpi_nfit_desc *acpi_desc,
20985164 499 struct nfit_table_prev *prev,
c2ad2954
RZ		 500 struct acpi_nfit_flush_address *flush)
501{
826c416f 502 size_t length = min_t(size_t, sizeof(*flush), flush->header.length);
c2ad2954 503 struct device *dev = acpi_desc->dev;
20985164 504 struct nfit_flush *nfit_flush;
c2ad2954 505
20985164 506 list_for_each_entry(nfit_flush, &prev->flushes, list)
826c416f 507 if (memcmp(nfit_flush->flush, flush, length) == 0) {
20985164
VV		 508 list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
509 return true;
510 }
511
512 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush), GFP_KERNEL);
c2ad2954
RZ		 513 if (!nfit_flush)
514 return false;
515 INIT_LIST_HEAD(&nfit_flush->list);
516 nfit_flush->flush = flush;
517 list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
518 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__,
519 flush->device_handle, flush->hint_count);
520 return true;
521}
522
20985164
VV		 523static void *add_table(struct acpi_nfit_desc *acpi_desc,
524 struct nfit_table_prev *prev, void *table, const void *end)
b94d5230
DW		 525{
526 struct device *dev = acpi_desc->dev;
527 struct acpi_nfit_header *hdr;
528 void *err = ERR_PTR(-ENOMEM);
529
530 if (table >= end)
531 return NULL;
532
533 hdr = table;
564d5011
VV		 534 if (!hdr->length) {
535 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
536 hdr->type);
537 return NULL;
538 }
539
b94d5230
DW		 540 switch (hdr->type) {
541 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
20985164 542 if (!add_spa(acpi_desc, prev, table))
b94d5230
DW		 543 return err;
544 break;
545 case ACPI_NFIT_TYPE_MEMORY_MAP:
20985164 546 if (!add_memdev(acpi_desc, prev, table))
b94d5230
DW		 547 return err;
548 break;
549 case ACPI_NFIT_TYPE_CONTROL_REGION:
20985164 550 if (!add_dcr(acpi_desc, prev, table))
b94d5230
DW		 551 return err;
552 break;
553 case ACPI_NFIT_TYPE_DATA_REGION:
20985164 554 if (!add_bdw(acpi_desc, prev, table))
b94d5230
DW		 555 return err;
556 break;
b94d5230 557 case ACPI_NFIT_TYPE_INTERLEAVE:
20985164 558 if (!add_idt(acpi_desc, prev, table))
047fc8a1 559 return err;
b94d5230
DW		 560 break;
561 case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
20985164 562 if (!add_flush(acpi_desc, prev, table))
c2ad2954 563 return err;
b94d5230
DW		 564 break;
565 case ACPI_NFIT_TYPE_SMBIOS:
566 dev_dbg(dev, "%s: smbios\n", __func__);
567 break;
568 default:
569 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
570 break;
571 }
572
573 return table + hdr->length;
574}
575
576static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
577 struct nfit_mem *nfit_mem)
578{
579 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
580 u16 dcr = nfit_mem->dcr->region_index;
581 struct nfit_spa *nfit_spa;
582
583 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
584 u16 range_index = nfit_spa->spa->range_index;
585 int type = nfit_spa_type(nfit_spa->spa);
586 struct nfit_memdev *nfit_memdev;
587
588 if (type != NFIT_SPA_BDW)
589 continue;
590
591 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
592 if (nfit_memdev->memdev->range_index != range_index)
593 continue;
594 if (nfit_memdev->memdev->device_handle != device_handle)
595 continue;
596 if (nfit_memdev->memdev->region_index != dcr)
597 continue;
598
599 nfit_mem->spa_bdw = nfit_spa->spa;
600 return;
601 }
602 }
603
604 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
605 nfit_mem->spa_dcr->range_index);
606 nfit_mem->bdw = NULL;
607}
608
6697b2cf 609static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
b94d5230
DW		 610 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
611{
612 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
047fc8a1 613 struct nfit_memdev *nfit_memdev;
c2ad2954 614 struct nfit_flush *nfit_flush;
b94d5230 615 struct nfit_bdw *nfit_bdw;
047fc8a1
RZ		 616 struct nfit_idt *nfit_idt;
617 u16 idt_idx, range_index;
b94d5230 618
b94d5230
DW		 619 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
620 if (nfit_bdw->bdw->region_index != dcr)
621 continue;
622 nfit_mem->bdw = nfit_bdw->bdw;
623 break;
624 }
625
626 if (!nfit_mem->bdw)
6697b2cf 627 return;
b94d5230
DW		 628
629 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
047fc8a1
RZ		 630
631 if (!nfit_mem->spa_bdw)
6697b2cf 632 return;
047fc8a1
RZ		 633
634 range_index = nfit_mem->spa_bdw->range_index;
635 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
636 if (nfit_memdev->memdev->range_index != range_index ||
637 nfit_memdev->memdev->region_index != dcr)
638 continue;
639 nfit_mem->memdev_bdw = nfit_memdev->memdev;
640 idt_idx = nfit_memdev->memdev->interleave_index;
641 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
642 if (nfit_idt->idt->interleave_index != idt_idx)
643 continue;
644 nfit_mem->idt_bdw = nfit_idt->idt;
645 break;
646 }
c2ad2954
RZ		 647
648 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
649 if (nfit_flush->flush->device_handle !=
650 nfit_memdev->memdev->device_handle)
651 continue;
652 nfit_mem->nfit_flush = nfit_flush;
653 break;
654 }
047fc8a1
RZ		 655 break;
656 }
b94d5230
DW		 657}
658
659static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
660 struct acpi_nfit_system_address *spa)
661{
662 struct nfit_mem *nfit_mem, *found;
663 struct nfit_memdev *nfit_memdev;
664 int type = nfit_spa_type(spa);
b94d5230
DW		 665
666 switch (type) {
667 case NFIT_SPA_DCR:
668 case NFIT_SPA_PM:
669 break;
670 default:
671 return 0;
672 }
673
674 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
6697b2cf
DW		 675 struct nfit_dcr *nfit_dcr;
676 u32 device_handle;
677 u16 dcr;
b94d5230
DW		 678
679 if (nfit_memdev->memdev->range_index != spa->range_index)
680 continue;
681 found = NULL;
682 dcr = nfit_memdev->memdev->region_index;
6697b2cf 683 device_handle = nfit_memdev->memdev->device_handle;
b94d5230 684 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
6697b2cf
DW		 685 if (__to_nfit_memdev(nfit_mem)->device_handle
686 == device_handle) {
b94d5230
DW		 687 found = nfit_mem;
688 break;
689 }
690
691 if (found)
692 nfit_mem = found;
693 else {
694 nfit_mem = devm_kzalloc(acpi_desc->dev,
695 sizeof(*nfit_mem), GFP_KERNEL);
696 if (!nfit_mem)
697 return -ENOMEM;
698 INIT_LIST_HEAD(&nfit_mem->list);
6697b2cf
DW		 699 list_add(&nfit_mem->list, &acpi_desc->dimms);
700 }
701
702 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
703 if (nfit_dcr->dcr->region_index != dcr)
704 continue;
705 /*
706 * Record the control region for the dimm. For
707 * the ACPI 6.1 case, where there are separate
708 * control regions for the pmem vs blk
709 * interfaces, be sure to record the extended
710 * blk details.
711 */
712 if (!nfit_mem->dcr)
713 nfit_mem->dcr = nfit_dcr->dcr;
714 else if (nfit_mem->dcr->windows == 0
715 && nfit_dcr->dcr->windows)
716 nfit_mem->dcr = nfit_dcr->dcr;
717 break;
718 }
719
720 if (dcr && !nfit_mem->dcr) {
721 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
722 spa->range_index, dcr);
723 return -ENODEV;
b94d5230
DW		 724 }
725
726 if (type == NFIT_SPA_DCR) {
047fc8a1
RZ		 727 struct nfit_idt *nfit_idt;
728 u16 idt_idx;
729
b94d5230
DW		 730 /* multiple dimms may share a SPA when interleaved */
731 nfit_mem->spa_dcr = spa;
732 nfit_mem->memdev_dcr = nfit_memdev->memdev;
047fc8a1
RZ		 733 idt_idx = nfit_memdev->memdev->interleave_index;
734 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
735 if (nfit_idt->idt->interleave_index != idt_idx)
736 continue;
737 nfit_mem->idt_dcr = nfit_idt->idt;
738 break;
739 }
6697b2cf 740 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
b94d5230
DW		 741 } else {
742 /*
743 * A single dimm may belong to multiple SPA-PM
744 * ranges, record at least one in addition to
745 * any SPA-DCR range.
746 */
747 nfit_mem->memdev_pmem = nfit_memdev->memdev;
748 }
b94d5230
DW		 749 }
750
751 return 0;
752}
753
754static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
755{
756 struct nfit_mem *a = container_of(_a, typeof(*a), list);
757 struct nfit_mem *b = container_of(_b, typeof(*b), list);
758 u32 handleA, handleB;
759
760 handleA = __to_nfit_memdev(a)->device_handle;
761 handleB = __to_nfit_memdev(b)->device_handle;
762 if (handleA < handleB)
763 return -1;
764 else if (handleA > handleB)
765 return 1;
766 return 0;
767}
768
769static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
770{
771 struct nfit_spa *nfit_spa;
772
773 /*
774 * For each SPA-DCR or SPA-PMEM address range find its
775 * corresponding MEMDEV(s). From each MEMDEV find the
776 * corresponding DCR. Then, if we're operating on a SPA-DCR,
777 * try to find a SPA-BDW and a corresponding BDW that references
778 * the DCR. Throw it all into an nfit_mem object. Note, that
779 * BDWs are optional.
780 */
781 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
782 int rc;
783
784 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
785 if (rc)
786 return rc;
787 }
788
789 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
790
791 return 0;
792}
793
45def22c
DW		 794static ssize_t revision_show(struct device *dev,
795 struct device_attribute *attr, char *buf)
796{
797 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
798 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
799 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
800
6b577c9d 801 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
45def22c
DW		 802}
803static DEVICE_ATTR_RO(revision);
804
805static struct attribute *acpi_nfit_attributes[] = {
806 &dev_attr_revision.attr,
807 NULL,
808};
809
810static struct attribute_group acpi_nfit_attribute_group = {
811 .name = "nfit",
812 .attrs = acpi_nfit_attributes,
813};
814
a61fe6f7 815static const struct attribute_group *acpi_nfit_attribute_groups[] = {
45def22c
DW		 816 &nvdimm_bus_attribute_group,
817 &acpi_nfit_attribute_group,
818 NULL,
819};
820
e6dfb2de
DW		 821static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
822{
823 struct nvdimm *nvdimm = to_nvdimm(dev);
824 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
825
826 return __to_nfit_memdev(nfit_mem);
827}
828
829static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
830{
831 struct nvdimm *nvdimm = to_nvdimm(dev);
832 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
833
834 return nfit_mem->dcr;
835}
836
837static ssize_t handle_show(struct device *dev,
838 struct device_attribute *attr, char *buf)
839{
840 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
841
842 return sprintf(buf, "%#x\n", memdev->device_handle);
843}
844static DEVICE_ATTR_RO(handle);
845
846static ssize_t phys_id_show(struct device *dev,
847 struct device_attribute *attr, char *buf)
848{
849 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
850
851 return sprintf(buf, "%#x\n", memdev->physical_id);
852}
853static DEVICE_ATTR_RO(phys_id);
854
855static ssize_t vendor_show(struct device *dev,
856 struct device_attribute *attr, char *buf)
857{
858 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
859
860 return sprintf(buf, "%#x\n", dcr->vendor_id);
861}
862static DEVICE_ATTR_RO(vendor);
863
864static ssize_t rev_id_show(struct device *dev,
865 struct device_attribute *attr, char *buf)
866{
867 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
868
869 return sprintf(buf, "%#x\n", dcr->revision_id);
870}
871static DEVICE_ATTR_RO(rev_id);
872
873static ssize_t device_show(struct device *dev,
874 struct device_attribute *attr, char *buf)
875{
876 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
877
878 return sprintf(buf, "%#x\n", dcr->device_id);
879}
880static DEVICE_ATTR_RO(device);
881
882static ssize_t format_show(struct device *dev,
883 struct device_attribute *attr, char *buf)
884{
885 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
886
887 return sprintf(buf, "%#x\n", dcr->code);
888}
889static DEVICE_ATTR_RO(format);
890
891static ssize_t serial_show(struct device *dev,
892 struct device_attribute *attr, char *buf)
893{
894 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
895
896 return sprintf(buf, "%#x\n", dcr->serial_number);
897}
898static DEVICE_ATTR_RO(serial);
899
58138820
DW		 900static ssize_t flags_show(struct device *dev,
901 struct device_attribute *attr, char *buf)
902{
903 u16 flags = to_nfit_memdev(dev)->flags;
904
905 return sprintf(buf, "%s%s%s%s%s\n",
402bae59
TK		 906 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
907 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
908 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
ca321d1c 909 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
402bae59 910 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "");
58138820
DW		 911}
912static DEVICE_ATTR_RO(flags);
913
e6dfb2de
DW		 914static struct attribute *acpi_nfit_dimm_attributes[] = {
915 &dev_attr_handle.attr,
916 &dev_attr_phys_id.attr,
917 &dev_attr_vendor.attr,
918 &dev_attr_device.attr,
919 &dev_attr_format.attr,
920 &dev_attr_serial.attr,
921 &dev_attr_rev_id.attr,
58138820 922 &dev_attr_flags.attr,
e6dfb2de
DW		 923 NULL,
924};
925
926static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
927 struct attribute *a, int n)
928{
929 struct device *dev = container_of(kobj, struct device, kobj);
930
931 if (to_nfit_dcr(dev))
932 return a->mode;
933 else
934 return 0;
935}
936
937static struct attribute_group acpi_nfit_dimm_attribute_group = {
938 .name = "nfit",
939 .attrs = acpi_nfit_dimm_attributes,
940 .is_visible = acpi_nfit_dimm_attr_visible,
941};
942
943static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
62232e45 944 &nvdimm_attribute_group,
4d88a97a 945 &nd_device_attribute_group,
e6dfb2de
DW		 946 &acpi_nfit_dimm_attribute_group,
947 NULL,
948};
949
950static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
951 u32 device_handle)
952{
953 struct nfit_mem *nfit_mem;
954
955 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
956 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
957 return nfit_mem->nvdimm;
958
959 return NULL;
960}
961
62232e45
DW		 962static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
963 struct nfit_mem *nfit_mem, u32 device_handle)
964{
965 struct acpi_device *adev, *adev_dimm;
966 struct device *dev = acpi_desc->dev;
31eca76b
DW		 967 unsigned long dsm_mask;
968 const u8 *uuid;
60e95f43 969 int i;
62232e45 970
e3654eca
DW		 971 /* nfit test assumes 1:1 relationship between commands and dsms */
972 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
31eca76b 973 nfit_mem->family = NVDIMM_FAMILY_INTEL;
62232e45
DW		 974 adev = to_acpi_dev(acpi_desc);
975 if (!adev)
976 return 0;
977
978 adev_dimm = acpi_find_child_device(adev, device_handle, false);
979 nfit_mem->adev = adev_dimm;
980 if (!adev_dimm) {
981 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
982 device_handle);
4d88a97a 983 return force_enable_dimms ? 0 : -ENODEV;
62232e45
DW		 984 }
985
31eca76b
DW		 986 /*
987 * Until standardization materializes we need to consider up to 3
988 * different command sets. Note, that checking for function0 (bit0)
989 * tells us if any commands are reachable through this uuid.
990 */
991 for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_HPE2; i++)
992 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
993 break;
994
995 /* limit the supported commands to those that are publicly documented */
996 nfit_mem->family = i;
87554098 997 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
31eca76b 998 dsm_mask = 0x3fe;
87554098
DW		 999 if (disable_vendor_specific)
1000 dsm_mask &= ~(1 << ND_CMD_VENDOR);
1001 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1)
31eca76b 1002 dsm_mask = 0x1c3c76;
87554098 1003 else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
31eca76b 1004 dsm_mask = 0x1fe;
87554098
DW		 1005 if (disable_vendor_specific)
1006 dsm_mask &= ~(1 << 8);
1007 } else {
31eca76b
DW		 1008 dev_err(dev, "unknown dimm command family\n");
1009 nfit_mem->family = -1;
1010 return force_enable_dimms ? 0 : -ENODEV;
1011 }
1012
1013 uuid = to_nfit_uuid(nfit_mem->family);
1014 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
62232e45
DW		 1015 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i))
1016 set_bit(i, &nfit_mem->dsm_mask);
1017
60e95f43 1018 return 0;
62232e45
DW		 1019}
1020
e6dfb2de
DW		 1021static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
1022{
1023 struct nfit_mem *nfit_mem;
4d88a97a 1024 int dimm_count = 0;
e6dfb2de
DW		 1025
1026 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
31eca76b 1027 unsigned long flags = 0, cmd_mask;
e6dfb2de 1028 struct nvdimm *nvdimm;
e6dfb2de 1029 u32 device_handle;
58138820 1030 u16 mem_flags;
62232e45 1031 int rc;
e6dfb2de
DW		 1032
1033 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
1034 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
1035 if (nvdimm) {
20985164 1036 dimm_count++;
e6dfb2de
DW		 1037 continue;
1038 }
1039
1040 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
1041 flags |= NDD_ALIASING;
1042
58138820 1043 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
ca321d1c 1044 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
58138820
DW		 1045 flags |= NDD_UNARMED;
1046
62232e45
DW		 1047 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
1048 if (rc)
1049 continue;
1050
e3654eca 1051 /*
31eca76b
DW		 1052 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
1053 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
1054 * userspace interface.
e3654eca 1055 */
31eca76b
DW		 1056 cmd_mask = 1UL << ND_CMD_CALL;
1057 if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1058 cmd_mask |= nfit_mem->dsm_mask;
1059
e6dfb2de 1060 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
62232e45 1061 acpi_nfit_dimm_attribute_groups,
31eca76b 1062 flags, cmd_mask);
e6dfb2de
DW		 1063 if (!nvdimm)
1064 return -ENOMEM;
1065
1066 nfit_mem->nvdimm = nvdimm;
4d88a97a 1067 dimm_count++;
58138820
DW		 1068
1069 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
1070 continue;
1071
402bae59 1072 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n",
58138820 1073 nvdimm_name(nvdimm),
402bae59
TK		 1074 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
1075 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
1076 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
ca321d1c 1077 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "");
58138820 1078
e6dfb2de
DW		 1079 }
1080
4d88a97a 1081 return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
e6dfb2de
DW		 1082}
1083
62232e45
DW		 1084static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
1085{
1086 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1087 const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS);
1088 struct acpi_device *adev;
1089 int i;
1090
e3654eca 1091 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
62232e45
DW		 1092 adev = to_acpi_dev(acpi_desc);
1093 if (!adev)
1094 return;
1095
d4f32367 1096 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
62232e45 1097 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i))
e3654eca 1098 set_bit(i, &nd_desc->cmd_mask);
62232e45
DW		 1099}
1100
1f7df6f8
DW		 1101static ssize_t range_index_show(struct device *dev,
1102 struct device_attribute *attr, char *buf)
1103{
1104 struct nd_region *nd_region = to_nd_region(dev);
1105 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
1106
1107 return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
1108}
1109static DEVICE_ATTR_RO(range_index);
1110
1111static struct attribute *acpi_nfit_region_attributes[] = {
1112 &dev_attr_range_index.attr,
1113 NULL,
1114};
1115
1116static struct attribute_group acpi_nfit_region_attribute_group = {
1117 .name = "nfit",
1118 .attrs = acpi_nfit_region_attributes,
1119};
1120
1121static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
1122 &nd_region_attribute_group,
1123 &nd_mapping_attribute_group,
3d88002e 1124 &nd_device_attribute_group,
74ae66c3 1125 &nd_numa_attribute_group,
1f7df6f8
DW		 1126 &acpi_nfit_region_attribute_group,
1127 NULL,
1128};
1129
eaf96153
DW		 1130/* enough info to uniquely specify an interleave set */
1131struct nfit_set_info {
1132 struct nfit_set_info_map {
1133 u64 region_offset;
1134 u32 serial_number;
1135 u32 pad;
1136 } mapping[0];
1137};
1138
1139static size_t sizeof_nfit_set_info(int num_mappings)
1140{
1141 return sizeof(struct nfit_set_info)
1142 + num_mappings * sizeof(struct nfit_set_info_map);
1143}
1144
1145static int cmp_map(const void *m0, const void *m1)
1146{
1147 const struct nfit_set_info_map *map0 = m0;
1148 const struct nfit_set_info_map *map1 = m1;
1149
1150 return memcmp(&map0->region_offset, &map1->region_offset,
1151 sizeof(u64));
1152}
1153
1154/* Retrieve the nth entry referencing this spa */
1155static struct acpi_nfit_memory_map *memdev_from_spa(
1156 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
1157{
1158 struct nfit_memdev *nfit_memdev;
1159
1160 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
1161 if (nfit_memdev->memdev->range_index == range_index)
1162 if (n-- == 0)
1163 return nfit_memdev->memdev;
1164 return NULL;
1165}
1166
1167static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
1168 struct nd_region_desc *ndr_desc,
1169 struct acpi_nfit_system_address *spa)
1170{
1171 int i, spa_type = nfit_spa_type(spa);
1172 struct device *dev = acpi_desc->dev;
1173 struct nd_interleave_set *nd_set;
1174 u16 nr = ndr_desc->num_mappings;
1175 struct nfit_set_info *info;
1176
1177 if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE)
1178 /* pass */;
1179 else
1180 return 0;
1181
1182 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
1183 if (!nd_set)
1184 return -ENOMEM;
1185
1186 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
1187 if (!info)
1188 return -ENOMEM;
1189 for (i = 0; i < nr; i++) {
1190 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
1191 struct nfit_set_info_map *map = &info->mapping[i];
1192 struct nvdimm *nvdimm = nd_mapping->nvdimm;
1193 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1194 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
1195 spa->range_index, i);
1196
1197 if (!memdev || !nfit_mem->dcr) {
1198 dev_err(dev, "%s: failed to find DCR\n", __func__);
1199 return -ENODEV;
1200 }
1201
1202 map->region_offset = memdev->region_offset;
1203 map->serial_number = nfit_mem->dcr->serial_number;
1204 }
1205
1206 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
1207 cmp_map, NULL);
1208 nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
1209 ndr_desc->nd_set = nd_set;
1210 devm_kfree(dev, info);
1211
1212 return 0;
1213}
1214
047fc8a1
RZ		 1215static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
1216{
1217 struct acpi_nfit_interleave *idt = mmio->idt;
1218 u32 sub_line_offset, line_index, line_offset;
1219 u64 line_no, table_skip_count, table_offset;
1220
1221 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
1222 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
1223 line_offset = idt->line_offset[line_index]
1224 * mmio->line_size;
1225 table_offset = table_skip_count * mmio->table_size;
1226
1227 return mmio->base_offset + line_offset + table_offset + sub_line_offset;
1228}
1229
c2ad2954
RZ		 1230static void wmb_blk(struct nfit_blk *nfit_blk)
1231{
1232
1233 if (nfit_blk->nvdimm_flush) {
1234 /*
1235 * The first wmb() is needed to 'sfence' all previous writes
1236 * such that they are architecturally visible for the platform
1237 * buffer flush. Note that we've already arranged for pmem
1238 * writes to avoid the cache via arch_memcpy_to_pmem(). The
1239 * final wmb() ensures ordering for the NVDIMM flush write.
1240 */
1241 wmb();
1242 writeq(1, nfit_blk->nvdimm_flush);
1243 wmb();
1244 } else
1245 wmb_pmem();
1246}
1247
de4a196c 1248static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
047fc8a1
RZ		 1249{
1250 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1251 u64 offset = nfit_blk->stat_offset + mmio->size * bw;
1252
1253 if (mmio->num_lines)
1254 offset = to_interleave_offset(offset, mmio);
1255
12f03ee6 1256 return readl(mmio->addr.base + offset);
047fc8a1
RZ		 1257}
1258
1259static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
1260 resource_size_t dpa, unsigned int len, unsigned int write)
1261{
1262 u64 cmd, offset;
1263 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1264
1265 enum {
1266 BCW_OFFSET_MASK = (1ULL << 48)-1,
1267 BCW_LEN_SHIFT = 48,
1268 BCW_LEN_MASK = (1ULL << 8) - 1,
1269 BCW_CMD_SHIFT = 56,
1270 };
1271
1272 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
1273 len = len >> L1_CACHE_SHIFT;
1274 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
1275 cmd |= ((u64) write) << BCW_CMD_SHIFT;
1276
1277 offset = nfit_blk->cmd_offset + mmio->size * bw;
1278 if (mmio->num_lines)
1279 offset = to_interleave_offset(offset, mmio);
1280
67a3e8fe 1281 writeq(cmd, mmio->addr.base + offset);
c2ad2954 1282 wmb_blk(nfit_blk);
f0f2c072 1283
aef25338 1284 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
67a3e8fe 1285 readq(mmio->addr.base + offset);
047fc8a1
RZ		 1286}
1287
1288static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
1289 resource_size_t dpa, void *iobuf, size_t len, int rw,
1290 unsigned int lane)
1291{
1292 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1293 unsigned int copied = 0;
1294 u64 base_offset;
1295 int rc;
1296
1297 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
1298 + lane * mmio->size;
047fc8a1
RZ		 1299 write_blk_ctl(nfit_blk, lane, dpa, len, rw);
1300 while (len) {
1301 unsigned int c;
1302 u64 offset;
1303
1304 if (mmio->num_lines) {
1305 u32 line_offset;
1306
1307 offset = to_interleave_offset(base_offset + copied,
1308 mmio);
1309 div_u64_rem(offset, mmio->line_size, &line_offset);
1310 c = min_t(size_t, len, mmio->line_size - line_offset);
1311 } else {
1312 offset = base_offset + nfit_blk->bdw_offset;
1313 c = len;
1314 }
1315
1316 if (rw)
67a3e8fe 1317 memcpy_to_pmem(mmio->addr.aperture + offset,
c2ad2954 1318 iobuf + copied, c);
67a3e8fe 1319 else {
aef25338 1320 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
67a3e8fe
RZ		 1321 mmio_flush_range((void __force *)
1322 mmio->addr.aperture + offset, c);
1323
c2ad2954 1324 memcpy_from_pmem(iobuf + copied,
67a3e8fe
RZ		 1325 mmio->addr.aperture + offset, c);
1326 }
047fc8a1
RZ		 1327
1328 copied += c;
1329 len -= c;
1330 }
c2ad2954
RZ		 1331
1332 if (rw)
1333 wmb_blk(nfit_blk);
1334
047fc8a1
RZ		 1335 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
1336 return rc;
1337}
1338
1339static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
1340 resource_size_t dpa, void *iobuf, u64 len, int rw)
1341{
1342 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1343 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1344 struct nd_region *nd_region = nfit_blk->nd_region;
1345 unsigned int lane, copied = 0;
1346 int rc = 0;
1347
1348 lane = nd_region_acquire_lane(nd_region);
1349 while (len) {
1350 u64 c = min(len, mmio->size);
1351
1352 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
1353 iobuf + copied, c, rw, lane);
1354 if (rc)
1355 break;
1356
1357 copied += c;
1358 len -= c;
1359 }
1360 nd_region_release_lane(nd_region, lane);
1361
1362 return rc;
1363}
1364
1365static void nfit_spa_mapping_release(struct kref *kref)
1366{
1367 struct nfit_spa_mapping *spa_map = to_spa_map(kref);
1368 struct acpi_nfit_system_address *spa = spa_map->spa;
1369 struct acpi_nfit_desc *acpi_desc = spa_map->acpi_desc;
1370
1371 WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1372 dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index);
67a3e8fe
RZ		 1373 if (spa_map->type == SPA_MAP_APERTURE)
1374 memunmap((void __force *)spa_map->addr.aperture);
1375 else
1376 iounmap(spa_map->addr.base);
047fc8a1
RZ		 1377 release_mem_region(spa->address, spa->length);
1378 list_del(&spa_map->list);
1379 kfree(spa_map);
1380}
1381
1382static struct nfit_spa_mapping *find_spa_mapping(
1383 struct acpi_nfit_desc *acpi_desc,
1384 struct acpi_nfit_system_address *spa)
1385{
1386 struct nfit_spa_mapping *spa_map;
1387
1388 WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1389 list_for_each_entry(spa_map, &acpi_desc->spa_maps, list)
1390 if (spa_map->spa == spa)
1391 return spa_map;
1392
1393 return NULL;
1394}
1395
1396static void nfit_spa_unmap(struct acpi_nfit_desc *acpi_desc,
1397 struct acpi_nfit_system_address *spa)
1398{
1399 struct nfit_spa_mapping *spa_map;
1400
1401 mutex_lock(&acpi_desc->spa_map_mutex);
1402 spa_map = find_spa_mapping(acpi_desc, spa);
1403
1404 if (spa_map)
1405 kref_put(&spa_map->kref, nfit_spa_mapping_release);
1406 mutex_unlock(&acpi_desc->spa_map_mutex);
1407}
1408
1409static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
c2ad2954 1410 struct acpi_nfit_system_address *spa, enum spa_map_type type)
047fc8a1
RZ		 1411{
1412 resource_size_t start = spa->address;
1413 resource_size_t n = spa->length;
1414 struct nfit_spa_mapping *spa_map;
1415 struct resource *res;
1416
1417 WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1418
1419 spa_map = find_spa_mapping(acpi_desc, spa);
1420 if (spa_map) {
1421 kref_get(&spa_map->kref);
67a3e8fe 1422 return spa_map->addr.base;
047fc8a1
RZ		 1423 }
1424
1425 spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL);
1426 if (!spa_map)
1427 return NULL;
1428
1429 INIT_LIST_HEAD(&spa_map->list);
1430 spa_map->spa = spa;
1431 kref_init(&spa_map->kref);
1432 spa_map->acpi_desc = acpi_desc;
1433
1434 res = request_mem_region(start, n, dev_name(acpi_desc->dev));
1435 if (!res)
1436 goto err_mem;
1437
67a3e8fe
RZ		 1438 spa_map->type = type;
1439 if (type == SPA_MAP_APERTURE)
1440 spa_map->addr.aperture = (void __pmem *)memremap(start, n,
1441 ARCH_MEMREMAP_PMEM);
1442 else
1443 spa_map->addr.base = ioremap_nocache(start, n);
1444
c2ad2954 1445
67a3e8fe 1446 if (!spa_map->addr.base)
047fc8a1
RZ		 1447 goto err_map;
1448
1449 list_add_tail(&spa_map->list, &acpi_desc->spa_maps);
67a3e8fe 1450 return spa_map->addr.base;
047fc8a1
RZ		 1451
1452 err_map:
1453 release_mem_region(start, n);
1454 err_mem:
1455 kfree(spa_map);
1456 return NULL;
1457}
1458
1459/**
1460 * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1461 * @nvdimm_bus: NFIT-bus that provided the spa table entry
1462 * @nfit_spa: spa table to map
c2ad2954 1463 * @type: aperture or control region
047fc8a1
RZ		 1464 *
1465 * In the case where block-data-window apertures and
1466 * dimm-control-regions are interleaved they will end up sharing a
1467 * single request_mem_region() + ioremap() for the address range. In
1468 * the style of devm nfit_spa_map() mappings are automatically dropped
1469 * when all region devices referencing the same mapping are disabled /
1470 * unbound.
1471 */
1472static void __iomem *nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
c2ad2954 1473 struct acpi_nfit_system_address *spa, enum spa_map_type type)
047fc8a1
RZ		 1474{
1475 void __iomem *iomem;
1476
1477 mutex_lock(&acpi_desc->spa_map_mutex);
c2ad2954 1478 iomem = __nfit_spa_map(acpi_desc, spa, type);
047fc8a1
RZ		 1479 mutex_unlock(&acpi_desc->spa_map_mutex);
1480
1481 return iomem;
1482}
1483
1484static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
1485 struct acpi_nfit_interleave *idt, u16 interleave_ways)
1486{
1487 if (idt) {
1488 mmio->num_lines = idt->line_count;
1489 mmio->line_size = idt->line_size;
1490 if (interleave_ways == 0)
1491 return -ENXIO;
1492 mmio->table_size = mmio->num_lines * interleave_ways
1493 * mmio->line_size;
1494 }
1495
1496 return 0;
1497}
1498
f0f2c072
RZ		 1499static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
1500 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
1501{
1502 struct nd_cmd_dimm_flags flags;
1503 int rc;
1504
1505 memset(&flags, 0, sizeof(flags));
1506 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
aef25338 1507 sizeof(flags), NULL);
f0f2c072
RZ		 1508
1509 if (rc >= 0 && flags.status == 0)
1510 nfit_blk->dimm_flags = flags.flags;
1511 else if (rc == -ENOTTY) {
1512 /* fall back to a conservative default */
aef25338 1513 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
f0f2c072
RZ		 1514 rc = 0;
1515 } else
1516 rc = -ENXIO;
1517
1518 return rc;
1519}
1520
047fc8a1
RZ		 1521static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
1522 struct device *dev)
1523{
1524 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1525 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1526 struct nd_blk_region *ndbr = to_nd_blk_region(dev);
c2ad2954 1527 struct nfit_flush *nfit_flush;
047fc8a1
RZ		 1528 struct nfit_blk_mmio *mmio;
1529 struct nfit_blk *nfit_blk;
1530 struct nfit_mem *nfit_mem;
1531 struct nvdimm *nvdimm;
1532 int rc;
1533
1534 nvdimm = nd_blk_region_to_dimm(ndbr);
1535 nfit_mem = nvdimm_provider_data(nvdimm);
1536 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
1537 dev_dbg(dev, "%s: missing%s%s%s\n", __func__,
1538 nfit_mem ? "" : " nfit_mem",
193ccca4
DW		 1539 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
1540 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
047fc8a1
RZ		 1541 return -ENXIO;
1542 }
1543
1544 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
1545 if (!nfit_blk)
1546 return -ENOMEM;
1547 nd_blk_region_set_provider_data(ndbr, nfit_blk);
1548 nfit_blk->nd_region = to_nd_region(dev);
1549
1550 /* map block aperture memory */
1551 nfit_blk->bdw_offset = nfit_mem->bdw->offset;
1552 mmio = &nfit_blk->mmio[BDW];
67a3e8fe 1553 mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw,
c2ad2954 1554 SPA_MAP_APERTURE);
67a3e8fe 1555 if (!mmio->addr.base) {
047fc8a1
RZ		 1556 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__,
1557 nvdimm_name(nvdimm));
1558 return -ENOMEM;
1559 }
1560 mmio->size = nfit_mem->bdw->size;
1561 mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
1562 mmio->idt = nfit_mem->idt_bdw;
1563 mmio->spa = nfit_mem->spa_bdw;
1564 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
1565 nfit_mem->memdev_bdw->interleave_ways);
1566 if (rc) {
1567 dev_dbg(dev, "%s: %s failed to init bdw interleave\n",
1568 __func__, nvdimm_name(nvdimm));
1569 return rc;
1570 }
1571
1572 /* map block control memory */
1573 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
1574 nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
1575 mmio = &nfit_blk->mmio[DCR];
67a3e8fe 1576 mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr,
c2ad2954 1577 SPA_MAP_CONTROL);
67a3e8fe 1578 if (!mmio->addr.base) {
047fc8a1
RZ		 1579 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__,
1580 nvdimm_name(nvdimm));
1581 return -ENOMEM;
1582 }
1583 mmio->size = nfit_mem->dcr->window_size;
1584 mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
1585 mmio->idt = nfit_mem->idt_dcr;
1586 mmio->spa = nfit_mem->spa_dcr;
1587 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
1588 nfit_mem->memdev_dcr->interleave_ways);
1589 if (rc) {
1590 dev_dbg(dev, "%s: %s failed to init dcr interleave\n",
1591 __func__, nvdimm_name(nvdimm));
1592 return rc;
1593 }
1594
f0f2c072
RZ		 1595 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
1596 if (rc < 0) {
1597 dev_dbg(dev, "%s: %s failed get DIMM flags\n",
1598 __func__, nvdimm_name(nvdimm));
1599 return rc;
1600 }
1601
c2ad2954
RZ		 1602 nfit_flush = nfit_mem->nfit_flush;
1603 if (nfit_flush && nfit_flush->flush->hint_count != 0) {
1604 nfit_blk->nvdimm_flush = devm_ioremap_nocache(dev,
1605 nfit_flush->flush->hint_address[0], 8);
1606 if (!nfit_blk->nvdimm_flush)
1607 return -ENOMEM;
1608 }
1609
96601adb 1610 if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush)
c2ad2954
RZ		 1611 dev_warn(dev, "unable to guarantee persistence of writes\n");
1612
047fc8a1
RZ		 1613 if (mmio->line_size == 0)
1614 return 0;
1615
1616 if ((u32) nfit_blk->cmd_offset % mmio->line_size
1617 + 8 > mmio->line_size) {
1618 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
1619 return -ENXIO;
1620 } else if ((u32) nfit_blk->stat_offset % mmio->line_size
1621 + 8 > mmio->line_size) {
1622 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
1623 return -ENXIO;
1624 }
1625
1626 return 0;
1627}
1628
1629static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus,
1630 struct device *dev)
1631{
1632 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1633 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1634 struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1635 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1636 int i;
1637
1638 if (!nfit_blk)
1639 return; /* never enabled */
1640
1641 /* auto-free BLK spa mappings */
1642 for (i = 0; i < 2; i++) {
1643 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i];
1644
67a3e8fe 1645 if (mmio->addr.base)
047fc8a1
RZ		 1646 nfit_spa_unmap(acpi_desc, mmio->spa);
1647 }
1648 nd_blk_region_set_provider_data(ndbr, NULL);
1649 /* devm will free nfit_blk */
1650}
1651
aef25338 1652static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
1cf03c00 1653 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
0caeef63 1654{
aef25338 1655 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1cf03c00 1656 struct acpi_nfit_system_address *spa = nfit_spa->spa;
aef25338
DW		 1657 int cmd_rc, rc;
1658
1cf03c00
DW		 1659 cmd->address = spa->address;
1660 cmd->length = spa->length;
aef25338
DW		 1661 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
1662 sizeof(*cmd), &cmd_rc);
1663 if (rc < 0)
1664 return rc;
1cf03c00 1665 return cmd_rc;
0caeef63
VV		 1666}
1667
1cf03c00 1668static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa)
0caeef63
VV		 1669{
1670 int rc;
1cf03c00
DW		 1671 int cmd_rc;
1672 struct nd_cmd_ars_start ars_start;
1673 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1674 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
0caeef63 1675
1cf03c00
DW		 1676 memset(&ars_start, 0, sizeof(ars_start));
1677 ars_start.address = spa->address;
1678 ars_start.length = spa->length;
1679 if (nfit_spa_type(spa) == NFIT_SPA_PM)
1680 ars_start.type = ND_ARS_PERSISTENT;
1681 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
1682 ars_start.type = ND_ARS_VOLATILE;
1683 else
1684 return -ENOTTY;
aef25338 1685
1cf03c00
DW		 1686 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
1687 sizeof(ars_start), &cmd_rc);
aef25338 1688
1cf03c00
DW		 1689 if (rc < 0)
1690 return rc;
1691 return cmd_rc;
0caeef63
VV		 1692}
1693
1cf03c00 1694static int ars_continue(struct acpi_nfit_desc *acpi_desc)
0caeef63 1695{
aef25338 1696 int rc, cmd_rc;
1cf03c00
DW		 1697 struct nd_cmd_ars_start ars_start;
1698 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1699 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
1700
1701 memset(&ars_start, 0, sizeof(ars_start));
1702 ars_start.address = ars_status->restart_address;
1703 ars_start.length = ars_status->restart_length;
1704 ars_start.type = ars_status->type;
1705 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
1706 sizeof(ars_start), &cmd_rc);
1707 if (rc < 0)
1708 return rc;
1709 return cmd_rc;
1710}
0caeef63 1711
1cf03c00
DW		 1712static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
1713{
1714 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1715 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
1716 int rc, cmd_rc;
aef25338 1717
1cf03c00
DW		 1718 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
1719 acpi_desc->ars_status_size, &cmd_rc);
1720 if (rc < 0)
1721 return rc;
1722 return cmd_rc;
0caeef63
VV		 1723}
1724
1725static int ars_status_process_records(struct nvdimm_bus *nvdimm_bus,
1cf03c00 1726 struct nd_cmd_ars_status *ars_status)
0caeef63
VV		 1727{
1728 int rc;
1729 u32 i;
1730
0caeef63
VV		 1731 for (i = 0; i < ars_status->num_records; i++) {
1732 rc = nvdimm_bus_add_poison(nvdimm_bus,
1733 ars_status->records[i].err_address,
1734 ars_status->records[i].length);
1735 if (rc)
1736 return rc;
1737 }
1738
1739 return 0;
1740}
1741
af1996ef
TK		 1742static void acpi_nfit_remove_resource(void *data)
1743{
1744 struct resource *res = data;
1745
1746 remove_resource(res);
1747}
1748
1749static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
1750 struct nd_region_desc *ndr_desc)
1751{
1752 struct resource *res, *nd_res = ndr_desc->res;
1753 int is_pmem, ret;
1754
1755 /* No operation if the region is already registered as PMEM */
1756 is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
1757 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
1758 if (is_pmem == REGION_INTERSECTS)
1759 return 0;
1760
1761 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
1762 if (!res)
1763 return -ENOMEM;
1764
1765 res->name = "Persistent Memory";
1766 res->start = nd_res->start;
1767 res->end = nd_res->end;
1768 res->flags = IORESOURCE_MEM;
1769 res->desc = IORES_DESC_PERSISTENT_MEMORY;
1770
1771 ret = insert_resource(&iomem_resource, res);
1772 if (ret)
1773 return ret;
1774
1775 ret = devm_add_action(acpi_desc->dev, acpi_nfit_remove_resource, res);
1776 if (ret) {
1777 remove_resource(res);
1778 return ret;
1779 }
1780
1781 return 0;
1782}
1783
1f7df6f8
DW		 1784static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
1785 struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc,
1786 struct acpi_nfit_memory_map *memdev,
1cf03c00 1787 struct nfit_spa *nfit_spa)
1f7df6f8
DW		 1788{
1789 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
1790 memdev->device_handle);
1cf03c00 1791 struct acpi_nfit_system_address *spa = nfit_spa->spa;
047fc8a1 1792 struct nd_blk_region_desc *ndbr_desc;
1f7df6f8
DW		 1793 struct nfit_mem *nfit_mem;
1794 int blk_valid = 0;
1795
1796 if (!nvdimm) {
1797 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
1798 spa->range_index, memdev->device_handle);
1799 return -ENODEV;
1800 }
1801
1802 nd_mapping->nvdimm = nvdimm;
1803 switch (nfit_spa_type(spa)) {
1804 case NFIT_SPA_PM:
1805 case NFIT_SPA_VOLATILE:
1806 nd_mapping->start = memdev->address;
1807 nd_mapping->size = memdev->region_size;
1808 break;
1809 case NFIT_SPA_DCR:
1810 nfit_mem = nvdimm_provider_data(nvdimm);
1811 if (!nfit_mem || !nfit_mem->bdw) {
1812 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
1813 spa->range_index, nvdimm_name(nvdimm));
1814 } else {
1815 nd_mapping->size = nfit_mem->bdw->capacity;
1816 nd_mapping->start = nfit_mem->bdw->start_address;
5212e11f 1817 ndr_desc->num_lanes = nfit_mem->bdw->windows;
1f7df6f8
DW		 1818 blk_valid = 1;
1819 }
1820
1821 ndr_desc->nd_mapping = nd_mapping;
1822 ndr_desc->num_mappings = blk_valid;
047fc8a1
RZ		 1823 ndbr_desc = to_blk_region_desc(ndr_desc);
1824 ndbr_desc->enable = acpi_nfit_blk_region_enable;
1825 ndbr_desc->disable = acpi_nfit_blk_region_disable;
6bc75619 1826 ndbr_desc->do_io = acpi_desc->blk_do_io;
1cf03c00
DW		 1827 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
1828 ndr_desc);
1829 if (!nfit_spa->nd_region)
1f7df6f8
DW		 1830 return -ENOMEM;
1831 break;
1832 }
1833
1834 return 0;
1835}
1836
1837static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
1838 struct nfit_spa *nfit_spa)
1839{
1840 static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS];
1841 struct acpi_nfit_system_address *spa = nfit_spa->spa;
047fc8a1
RZ		 1842 struct nd_blk_region_desc ndbr_desc;
1843 struct nd_region_desc *ndr_desc;
1f7df6f8 1844 struct nfit_memdev *nfit_memdev;
1f7df6f8
DW		 1845 struct nvdimm_bus *nvdimm_bus;
1846 struct resource res;
eaf96153 1847 int count = 0, rc;
1f7df6f8 1848
1cf03c00 1849 if (nfit_spa->nd_region)
20985164
VV		 1850 return 0;
1851
1f7df6f8
DW		 1852 if (spa->range_index == 0) {
1853 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n",
1854 __func__);
1855 return 0;
1856 }
1857
1858 memset(&res, 0, sizeof(res));
1859 memset(&nd_mappings, 0, sizeof(nd_mappings));
047fc8a1 1860 memset(&ndbr_desc, 0, sizeof(ndbr_desc));
1f7df6f8
DW		 1861 res.start = spa->address;
1862 res.end = res.start + spa->length - 1;
047fc8a1
RZ		 1863 ndr_desc = &ndbr_desc.ndr_desc;
1864 ndr_desc->res = &res;
1865 ndr_desc->provider_data = nfit_spa;
1866 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
41d7a6d6
TK		 1867 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
1868 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
1869 spa->proximity_domain);
1870 else
1871 ndr_desc->numa_node = NUMA_NO_NODE;
1872
1f7df6f8
DW		 1873 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1874 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1875 struct nd_mapping *nd_mapping;
1f7df6f8
DW		 1876
1877 if (memdev->range_index != spa->range_index)
1878 continue;
1879 if (count >= ND_MAX_MAPPINGS) {
1880 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
1881 spa->range_index, ND_MAX_MAPPINGS);
1882 return -ENXIO;
1883 }
1884 nd_mapping = &nd_mappings[count++];
047fc8a1 1885 rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc,
1cf03c00 1886 memdev, nfit_spa);
1f7df6f8 1887 if (rc)
1cf03c00 1888 goto out;
1f7df6f8
DW		 1889 }
1890
047fc8a1
RZ		 1891 ndr_desc->nd_mapping = nd_mappings;
1892 ndr_desc->num_mappings = count;
1893 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
eaf96153 1894 if (rc)
1cf03c00 1895 goto out;
eaf96153 1896
1f7df6f8
DW		 1897 nvdimm_bus = acpi_desc->nvdimm_bus;
1898 if (nfit_spa_type(spa) == NFIT_SPA_PM) {
af1996ef 1899 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
48901165 1900 if (rc) {
af1996ef
TK		 1901 dev_warn(acpi_desc->dev,
1902 "failed to insert pmem resource to iomem: %d\n",
1903 rc);
48901165 1904 goto out;
0caeef63 1905 }
48901165 1906
1cf03c00
DW		 1907 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
1908 ndr_desc);
1909 if (!nfit_spa->nd_region)
1910 rc = -ENOMEM;
1f7df6f8 1911 } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) {
1cf03c00
DW		 1912 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
1913 ndr_desc);
1914 if (!nfit_spa->nd_region)
1915 rc = -ENOMEM;
1f7df6f8 1916 }
20985164 1917
1cf03c00
DW		 1918 out:
1919 if (rc)
1920 dev_err(acpi_desc->dev, "failed to register spa range %d\n",
1921 nfit_spa->spa->range_index);
1922 return rc;
1923}
1924
1925static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc,
1926 u32 max_ars)
1927{
1928 struct device *dev = acpi_desc->dev;
1929 struct nd_cmd_ars_status *ars_status;
1930
1931 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) {
1932 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size);
1933 return 0;
1934 }
1935
1936 if (acpi_desc->ars_status)
1937 devm_kfree(dev, acpi_desc->ars_status);
1938 acpi_desc->ars_status = NULL;
1939 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL);
1940 if (!ars_status)
1941 return -ENOMEM;
1942 acpi_desc->ars_status = ars_status;
1943 acpi_desc->ars_status_size = max_ars;
1f7df6f8
DW		 1944 return 0;
1945}
1946
1cf03c00
DW		 1947static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc,
1948 struct nfit_spa *nfit_spa)
1949{
1950 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1951 int rc;
1952
1953 if (!nfit_spa->max_ars) {
1954 struct nd_cmd_ars_cap ars_cap;
1955
1956 memset(&ars_cap, 0, sizeof(ars_cap));
1957 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
1958 if (rc < 0)
1959 return rc;
1960 nfit_spa->max_ars = ars_cap.max_ars_out;
1961 nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
1962 /* check that the supported scrub types match the spa type */
1963 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE &&
1964 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0)
1965 return -ENOTTY;
1966 else if (nfit_spa_type(spa) == NFIT_SPA_PM &&
1967 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0)
1968 return -ENOTTY;
1969 }
1970
1971 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars))
1972 return -ENOMEM;
1973
1974 rc = ars_get_status(acpi_desc);
1975 if (rc < 0 && rc != -ENOSPC)
1976 return rc;
1977
1978 if (ars_status_process_records(acpi_desc->nvdimm_bus,
1979 acpi_desc->ars_status))
1980 return -ENOMEM;
1981
1982 return 0;
1983}
1984
1985static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc,
1986 struct nfit_spa *nfit_spa)
1987{
1988 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1989 unsigned int overflow_retry = scrub_overflow_abort;
1990 u64 init_ars_start = 0, init_ars_len = 0;
1991 struct device *dev = acpi_desc->dev;
1992 unsigned int tmo = scrub_timeout;
1993 int rc;
1994
1995 if (nfit_spa->ars_done || !nfit_spa->nd_region)
1996 return;
1997
1998 rc = ars_start(acpi_desc, nfit_spa);
1999 /*
2000 * If we timed out the initial scan we'll still be busy here,
2001 * and will wait another timeout before giving up permanently.
2002 */
2003 if (rc < 0 && rc != -EBUSY)
2004 return;
2005
2006 do {
2007 u64 ars_start, ars_len;
2008
2009 if (acpi_desc->cancel)
2010 break;
2011 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa);
2012 if (rc == -ENOTTY)
2013 break;
2014 if (rc == -EBUSY && !tmo) {
2015 dev_warn(dev, "range %d ars timeout, aborting\n",
2016 spa->range_index);
2017 break;
2018 }
2019
2020 if (rc == -EBUSY) {
2021 /*
2022 * Note, entries may be appended to the list
2023 * while the lock is dropped, but the workqueue
2024 * being active prevents entries being deleted /
2025 * freed.
2026 */
2027 mutex_unlock(&acpi_desc->init_mutex);
2028 ssleep(1);
2029 tmo--;
2030 mutex_lock(&acpi_desc->init_mutex);
2031 continue;
2032 }
2033
2034 /* we got some results, but there are more pending... */
2035 if (rc == -ENOSPC && overflow_retry--) {
2036 if (!init_ars_len) {
2037 init_ars_len = acpi_desc->ars_status->length;
2038 init_ars_start = acpi_desc->ars_status->address;
2039 }
2040 rc = ars_continue(acpi_desc);
2041 }
2042
2043 if (rc < 0) {
2044 dev_warn(dev, "range %d ars continuation failed\n",
2045 spa->range_index);
2046 break;
2047 }
2048
2049 if (init_ars_len) {
2050 ars_start = init_ars_start;
2051 ars_len = init_ars_len;
2052 } else {
2053 ars_start = acpi_desc->ars_status->address;
2054 ars_len = acpi_desc->ars_status->length;
2055 }
2056 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n",
2057 spa->range_index, ars_start, ars_len);
2058 /* notify the region about new poison entries */
2059 nvdimm_region_notify(nfit_spa->nd_region,
2060 NVDIMM_REVALIDATE_POISON);
2061 break;
2062 } while (1);
2063}
2064
2065static void acpi_nfit_scrub(struct work_struct *work)
1f7df6f8 2066{
1cf03c00
DW		 2067 struct device *dev;
2068 u64 init_scrub_length = 0;
1f7df6f8 2069 struct nfit_spa *nfit_spa;
1cf03c00
DW		 2070 u64 init_scrub_address = 0;
2071 bool init_ars_done = false;
2072 struct acpi_nfit_desc *acpi_desc;
2073 unsigned int tmo = scrub_timeout;
2074 unsigned int overflow_retry = scrub_overflow_abort;
2075
2076 acpi_desc = container_of(work, typeof(*acpi_desc), work);
2077 dev = acpi_desc->dev;
1f7df6f8 2078
1cf03c00
DW		 2079 /*
2080 * We scrub in 2 phases. The first phase waits for any platform
2081 * firmware initiated scrubs to complete and then we go search for the
2082 * affected spa regions to mark them scanned. In the second phase we
2083 * initiate a directed scrub for every range that was not scrubbed in
2084 * phase 1.
2085 */
2086
2087 /* process platform firmware initiated scrubs */
2088 retry:
2089 mutex_lock(&acpi_desc->init_mutex);
1f7df6f8 2090 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1cf03c00
DW		 2091 struct nd_cmd_ars_status *ars_status;
2092 struct acpi_nfit_system_address *spa;
2093 u64 ars_start, ars_len;
2094 int rc;
1f7df6f8 2095
1cf03c00
DW		 2096 if (acpi_desc->cancel)
2097 break;
2098
2099 if (nfit_spa->nd_region)
2100 continue;
2101
2102 if (init_ars_done) {
2103 /*
2104 * No need to re-query, we're now just
2105 * reconciling all the ranges covered by the
2106 * initial scrub
2107 */
2108 rc = 0;
2109 } else
2110 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa);
2111
2112 if (rc == -ENOTTY) {
2113 /* no ars capability, just register spa and move on */
2114 acpi_nfit_register_region(acpi_desc, nfit_spa);
2115 continue;
2116 }
2117
2118 if (rc == -EBUSY && !tmo) {
2119 /* fallthrough to directed scrub in phase 2 */
2120 dev_warn(dev, "timeout awaiting ars results, continuing...\n");
2121 break;
2122 } else if (rc == -EBUSY) {
2123 mutex_unlock(&acpi_desc->init_mutex);
2124 ssleep(1);
2125 tmo--;
2126 goto retry;
2127 }
2128
2129 /* we got some results, but there are more pending... */
2130 if (rc == -ENOSPC && overflow_retry--) {
2131 ars_status = acpi_desc->ars_status;
2132 /*
2133 * Record the original scrub range, so that we
2134 * can recall all the ranges impacted by the
2135 * initial scrub.
2136 */
2137 if (!init_scrub_length) {
2138 init_scrub_length = ars_status->length;
2139 init_scrub_address = ars_status->address;
2140 }
2141 rc = ars_continue(acpi_desc);
2142 if (rc == 0) {
2143 mutex_unlock(&acpi_desc->init_mutex);
2144 goto retry;
2145 }
2146 }
2147
2148 if (rc < 0) {
2149 /*
2150 * Initial scrub failed, we'll give it one more
2151 * try below...
2152 */
2153 break;
2154 }
2155
2156 /* We got some final results, record completed ranges */
2157 ars_status = acpi_desc->ars_status;
2158 if (init_scrub_length) {
2159 ars_start = init_scrub_address;
2160 ars_len = ars_start + init_scrub_length;
2161 } else {
2162 ars_start = ars_status->address;
2163 ars_len = ars_status->length;
2164 }
2165 spa = nfit_spa->spa;
2166
2167 if (!init_ars_done) {
2168 init_ars_done = true;
2169 dev_dbg(dev, "init scrub %#llx + %#llx complete\n",
2170 ars_start, ars_len);
2171 }
2172 if (ars_start <= spa->address && ars_start + ars_len
2173 >= spa->address + spa->length)
2174 acpi_nfit_register_region(acpi_desc, nfit_spa);
1f7df6f8 2175 }
1cf03c00
DW		 2176
2177 /*
2178 * For all the ranges not covered by an initial scrub we still
2179 * want to see if there are errors, but it's ok to discover them
2180 * asynchronously.
2181 */
2182 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2183 /*
2184 * Flag all the ranges that still need scrubbing, but
2185 * register them now to make data available.
2186 */
2187 if (nfit_spa->nd_region)
2188 nfit_spa->ars_done = 1;
2189 else
2190 acpi_nfit_register_region(acpi_desc, nfit_spa);
2191 }
2192
2193 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
2194 acpi_nfit_async_scrub(acpi_desc, nfit_spa);
2195 mutex_unlock(&acpi_desc->init_mutex);
2196}
2197
2198static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
2199{
2200 struct nfit_spa *nfit_spa;
2201 int rc;
2202
2203 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
2204 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) {
2205 /* BLK regions don't need to wait for ars results */
2206 rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
2207 if (rc)
2208 return rc;
2209 }
2210
2211 queue_work(nfit_wq, &acpi_desc->work);
1f7df6f8
DW		 2212 return 0;
2213}
2214
20985164
VV		 2215static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
2216 struct nfit_table_prev *prev)
2217{
2218 struct device *dev = acpi_desc->dev;
2219
2220 if (!list_empty(&prev->spas) ||
2221 !list_empty(&prev->memdevs) ||
2222 !list_empty(&prev->dcrs) ||
2223 !list_empty(&prev->bdws) ||
2224 !list_empty(&prev->idts) ||
2225 !list_empty(&prev->flushes)) {
2226 dev_err(dev, "new nfit deletes entries (unsupported)\n");
2227 return -ENXIO;
2228 }
2229 return 0;
2230}
2231
6bc75619 2232int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, acpi_size sz)
b94d5230
DW		 2233{
2234 struct device *dev = acpi_desc->dev;
20985164 2235 struct nfit_table_prev prev;
b94d5230
DW		 2236 const void *end;
2237 u8 *data;
1f7df6f8 2238 int rc;
b94d5230 2239
20985164
VV		 2240 mutex_lock(&acpi_desc->init_mutex);
2241
2242 INIT_LIST_HEAD(&prev.spas);
2243 INIT_LIST_HEAD(&prev.memdevs);
2244 INIT_LIST_HEAD(&prev.dcrs);
2245 INIT_LIST_HEAD(&prev.bdws);
2246 INIT_LIST_HEAD(&prev.idts);
2247 INIT_LIST_HEAD(&prev.flushes);
2248
2249 list_cut_position(&prev.spas, &acpi_desc->spas,
2250 acpi_desc->spas.prev);
2251 list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
2252 acpi_desc->memdevs.prev);
2253 list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
2254 acpi_desc->dcrs.prev);
2255 list_cut_position(&prev.bdws, &acpi_desc->bdws,
2256 acpi_desc->bdws.prev);
2257 list_cut_position(&prev.idts, &acpi_desc->idts,
2258 acpi_desc->idts.prev);
2259 list_cut_position(&prev.flushes, &acpi_desc->flushes,
2260 acpi_desc->flushes.prev);
b94d5230
DW		 2261
2262 data = (u8 *) acpi_desc->nfit;
2263 end = data + sz;
b94d5230 2264 while (!IS_ERR_OR_NULL(data))
20985164 2265 data = add_table(acpi_desc, &prev, data, end);
b94d5230
DW		 2266
2267 if (IS_ERR(data)) {
2268 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__,
2269 PTR_ERR(data));
20985164
VV		 2270 rc = PTR_ERR(data);
2271 goto out_unlock;
b94d5230
DW		 2272 }
2273
20985164
VV		 2274 rc = acpi_nfit_check_deletions(acpi_desc, &prev);
2275 if (rc)
2276 goto out_unlock;
2277
2278 if (nfit_mem_init(acpi_desc) != 0) {
2279 rc = -ENOMEM;
2280 goto out_unlock;
2281 }
b94d5230 2282
62232e45
DW		 2283 acpi_nfit_init_dsms(acpi_desc);
2284
1f7df6f8
DW		 2285 rc = acpi_nfit_register_dimms(acpi_desc);
2286 if (rc)
20985164
VV		 2287 goto out_unlock;
2288
2289 rc = acpi_nfit_register_regions(acpi_desc);
1f7df6f8 2290
20985164
VV		 2291 out_unlock:
2292 mutex_unlock(&acpi_desc->init_mutex);
2293 return rc;
b94d5230 2294}
6bc75619 2295EXPORT_SYMBOL_GPL(acpi_nfit_init);
b94d5230 2296
7ae0fa43
DW		 2297struct acpi_nfit_flush_work {
2298 struct work_struct work;
2299 struct completion cmp;
2300};
2301
2302static void flush_probe(struct work_struct *work)
2303{
2304 struct acpi_nfit_flush_work *flush;
2305
2306 flush = container_of(work, typeof(*flush), work);
2307 complete(&flush->cmp);
2308}
2309
2310static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
2311{
2312 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
2313 struct device *dev = acpi_desc->dev;
2314 struct acpi_nfit_flush_work flush;
2315
2316 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
2317 device_lock(dev);
2318 device_unlock(dev);
2319
2320 /*
2321 * Scrub work could take 10s of seconds, userspace may give up so we
2322 * need to be interruptible while waiting.
2323 */
2324 INIT_WORK_ONSTACK(&flush.work, flush_probe);
2325 COMPLETION_INITIALIZER_ONSTACK(flush.cmp);
2326 queue_work(nfit_wq, &flush.work);
2327 return wait_for_completion_interruptible(&flush.cmp);
2328}
2329
87bf572e
DW		 2330static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
2331 struct nvdimm *nvdimm, unsigned int cmd)
2332{
2333 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
2334
2335 if (nvdimm)
2336 return 0;
2337 if (cmd != ND_CMD_ARS_START)
2338 return 0;
2339
2340 /*
2341 * The kernel and userspace may race to initiate a scrub, but
2342 * the scrub thread is prepared to lose that initial race. It
2343 * just needs guarantees that any ars it initiates are not
2344 * interrupted by any intervening start reqeusts from userspace.
2345 */
2346 if (work_busy(&acpi_desc->work))
2347 return -EBUSY;
2348
2349 return 0;
2350}
2351
a61fe6f7 2352void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
b94d5230
DW		 2353{
2354 struct nvdimm_bus_descriptor *nd_desc;
b94d5230
DW		 2355
2356 dev_set_drvdata(dev, acpi_desc);
2357 acpi_desc->dev = dev;
6bc75619 2358 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
b94d5230
DW		 2359 nd_desc = &acpi_desc->nd_desc;
2360 nd_desc->provider_name = "ACPI.NFIT";
2361 nd_desc->ndctl = acpi_nfit_ctl;
7ae0fa43 2362 nd_desc->flush_probe = acpi_nfit_flush_probe;
87bf572e 2363 nd_desc->clear_to_send = acpi_nfit_clear_to_send;
45def22c 2364 nd_desc->attr_groups = acpi_nfit_attribute_groups;
b94d5230 2365
20985164
VV		 2366 INIT_LIST_HEAD(&acpi_desc->spa_maps);
2367 INIT_LIST_HEAD(&acpi_desc->spas);
2368 INIT_LIST_HEAD(&acpi_desc->dcrs);
2369 INIT_LIST_HEAD(&acpi_desc->bdws);
2370 INIT_LIST_HEAD(&acpi_desc->idts);
2371 INIT_LIST_HEAD(&acpi_desc->flushes);
2372 INIT_LIST_HEAD(&acpi_desc->memdevs);
2373 INIT_LIST_HEAD(&acpi_desc->dimms);
2374 mutex_init(&acpi_desc->spa_map_mutex);
2375 mutex_init(&acpi_desc->init_mutex);
1cf03c00 2376 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub);
20985164 2377}
a61fe6f7 2378EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
20985164
VV		 2379
2380static int acpi_nfit_add(struct acpi_device *adev)
2381{
2382 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
2383 struct acpi_nfit_desc *acpi_desc;
2384 struct device *dev = &adev->dev;
2385 struct acpi_table_header *tbl;
2386 acpi_status status = AE_OK;
2387 acpi_size sz;
2388 int rc;
2389
2390 status = acpi_get_table_with_size("NFIT", 0, &tbl, &sz);
2391 if (ACPI_FAILURE(status)) {
2392 /* This is ok, we could have an nvdimm hotplugged later */
2393 dev_dbg(dev, "failed to find NFIT at startup\n");
2394 return 0;
2395 }
2396
a61fe6f7
DW		 2397 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
2398 if (!acpi_desc)
2399 return -ENOMEM;
2400 acpi_nfit_desc_init(acpi_desc, &adev->dev);
2401 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, &acpi_desc->nd_desc);
2402 if (!acpi_desc->nvdimm_bus)
2403 return -ENOMEM;
20985164 2404
6b577c9d
LK		 2405 /*
2406 * Save the acpi header for later and then skip it,
2407 * making nfit point to the first nfit table header.
2408 */
2409 acpi_desc->acpi_header = *tbl;
2410 acpi_desc->nfit = (void *) tbl + sizeof(struct acpi_table_nfit);
2411 sz -= sizeof(struct acpi_table_nfit);
20985164
VV		 2412
2413 /* Evaluate _FIT and override with that if present */
2414 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
2415 if (ACPI_SUCCESS(status) && buf.length > 0) {
6b577c9d
LK		 2416 union acpi_object *obj;
2417 /*
2418 * Adjust for the acpi_object header of the _FIT
2419 */
2420 obj = buf.pointer;
2421 if (obj->type == ACPI_TYPE_BUFFER) {
2422 acpi_desc->nfit =
2423 (struct acpi_nfit_header *)obj->buffer.pointer;
2424 sz = obj->buffer.length;
2425 } else
2426 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n",
2427 __func__, (int) obj->type);
20985164 2428 }
b94d5230
DW		 2429
2430 rc = acpi_nfit_init(acpi_desc, sz);
2431 if (rc) {
2432 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
2433 return rc;
2434 }
2435 return 0;
2436}
2437
2438static int acpi_nfit_remove(struct acpi_device *adev)
2439{
2440 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
2441
7ae0fa43
DW		 2442 acpi_desc->cancel = 1;
2443 flush_workqueue(nfit_wq);
b94d5230
DW		 2444 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
2445 return 0;
2446}
2447
20985164
VV		 2448static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
2449{
2450 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
2451 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
6b577c9d
LK		 2452 struct acpi_nfit_header *nfit_saved;
2453 union acpi_object *obj;
20985164
VV		 2454 struct device *dev = &adev->dev;
2455 acpi_status status;
2456 int ret;
2457
2458 dev_dbg(dev, "%s: event: %d\n", __func__, event);
2459
2460 device_lock(dev);
2461 if (!dev->driver) {
2462 /* dev->driver may be null if we're being removed */
2463 dev_dbg(dev, "%s: no driver found for dev\n", __func__);
d91e8928 2464 goto out_unlock;
20985164
VV		 2465 }
2466
2467 if (!acpi_desc) {
a61fe6f7
DW		 2468 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
2469 if (!acpi_desc)
2470 goto out_unlock;
2471 acpi_nfit_desc_init(acpi_desc, &adev->dev);
2472 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, &acpi_desc->nd_desc);
2473 if (!acpi_desc->nvdimm_bus)
20985164 2474 goto out_unlock;
7ae0fa43
DW		 2475 } else {
2476 /*
2477 * Finish previous registration before considering new
2478 * regions.
2479 */
2480 flush_workqueue(nfit_wq);
20985164
VV		 2481 }
2482
2483 /* Evaluate _FIT */
2484 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
2485 if (ACPI_FAILURE(status)) {
2486 dev_err(dev, "failed to evaluate _FIT\n");
2487 goto out_unlock;
2488 }
2489
2490 nfit_saved = acpi_desc->nfit;
6b577c9d
LK		 2491 obj = buf.pointer;
2492 if (obj->type == ACPI_TYPE_BUFFER) {
2493 acpi_desc->nfit =
2494 (struct acpi_nfit_header *)obj->buffer.pointer;
2495 ret = acpi_nfit_init(acpi_desc, obj->buffer.length);
2496 if (ret) {
2497 /* Merge failed, restore old nfit, and exit */
2498 acpi_desc->nfit = nfit_saved;
2499 dev_err(dev, "failed to merge updated NFIT\n");
2500 }
2501 } else {
2502 /* Bad _FIT, restore old nfit */
2503 dev_err(dev, "Invalid _FIT\n");
20985164
VV		 2504 }
2505 kfree(buf.pointer);
2506
2507 out_unlock:
2508 device_unlock(dev);
2509}
2510
b94d5230
DW		 2511static const struct acpi_device_id acpi_nfit_ids[] = {
2512 { "ACPI0012", 0 },
2513 { "", 0 },
2514};
2515MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
2516
2517static struct acpi_driver acpi_nfit_driver = {
2518 .name = KBUILD_MODNAME,
2519 .ids = acpi_nfit_ids,
2520 .ops = {
2521 .add = acpi_nfit_add,
2522 .remove = acpi_nfit_remove,
20985164 2523 .notify = acpi_nfit_notify,
b94d5230
DW		 2524 },
2525};
2526
2527static __init int nfit_init(void)
2528{
2529 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
2530 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
2531 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
2532 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
2533 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
2534 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
2535 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
2536
2537 acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]);
2538 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]);
2539 acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]);
2540 acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]);
2541 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]);
2542 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]);
2543 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]);
2544 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]);
2545 acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]);
2546 acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]);
31eca76b
DW		 2547 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1, nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
2548 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2, nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
b94d5230 2549
7ae0fa43
DW		 2550 nfit_wq = create_singlethread_workqueue("nfit");
2551 if (!nfit_wq)
2552 return -ENOMEM;
2553
b94d5230
DW		 2554 return acpi_bus_register_driver(&acpi_nfit_driver);
2555}
2556
2557static __exit void nfit_exit(void)
2558{
2559 acpi_bus_unregister_driver(&acpi_nfit_driver);
7ae0fa43 2560 destroy_workqueue(nfit_wq);
b94d5230
DW		 2561}
2562
2563module_init(nfit_init);
2564module_exit(nfit_exit);
2565MODULE_LICENSE("GPL v2");
2566MODULE_AUTHOR("Intel Corporation");
Ubuntu Artful kernel mirror