2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/list_sort.h>
14 #include <linux/libnvdimm.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/ndctl.h>
18 #include <linux/delay.h>
19 #include <linux/list.h>
20 #include <linux/acpi.h>
21 #include <linux/sort.h>
22 #include <linux/pmem.h>
25 #include <asm/cacheflush.h>
29 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
32 #include <linux/io-64-nonatomic-hi-lo.h>
34 static bool force_enable_dimms
;
35 module_param(force_enable_dimms
, bool, S_IRUGO
|S_IWUSR
);
36 MODULE_PARM_DESC(force_enable_dimms
, "Ignore _STA (ACPI DIMM device) status");
38 static unsigned int scrub_timeout
= NFIT_ARS_TIMEOUT
;
39 module_param(scrub_timeout
, uint
, S_IRUGO
|S_IWUSR
);
40 MODULE_PARM_DESC(scrub_timeout
, "Initial scrub timeout in seconds");
42 /* after three payloads of overflow, it's dead jim */
43 static unsigned int scrub_overflow_abort
= 3;
44 module_param(scrub_overflow_abort
, uint
, S_IRUGO
|S_IWUSR
);
45 MODULE_PARM_DESC(scrub_overflow_abort
,
46 "Number of times we overflow ARS results before abort");
48 static struct workqueue_struct
*nfit_wq
;
50 struct nfit_table_prev
{
51 struct list_head spas
;
52 struct list_head memdevs
;
53 struct list_head dcrs
;
54 struct list_head bdws
;
55 struct list_head idts
;
56 struct list_head flushes
;
59 static u8 nfit_uuid
[NFIT_UUID_MAX
][16];
61 const u8
*to_nfit_uuid(enum nfit_uuids id
)
65 EXPORT_SYMBOL(to_nfit_uuid
);
67 static struct acpi_nfit_desc
*to_acpi_nfit_desc(
68 struct nvdimm_bus_descriptor
*nd_desc
)
70 return container_of(nd_desc
, struct acpi_nfit_desc
, nd_desc
);
73 static struct acpi_device
*to_acpi_dev(struct acpi_nfit_desc
*acpi_desc
)
75 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
78 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
81 if (!nd_desc
->provider_name
82 || strcmp(nd_desc
->provider_name
, "ACPI.NFIT") != 0)
85 return to_acpi_device(acpi_desc
->dev
);
88 static int xlat_status(void *buf
, unsigned int cmd
)
90 struct nd_cmd_clear_error
*clear_err
;
91 struct nd_cmd_ars_status
*ars_status
;
92 struct nd_cmd_ars_start
*ars_start
;
93 struct nd_cmd_ars_cap
*ars_cap
;
99 if ((ars_cap
->status
& 0xffff) == NFIT_ARS_CAP_NONE
)
103 if (ars_cap
->status
& 0xffff)
106 /* No supported scan types for this range */
107 flags
= ND_ARS_PERSISTENT
| ND_ARS_VOLATILE
;
108 if ((ars_cap
->status
>> 16 & flags
) == 0)
111 case ND_CMD_ARS_START
:
113 /* ARS is in progress */
114 if ((ars_start
->status
& 0xffff) == NFIT_ARS_START_BUSY
)
118 if (ars_start
->status
& 0xffff)
121 case ND_CMD_ARS_STATUS
:
124 if (ars_status
->status
& 0xffff)
126 /* Check extended status (Upper two bytes) */
127 if (ars_status
->status
== NFIT_ARS_STATUS_DONE
)
130 /* ARS is in progress */
131 if (ars_status
->status
== NFIT_ARS_STATUS_BUSY
)
134 /* No ARS performed for the current boot */
135 if (ars_status
->status
== NFIT_ARS_STATUS_NONE
)
139 * ARS interrupted, either we overflowed or some other
140 * agent wants the scan to stop. If we didn't overflow
141 * then just continue with the returned results.
143 if (ars_status
->status
== NFIT_ARS_STATUS_INTR
) {
144 if (ars_status
->flags
& NFIT_ARS_F_OVERFLOW
)
150 if (ars_status
->status
>> 16)
153 case ND_CMD_CLEAR_ERROR
:
155 if (clear_err
->status
& 0xffff)
157 if (!clear_err
->cleared
)
159 if (clear_err
->length
> clear_err
->cleared
)
160 return clear_err
->cleared
;
169 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor
*nd_desc
,
170 struct nvdimm
*nvdimm
, unsigned int cmd
, void *buf
,
171 unsigned int buf_len
, int *cmd_rc
)
173 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
174 const struct nd_cmd_desc
*desc
= NULL
;
175 union acpi_object in_obj
, in_buf
, *out_obj
;
176 struct device
*dev
= acpi_desc
->dev
;
177 const char *cmd_name
, *dimm_name
;
178 unsigned long dsm_mask
;
185 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
186 struct acpi_device
*adev
= nfit_mem
->adev
;
190 dimm_name
= nvdimm_name(nvdimm
);
191 cmd_name
= nvdimm_cmd_name(cmd
);
192 dsm_mask
= nfit_mem
->dsm_mask
;
193 desc
= nd_cmd_dimm_desc(cmd
);
194 uuid
= to_nfit_uuid(NFIT_DEV_DIMM
);
195 handle
= adev
->handle
;
197 struct acpi_device
*adev
= to_acpi_dev(acpi_desc
);
199 cmd_name
= nvdimm_bus_cmd_name(cmd
);
200 dsm_mask
= nd_desc
->dsm_mask
;
201 desc
= nd_cmd_bus_desc(cmd
);
202 uuid
= to_nfit_uuid(NFIT_DEV_BUS
);
203 handle
= adev
->handle
;
207 if (!desc
|| (cmd
&& (desc
->out_num
+ desc
->in_num
== 0)))
210 if (!test_bit(cmd
, &dsm_mask
))
213 in_obj
.type
= ACPI_TYPE_PACKAGE
;
214 in_obj
.package
.count
= 1;
215 in_obj
.package
.elements
= &in_buf
;
216 in_buf
.type
= ACPI_TYPE_BUFFER
;
217 in_buf
.buffer
.pointer
= buf
;
218 in_buf
.buffer
.length
= 0;
220 /* libnvdimm has already validated the input envelope */
221 for (i
= 0; i
< desc
->in_num
; i
++)
222 in_buf
.buffer
.length
+= nd_cmd_in_size(nvdimm
, cmd
, desc
,
225 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG
)) {
226 dev_dbg(dev
, "%s:%s cmd: %s input length: %d\n", __func__
,
227 dimm_name
, cmd_name
, in_buf
.buffer
.length
);
228 print_hex_dump_debug(cmd_name
, DUMP_PREFIX_OFFSET
, 4,
229 4, in_buf
.buffer
.pointer
, min_t(u32
, 128,
230 in_buf
.buffer
.length
), true);
233 out_obj
= acpi_evaluate_dsm(handle
, uuid
, 1, cmd
, &in_obj
);
235 dev_dbg(dev
, "%s:%s _DSM failed cmd: %s\n", __func__
, dimm_name
,
240 if (out_obj
->package
.type
!= ACPI_TYPE_BUFFER
) {
241 dev_dbg(dev
, "%s:%s unexpected output object type cmd: %s type: %d\n",
242 __func__
, dimm_name
, cmd_name
, out_obj
->type
);
247 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG
)) {
248 dev_dbg(dev
, "%s:%s cmd: %s output length: %d\n", __func__
,
249 dimm_name
, cmd_name
, out_obj
->buffer
.length
);
250 print_hex_dump_debug(cmd_name
, DUMP_PREFIX_OFFSET
, 4,
251 4, out_obj
->buffer
.pointer
, min_t(u32
, 128,
252 out_obj
->buffer
.length
), true);
255 for (i
= 0, offset
= 0; i
< desc
->out_num
; i
++) {
256 u32 out_size
= nd_cmd_out_size(nvdimm
, cmd
, desc
, i
, buf
,
257 (u32
*) out_obj
->buffer
.pointer
);
259 if (offset
+ out_size
> out_obj
->buffer
.length
) {
260 dev_dbg(dev
, "%s:%s output object underflow cmd: %s field: %d\n",
261 __func__
, dimm_name
, cmd_name
, i
);
265 if (in_buf
.buffer
.length
+ offset
+ out_size
> buf_len
) {
266 dev_dbg(dev
, "%s:%s output overrun cmd: %s field: %d\n",
267 __func__
, dimm_name
, cmd_name
, i
);
271 memcpy(buf
+ in_buf
.buffer
.length
+ offset
,
272 out_obj
->buffer
.pointer
+ offset
, out_size
);
275 if (offset
+ in_buf
.buffer
.length
< buf_len
) {
278 * status valid, return the number of bytes left
279 * unfilled in the output buffer
281 rc
= buf_len
- offset
- in_buf
.buffer
.length
;
283 *cmd_rc
= xlat_status(buf
, cmd
);
285 dev_err(dev
, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
286 __func__
, dimm_name
, cmd_name
, buf_len
,
293 *cmd_rc
= xlat_status(buf
, cmd
);
302 static const char *spa_type_name(u16 type
)
304 static const char *to_name
[] = {
305 [NFIT_SPA_VOLATILE
] = "volatile",
306 [NFIT_SPA_PM
] = "pmem",
307 [NFIT_SPA_DCR
] = "dimm-control-region",
308 [NFIT_SPA_BDW
] = "block-data-window",
309 [NFIT_SPA_VDISK
] = "volatile-disk",
310 [NFIT_SPA_VCD
] = "volatile-cd",
311 [NFIT_SPA_PDISK
] = "persistent-disk",
312 [NFIT_SPA_PCD
] = "persistent-cd",
316 if (type
> NFIT_SPA_PCD
)
319 return to_name
[type
];
322 static int nfit_spa_type(struct acpi_nfit_system_address
*spa
)
326 for (i
= 0; i
< NFIT_UUID_MAX
; i
++)
327 if (memcmp(to_nfit_uuid(i
), spa
->range_guid
, 16) == 0)
332 static bool add_spa(struct acpi_nfit_desc
*acpi_desc
,
333 struct nfit_table_prev
*prev
,
334 struct acpi_nfit_system_address
*spa
)
336 size_t length
= min_t(size_t, sizeof(*spa
), spa
->header
.length
);
337 struct device
*dev
= acpi_desc
->dev
;
338 struct nfit_spa
*nfit_spa
;
340 list_for_each_entry(nfit_spa
, &prev
->spas
, list
) {
341 if (memcmp(nfit_spa
->spa
, spa
, length
) == 0) {
342 list_move_tail(&nfit_spa
->list
, &acpi_desc
->spas
);
347 nfit_spa
= devm_kzalloc(dev
, sizeof(*nfit_spa
), GFP_KERNEL
);
350 INIT_LIST_HEAD(&nfit_spa
->list
);
352 list_add_tail(&nfit_spa
->list
, &acpi_desc
->spas
);
353 dev_dbg(dev
, "%s: spa index: %d type: %s\n", __func__
,
355 spa_type_name(nfit_spa_type(spa
)));
359 static bool add_memdev(struct acpi_nfit_desc
*acpi_desc
,
360 struct nfit_table_prev
*prev
,
361 struct acpi_nfit_memory_map
*memdev
)
363 size_t length
= min_t(size_t, sizeof(*memdev
), memdev
->header
.length
);
364 struct device
*dev
= acpi_desc
->dev
;
365 struct nfit_memdev
*nfit_memdev
;
367 list_for_each_entry(nfit_memdev
, &prev
->memdevs
, list
)
368 if (memcmp(nfit_memdev
->memdev
, memdev
, length
) == 0) {
369 list_move_tail(&nfit_memdev
->list
, &acpi_desc
->memdevs
);
373 nfit_memdev
= devm_kzalloc(dev
, sizeof(*nfit_memdev
), GFP_KERNEL
);
376 INIT_LIST_HEAD(&nfit_memdev
->list
);
377 nfit_memdev
->memdev
= memdev
;
378 list_add_tail(&nfit_memdev
->list
, &acpi_desc
->memdevs
);
379 dev_dbg(dev
, "%s: memdev handle: %#x spa: %d dcr: %d\n",
380 __func__
, memdev
->device_handle
, memdev
->range_index
,
381 memdev
->region_index
);
385 static bool add_dcr(struct acpi_nfit_desc
*acpi_desc
,
386 struct nfit_table_prev
*prev
,
387 struct acpi_nfit_control_region
*dcr
)
389 size_t length
= min_t(size_t, sizeof(*dcr
), dcr
->header
.length
);
390 struct device
*dev
= acpi_desc
->dev
;
391 struct nfit_dcr
*nfit_dcr
;
393 list_for_each_entry(nfit_dcr
, &prev
->dcrs
, list
)
394 if (memcmp(nfit_dcr
->dcr
, dcr
, length
) == 0) {
395 list_move_tail(&nfit_dcr
->list
, &acpi_desc
->dcrs
);
399 nfit_dcr
= devm_kzalloc(dev
, sizeof(*nfit_dcr
), GFP_KERNEL
);
402 INIT_LIST_HEAD(&nfit_dcr
->list
);
404 list_add_tail(&nfit_dcr
->list
, &acpi_desc
->dcrs
);
405 dev_dbg(dev
, "%s: dcr index: %d windows: %d\n", __func__
,
406 dcr
->region_index
, dcr
->windows
);
410 static bool add_bdw(struct acpi_nfit_desc
*acpi_desc
,
411 struct nfit_table_prev
*prev
,
412 struct acpi_nfit_data_region
*bdw
)
414 size_t length
= min_t(size_t, sizeof(*bdw
), bdw
->header
.length
);
415 struct device
*dev
= acpi_desc
->dev
;
416 struct nfit_bdw
*nfit_bdw
;
418 list_for_each_entry(nfit_bdw
, &prev
->bdws
, list
)
419 if (memcmp(nfit_bdw
->bdw
, bdw
, length
) == 0) {
420 list_move_tail(&nfit_bdw
->list
, &acpi_desc
->bdws
);
424 nfit_bdw
= devm_kzalloc(dev
, sizeof(*nfit_bdw
), GFP_KERNEL
);
427 INIT_LIST_HEAD(&nfit_bdw
->list
);
429 list_add_tail(&nfit_bdw
->list
, &acpi_desc
->bdws
);
430 dev_dbg(dev
, "%s: bdw dcr: %d windows: %d\n", __func__
,
431 bdw
->region_index
, bdw
->windows
);
435 static bool add_idt(struct acpi_nfit_desc
*acpi_desc
,
436 struct nfit_table_prev
*prev
,
437 struct acpi_nfit_interleave
*idt
)
439 size_t length
= min_t(size_t, sizeof(*idt
), idt
->header
.length
);
440 struct device
*dev
= acpi_desc
->dev
;
441 struct nfit_idt
*nfit_idt
;
443 list_for_each_entry(nfit_idt
, &prev
->idts
, list
)
444 if (memcmp(nfit_idt
->idt
, idt
, length
) == 0) {
445 list_move_tail(&nfit_idt
->list
, &acpi_desc
->idts
);
449 nfit_idt
= devm_kzalloc(dev
, sizeof(*nfit_idt
), GFP_KERNEL
);
452 INIT_LIST_HEAD(&nfit_idt
->list
);
454 list_add_tail(&nfit_idt
->list
, &acpi_desc
->idts
);
455 dev_dbg(dev
, "%s: idt index: %d num_lines: %d\n", __func__
,
456 idt
->interleave_index
, idt
->line_count
);
460 static bool add_flush(struct acpi_nfit_desc
*acpi_desc
,
461 struct nfit_table_prev
*prev
,
462 struct acpi_nfit_flush_address
*flush
)
464 size_t length
= min_t(size_t, sizeof(*flush
), flush
->header
.length
);
465 struct device
*dev
= acpi_desc
->dev
;
466 struct nfit_flush
*nfit_flush
;
468 list_for_each_entry(nfit_flush
, &prev
->flushes
, list
)
469 if (memcmp(nfit_flush
->flush
, flush
, length
) == 0) {
470 list_move_tail(&nfit_flush
->list
, &acpi_desc
->flushes
);
474 nfit_flush
= devm_kzalloc(dev
, sizeof(*nfit_flush
), GFP_KERNEL
);
477 INIT_LIST_HEAD(&nfit_flush
->list
);
478 nfit_flush
->flush
= flush
;
479 list_add_tail(&nfit_flush
->list
, &acpi_desc
->flushes
);
480 dev_dbg(dev
, "%s: nfit_flush handle: %d hint_count: %d\n", __func__
,
481 flush
->device_handle
, flush
->hint_count
);
485 static void *add_table(struct acpi_nfit_desc
*acpi_desc
,
486 struct nfit_table_prev
*prev
, void *table
, const void *end
)
488 struct device
*dev
= acpi_desc
->dev
;
489 struct acpi_nfit_header
*hdr
;
490 void *err
= ERR_PTR(-ENOMEM
);
497 dev_warn(dev
, "found a zero length table '%d' parsing nfit\n",
503 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS
:
504 if (!add_spa(acpi_desc
, prev
, table
))
507 case ACPI_NFIT_TYPE_MEMORY_MAP
:
508 if (!add_memdev(acpi_desc
, prev
, table
))
511 case ACPI_NFIT_TYPE_CONTROL_REGION
:
512 if (!add_dcr(acpi_desc
, prev
, table
))
515 case ACPI_NFIT_TYPE_DATA_REGION
:
516 if (!add_bdw(acpi_desc
, prev
, table
))
519 case ACPI_NFIT_TYPE_INTERLEAVE
:
520 if (!add_idt(acpi_desc
, prev
, table
))
523 case ACPI_NFIT_TYPE_FLUSH_ADDRESS
:
524 if (!add_flush(acpi_desc
, prev
, table
))
527 case ACPI_NFIT_TYPE_SMBIOS
:
528 dev_dbg(dev
, "%s: smbios\n", __func__
);
531 dev_err(dev
, "unknown table '%d' parsing nfit\n", hdr
->type
);
535 return table
+ hdr
->length
;
538 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc
*acpi_desc
,
539 struct nfit_mem
*nfit_mem
)
541 u32 device_handle
= __to_nfit_memdev(nfit_mem
)->device_handle
;
542 u16 dcr
= nfit_mem
->dcr
->region_index
;
543 struct nfit_spa
*nfit_spa
;
545 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
546 u16 range_index
= nfit_spa
->spa
->range_index
;
547 int type
= nfit_spa_type(nfit_spa
->spa
);
548 struct nfit_memdev
*nfit_memdev
;
550 if (type
!= NFIT_SPA_BDW
)
553 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
554 if (nfit_memdev
->memdev
->range_index
!= range_index
)
556 if (nfit_memdev
->memdev
->device_handle
!= device_handle
)
558 if (nfit_memdev
->memdev
->region_index
!= dcr
)
561 nfit_mem
->spa_bdw
= nfit_spa
->spa
;
566 dev_dbg(acpi_desc
->dev
, "SPA-BDW not found for SPA-DCR %d\n",
567 nfit_mem
->spa_dcr
->range_index
);
568 nfit_mem
->bdw
= NULL
;
571 static void nfit_mem_init_bdw(struct acpi_nfit_desc
*acpi_desc
,
572 struct nfit_mem
*nfit_mem
, struct acpi_nfit_system_address
*spa
)
574 u16 dcr
= __to_nfit_memdev(nfit_mem
)->region_index
;
575 struct nfit_memdev
*nfit_memdev
;
576 struct nfit_flush
*nfit_flush
;
577 struct nfit_bdw
*nfit_bdw
;
578 struct nfit_idt
*nfit_idt
;
579 u16 idt_idx
, range_index
;
581 list_for_each_entry(nfit_bdw
, &acpi_desc
->bdws
, list
) {
582 if (nfit_bdw
->bdw
->region_index
!= dcr
)
584 nfit_mem
->bdw
= nfit_bdw
->bdw
;
591 nfit_mem_find_spa_bdw(acpi_desc
, nfit_mem
);
593 if (!nfit_mem
->spa_bdw
)
596 range_index
= nfit_mem
->spa_bdw
->range_index
;
597 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
598 if (nfit_memdev
->memdev
->range_index
!= range_index
||
599 nfit_memdev
->memdev
->region_index
!= dcr
)
601 nfit_mem
->memdev_bdw
= nfit_memdev
->memdev
;
602 idt_idx
= nfit_memdev
->memdev
->interleave_index
;
603 list_for_each_entry(nfit_idt
, &acpi_desc
->idts
, list
) {
604 if (nfit_idt
->idt
->interleave_index
!= idt_idx
)
606 nfit_mem
->idt_bdw
= nfit_idt
->idt
;
610 list_for_each_entry(nfit_flush
, &acpi_desc
->flushes
, list
) {
611 if (nfit_flush
->flush
->device_handle
!=
612 nfit_memdev
->memdev
->device_handle
)
614 nfit_mem
->nfit_flush
= nfit_flush
;
621 static int nfit_mem_dcr_init(struct acpi_nfit_desc
*acpi_desc
,
622 struct acpi_nfit_system_address
*spa
)
624 struct nfit_mem
*nfit_mem
, *found
;
625 struct nfit_memdev
*nfit_memdev
;
626 int type
= nfit_spa_type(spa
);
636 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
637 struct nfit_dcr
*nfit_dcr
;
641 if (nfit_memdev
->memdev
->range_index
!= spa
->range_index
)
644 dcr
= nfit_memdev
->memdev
->region_index
;
645 device_handle
= nfit_memdev
->memdev
->device_handle
;
646 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
)
647 if (__to_nfit_memdev(nfit_mem
)->device_handle
656 nfit_mem
= devm_kzalloc(acpi_desc
->dev
,
657 sizeof(*nfit_mem
), GFP_KERNEL
);
660 INIT_LIST_HEAD(&nfit_mem
->list
);
661 list_add(&nfit_mem
->list
, &acpi_desc
->dimms
);
664 list_for_each_entry(nfit_dcr
, &acpi_desc
->dcrs
, list
) {
665 if (nfit_dcr
->dcr
->region_index
!= dcr
)
668 * Record the control region for the dimm. For
669 * the ACPI 6.1 case, where there are separate
670 * control regions for the pmem vs blk
671 * interfaces, be sure to record the extended
675 nfit_mem
->dcr
= nfit_dcr
->dcr
;
676 else if (nfit_mem
->dcr
->windows
== 0
677 && nfit_dcr
->dcr
->windows
)
678 nfit_mem
->dcr
= nfit_dcr
->dcr
;
682 if (dcr
&& !nfit_mem
->dcr
) {
683 dev_err(acpi_desc
->dev
, "SPA %d missing DCR %d\n",
684 spa
->range_index
, dcr
);
688 if (type
== NFIT_SPA_DCR
) {
689 struct nfit_idt
*nfit_idt
;
692 /* multiple dimms may share a SPA when interleaved */
693 nfit_mem
->spa_dcr
= spa
;
694 nfit_mem
->memdev_dcr
= nfit_memdev
->memdev
;
695 idt_idx
= nfit_memdev
->memdev
->interleave_index
;
696 list_for_each_entry(nfit_idt
, &acpi_desc
->idts
, list
) {
697 if (nfit_idt
->idt
->interleave_index
!= idt_idx
)
699 nfit_mem
->idt_dcr
= nfit_idt
->idt
;
702 nfit_mem_init_bdw(acpi_desc
, nfit_mem
, spa
);
705 * A single dimm may belong to multiple SPA-PM
706 * ranges, record at least one in addition to
709 nfit_mem
->memdev_pmem
= nfit_memdev
->memdev
;
716 static int nfit_mem_cmp(void *priv
, struct list_head
*_a
, struct list_head
*_b
)
718 struct nfit_mem
*a
= container_of(_a
, typeof(*a
), list
);
719 struct nfit_mem
*b
= container_of(_b
, typeof(*b
), list
);
720 u32 handleA
, handleB
;
722 handleA
= __to_nfit_memdev(a
)->device_handle
;
723 handleB
= __to_nfit_memdev(b
)->device_handle
;
724 if (handleA
< handleB
)
726 else if (handleA
> handleB
)
731 static int nfit_mem_init(struct acpi_nfit_desc
*acpi_desc
)
733 struct nfit_spa
*nfit_spa
;
736 * For each SPA-DCR or SPA-PMEM address range find its
737 * corresponding MEMDEV(s). From each MEMDEV find the
738 * corresponding DCR. Then, if we're operating on a SPA-DCR,
739 * try to find a SPA-BDW and a corresponding BDW that references
740 * the DCR. Throw it all into an nfit_mem object. Note, that
743 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
746 rc
= nfit_mem_dcr_init(acpi_desc
, nfit_spa
->spa
);
751 list_sort(NULL
, &acpi_desc
->dimms
, nfit_mem_cmp
);
756 static ssize_t
revision_show(struct device
*dev
,
757 struct device_attribute
*attr
, char *buf
)
759 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
760 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
761 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
763 return sprintf(buf
, "%d\n", acpi_desc
->acpi_header
.revision
);
765 static DEVICE_ATTR_RO(revision
);
767 static struct attribute
*acpi_nfit_attributes
[] = {
768 &dev_attr_revision
.attr
,
772 static struct attribute_group acpi_nfit_attribute_group
= {
774 .attrs
= acpi_nfit_attributes
,
777 static const struct attribute_group
*acpi_nfit_attribute_groups
[] = {
778 &nvdimm_bus_attribute_group
,
779 &acpi_nfit_attribute_group
,
783 static struct acpi_nfit_memory_map
*to_nfit_memdev(struct device
*dev
)
785 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
786 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
788 return __to_nfit_memdev(nfit_mem
);
791 static struct acpi_nfit_control_region
*to_nfit_dcr(struct device
*dev
)
793 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
794 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
796 return nfit_mem
->dcr
;
799 static ssize_t
handle_show(struct device
*dev
,
800 struct device_attribute
*attr
, char *buf
)
802 struct acpi_nfit_memory_map
*memdev
= to_nfit_memdev(dev
);
804 return sprintf(buf
, "%#x\n", memdev
->device_handle
);
806 static DEVICE_ATTR_RO(handle
);
808 static ssize_t
phys_id_show(struct device
*dev
,
809 struct device_attribute
*attr
, char *buf
)
811 struct acpi_nfit_memory_map
*memdev
= to_nfit_memdev(dev
);
813 return sprintf(buf
, "%#x\n", memdev
->physical_id
);
815 static DEVICE_ATTR_RO(phys_id
);
817 static ssize_t
vendor_show(struct device
*dev
,
818 struct device_attribute
*attr
, char *buf
)
820 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
822 return sprintf(buf
, "%#x\n", dcr
->vendor_id
);
824 static DEVICE_ATTR_RO(vendor
);
826 static ssize_t
rev_id_show(struct device
*dev
,
827 struct device_attribute
*attr
, char *buf
)
829 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
831 return sprintf(buf
, "%#x\n", dcr
->revision_id
);
833 static DEVICE_ATTR_RO(rev_id
);
835 static ssize_t
device_show(struct device
*dev
,
836 struct device_attribute
*attr
, char *buf
)
838 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
840 return sprintf(buf
, "%#x\n", dcr
->device_id
);
842 static DEVICE_ATTR_RO(device
);
844 static ssize_t
format_show(struct device
*dev
,
845 struct device_attribute
*attr
, char *buf
)
847 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
849 return sprintf(buf
, "%#x\n", dcr
->code
);
851 static DEVICE_ATTR_RO(format
);
853 static ssize_t
serial_show(struct device
*dev
,
854 struct device_attribute
*attr
, char *buf
)
856 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
858 return sprintf(buf
, "%#x\n", dcr
->serial_number
);
860 static DEVICE_ATTR_RO(serial
);
862 static ssize_t
flags_show(struct device
*dev
,
863 struct device_attribute
*attr
, char *buf
)
865 u16 flags
= to_nfit_memdev(dev
)->flags
;
867 return sprintf(buf
, "%s%s%s%s%s\n",
868 flags
& ACPI_NFIT_MEM_SAVE_FAILED
? "save_fail " : "",
869 flags
& ACPI_NFIT_MEM_RESTORE_FAILED
? "restore_fail " : "",
870 flags
& ACPI_NFIT_MEM_FLUSH_FAILED
? "flush_fail " : "",
871 flags
& ACPI_NFIT_MEM_NOT_ARMED
? "not_armed " : "",
872 flags
& ACPI_NFIT_MEM_HEALTH_OBSERVED
? "smart_event " : "");
874 static DEVICE_ATTR_RO(flags
);
876 static struct attribute
*acpi_nfit_dimm_attributes
[] = {
877 &dev_attr_handle
.attr
,
878 &dev_attr_phys_id
.attr
,
879 &dev_attr_vendor
.attr
,
880 &dev_attr_device
.attr
,
881 &dev_attr_format
.attr
,
882 &dev_attr_serial
.attr
,
883 &dev_attr_rev_id
.attr
,
884 &dev_attr_flags
.attr
,
888 static umode_t
acpi_nfit_dimm_attr_visible(struct kobject
*kobj
,
889 struct attribute
*a
, int n
)
891 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
893 if (to_nfit_dcr(dev
))
899 static struct attribute_group acpi_nfit_dimm_attribute_group
= {
901 .attrs
= acpi_nfit_dimm_attributes
,
902 .is_visible
= acpi_nfit_dimm_attr_visible
,
905 static const struct attribute_group
*acpi_nfit_dimm_attribute_groups
[] = {
906 &nvdimm_attribute_group
,
907 &nd_device_attribute_group
,
908 &acpi_nfit_dimm_attribute_group
,
912 static struct nvdimm
*acpi_nfit_dimm_by_handle(struct acpi_nfit_desc
*acpi_desc
,
915 struct nfit_mem
*nfit_mem
;
917 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
)
918 if (__to_nfit_memdev(nfit_mem
)->device_handle
== device_handle
)
919 return nfit_mem
->nvdimm
;
924 static int acpi_nfit_add_dimm(struct acpi_nfit_desc
*acpi_desc
,
925 struct nfit_mem
*nfit_mem
, u32 device_handle
)
927 struct acpi_device
*adev
, *adev_dimm
;
928 struct device
*dev
= acpi_desc
->dev
;
929 const u8
*uuid
= to_nfit_uuid(NFIT_DEV_DIMM
);
932 nfit_mem
->dsm_mask
= acpi_desc
->dimm_dsm_force_en
;
933 adev
= to_acpi_dev(acpi_desc
);
937 adev_dimm
= acpi_find_child_device(adev
, device_handle
, false);
938 nfit_mem
->adev
= adev_dimm
;
940 dev_err(dev
, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
942 return force_enable_dimms
? 0 : -ENODEV
;
945 for (i
= ND_CMD_SMART
; i
<= ND_CMD_VENDOR
; i
++)
946 if (acpi_check_dsm(adev_dimm
->handle
, uuid
, 1, 1ULL << i
))
947 set_bit(i
, &nfit_mem
->dsm_mask
);
952 static int acpi_nfit_register_dimms(struct acpi_nfit_desc
*acpi_desc
)
954 struct nfit_mem
*nfit_mem
;
957 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
) {
958 struct nvdimm
*nvdimm
;
959 unsigned long flags
= 0;
964 device_handle
= __to_nfit_memdev(nfit_mem
)->device_handle
;
965 nvdimm
= acpi_nfit_dimm_by_handle(acpi_desc
, device_handle
);
971 if (nfit_mem
->bdw
&& nfit_mem
->memdev_pmem
)
972 flags
|= NDD_ALIASING
;
974 mem_flags
= __to_nfit_memdev(nfit_mem
)->flags
;
975 if (mem_flags
& ACPI_NFIT_MEM_NOT_ARMED
)
976 flags
|= NDD_UNARMED
;
978 rc
= acpi_nfit_add_dimm(acpi_desc
, nfit_mem
, device_handle
);
982 nvdimm
= nvdimm_create(acpi_desc
->nvdimm_bus
, nfit_mem
,
983 acpi_nfit_dimm_attribute_groups
,
984 flags
, &nfit_mem
->dsm_mask
);
988 nfit_mem
->nvdimm
= nvdimm
;
991 if ((mem_flags
& ACPI_NFIT_MEM_FAILED_MASK
) == 0)
994 dev_info(acpi_desc
->dev
, "%s flags:%s%s%s%s\n",
996 mem_flags
& ACPI_NFIT_MEM_SAVE_FAILED
? " save_fail" : "",
997 mem_flags
& ACPI_NFIT_MEM_RESTORE_FAILED
? " restore_fail":"",
998 mem_flags
& ACPI_NFIT_MEM_FLUSH_FAILED
? " flush_fail" : "",
999 mem_flags
& ACPI_NFIT_MEM_NOT_ARMED
? " not_armed" : "");
1003 return nvdimm_bus_check_dimm_count(acpi_desc
->nvdimm_bus
, dimm_count
);
1006 static void acpi_nfit_init_dsms(struct acpi_nfit_desc
*acpi_desc
)
1008 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1009 const u8
*uuid
= to_nfit_uuid(NFIT_DEV_BUS
);
1010 struct acpi_device
*adev
;
1013 nd_desc
->dsm_mask
= acpi_desc
->bus_dsm_force_en
;
1014 adev
= to_acpi_dev(acpi_desc
);
1018 for (i
= ND_CMD_ARS_CAP
; i
<= ND_CMD_CLEAR_ERROR
; i
++)
1019 if (acpi_check_dsm(adev
->handle
, uuid
, 1, 1ULL << i
))
1020 set_bit(i
, &nd_desc
->dsm_mask
);
1023 static ssize_t
range_index_show(struct device
*dev
,
1024 struct device_attribute
*attr
, char *buf
)
1026 struct nd_region
*nd_region
= to_nd_region(dev
);
1027 struct nfit_spa
*nfit_spa
= nd_region_provider_data(nd_region
);
1029 return sprintf(buf
, "%d\n", nfit_spa
->spa
->range_index
);
1031 static DEVICE_ATTR_RO(range_index
);
1033 static struct attribute
*acpi_nfit_region_attributes
[] = {
1034 &dev_attr_range_index
.attr
,
1038 static struct attribute_group acpi_nfit_region_attribute_group
= {
1040 .attrs
= acpi_nfit_region_attributes
,
1043 static const struct attribute_group
*acpi_nfit_region_attribute_groups
[] = {
1044 &nd_region_attribute_group
,
1045 &nd_mapping_attribute_group
,
1046 &nd_device_attribute_group
,
1047 &nd_numa_attribute_group
,
1048 &acpi_nfit_region_attribute_group
,
1052 /* enough info to uniquely specify an interleave set */
1053 struct nfit_set_info
{
1054 struct nfit_set_info_map
{
1061 static size_t sizeof_nfit_set_info(int num_mappings
)
1063 return sizeof(struct nfit_set_info
)
1064 + num_mappings
* sizeof(struct nfit_set_info_map
);
1067 static int cmp_map(const void *m0
, const void *m1
)
1069 const struct nfit_set_info_map
*map0
= m0
;
1070 const struct nfit_set_info_map
*map1
= m1
;
1072 return memcmp(&map0
->region_offset
, &map1
->region_offset
,
1076 /* Retrieve the nth entry referencing this spa */
1077 static struct acpi_nfit_memory_map
*memdev_from_spa(
1078 struct acpi_nfit_desc
*acpi_desc
, u16 range_index
, int n
)
1080 struct nfit_memdev
*nfit_memdev
;
1082 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
)
1083 if (nfit_memdev
->memdev
->range_index
== range_index
)
1085 return nfit_memdev
->memdev
;
1089 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc
*acpi_desc
,
1090 struct nd_region_desc
*ndr_desc
,
1091 struct acpi_nfit_system_address
*spa
)
1093 int i
, spa_type
= nfit_spa_type(spa
);
1094 struct device
*dev
= acpi_desc
->dev
;
1095 struct nd_interleave_set
*nd_set
;
1096 u16 nr
= ndr_desc
->num_mappings
;
1097 struct nfit_set_info
*info
;
1099 if (spa_type
== NFIT_SPA_PM
|| spa_type
== NFIT_SPA_VOLATILE
)
1104 nd_set
= devm_kzalloc(dev
, sizeof(*nd_set
), GFP_KERNEL
);
1108 info
= devm_kzalloc(dev
, sizeof_nfit_set_info(nr
), GFP_KERNEL
);
1111 for (i
= 0; i
< nr
; i
++) {
1112 struct nd_mapping
*nd_mapping
= &ndr_desc
->nd_mapping
[i
];
1113 struct nfit_set_info_map
*map
= &info
->mapping
[i
];
1114 struct nvdimm
*nvdimm
= nd_mapping
->nvdimm
;
1115 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1116 struct acpi_nfit_memory_map
*memdev
= memdev_from_spa(acpi_desc
,
1117 spa
->range_index
, i
);
1119 if (!memdev
|| !nfit_mem
->dcr
) {
1120 dev_err(dev
, "%s: failed to find DCR\n", __func__
);
1124 map
->region_offset
= memdev
->region_offset
;
1125 map
->serial_number
= nfit_mem
->dcr
->serial_number
;
1128 sort(&info
->mapping
[0], nr
, sizeof(struct nfit_set_info_map
),
1130 nd_set
->cookie
= nd_fletcher64(info
, sizeof_nfit_set_info(nr
), 0);
1131 ndr_desc
->nd_set
= nd_set
;
1132 devm_kfree(dev
, info
);
1137 static u64
to_interleave_offset(u64 offset
, struct nfit_blk_mmio
*mmio
)
1139 struct acpi_nfit_interleave
*idt
= mmio
->idt
;
1140 u32 sub_line_offset
, line_index
, line_offset
;
1141 u64 line_no
, table_skip_count
, table_offset
;
1143 line_no
= div_u64_rem(offset
, mmio
->line_size
, &sub_line_offset
);
1144 table_skip_count
= div_u64_rem(line_no
, mmio
->num_lines
, &line_index
);
1145 line_offset
= idt
->line_offset
[line_index
]
1147 table_offset
= table_skip_count
* mmio
->table_size
;
1149 return mmio
->base_offset
+ line_offset
+ table_offset
+ sub_line_offset
;
1152 static void wmb_blk(struct nfit_blk
*nfit_blk
)
1155 if (nfit_blk
->nvdimm_flush
) {
1157 * The first wmb() is needed to 'sfence' all previous writes
1158 * such that they are architecturally visible for the platform
1159 * buffer flush. Note that we've already arranged for pmem
1160 * writes to avoid the cache via arch_memcpy_to_pmem(). The
1161 * final wmb() ensures ordering for the NVDIMM flush write.
1164 writeq(1, nfit_blk
->nvdimm_flush
);
1170 static u32
read_blk_stat(struct nfit_blk
*nfit_blk
, unsigned int bw
)
1172 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[DCR
];
1173 u64 offset
= nfit_blk
->stat_offset
+ mmio
->size
* bw
;
1175 if (mmio
->num_lines
)
1176 offset
= to_interleave_offset(offset
, mmio
);
1178 return readl(mmio
->addr
.base
+ offset
);
1181 static void write_blk_ctl(struct nfit_blk
*nfit_blk
, unsigned int bw
,
1182 resource_size_t dpa
, unsigned int len
, unsigned int write
)
1185 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[DCR
];
1188 BCW_OFFSET_MASK
= (1ULL << 48)-1,
1190 BCW_LEN_MASK
= (1ULL << 8) - 1,
1194 cmd
= (dpa
>> L1_CACHE_SHIFT
) & BCW_OFFSET_MASK
;
1195 len
= len
>> L1_CACHE_SHIFT
;
1196 cmd
|= ((u64
) len
& BCW_LEN_MASK
) << BCW_LEN_SHIFT
;
1197 cmd
|= ((u64
) write
) << BCW_CMD_SHIFT
;
1199 offset
= nfit_blk
->cmd_offset
+ mmio
->size
* bw
;
1200 if (mmio
->num_lines
)
1201 offset
= to_interleave_offset(offset
, mmio
);
1203 writeq(cmd
, mmio
->addr
.base
+ offset
);
1206 if (nfit_blk
->dimm_flags
& NFIT_BLK_DCR_LATCH
)
1207 readq(mmio
->addr
.base
+ offset
);
1210 static int acpi_nfit_blk_single_io(struct nfit_blk
*nfit_blk
,
1211 resource_size_t dpa
, void *iobuf
, size_t len
, int rw
,
1214 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[BDW
];
1215 unsigned int copied
= 0;
1219 base_offset
= nfit_blk
->bdw_offset
+ dpa
% L1_CACHE_BYTES
1220 + lane
* mmio
->size
;
1221 write_blk_ctl(nfit_blk
, lane
, dpa
, len
, rw
);
1226 if (mmio
->num_lines
) {
1229 offset
= to_interleave_offset(base_offset
+ copied
,
1231 div_u64_rem(offset
, mmio
->line_size
, &line_offset
);
1232 c
= min_t(size_t, len
, mmio
->line_size
- line_offset
);
1234 offset
= base_offset
+ nfit_blk
->bdw_offset
;
1239 memcpy_to_pmem(mmio
->addr
.aperture
+ offset
,
1242 if (nfit_blk
->dimm_flags
& NFIT_BLK_READ_FLUSH
)
1243 mmio_flush_range((void __force
*)
1244 mmio
->addr
.aperture
+ offset
, c
);
1246 memcpy_from_pmem(iobuf
+ copied
,
1247 mmio
->addr
.aperture
+ offset
, c
);
1257 rc
= read_blk_stat(nfit_blk
, lane
) ? -EIO
: 0;
1261 static int acpi_nfit_blk_region_do_io(struct nd_blk_region
*ndbr
,
1262 resource_size_t dpa
, void *iobuf
, u64 len
, int rw
)
1264 struct nfit_blk
*nfit_blk
= nd_blk_region_provider_data(ndbr
);
1265 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[BDW
];
1266 struct nd_region
*nd_region
= nfit_blk
->nd_region
;
1267 unsigned int lane
, copied
= 0;
1270 lane
= nd_region_acquire_lane(nd_region
);
1272 u64 c
= min(len
, mmio
->size
);
1274 rc
= acpi_nfit_blk_single_io(nfit_blk
, dpa
+ copied
,
1275 iobuf
+ copied
, c
, rw
, lane
);
1282 nd_region_release_lane(nd_region
, lane
);
1287 static void nfit_spa_mapping_release(struct kref
*kref
)
1289 struct nfit_spa_mapping
*spa_map
= to_spa_map(kref
);
1290 struct acpi_nfit_system_address
*spa
= spa_map
->spa
;
1291 struct acpi_nfit_desc
*acpi_desc
= spa_map
->acpi_desc
;
1293 WARN_ON(!mutex_is_locked(&acpi_desc
->spa_map_mutex
));
1294 dev_dbg(acpi_desc
->dev
, "%s: SPA%d\n", __func__
, spa
->range_index
);
1295 if (spa_map
->type
== SPA_MAP_APERTURE
)
1296 memunmap((void __force
*)spa_map
->addr
.aperture
);
1298 iounmap(spa_map
->addr
.base
);
1299 release_mem_region(spa
->address
, spa
->length
);
1300 list_del(&spa_map
->list
);
1304 static struct nfit_spa_mapping
*find_spa_mapping(
1305 struct acpi_nfit_desc
*acpi_desc
,
1306 struct acpi_nfit_system_address
*spa
)
1308 struct nfit_spa_mapping
*spa_map
;
1310 WARN_ON(!mutex_is_locked(&acpi_desc
->spa_map_mutex
));
1311 list_for_each_entry(spa_map
, &acpi_desc
->spa_maps
, list
)
1312 if (spa_map
->spa
== spa
)
1318 static void nfit_spa_unmap(struct acpi_nfit_desc
*acpi_desc
,
1319 struct acpi_nfit_system_address
*spa
)
1321 struct nfit_spa_mapping
*spa_map
;
1323 mutex_lock(&acpi_desc
->spa_map_mutex
);
1324 spa_map
= find_spa_mapping(acpi_desc
, spa
);
1327 kref_put(&spa_map
->kref
, nfit_spa_mapping_release
);
1328 mutex_unlock(&acpi_desc
->spa_map_mutex
);
1331 static void __iomem
*__nfit_spa_map(struct acpi_nfit_desc
*acpi_desc
,
1332 struct acpi_nfit_system_address
*spa
, enum spa_map_type type
)
1334 resource_size_t start
= spa
->address
;
1335 resource_size_t n
= spa
->length
;
1336 struct nfit_spa_mapping
*spa_map
;
1337 struct resource
*res
;
1339 WARN_ON(!mutex_is_locked(&acpi_desc
->spa_map_mutex
));
1341 spa_map
= find_spa_mapping(acpi_desc
, spa
);
1343 kref_get(&spa_map
->kref
);
1344 return spa_map
->addr
.base
;
1347 spa_map
= kzalloc(sizeof(*spa_map
), GFP_KERNEL
);
1351 INIT_LIST_HEAD(&spa_map
->list
);
1353 kref_init(&spa_map
->kref
);
1354 spa_map
->acpi_desc
= acpi_desc
;
1356 res
= request_mem_region(start
, n
, dev_name(acpi_desc
->dev
));
1360 spa_map
->type
= type
;
1361 if (type
== SPA_MAP_APERTURE
)
1362 spa_map
->addr
.aperture
= (void __pmem
*)memremap(start
, n
,
1363 ARCH_MEMREMAP_PMEM
);
1365 spa_map
->addr
.base
= ioremap_nocache(start
, n
);
1368 if (!spa_map
->addr
.base
)
1371 list_add_tail(&spa_map
->list
, &acpi_desc
->spa_maps
);
1372 return spa_map
->addr
.base
;
1375 release_mem_region(start
, n
);
1382 * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1383 * @nvdimm_bus: NFIT-bus that provided the spa table entry
1384 * @nfit_spa: spa table to map
1385 * @type: aperture or control region
1387 * In the case where block-data-window apertures and
1388 * dimm-control-regions are interleaved they will end up sharing a
1389 * single request_mem_region() + ioremap() for the address range. In
1390 * the style of devm nfit_spa_map() mappings are automatically dropped
1391 * when all region devices referencing the same mapping are disabled /
1394 static void __iomem
*nfit_spa_map(struct acpi_nfit_desc
*acpi_desc
,
1395 struct acpi_nfit_system_address
*spa
, enum spa_map_type type
)
1397 void __iomem
*iomem
;
1399 mutex_lock(&acpi_desc
->spa_map_mutex
);
1400 iomem
= __nfit_spa_map(acpi_desc
, spa
, type
);
1401 mutex_unlock(&acpi_desc
->spa_map_mutex
);
1406 static int nfit_blk_init_interleave(struct nfit_blk_mmio
*mmio
,
1407 struct acpi_nfit_interleave
*idt
, u16 interleave_ways
)
1410 mmio
->num_lines
= idt
->line_count
;
1411 mmio
->line_size
= idt
->line_size
;
1412 if (interleave_ways
== 0)
1414 mmio
->table_size
= mmio
->num_lines
* interleave_ways
1421 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor
*nd_desc
,
1422 struct nvdimm
*nvdimm
, struct nfit_blk
*nfit_blk
)
1424 struct nd_cmd_dimm_flags flags
;
1427 memset(&flags
, 0, sizeof(flags
));
1428 rc
= nd_desc
->ndctl(nd_desc
, nvdimm
, ND_CMD_DIMM_FLAGS
, &flags
,
1429 sizeof(flags
), NULL
);
1431 if (rc
>= 0 && flags
.status
== 0)
1432 nfit_blk
->dimm_flags
= flags
.flags
;
1433 else if (rc
== -ENOTTY
) {
1434 /* fall back to a conservative default */
1435 nfit_blk
->dimm_flags
= NFIT_BLK_DCR_LATCH
| NFIT_BLK_READ_FLUSH
;
1443 static int acpi_nfit_blk_region_enable(struct nvdimm_bus
*nvdimm_bus
,
1446 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
1447 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
1448 struct nd_blk_region
*ndbr
= to_nd_blk_region(dev
);
1449 struct nfit_flush
*nfit_flush
;
1450 struct nfit_blk_mmio
*mmio
;
1451 struct nfit_blk
*nfit_blk
;
1452 struct nfit_mem
*nfit_mem
;
1453 struct nvdimm
*nvdimm
;
1456 nvdimm
= nd_blk_region_to_dimm(ndbr
);
1457 nfit_mem
= nvdimm_provider_data(nvdimm
);
1458 if (!nfit_mem
|| !nfit_mem
->dcr
|| !nfit_mem
->bdw
) {
1459 dev_dbg(dev
, "%s: missing%s%s%s\n", __func__
,
1460 nfit_mem
? "" : " nfit_mem",
1461 (nfit_mem
&& nfit_mem
->dcr
) ? "" : " dcr",
1462 (nfit_mem
&& nfit_mem
->bdw
) ? "" : " bdw");
1466 nfit_blk
= devm_kzalloc(dev
, sizeof(*nfit_blk
), GFP_KERNEL
);
1469 nd_blk_region_set_provider_data(ndbr
, nfit_blk
);
1470 nfit_blk
->nd_region
= to_nd_region(dev
);
1472 /* map block aperture memory */
1473 nfit_blk
->bdw_offset
= nfit_mem
->bdw
->offset
;
1474 mmio
= &nfit_blk
->mmio
[BDW
];
1475 mmio
->addr
.base
= nfit_spa_map(acpi_desc
, nfit_mem
->spa_bdw
,
1477 if (!mmio
->addr
.base
) {
1478 dev_dbg(dev
, "%s: %s failed to map bdw\n", __func__
,
1479 nvdimm_name(nvdimm
));
1482 mmio
->size
= nfit_mem
->bdw
->size
;
1483 mmio
->base_offset
= nfit_mem
->memdev_bdw
->region_offset
;
1484 mmio
->idt
= nfit_mem
->idt_bdw
;
1485 mmio
->spa
= nfit_mem
->spa_bdw
;
1486 rc
= nfit_blk_init_interleave(mmio
, nfit_mem
->idt_bdw
,
1487 nfit_mem
->memdev_bdw
->interleave_ways
);
1489 dev_dbg(dev
, "%s: %s failed to init bdw interleave\n",
1490 __func__
, nvdimm_name(nvdimm
));
1494 /* map block control memory */
1495 nfit_blk
->cmd_offset
= nfit_mem
->dcr
->command_offset
;
1496 nfit_blk
->stat_offset
= nfit_mem
->dcr
->status_offset
;
1497 mmio
= &nfit_blk
->mmio
[DCR
];
1498 mmio
->addr
.base
= nfit_spa_map(acpi_desc
, nfit_mem
->spa_dcr
,
1500 if (!mmio
->addr
.base
) {
1501 dev_dbg(dev
, "%s: %s failed to map dcr\n", __func__
,
1502 nvdimm_name(nvdimm
));
1505 mmio
->size
= nfit_mem
->dcr
->window_size
;
1506 mmio
->base_offset
= nfit_mem
->memdev_dcr
->region_offset
;
1507 mmio
->idt
= nfit_mem
->idt_dcr
;
1508 mmio
->spa
= nfit_mem
->spa_dcr
;
1509 rc
= nfit_blk_init_interleave(mmio
, nfit_mem
->idt_dcr
,
1510 nfit_mem
->memdev_dcr
->interleave_ways
);
1512 dev_dbg(dev
, "%s: %s failed to init dcr interleave\n",
1513 __func__
, nvdimm_name(nvdimm
));
1517 rc
= acpi_nfit_blk_get_flags(nd_desc
, nvdimm
, nfit_blk
);
1519 dev_dbg(dev
, "%s: %s failed get DIMM flags\n",
1520 __func__
, nvdimm_name(nvdimm
));
1524 nfit_flush
= nfit_mem
->nfit_flush
;
1525 if (nfit_flush
&& nfit_flush
->flush
->hint_count
!= 0) {
1526 nfit_blk
->nvdimm_flush
= devm_ioremap_nocache(dev
,
1527 nfit_flush
->flush
->hint_address
[0], 8);
1528 if (!nfit_blk
->nvdimm_flush
)
1532 if (!arch_has_wmb_pmem() && !nfit_blk
->nvdimm_flush
)
1533 dev_warn(dev
, "unable to guarantee persistence of writes\n");
1535 if (mmio
->line_size
== 0)
1538 if ((u32
) nfit_blk
->cmd_offset
% mmio
->line_size
1539 + 8 > mmio
->line_size
) {
1540 dev_dbg(dev
, "cmd_offset crosses interleave boundary\n");
1542 } else if ((u32
) nfit_blk
->stat_offset
% mmio
->line_size
1543 + 8 > mmio
->line_size
) {
1544 dev_dbg(dev
, "stat_offset crosses interleave boundary\n");
1551 static void acpi_nfit_blk_region_disable(struct nvdimm_bus
*nvdimm_bus
,
1554 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
1555 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
1556 struct nd_blk_region
*ndbr
= to_nd_blk_region(dev
);
1557 struct nfit_blk
*nfit_blk
= nd_blk_region_provider_data(ndbr
);
1561 return; /* never enabled */
1563 /* auto-free BLK spa mappings */
1564 for (i
= 0; i
< 2; i
++) {
1565 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[i
];
1567 if (mmio
->addr
.base
)
1568 nfit_spa_unmap(acpi_desc
, mmio
->spa
);
1570 nd_blk_region_set_provider_data(ndbr
, NULL
);
1571 /* devm will free nfit_blk */
1574 static int ars_get_cap(struct acpi_nfit_desc
*acpi_desc
,
1575 struct nd_cmd_ars_cap
*cmd
, struct nfit_spa
*nfit_spa
)
1577 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1578 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1581 cmd
->address
= spa
->address
;
1582 cmd
->length
= spa
->length
;
1583 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_CAP
, cmd
,
1584 sizeof(*cmd
), &cmd_rc
);
1590 static int ars_start(struct acpi_nfit_desc
*acpi_desc
, struct nfit_spa
*nfit_spa
)
1594 struct nd_cmd_ars_start ars_start
;
1595 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1596 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1598 memset(&ars_start
, 0, sizeof(ars_start
));
1599 ars_start
.address
= spa
->address
;
1600 ars_start
.length
= spa
->length
;
1601 if (nfit_spa_type(spa
) == NFIT_SPA_PM
)
1602 ars_start
.type
= ND_ARS_PERSISTENT
;
1603 else if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
)
1604 ars_start
.type
= ND_ARS_VOLATILE
;
1608 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_START
, &ars_start
,
1609 sizeof(ars_start
), &cmd_rc
);
1616 static int ars_continue(struct acpi_nfit_desc
*acpi_desc
)
1619 struct nd_cmd_ars_start ars_start
;
1620 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1621 struct nd_cmd_ars_status
*ars_status
= acpi_desc
->ars_status
;
1623 memset(&ars_start
, 0, sizeof(ars_start
));
1624 ars_start
.address
= ars_status
->restart_address
;
1625 ars_start
.length
= ars_status
->restart_length
;
1626 ars_start
.type
= ars_status
->type
;
1627 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_START
, &ars_start
,
1628 sizeof(ars_start
), &cmd_rc
);
1634 static int ars_get_status(struct acpi_nfit_desc
*acpi_desc
)
1636 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1637 struct nd_cmd_ars_status
*ars_status
= acpi_desc
->ars_status
;
1640 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_STATUS
, ars_status
,
1641 acpi_desc
->ars_status_size
, &cmd_rc
);
1647 static int ars_status_process_records(struct nvdimm_bus
*nvdimm_bus
,
1648 struct nd_cmd_ars_status
*ars_status
)
1653 for (i
= 0; i
< ars_status
->num_records
; i
++) {
1654 rc
= nvdimm_bus_add_poison(nvdimm_bus
,
1655 ars_status
->records
[i
].err_address
,
1656 ars_status
->records
[i
].length
);
1664 static void acpi_nfit_remove_resource(void *data
)
1666 struct resource
*res
= data
;
1668 remove_resource(res
);
1671 static int acpi_nfit_insert_resource(struct acpi_nfit_desc
*acpi_desc
,
1672 struct nd_region_desc
*ndr_desc
)
1674 struct resource
*res
, *nd_res
= ndr_desc
->res
;
1677 /* No operation if the region is already registered as PMEM */
1678 is_pmem
= region_intersects(nd_res
->start
, resource_size(nd_res
),
1679 IORESOURCE_MEM
, IORES_DESC_PERSISTENT_MEMORY
);
1680 if (is_pmem
== REGION_INTERSECTS
)
1683 res
= devm_kzalloc(acpi_desc
->dev
, sizeof(*res
), GFP_KERNEL
);
1687 res
->name
= "Persistent Memory";
1688 res
->start
= nd_res
->start
;
1689 res
->end
= nd_res
->end
;
1690 res
->flags
= IORESOURCE_MEM
;
1691 res
->desc
= IORES_DESC_PERSISTENT_MEMORY
;
1693 ret
= insert_resource(&iomem_resource
, res
);
1697 ret
= devm_add_action(acpi_desc
->dev
, acpi_nfit_remove_resource
, res
);
1699 remove_resource(res
);
1706 static int acpi_nfit_init_mapping(struct acpi_nfit_desc
*acpi_desc
,
1707 struct nd_mapping
*nd_mapping
, struct nd_region_desc
*ndr_desc
,
1708 struct acpi_nfit_memory_map
*memdev
,
1709 struct nfit_spa
*nfit_spa
)
1711 struct nvdimm
*nvdimm
= acpi_nfit_dimm_by_handle(acpi_desc
,
1712 memdev
->device_handle
);
1713 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1714 struct nd_blk_region_desc
*ndbr_desc
;
1715 struct nfit_mem
*nfit_mem
;
1719 dev_err(acpi_desc
->dev
, "spa%d dimm: %#x not found\n",
1720 spa
->range_index
, memdev
->device_handle
);
1724 nd_mapping
->nvdimm
= nvdimm
;
1725 switch (nfit_spa_type(spa
)) {
1727 case NFIT_SPA_VOLATILE
:
1728 nd_mapping
->start
= memdev
->address
;
1729 nd_mapping
->size
= memdev
->region_size
;
1732 nfit_mem
= nvdimm_provider_data(nvdimm
);
1733 if (!nfit_mem
|| !nfit_mem
->bdw
) {
1734 dev_dbg(acpi_desc
->dev
, "spa%d %s missing bdw\n",
1735 spa
->range_index
, nvdimm_name(nvdimm
));
1737 nd_mapping
->size
= nfit_mem
->bdw
->capacity
;
1738 nd_mapping
->start
= nfit_mem
->bdw
->start_address
;
1739 ndr_desc
->num_lanes
= nfit_mem
->bdw
->windows
;
1743 ndr_desc
->nd_mapping
= nd_mapping
;
1744 ndr_desc
->num_mappings
= blk_valid
;
1745 ndbr_desc
= to_blk_region_desc(ndr_desc
);
1746 ndbr_desc
->enable
= acpi_nfit_blk_region_enable
;
1747 ndbr_desc
->disable
= acpi_nfit_blk_region_disable
;
1748 ndbr_desc
->do_io
= acpi_desc
->blk_do_io
;
1749 nfit_spa
->nd_region
= nvdimm_blk_region_create(acpi_desc
->nvdimm_bus
,
1751 if (!nfit_spa
->nd_region
)
1759 static int acpi_nfit_register_region(struct acpi_nfit_desc
*acpi_desc
,
1760 struct nfit_spa
*nfit_spa
)
1762 static struct nd_mapping nd_mappings
[ND_MAX_MAPPINGS
];
1763 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1764 struct nd_blk_region_desc ndbr_desc
;
1765 struct nd_region_desc
*ndr_desc
;
1766 struct nfit_memdev
*nfit_memdev
;
1767 struct nvdimm_bus
*nvdimm_bus
;
1768 struct resource res
;
1771 if (nfit_spa
->nd_region
)
1774 if (spa
->range_index
== 0) {
1775 dev_dbg(acpi_desc
->dev
, "%s: detected invalid spa index\n",
1780 memset(&res
, 0, sizeof(res
));
1781 memset(&nd_mappings
, 0, sizeof(nd_mappings
));
1782 memset(&ndbr_desc
, 0, sizeof(ndbr_desc
));
1783 res
.start
= spa
->address
;
1784 res
.end
= res
.start
+ spa
->length
- 1;
1785 ndr_desc
= &ndbr_desc
.ndr_desc
;
1786 ndr_desc
->res
= &res
;
1787 ndr_desc
->provider_data
= nfit_spa
;
1788 ndr_desc
->attr_groups
= acpi_nfit_region_attribute_groups
;
1789 if (spa
->flags
& ACPI_NFIT_PROXIMITY_VALID
)
1790 ndr_desc
->numa_node
= acpi_map_pxm_to_online_node(
1791 spa
->proximity_domain
);
1793 ndr_desc
->numa_node
= NUMA_NO_NODE
;
1795 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
1796 struct acpi_nfit_memory_map
*memdev
= nfit_memdev
->memdev
;
1797 struct nd_mapping
*nd_mapping
;
1799 if (memdev
->range_index
!= spa
->range_index
)
1801 if (count
>= ND_MAX_MAPPINGS
) {
1802 dev_err(acpi_desc
->dev
, "spa%d exceeds max mappings %d\n",
1803 spa
->range_index
, ND_MAX_MAPPINGS
);
1806 nd_mapping
= &nd_mappings
[count
++];
1807 rc
= acpi_nfit_init_mapping(acpi_desc
, nd_mapping
, ndr_desc
,
1813 ndr_desc
->nd_mapping
= nd_mappings
;
1814 ndr_desc
->num_mappings
= count
;
1815 rc
= acpi_nfit_init_interleave_set(acpi_desc
, ndr_desc
, spa
);
1819 nvdimm_bus
= acpi_desc
->nvdimm_bus
;
1820 if (nfit_spa_type(spa
) == NFIT_SPA_PM
) {
1821 rc
= acpi_nfit_insert_resource(acpi_desc
, ndr_desc
);
1823 dev_warn(acpi_desc
->dev
,
1824 "failed to insert pmem resource to iomem: %d\n",
1829 nfit_spa
->nd_region
= nvdimm_pmem_region_create(nvdimm_bus
,
1831 if (!nfit_spa
->nd_region
)
1833 } else if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
) {
1834 nfit_spa
->nd_region
= nvdimm_volatile_region_create(nvdimm_bus
,
1836 if (!nfit_spa
->nd_region
)
1842 dev_err(acpi_desc
->dev
, "failed to register spa range %d\n",
1843 nfit_spa
->spa
->range_index
);
1847 static int ars_status_alloc(struct acpi_nfit_desc
*acpi_desc
,
1850 struct device
*dev
= acpi_desc
->dev
;
1851 struct nd_cmd_ars_status
*ars_status
;
1853 if (acpi_desc
->ars_status
&& acpi_desc
->ars_status_size
>= max_ars
) {
1854 memset(acpi_desc
->ars_status
, 0, acpi_desc
->ars_status_size
);
1858 if (acpi_desc
->ars_status
)
1859 devm_kfree(dev
, acpi_desc
->ars_status
);
1860 acpi_desc
->ars_status
= NULL
;
1861 ars_status
= devm_kzalloc(dev
, max_ars
, GFP_KERNEL
);
1864 acpi_desc
->ars_status
= ars_status
;
1865 acpi_desc
->ars_status_size
= max_ars
;
1869 static int acpi_nfit_query_poison(struct acpi_nfit_desc
*acpi_desc
,
1870 struct nfit_spa
*nfit_spa
)
1872 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1875 if (!nfit_spa
->max_ars
) {
1876 struct nd_cmd_ars_cap ars_cap
;
1878 memset(&ars_cap
, 0, sizeof(ars_cap
));
1879 rc
= ars_get_cap(acpi_desc
, &ars_cap
, nfit_spa
);
1882 nfit_spa
->max_ars
= ars_cap
.max_ars_out
;
1883 nfit_spa
->clear_err_unit
= ars_cap
.clear_err_unit
;
1884 /* check that the supported scrub types match the spa type */
1885 if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
&&
1886 ((ars_cap
.status
>> 16) & ND_ARS_VOLATILE
) == 0)
1888 else if (nfit_spa_type(spa
) == NFIT_SPA_PM
&&
1889 ((ars_cap
.status
>> 16) & ND_ARS_PERSISTENT
) == 0)
1893 if (ars_status_alloc(acpi_desc
, nfit_spa
->max_ars
))
1896 rc
= ars_get_status(acpi_desc
);
1897 if (rc
< 0 && rc
!= -ENOSPC
)
1900 if (ars_status_process_records(acpi_desc
->nvdimm_bus
,
1901 acpi_desc
->ars_status
))
1907 static void acpi_nfit_async_scrub(struct acpi_nfit_desc
*acpi_desc
,
1908 struct nfit_spa
*nfit_spa
)
1910 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1911 unsigned int overflow_retry
= scrub_overflow_abort
;
1912 u64 init_ars_start
= 0, init_ars_len
= 0;
1913 struct device
*dev
= acpi_desc
->dev
;
1914 unsigned int tmo
= scrub_timeout
;
1917 if (nfit_spa
->ars_done
|| !nfit_spa
->nd_region
)
1920 rc
= ars_start(acpi_desc
, nfit_spa
);
1922 * If we timed out the initial scan we'll still be busy here,
1923 * and will wait another timeout before giving up permanently.
1925 if (rc
< 0 && rc
!= -EBUSY
)
1929 u64 ars_start
, ars_len
;
1931 if (acpi_desc
->cancel
)
1933 rc
= acpi_nfit_query_poison(acpi_desc
, nfit_spa
);
1936 if (rc
== -EBUSY
&& !tmo
) {
1937 dev_warn(dev
, "range %d ars timeout, aborting\n",
1944 * Note, entries may be appended to the list
1945 * while the lock is dropped, but the workqueue
1946 * being active prevents entries being deleted /
1949 mutex_unlock(&acpi_desc
->init_mutex
);
1952 mutex_lock(&acpi_desc
->init_mutex
);
1956 /* we got some results, but there are more pending... */
1957 if (rc
== -ENOSPC
&& overflow_retry
--) {
1958 if (!init_ars_len
) {
1959 init_ars_len
= acpi_desc
->ars_status
->length
;
1960 init_ars_start
= acpi_desc
->ars_status
->address
;
1962 rc
= ars_continue(acpi_desc
);
1966 dev_warn(dev
, "range %d ars continuation failed\n",
1972 ars_start
= init_ars_start
;
1973 ars_len
= init_ars_len
;
1975 ars_start
= acpi_desc
->ars_status
->address
;
1976 ars_len
= acpi_desc
->ars_status
->length
;
1978 dev_dbg(dev
, "spa range: %d ars from %#llx + %#llx complete\n",
1979 spa
->range_index
, ars_start
, ars_len
);
1980 /* notify the region about new poison entries */
1981 nvdimm_region_notify(nfit_spa
->nd_region
,
1982 NVDIMM_REVALIDATE_POISON
);
1987 static void acpi_nfit_scrub(struct work_struct
*work
)
1990 u64 init_scrub_length
= 0;
1991 struct nfit_spa
*nfit_spa
;
1992 u64 init_scrub_address
= 0;
1993 bool init_ars_done
= false;
1994 struct acpi_nfit_desc
*acpi_desc
;
1995 unsigned int tmo
= scrub_timeout
;
1996 unsigned int overflow_retry
= scrub_overflow_abort
;
1998 acpi_desc
= container_of(work
, typeof(*acpi_desc
), work
);
1999 dev
= acpi_desc
->dev
;
2002 * We scrub in 2 phases. The first phase waits for any platform
2003 * firmware initiated scrubs to complete and then we go search for the
2004 * affected spa regions to mark them scanned. In the second phase we
2005 * initiate a directed scrub for every range that was not scrubbed in
2009 /* process platform firmware initiated scrubs */
2011 mutex_lock(&acpi_desc
->init_mutex
);
2012 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
2013 struct nd_cmd_ars_status
*ars_status
;
2014 struct acpi_nfit_system_address
*spa
;
2015 u64 ars_start
, ars_len
;
2018 if (acpi_desc
->cancel
)
2021 if (nfit_spa
->nd_region
)
2024 if (init_ars_done
) {
2026 * No need to re-query, we're now just
2027 * reconciling all the ranges covered by the
2032 rc
= acpi_nfit_query_poison(acpi_desc
, nfit_spa
);
2034 if (rc
== -ENOTTY
) {
2035 /* no ars capability, just register spa and move on */
2036 acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2040 if (rc
== -EBUSY
&& !tmo
) {
2041 /* fallthrough to directed scrub in phase 2 */
2042 dev_warn(dev
, "timeout awaiting ars results, continuing...\n");
2044 } else if (rc
== -EBUSY
) {
2045 mutex_unlock(&acpi_desc
->init_mutex
);
2051 /* we got some results, but there are more pending... */
2052 if (rc
== -ENOSPC
&& overflow_retry
--) {
2053 ars_status
= acpi_desc
->ars_status
;
2055 * Record the original scrub range, so that we
2056 * can recall all the ranges impacted by the
2059 if (!init_scrub_length
) {
2060 init_scrub_length
= ars_status
->length
;
2061 init_scrub_address
= ars_status
->address
;
2063 rc
= ars_continue(acpi_desc
);
2065 mutex_unlock(&acpi_desc
->init_mutex
);
2072 * Initial scrub failed, we'll give it one more
2078 /* We got some final results, record completed ranges */
2079 ars_status
= acpi_desc
->ars_status
;
2080 if (init_scrub_length
) {
2081 ars_start
= init_scrub_address
;
2082 ars_len
= ars_start
+ init_scrub_length
;
2084 ars_start
= ars_status
->address
;
2085 ars_len
= ars_status
->length
;
2087 spa
= nfit_spa
->spa
;
2089 if (!init_ars_done
) {
2090 init_ars_done
= true;
2091 dev_dbg(dev
, "init scrub %#llx + %#llx complete\n",
2092 ars_start
, ars_len
);
2094 if (ars_start
<= spa
->address
&& ars_start
+ ars_len
2095 >= spa
->address
+ spa
->length
)
2096 acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2100 * For all the ranges not covered by an initial scrub we still
2101 * want to see if there are errors, but it's ok to discover them
2104 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
2106 * Flag all the ranges that still need scrubbing, but
2107 * register them now to make data available.
2109 if (nfit_spa
->nd_region
)
2110 nfit_spa
->ars_done
= 1;
2112 acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2115 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
)
2116 acpi_nfit_async_scrub(acpi_desc
, nfit_spa
);
2117 mutex_unlock(&acpi_desc
->init_mutex
);
2120 static int acpi_nfit_register_regions(struct acpi_nfit_desc
*acpi_desc
)
2122 struct nfit_spa
*nfit_spa
;
2125 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
)
2126 if (nfit_spa_type(nfit_spa
->spa
) == NFIT_SPA_DCR
) {
2127 /* BLK regions don't need to wait for ars results */
2128 rc
= acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2133 queue_work(nfit_wq
, &acpi_desc
->work
);
2137 static int acpi_nfit_check_deletions(struct acpi_nfit_desc
*acpi_desc
,
2138 struct nfit_table_prev
*prev
)
2140 struct device
*dev
= acpi_desc
->dev
;
2142 if (!list_empty(&prev
->spas
) ||
2143 !list_empty(&prev
->memdevs
) ||
2144 !list_empty(&prev
->dcrs
) ||
2145 !list_empty(&prev
->bdws
) ||
2146 !list_empty(&prev
->idts
) ||
2147 !list_empty(&prev
->flushes
)) {
2148 dev_err(dev
, "new nfit deletes entries (unsupported)\n");
2154 int acpi_nfit_init(struct acpi_nfit_desc
*acpi_desc
, acpi_size sz
)
2156 struct device
*dev
= acpi_desc
->dev
;
2157 struct nfit_table_prev prev
;
2162 mutex_lock(&acpi_desc
->init_mutex
);
2164 INIT_LIST_HEAD(&prev
.spas
);
2165 INIT_LIST_HEAD(&prev
.memdevs
);
2166 INIT_LIST_HEAD(&prev
.dcrs
);
2167 INIT_LIST_HEAD(&prev
.bdws
);
2168 INIT_LIST_HEAD(&prev
.idts
);
2169 INIT_LIST_HEAD(&prev
.flushes
);
2171 list_cut_position(&prev
.spas
, &acpi_desc
->spas
,
2172 acpi_desc
->spas
.prev
);
2173 list_cut_position(&prev
.memdevs
, &acpi_desc
->memdevs
,
2174 acpi_desc
->memdevs
.prev
);
2175 list_cut_position(&prev
.dcrs
, &acpi_desc
->dcrs
,
2176 acpi_desc
->dcrs
.prev
);
2177 list_cut_position(&prev
.bdws
, &acpi_desc
->bdws
,
2178 acpi_desc
->bdws
.prev
);
2179 list_cut_position(&prev
.idts
, &acpi_desc
->idts
,
2180 acpi_desc
->idts
.prev
);
2181 list_cut_position(&prev
.flushes
, &acpi_desc
->flushes
,
2182 acpi_desc
->flushes
.prev
);
2184 data
= (u8
*) acpi_desc
->nfit
;
2186 while (!IS_ERR_OR_NULL(data
))
2187 data
= add_table(acpi_desc
, &prev
, data
, end
);
2190 dev_dbg(dev
, "%s: nfit table parsing error: %ld\n", __func__
,
2196 rc
= acpi_nfit_check_deletions(acpi_desc
, &prev
);
2200 if (nfit_mem_init(acpi_desc
) != 0) {
2205 acpi_nfit_init_dsms(acpi_desc
);
2207 rc
= acpi_nfit_register_dimms(acpi_desc
);
2211 rc
= acpi_nfit_register_regions(acpi_desc
);
2214 mutex_unlock(&acpi_desc
->init_mutex
);
2217 EXPORT_SYMBOL_GPL(acpi_nfit_init
);
2219 struct acpi_nfit_flush_work
{
2220 struct work_struct work
;
2221 struct completion cmp
;
2224 static void flush_probe(struct work_struct
*work
)
2226 struct acpi_nfit_flush_work
*flush
;
2228 flush
= container_of(work
, typeof(*flush
), work
);
2229 complete(&flush
->cmp
);
2232 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor
*nd_desc
)
2234 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
2235 struct device
*dev
= acpi_desc
->dev
;
2236 struct acpi_nfit_flush_work flush
;
2238 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
2243 * Scrub work could take 10s of seconds, userspace may give up so we
2244 * need to be interruptible while waiting.
2246 INIT_WORK_ONSTACK(&flush
.work
, flush_probe
);
2247 COMPLETION_INITIALIZER_ONSTACK(flush
.cmp
);
2248 queue_work(nfit_wq
, &flush
.work
);
2249 return wait_for_completion_interruptible(&flush
.cmp
);
2252 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor
*nd_desc
,
2253 struct nvdimm
*nvdimm
, unsigned int cmd
)
2255 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
2259 if (cmd
!= ND_CMD_ARS_START
)
2263 * The kernel and userspace may race to initiate a scrub, but
2264 * the scrub thread is prepared to lose that initial race. It
2265 * just needs guarantees that any ars it initiates are not
2266 * interrupted by any intervening start reqeusts from userspace.
2268 if (work_busy(&acpi_desc
->work
))
2274 void acpi_nfit_desc_init(struct acpi_nfit_desc
*acpi_desc
, struct device
*dev
)
2276 struct nvdimm_bus_descriptor
*nd_desc
;
2278 dev_set_drvdata(dev
, acpi_desc
);
2279 acpi_desc
->dev
= dev
;
2280 acpi_desc
->blk_do_io
= acpi_nfit_blk_region_do_io
;
2281 nd_desc
= &acpi_desc
->nd_desc
;
2282 nd_desc
->provider_name
= "ACPI.NFIT";
2283 nd_desc
->ndctl
= acpi_nfit_ctl
;
2284 nd_desc
->flush_probe
= acpi_nfit_flush_probe
;
2285 nd_desc
->clear_to_send
= acpi_nfit_clear_to_send
;
2286 nd_desc
->attr_groups
= acpi_nfit_attribute_groups
;
2288 INIT_LIST_HEAD(&acpi_desc
->spa_maps
);
2289 INIT_LIST_HEAD(&acpi_desc
->spas
);
2290 INIT_LIST_HEAD(&acpi_desc
->dcrs
);
2291 INIT_LIST_HEAD(&acpi_desc
->bdws
);
2292 INIT_LIST_HEAD(&acpi_desc
->idts
);
2293 INIT_LIST_HEAD(&acpi_desc
->flushes
);
2294 INIT_LIST_HEAD(&acpi_desc
->memdevs
);
2295 INIT_LIST_HEAD(&acpi_desc
->dimms
);
2296 mutex_init(&acpi_desc
->spa_map_mutex
);
2297 mutex_init(&acpi_desc
->init_mutex
);
2298 INIT_WORK(&acpi_desc
->work
, acpi_nfit_scrub
);
2300 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init
);
2302 static int acpi_nfit_add(struct acpi_device
*adev
)
2304 struct acpi_buffer buf
= { ACPI_ALLOCATE_BUFFER
, NULL
};
2305 struct acpi_nfit_desc
*acpi_desc
;
2306 struct device
*dev
= &adev
->dev
;
2307 struct acpi_table_header
*tbl
;
2308 acpi_status status
= AE_OK
;
2312 status
= acpi_get_table_with_size("NFIT", 0, &tbl
, &sz
);
2313 if (ACPI_FAILURE(status
)) {
2314 /* This is ok, we could have an nvdimm hotplugged later */
2315 dev_dbg(dev
, "failed to find NFIT at startup\n");
2319 acpi_desc
= devm_kzalloc(dev
, sizeof(*acpi_desc
), GFP_KERNEL
);
2322 acpi_nfit_desc_init(acpi_desc
, &adev
->dev
);
2323 acpi_desc
->nvdimm_bus
= nvdimm_bus_register(dev
, &acpi_desc
->nd_desc
);
2324 if (!acpi_desc
->nvdimm_bus
)
2328 * Save the acpi header for later and then skip it,
2329 * making nfit point to the first nfit table header.
2331 acpi_desc
->acpi_header
= *tbl
;
2332 acpi_desc
->nfit
= (void *) tbl
+ sizeof(struct acpi_table_nfit
);
2333 sz
-= sizeof(struct acpi_table_nfit
);
2335 /* Evaluate _FIT and override with that if present */
2336 status
= acpi_evaluate_object(adev
->handle
, "_FIT", NULL
, &buf
);
2337 if (ACPI_SUCCESS(status
) && buf
.length
> 0) {
2338 union acpi_object
*obj
;
2340 * Adjust for the acpi_object header of the _FIT
2343 if (obj
->type
== ACPI_TYPE_BUFFER
) {
2345 (struct acpi_nfit_header
*)obj
->buffer
.pointer
;
2346 sz
= obj
->buffer
.length
;
2348 dev_dbg(dev
, "%s invalid type %d, ignoring _FIT\n",
2349 __func__
, (int) obj
->type
);
2352 rc
= acpi_nfit_init(acpi_desc
, sz
);
2354 nvdimm_bus_unregister(acpi_desc
->nvdimm_bus
);
2360 static int acpi_nfit_remove(struct acpi_device
*adev
)
2362 struct acpi_nfit_desc
*acpi_desc
= dev_get_drvdata(&adev
->dev
);
2364 acpi_desc
->cancel
= 1;
2365 flush_workqueue(nfit_wq
);
2366 nvdimm_bus_unregister(acpi_desc
->nvdimm_bus
);
2370 static void acpi_nfit_notify(struct acpi_device
*adev
, u32 event
)
2372 struct acpi_nfit_desc
*acpi_desc
= dev_get_drvdata(&adev
->dev
);
2373 struct acpi_buffer buf
= { ACPI_ALLOCATE_BUFFER
, NULL
};
2374 struct acpi_nfit_header
*nfit_saved
;
2375 union acpi_object
*obj
;
2376 struct device
*dev
= &adev
->dev
;
2380 dev_dbg(dev
, "%s: event: %d\n", __func__
, event
);
2384 /* dev->driver may be null if we're being removed */
2385 dev_dbg(dev
, "%s: no driver found for dev\n", __func__
);
2390 acpi_desc
= devm_kzalloc(dev
, sizeof(*acpi_desc
), GFP_KERNEL
);
2393 acpi_nfit_desc_init(acpi_desc
, &adev
->dev
);
2394 acpi_desc
->nvdimm_bus
= nvdimm_bus_register(dev
, &acpi_desc
->nd_desc
);
2395 if (!acpi_desc
->nvdimm_bus
)
2399 * Finish previous registration before considering new
2402 flush_workqueue(nfit_wq
);
2406 status
= acpi_evaluate_object(adev
->handle
, "_FIT", NULL
, &buf
);
2407 if (ACPI_FAILURE(status
)) {
2408 dev_err(dev
, "failed to evaluate _FIT\n");
2412 nfit_saved
= acpi_desc
->nfit
;
2414 if (obj
->type
== ACPI_TYPE_BUFFER
) {
2416 (struct acpi_nfit_header
*)obj
->buffer
.pointer
;
2417 ret
= acpi_nfit_init(acpi_desc
, obj
->buffer
.length
);
2419 /* Merge failed, restore old nfit, and exit */
2420 acpi_desc
->nfit
= nfit_saved
;
2421 dev_err(dev
, "failed to merge updated NFIT\n");
2424 /* Bad _FIT, restore old nfit */
2425 dev_err(dev
, "Invalid _FIT\n");
2433 static const struct acpi_device_id acpi_nfit_ids
[] = {
2437 MODULE_DEVICE_TABLE(acpi
, acpi_nfit_ids
);
2439 static struct acpi_driver acpi_nfit_driver
= {
2440 .name
= KBUILD_MODNAME
,
2441 .ids
= acpi_nfit_ids
,
2443 .add
= acpi_nfit_add
,
2444 .remove
= acpi_nfit_remove
,
2445 .notify
= acpi_nfit_notify
,
2449 static __init
int nfit_init(void)
2451 BUILD_BUG_ON(sizeof(struct acpi_table_nfit
) != 40);
2452 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address
) != 56);
2453 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map
) != 48);
2454 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave
) != 20);
2455 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios
) != 9);
2456 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region
) != 80);
2457 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region
) != 40);
2459 acpi_str_to_uuid(UUID_VOLATILE_MEMORY
, nfit_uuid
[NFIT_SPA_VOLATILE
]);
2460 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY
, nfit_uuid
[NFIT_SPA_PM
]);
2461 acpi_str_to_uuid(UUID_CONTROL_REGION
, nfit_uuid
[NFIT_SPA_DCR
]);
2462 acpi_str_to_uuid(UUID_DATA_REGION
, nfit_uuid
[NFIT_SPA_BDW
]);
2463 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK
, nfit_uuid
[NFIT_SPA_VDISK
]);
2464 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD
, nfit_uuid
[NFIT_SPA_VCD
]);
2465 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK
, nfit_uuid
[NFIT_SPA_PDISK
]);
2466 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD
, nfit_uuid
[NFIT_SPA_PCD
]);
2467 acpi_str_to_uuid(UUID_NFIT_BUS
, nfit_uuid
[NFIT_DEV_BUS
]);
2468 acpi_str_to_uuid(UUID_NFIT_DIMM
, nfit_uuid
[NFIT_DEV_DIMM
]);
2470 nfit_wq
= create_singlethread_workqueue("nfit");
2474 return acpi_bus_register_driver(&acpi_nfit_driver
);
2477 static __exit
void nfit_exit(void)
2479 acpi_bus_unregister_driver(&acpi_nfit_driver
);
2480 destroy_workqueue(nfit_wq
);
2483 module_init(nfit_init
);
2484 module_exit(nfit_exit
);
2485 MODULE_LICENSE("GPL v2");
2486 MODULE_AUTHOR("Intel Corporation");