2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/list_sort.h>
14 #include <linux/libnvdimm.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/ndctl.h>
18 #include <linux/sysfs.h>
19 #include <linux/delay.h>
20 #include <linux/list.h>
21 #include <linux/acpi.h>
22 #include <linux/sort.h>
23 #include <linux/pmem.h>
26 #include <asm/cacheflush.h>
30 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
33 #include <linux/io-64-nonatomic-hi-lo.h>
35 static bool force_enable_dimms
;
36 module_param(force_enable_dimms
, bool, S_IRUGO
|S_IWUSR
);
37 MODULE_PARM_DESC(force_enable_dimms
, "Ignore _STA (ACPI DIMM device) status");
39 static unsigned int scrub_timeout
= NFIT_ARS_TIMEOUT
;
40 module_param(scrub_timeout
, uint
, S_IRUGO
|S_IWUSR
);
41 MODULE_PARM_DESC(scrub_timeout
, "Initial scrub timeout in seconds");
43 /* after three payloads of overflow, it's dead jim */
44 static unsigned int scrub_overflow_abort
= 3;
45 module_param(scrub_overflow_abort
, uint
, S_IRUGO
|S_IWUSR
);
46 MODULE_PARM_DESC(scrub_overflow_abort
,
47 "Number of times we overflow ARS results before abort");
49 static bool disable_vendor_specific
;
50 module_param(disable_vendor_specific
, bool, S_IRUGO
);
51 MODULE_PARM_DESC(disable_vendor_specific
,
52 "Limit commands to the publicly specified set\n");
54 LIST_HEAD(acpi_descs
);
55 DEFINE_MUTEX(acpi_desc_lock
);
57 static struct workqueue_struct
*nfit_wq
;
59 struct nfit_table_prev
{
60 struct list_head spas
;
61 struct list_head memdevs
;
62 struct list_head dcrs
;
63 struct list_head bdws
;
64 struct list_head idts
;
65 struct list_head flushes
;
68 static u8 nfit_uuid
[NFIT_UUID_MAX
][16];
70 const u8
*to_nfit_uuid(enum nfit_uuids id
)
74 EXPORT_SYMBOL(to_nfit_uuid
);
76 static struct acpi_nfit_desc
*to_acpi_nfit_desc(
77 struct nvdimm_bus_descriptor
*nd_desc
)
79 return container_of(nd_desc
, struct acpi_nfit_desc
, nd_desc
);
82 static struct acpi_device
*to_acpi_dev(struct acpi_nfit_desc
*acpi_desc
)
84 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
87 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
90 if (!nd_desc
->provider_name
91 || strcmp(nd_desc
->provider_name
, "ACPI.NFIT") != 0)
94 return to_acpi_device(acpi_desc
->dev
);
97 static int xlat_bus_status(void *buf
, unsigned int cmd
, u32 status
)
99 struct nd_cmd_clear_error
*clear_err
;
100 struct nd_cmd_ars_status
*ars_status
;
105 if ((status
& 0xffff) == NFIT_ARS_CAP_NONE
)
112 /* No supported scan types for this range */
113 flags
= ND_ARS_PERSISTENT
| ND_ARS_VOLATILE
;
114 if ((status
>> 16 & flags
) == 0)
117 case ND_CMD_ARS_START
:
118 /* ARS is in progress */
119 if ((status
& 0xffff) == NFIT_ARS_START_BUSY
)
126 case ND_CMD_ARS_STATUS
:
131 /* Check extended status (Upper two bytes) */
132 if (status
== NFIT_ARS_STATUS_DONE
)
135 /* ARS is in progress */
136 if (status
== NFIT_ARS_STATUS_BUSY
)
139 /* No ARS performed for the current boot */
140 if (status
== NFIT_ARS_STATUS_NONE
)
144 * ARS interrupted, either we overflowed or some other
145 * agent wants the scan to stop. If we didn't overflow
146 * then just continue with the returned results.
148 if (status
== NFIT_ARS_STATUS_INTR
) {
149 if (ars_status
->out_length
>= 40 && (ars_status
->flags
150 & NFIT_ARS_F_OVERFLOW
))
159 case ND_CMD_CLEAR_ERROR
:
163 if (!clear_err
->cleared
)
165 if (clear_err
->length
> clear_err
->cleared
)
166 return clear_err
->cleared
;
172 /* all other non-zero status results in an error */
178 static int xlat_status(struct nvdimm
*nvdimm
, void *buf
, unsigned int cmd
,
182 return xlat_bus_status(buf
, cmd
, status
);
188 int acpi_nfit_ctl(struct nvdimm_bus_descriptor
*nd_desc
, struct nvdimm
*nvdimm
,
189 unsigned int cmd
, void *buf
, unsigned int buf_len
, int *cmd_rc
)
191 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
192 union acpi_object in_obj
, in_buf
, *out_obj
;
193 const struct nd_cmd_desc
*desc
= NULL
;
194 struct device
*dev
= acpi_desc
->dev
;
195 struct nd_cmd_pkg
*call_pkg
= NULL
;
196 const char *cmd_name
, *dimm_name
;
197 unsigned long cmd_mask
, dsm_mask
;
198 u32 offset
, fw_status
= 0;
205 if (cmd
== ND_CMD_CALL
) {
207 func
= call_pkg
->nd_command
;
211 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
212 struct acpi_device
*adev
= nfit_mem
->adev
;
216 if (call_pkg
&& nfit_mem
->family
!= call_pkg
->nd_family
)
219 dimm_name
= nvdimm_name(nvdimm
);
220 cmd_name
= nvdimm_cmd_name(cmd
);
221 cmd_mask
= nvdimm_cmd_mask(nvdimm
);
222 dsm_mask
= nfit_mem
->dsm_mask
;
223 desc
= nd_cmd_dimm_desc(cmd
);
224 uuid
= to_nfit_uuid(nfit_mem
->family
);
225 handle
= adev
->handle
;
227 struct acpi_device
*adev
= to_acpi_dev(acpi_desc
);
229 cmd_name
= nvdimm_bus_cmd_name(cmd
);
230 cmd_mask
= nd_desc
->cmd_mask
;
232 desc
= nd_cmd_bus_desc(cmd
);
233 uuid
= to_nfit_uuid(NFIT_DEV_BUS
);
234 handle
= adev
->handle
;
238 if (!desc
|| (cmd
&& (desc
->out_num
+ desc
->in_num
== 0)))
241 if (!test_bit(cmd
, &cmd_mask
) || !test_bit(func
, &dsm_mask
))
244 in_obj
.type
= ACPI_TYPE_PACKAGE
;
245 in_obj
.package
.count
= 1;
246 in_obj
.package
.elements
= &in_buf
;
247 in_buf
.type
= ACPI_TYPE_BUFFER
;
248 in_buf
.buffer
.pointer
= buf
;
249 in_buf
.buffer
.length
= 0;
251 /* libnvdimm has already validated the input envelope */
252 for (i
= 0; i
< desc
->in_num
; i
++)
253 in_buf
.buffer
.length
+= nd_cmd_in_size(nvdimm
, cmd
, desc
,
257 /* skip over package wrapper */
258 in_buf
.buffer
.pointer
= (void *) &call_pkg
->nd_payload
;
259 in_buf
.buffer
.length
= call_pkg
->nd_size_in
;
262 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG
)) {
263 dev_dbg(dev
, "%s:%s cmd: %d: func: %d input length: %d\n",
264 __func__
, dimm_name
, cmd
, func
,
265 in_buf
.buffer
.length
);
266 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET
, 4, 4,
267 in_buf
.buffer
.pointer
,
268 min_t(u32
, 256, in_buf
.buffer
.length
), true);
271 out_obj
= acpi_evaluate_dsm(handle
, uuid
, 1, func
, &in_obj
);
273 dev_dbg(dev
, "%s:%s _DSM failed cmd: %s\n", __func__
, dimm_name
,
279 call_pkg
->nd_fw_size
= out_obj
->buffer
.length
;
280 memcpy(call_pkg
->nd_payload
+ call_pkg
->nd_size_in
,
281 out_obj
->buffer
.pointer
,
282 min(call_pkg
->nd_fw_size
, call_pkg
->nd_size_out
));
286 * Need to support FW function w/o known size in advance.
287 * Caller can determine required size based upon nd_fw_size.
288 * If we return an error (like elsewhere) then caller wouldn't
289 * be able to rely upon data returned to make calculation.
294 if (out_obj
->package
.type
!= ACPI_TYPE_BUFFER
) {
295 dev_dbg(dev
, "%s:%s unexpected output object type cmd: %s type: %d\n",
296 __func__
, dimm_name
, cmd_name
, out_obj
->type
);
301 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG
)) {
302 dev_dbg(dev
, "%s:%s cmd: %s output length: %d\n", __func__
,
303 dimm_name
, cmd_name
, out_obj
->buffer
.length
);
304 print_hex_dump_debug(cmd_name
, DUMP_PREFIX_OFFSET
, 4,
305 4, out_obj
->buffer
.pointer
, min_t(u32
, 128,
306 out_obj
->buffer
.length
), true);
309 for (i
= 0, offset
= 0; i
< desc
->out_num
; i
++) {
310 u32 out_size
= nd_cmd_out_size(nvdimm
, cmd
, desc
, i
, buf
,
311 (u32
*) out_obj
->buffer
.pointer
,
312 out_obj
->buffer
.length
- offset
);
314 if (offset
+ out_size
> out_obj
->buffer
.length
) {
315 dev_dbg(dev
, "%s:%s output object underflow cmd: %s field: %d\n",
316 __func__
, dimm_name
, cmd_name
, i
);
320 if (in_buf
.buffer
.length
+ offset
+ out_size
> buf_len
) {
321 dev_dbg(dev
, "%s:%s output overrun cmd: %s field: %d\n",
322 __func__
, dimm_name
, cmd_name
, i
);
326 memcpy(buf
+ in_buf
.buffer
.length
+ offset
,
327 out_obj
->buffer
.pointer
+ offset
, out_size
);
332 * Set fw_status for all the commands with a known format to be
333 * later interpreted by xlat_status().
335 if (i
>= 1 && ((cmd
>= ND_CMD_ARS_CAP
&& cmd
<= ND_CMD_CLEAR_ERROR
)
336 || (cmd
>= ND_CMD_SMART
&& cmd
<= ND_CMD_VENDOR
)))
337 fw_status
= *(u32
*) out_obj
->buffer
.pointer
;
339 if (offset
+ in_buf
.buffer
.length
< buf_len
) {
342 * status valid, return the number of bytes left
343 * unfilled in the output buffer
345 rc
= buf_len
- offset
- in_buf
.buffer
.length
;
347 *cmd_rc
= xlat_status(nvdimm
, buf
, cmd
,
350 dev_err(dev
, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
351 __func__
, dimm_name
, cmd_name
, buf_len
,
358 *cmd_rc
= xlat_status(nvdimm
, buf
, cmd
, fw_status
);
366 EXPORT_SYMBOL_GPL(acpi_nfit_ctl
);
368 static const char *spa_type_name(u16 type
)
370 static const char *to_name
[] = {
371 [NFIT_SPA_VOLATILE
] = "volatile",
372 [NFIT_SPA_PM
] = "pmem",
373 [NFIT_SPA_DCR
] = "dimm-control-region",
374 [NFIT_SPA_BDW
] = "block-data-window",
375 [NFIT_SPA_VDISK
] = "volatile-disk",
376 [NFIT_SPA_VCD
] = "volatile-cd",
377 [NFIT_SPA_PDISK
] = "persistent-disk",
378 [NFIT_SPA_PCD
] = "persistent-cd",
382 if (type
> NFIT_SPA_PCD
)
385 return to_name
[type
];
388 int nfit_spa_type(struct acpi_nfit_system_address
*spa
)
392 for (i
= 0; i
< NFIT_UUID_MAX
; i
++)
393 if (memcmp(to_nfit_uuid(i
), spa
->range_guid
, 16) == 0)
398 static bool add_spa(struct acpi_nfit_desc
*acpi_desc
,
399 struct nfit_table_prev
*prev
,
400 struct acpi_nfit_system_address
*spa
)
402 struct device
*dev
= acpi_desc
->dev
;
403 struct nfit_spa
*nfit_spa
;
405 if (spa
->header
.length
!= sizeof(*spa
))
408 list_for_each_entry(nfit_spa
, &prev
->spas
, list
) {
409 if (memcmp(nfit_spa
->spa
, spa
, sizeof(*spa
)) == 0) {
410 list_move_tail(&nfit_spa
->list
, &acpi_desc
->spas
);
415 nfit_spa
= devm_kzalloc(dev
, sizeof(*nfit_spa
) + sizeof(*spa
),
419 INIT_LIST_HEAD(&nfit_spa
->list
);
420 memcpy(nfit_spa
->spa
, spa
, sizeof(*spa
));
421 list_add_tail(&nfit_spa
->list
, &acpi_desc
->spas
);
422 dev_dbg(dev
, "%s: spa index: %d type: %s\n", __func__
,
424 spa_type_name(nfit_spa_type(spa
)));
428 static bool add_memdev(struct acpi_nfit_desc
*acpi_desc
,
429 struct nfit_table_prev
*prev
,
430 struct acpi_nfit_memory_map
*memdev
)
432 struct device
*dev
= acpi_desc
->dev
;
433 struct nfit_memdev
*nfit_memdev
;
435 if (memdev
->header
.length
!= sizeof(*memdev
))
438 list_for_each_entry(nfit_memdev
, &prev
->memdevs
, list
)
439 if (memcmp(nfit_memdev
->memdev
, memdev
, sizeof(*memdev
)) == 0) {
440 list_move_tail(&nfit_memdev
->list
, &acpi_desc
->memdevs
);
444 nfit_memdev
= devm_kzalloc(dev
, sizeof(*nfit_memdev
) + sizeof(*memdev
),
448 INIT_LIST_HEAD(&nfit_memdev
->list
);
449 memcpy(nfit_memdev
->memdev
, memdev
, sizeof(*memdev
));
450 list_add_tail(&nfit_memdev
->list
, &acpi_desc
->memdevs
);
451 dev_dbg(dev
, "%s: memdev handle: %#x spa: %d dcr: %d\n",
452 __func__
, memdev
->device_handle
, memdev
->range_index
,
453 memdev
->region_index
);
458 * An implementation may provide a truncated control region if no block windows
461 static size_t sizeof_dcr(struct acpi_nfit_control_region
*dcr
)
463 if (dcr
->header
.length
< offsetof(struct acpi_nfit_control_region
,
468 return offsetof(struct acpi_nfit_control_region
, window_size
);
471 static bool add_dcr(struct acpi_nfit_desc
*acpi_desc
,
472 struct nfit_table_prev
*prev
,
473 struct acpi_nfit_control_region
*dcr
)
475 struct device
*dev
= acpi_desc
->dev
;
476 struct nfit_dcr
*nfit_dcr
;
478 if (!sizeof_dcr(dcr
))
481 list_for_each_entry(nfit_dcr
, &prev
->dcrs
, list
)
482 if (memcmp(nfit_dcr
->dcr
, dcr
, sizeof_dcr(dcr
)) == 0) {
483 list_move_tail(&nfit_dcr
->list
, &acpi_desc
->dcrs
);
487 nfit_dcr
= devm_kzalloc(dev
, sizeof(*nfit_dcr
) + sizeof(*dcr
),
491 INIT_LIST_HEAD(&nfit_dcr
->list
);
492 memcpy(nfit_dcr
->dcr
, dcr
, sizeof_dcr(dcr
));
493 list_add_tail(&nfit_dcr
->list
, &acpi_desc
->dcrs
);
494 dev_dbg(dev
, "%s: dcr index: %d windows: %d\n", __func__
,
495 dcr
->region_index
, dcr
->windows
);
499 static bool add_bdw(struct acpi_nfit_desc
*acpi_desc
,
500 struct nfit_table_prev
*prev
,
501 struct acpi_nfit_data_region
*bdw
)
503 struct device
*dev
= acpi_desc
->dev
;
504 struct nfit_bdw
*nfit_bdw
;
506 if (bdw
->header
.length
!= sizeof(*bdw
))
508 list_for_each_entry(nfit_bdw
, &prev
->bdws
, list
)
509 if (memcmp(nfit_bdw
->bdw
, bdw
, sizeof(*bdw
)) == 0) {
510 list_move_tail(&nfit_bdw
->list
, &acpi_desc
->bdws
);
514 nfit_bdw
= devm_kzalloc(dev
, sizeof(*nfit_bdw
) + sizeof(*bdw
),
518 INIT_LIST_HEAD(&nfit_bdw
->list
);
519 memcpy(nfit_bdw
->bdw
, bdw
, sizeof(*bdw
));
520 list_add_tail(&nfit_bdw
->list
, &acpi_desc
->bdws
);
521 dev_dbg(dev
, "%s: bdw dcr: %d windows: %d\n", __func__
,
522 bdw
->region_index
, bdw
->windows
);
526 static size_t sizeof_idt(struct acpi_nfit_interleave
*idt
)
528 if (idt
->header
.length
< sizeof(*idt
))
530 return sizeof(*idt
) + sizeof(u32
) * (idt
->line_count
- 1);
533 static bool add_idt(struct acpi_nfit_desc
*acpi_desc
,
534 struct nfit_table_prev
*prev
,
535 struct acpi_nfit_interleave
*idt
)
537 struct device
*dev
= acpi_desc
->dev
;
538 struct nfit_idt
*nfit_idt
;
540 if (!sizeof_idt(idt
))
543 list_for_each_entry(nfit_idt
, &prev
->idts
, list
) {
544 if (sizeof_idt(nfit_idt
->idt
) != sizeof_idt(idt
))
547 if (memcmp(nfit_idt
->idt
, idt
, sizeof_idt(idt
)) == 0) {
548 list_move_tail(&nfit_idt
->list
, &acpi_desc
->idts
);
553 nfit_idt
= devm_kzalloc(dev
, sizeof(*nfit_idt
) + sizeof_idt(idt
),
557 INIT_LIST_HEAD(&nfit_idt
->list
);
558 memcpy(nfit_idt
->idt
, idt
, sizeof_idt(idt
));
559 list_add_tail(&nfit_idt
->list
, &acpi_desc
->idts
);
560 dev_dbg(dev
, "%s: idt index: %d num_lines: %d\n", __func__
,
561 idt
->interleave_index
, idt
->line_count
);
565 static size_t sizeof_flush(struct acpi_nfit_flush_address
*flush
)
567 if (flush
->header
.length
< sizeof(*flush
))
569 return sizeof(*flush
) + sizeof(u64
) * (flush
->hint_count
- 1);
572 static bool add_flush(struct acpi_nfit_desc
*acpi_desc
,
573 struct nfit_table_prev
*prev
,
574 struct acpi_nfit_flush_address
*flush
)
576 struct device
*dev
= acpi_desc
->dev
;
577 struct nfit_flush
*nfit_flush
;
579 if (!sizeof_flush(flush
))
582 list_for_each_entry(nfit_flush
, &prev
->flushes
, list
) {
583 if (sizeof_flush(nfit_flush
->flush
) != sizeof_flush(flush
))
586 if (memcmp(nfit_flush
->flush
, flush
,
587 sizeof_flush(flush
)) == 0) {
588 list_move_tail(&nfit_flush
->list
, &acpi_desc
->flushes
);
593 nfit_flush
= devm_kzalloc(dev
, sizeof(*nfit_flush
)
594 + sizeof_flush(flush
), GFP_KERNEL
);
597 INIT_LIST_HEAD(&nfit_flush
->list
);
598 memcpy(nfit_flush
->flush
, flush
, sizeof_flush(flush
));
599 list_add_tail(&nfit_flush
->list
, &acpi_desc
->flushes
);
600 dev_dbg(dev
, "%s: nfit_flush handle: %d hint_count: %d\n", __func__
,
601 flush
->device_handle
, flush
->hint_count
);
605 static void *add_table(struct acpi_nfit_desc
*acpi_desc
,
606 struct nfit_table_prev
*prev
, void *table
, const void *end
)
608 struct device
*dev
= acpi_desc
->dev
;
609 struct acpi_nfit_header
*hdr
;
610 void *err
= ERR_PTR(-ENOMEM
);
617 dev_warn(dev
, "found a zero length table '%d' parsing nfit\n",
623 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS
:
624 if (!add_spa(acpi_desc
, prev
, table
))
627 case ACPI_NFIT_TYPE_MEMORY_MAP
:
628 if (!add_memdev(acpi_desc
, prev
, table
))
631 case ACPI_NFIT_TYPE_CONTROL_REGION
:
632 if (!add_dcr(acpi_desc
, prev
, table
))
635 case ACPI_NFIT_TYPE_DATA_REGION
:
636 if (!add_bdw(acpi_desc
, prev
, table
))
639 case ACPI_NFIT_TYPE_INTERLEAVE
:
640 if (!add_idt(acpi_desc
, prev
, table
))
643 case ACPI_NFIT_TYPE_FLUSH_ADDRESS
:
644 if (!add_flush(acpi_desc
, prev
, table
))
647 case ACPI_NFIT_TYPE_SMBIOS
:
648 dev_dbg(dev
, "%s: smbios\n", __func__
);
651 dev_err(dev
, "unknown table '%d' parsing nfit\n", hdr
->type
);
655 return table
+ hdr
->length
;
658 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc
*acpi_desc
,
659 struct nfit_mem
*nfit_mem
)
661 u32 device_handle
= __to_nfit_memdev(nfit_mem
)->device_handle
;
662 u16 dcr
= nfit_mem
->dcr
->region_index
;
663 struct nfit_spa
*nfit_spa
;
665 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
666 u16 range_index
= nfit_spa
->spa
->range_index
;
667 int type
= nfit_spa_type(nfit_spa
->spa
);
668 struct nfit_memdev
*nfit_memdev
;
670 if (type
!= NFIT_SPA_BDW
)
673 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
674 if (nfit_memdev
->memdev
->range_index
!= range_index
)
676 if (nfit_memdev
->memdev
->device_handle
!= device_handle
)
678 if (nfit_memdev
->memdev
->region_index
!= dcr
)
681 nfit_mem
->spa_bdw
= nfit_spa
->spa
;
686 dev_dbg(acpi_desc
->dev
, "SPA-BDW not found for SPA-DCR %d\n",
687 nfit_mem
->spa_dcr
->range_index
);
688 nfit_mem
->bdw
= NULL
;
691 static void nfit_mem_init_bdw(struct acpi_nfit_desc
*acpi_desc
,
692 struct nfit_mem
*nfit_mem
, struct acpi_nfit_system_address
*spa
)
694 u16 dcr
= __to_nfit_memdev(nfit_mem
)->region_index
;
695 struct nfit_memdev
*nfit_memdev
;
696 struct nfit_bdw
*nfit_bdw
;
697 struct nfit_idt
*nfit_idt
;
698 u16 idt_idx
, range_index
;
700 list_for_each_entry(nfit_bdw
, &acpi_desc
->bdws
, list
) {
701 if (nfit_bdw
->bdw
->region_index
!= dcr
)
703 nfit_mem
->bdw
= nfit_bdw
->bdw
;
710 nfit_mem_find_spa_bdw(acpi_desc
, nfit_mem
);
712 if (!nfit_mem
->spa_bdw
)
715 range_index
= nfit_mem
->spa_bdw
->range_index
;
716 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
717 if (nfit_memdev
->memdev
->range_index
!= range_index
||
718 nfit_memdev
->memdev
->region_index
!= dcr
)
720 nfit_mem
->memdev_bdw
= nfit_memdev
->memdev
;
721 idt_idx
= nfit_memdev
->memdev
->interleave_index
;
722 list_for_each_entry(nfit_idt
, &acpi_desc
->idts
, list
) {
723 if (nfit_idt
->idt
->interleave_index
!= idt_idx
)
725 nfit_mem
->idt_bdw
= nfit_idt
->idt
;
732 static int nfit_mem_dcr_init(struct acpi_nfit_desc
*acpi_desc
,
733 struct acpi_nfit_system_address
*spa
)
735 struct nfit_mem
*nfit_mem
, *found
;
736 struct nfit_memdev
*nfit_memdev
;
737 int type
= nfit_spa_type(spa
);
747 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
748 struct nfit_flush
*nfit_flush
;
749 struct nfit_dcr
*nfit_dcr
;
753 if (nfit_memdev
->memdev
->range_index
!= spa
->range_index
)
756 dcr
= nfit_memdev
->memdev
->region_index
;
757 device_handle
= nfit_memdev
->memdev
->device_handle
;
758 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
)
759 if (__to_nfit_memdev(nfit_mem
)->device_handle
768 nfit_mem
= devm_kzalloc(acpi_desc
->dev
,
769 sizeof(*nfit_mem
), GFP_KERNEL
);
772 INIT_LIST_HEAD(&nfit_mem
->list
);
773 nfit_mem
->acpi_desc
= acpi_desc
;
774 list_add(&nfit_mem
->list
, &acpi_desc
->dimms
);
777 list_for_each_entry(nfit_dcr
, &acpi_desc
->dcrs
, list
) {
778 if (nfit_dcr
->dcr
->region_index
!= dcr
)
781 * Record the control region for the dimm. For
782 * the ACPI 6.1 case, where there are separate
783 * control regions for the pmem vs blk
784 * interfaces, be sure to record the extended
788 nfit_mem
->dcr
= nfit_dcr
->dcr
;
789 else if (nfit_mem
->dcr
->windows
== 0
790 && nfit_dcr
->dcr
->windows
)
791 nfit_mem
->dcr
= nfit_dcr
->dcr
;
795 list_for_each_entry(nfit_flush
, &acpi_desc
->flushes
, list
) {
796 struct acpi_nfit_flush_address
*flush
;
799 if (nfit_flush
->flush
->device_handle
!= device_handle
)
801 nfit_mem
->nfit_flush
= nfit_flush
;
802 flush
= nfit_flush
->flush
;
803 nfit_mem
->flush_wpq
= devm_kzalloc(acpi_desc
->dev
,
805 * sizeof(struct resource
), GFP_KERNEL
);
806 if (!nfit_mem
->flush_wpq
)
808 for (i
= 0; i
< flush
->hint_count
; i
++) {
809 struct resource
*res
= &nfit_mem
->flush_wpq
[i
];
811 res
->start
= flush
->hint_address
[i
];
812 res
->end
= res
->start
+ 8 - 1;
817 if (dcr
&& !nfit_mem
->dcr
) {
818 dev_err(acpi_desc
->dev
, "SPA %d missing DCR %d\n",
819 spa
->range_index
, dcr
);
823 if (type
== NFIT_SPA_DCR
) {
824 struct nfit_idt
*nfit_idt
;
827 /* multiple dimms may share a SPA when interleaved */
828 nfit_mem
->spa_dcr
= spa
;
829 nfit_mem
->memdev_dcr
= nfit_memdev
->memdev
;
830 idt_idx
= nfit_memdev
->memdev
->interleave_index
;
831 list_for_each_entry(nfit_idt
, &acpi_desc
->idts
, list
) {
832 if (nfit_idt
->idt
->interleave_index
!= idt_idx
)
834 nfit_mem
->idt_dcr
= nfit_idt
->idt
;
837 nfit_mem_init_bdw(acpi_desc
, nfit_mem
, spa
);
840 * A single dimm may belong to multiple SPA-PM
841 * ranges, record at least one in addition to
844 nfit_mem
->memdev_pmem
= nfit_memdev
->memdev
;
851 static int nfit_mem_cmp(void *priv
, struct list_head
*_a
, struct list_head
*_b
)
853 struct nfit_mem
*a
= container_of(_a
, typeof(*a
), list
);
854 struct nfit_mem
*b
= container_of(_b
, typeof(*b
), list
);
855 u32 handleA
, handleB
;
857 handleA
= __to_nfit_memdev(a
)->device_handle
;
858 handleB
= __to_nfit_memdev(b
)->device_handle
;
859 if (handleA
< handleB
)
861 else if (handleA
> handleB
)
866 static int nfit_mem_init(struct acpi_nfit_desc
*acpi_desc
)
868 struct nfit_spa
*nfit_spa
;
871 * For each SPA-DCR or SPA-PMEM address range find its
872 * corresponding MEMDEV(s). From each MEMDEV find the
873 * corresponding DCR. Then, if we're operating on a SPA-DCR,
874 * try to find a SPA-BDW and a corresponding BDW that references
875 * the DCR. Throw it all into an nfit_mem object. Note, that
878 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
881 rc
= nfit_mem_dcr_init(acpi_desc
, nfit_spa
->spa
);
886 list_sort(NULL
, &acpi_desc
->dimms
, nfit_mem_cmp
);
891 static ssize_t
revision_show(struct device
*dev
,
892 struct device_attribute
*attr
, char *buf
)
894 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
895 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
896 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
898 return sprintf(buf
, "%d\n", acpi_desc
->acpi_header
.revision
);
900 static DEVICE_ATTR_RO(revision
);
902 static ssize_t
hw_error_scrub_show(struct device
*dev
,
903 struct device_attribute
*attr
, char *buf
)
905 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
906 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
907 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
909 return sprintf(buf
, "%d\n", acpi_desc
->scrub_mode
);
913 * The 'hw_error_scrub' attribute can have the following values written to it:
914 * '0': Switch to the default mode where an exception will only insert
915 * the address of the memory error into the poison and badblocks lists.
916 * '1': Enable a full scrub to happen if an exception for a memory error is
919 static ssize_t
hw_error_scrub_store(struct device
*dev
,
920 struct device_attribute
*attr
, const char *buf
, size_t size
)
922 struct nvdimm_bus_descriptor
*nd_desc
;
926 rc
= kstrtol(buf
, 0, &val
);
931 nd_desc
= dev_get_drvdata(dev
);
933 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
936 case HW_ERROR_SCRUB_ON
:
937 acpi_desc
->scrub_mode
= HW_ERROR_SCRUB_ON
;
939 case HW_ERROR_SCRUB_OFF
:
940 acpi_desc
->scrub_mode
= HW_ERROR_SCRUB_OFF
;
952 static DEVICE_ATTR_RW(hw_error_scrub
);
955 * This shows the number of full Address Range Scrubs that have been
956 * completed since driver load time. Userspace can wait on this using
957 * select/poll etc. A '+' at the end indicates an ARS is in progress
959 static ssize_t
scrub_show(struct device
*dev
,
960 struct device_attribute
*attr
, char *buf
)
962 struct nvdimm_bus_descriptor
*nd_desc
;
966 nd_desc
= dev_get_drvdata(dev
);
968 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
970 rc
= sprintf(buf
, "%d%s", acpi_desc
->scrub_count
,
971 (work_busy(&acpi_desc
->work
)) ? "+\n" : "\n");
977 static ssize_t
scrub_store(struct device
*dev
,
978 struct device_attribute
*attr
, const char *buf
, size_t size
)
980 struct nvdimm_bus_descriptor
*nd_desc
;
984 rc
= kstrtol(buf
, 0, &val
);
991 nd_desc
= dev_get_drvdata(dev
);
993 struct acpi_nfit_desc
*acpi_desc
= to_acpi_desc(nd_desc
);
995 rc
= acpi_nfit_ars_rescan(acpi_desc
);
1002 static DEVICE_ATTR_RW(scrub
);
1004 static bool ars_supported(struct nvdimm_bus
*nvdimm_bus
)
1006 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
1007 const unsigned long mask
= 1 << ND_CMD_ARS_CAP
| 1 << ND_CMD_ARS_START
1008 | 1 << ND_CMD_ARS_STATUS
;
1010 return (nd_desc
->cmd_mask
& mask
) == mask
;
1013 static umode_t
nfit_visible(struct kobject
*kobj
, struct attribute
*a
, int n
)
1015 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
1016 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
1018 if (a
== &dev_attr_scrub
.attr
&& !ars_supported(nvdimm_bus
))
1023 static struct attribute
*acpi_nfit_attributes
[] = {
1024 &dev_attr_revision
.attr
,
1025 &dev_attr_scrub
.attr
,
1026 &dev_attr_hw_error_scrub
.attr
,
1030 static struct attribute_group acpi_nfit_attribute_group
= {
1032 .attrs
= acpi_nfit_attributes
,
1033 .is_visible
= nfit_visible
,
1036 static const struct attribute_group
*acpi_nfit_attribute_groups
[] = {
1037 &nvdimm_bus_attribute_group
,
1038 &acpi_nfit_attribute_group
,
1042 static struct acpi_nfit_memory_map
*to_nfit_memdev(struct device
*dev
)
1044 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1045 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1047 return __to_nfit_memdev(nfit_mem
);
1050 static struct acpi_nfit_control_region
*to_nfit_dcr(struct device
*dev
)
1052 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1053 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1055 return nfit_mem
->dcr
;
1058 static ssize_t
handle_show(struct device
*dev
,
1059 struct device_attribute
*attr
, char *buf
)
1061 struct acpi_nfit_memory_map
*memdev
= to_nfit_memdev(dev
);
1063 return sprintf(buf
, "%#x\n", memdev
->device_handle
);
1065 static DEVICE_ATTR_RO(handle
);
1067 static ssize_t
phys_id_show(struct device
*dev
,
1068 struct device_attribute
*attr
, char *buf
)
1070 struct acpi_nfit_memory_map
*memdev
= to_nfit_memdev(dev
);
1072 return sprintf(buf
, "%#x\n", memdev
->physical_id
);
1074 static DEVICE_ATTR_RO(phys_id
);
1076 static ssize_t
vendor_show(struct device
*dev
,
1077 struct device_attribute
*attr
, char *buf
)
1079 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1081 return sprintf(buf
, "0x%04x\n", be16_to_cpu(dcr
->vendor_id
));
1083 static DEVICE_ATTR_RO(vendor
);
1085 static ssize_t
rev_id_show(struct device
*dev
,
1086 struct device_attribute
*attr
, char *buf
)
1088 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1090 return sprintf(buf
, "0x%04x\n", be16_to_cpu(dcr
->revision_id
));
1092 static DEVICE_ATTR_RO(rev_id
);
1094 static ssize_t
device_show(struct device
*dev
,
1095 struct device_attribute
*attr
, char *buf
)
1097 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1099 return sprintf(buf
, "0x%04x\n", be16_to_cpu(dcr
->device_id
));
1101 static DEVICE_ATTR_RO(device
);
1103 static ssize_t
subsystem_vendor_show(struct device
*dev
,
1104 struct device_attribute
*attr
, char *buf
)
1106 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1108 return sprintf(buf
, "0x%04x\n", be16_to_cpu(dcr
->subsystem_vendor_id
));
1110 static DEVICE_ATTR_RO(subsystem_vendor
);
1112 static ssize_t
subsystem_rev_id_show(struct device
*dev
,
1113 struct device_attribute
*attr
, char *buf
)
1115 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1117 return sprintf(buf
, "0x%04x\n",
1118 be16_to_cpu(dcr
->subsystem_revision_id
));
1120 static DEVICE_ATTR_RO(subsystem_rev_id
);
1122 static ssize_t
subsystem_device_show(struct device
*dev
,
1123 struct device_attribute
*attr
, char *buf
)
1125 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1127 return sprintf(buf
, "0x%04x\n", be16_to_cpu(dcr
->subsystem_device_id
));
1129 static DEVICE_ATTR_RO(subsystem_device
);
1131 static int num_nvdimm_formats(struct nvdimm
*nvdimm
)
1133 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1136 if (nfit_mem
->memdev_pmem
)
1138 if (nfit_mem
->memdev_bdw
)
1143 static ssize_t
format_show(struct device
*dev
,
1144 struct device_attribute
*attr
, char *buf
)
1146 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1148 return sprintf(buf
, "0x%04x\n", le16_to_cpu(dcr
->code
));
1150 static DEVICE_ATTR_RO(format
);
1152 static ssize_t
format1_show(struct device
*dev
,
1153 struct device_attribute
*attr
, char *buf
)
1156 ssize_t rc
= -ENXIO
;
1157 struct nfit_mem
*nfit_mem
;
1158 struct nfit_memdev
*nfit_memdev
;
1159 struct acpi_nfit_desc
*acpi_desc
;
1160 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1161 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1163 nfit_mem
= nvdimm_provider_data(nvdimm
);
1164 acpi_desc
= nfit_mem
->acpi_desc
;
1165 handle
= to_nfit_memdev(dev
)->device_handle
;
1167 /* assumes DIMMs have at most 2 published interface codes */
1168 mutex_lock(&acpi_desc
->init_mutex
);
1169 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
1170 struct acpi_nfit_memory_map
*memdev
= nfit_memdev
->memdev
;
1171 struct nfit_dcr
*nfit_dcr
;
1173 if (memdev
->device_handle
!= handle
)
1176 list_for_each_entry(nfit_dcr
, &acpi_desc
->dcrs
, list
) {
1177 if (nfit_dcr
->dcr
->region_index
!= memdev
->region_index
)
1179 if (nfit_dcr
->dcr
->code
== dcr
->code
)
1181 rc
= sprintf(buf
, "0x%04x\n",
1182 le16_to_cpu(nfit_dcr
->dcr
->code
));
1188 mutex_unlock(&acpi_desc
->init_mutex
);
1191 static DEVICE_ATTR_RO(format1
);
1193 static ssize_t
formats_show(struct device
*dev
,
1194 struct device_attribute
*attr
, char *buf
)
1196 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1198 return sprintf(buf
, "%d\n", num_nvdimm_formats(nvdimm
));
1200 static DEVICE_ATTR_RO(formats
);
1202 static ssize_t
serial_show(struct device
*dev
,
1203 struct device_attribute
*attr
, char *buf
)
1205 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1207 return sprintf(buf
, "0x%08x\n", be32_to_cpu(dcr
->serial_number
));
1209 static DEVICE_ATTR_RO(serial
);
1211 static ssize_t
family_show(struct device
*dev
,
1212 struct device_attribute
*attr
, char *buf
)
1214 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1215 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1217 if (nfit_mem
->family
< 0)
1219 return sprintf(buf
, "%d\n", nfit_mem
->family
);
1221 static DEVICE_ATTR_RO(family
);
1223 static ssize_t
dsm_mask_show(struct device
*dev
,
1224 struct device_attribute
*attr
, char *buf
)
1226 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1227 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1229 if (nfit_mem
->family
< 0)
1231 return sprintf(buf
, "%#lx\n", nfit_mem
->dsm_mask
);
1233 static DEVICE_ATTR_RO(dsm_mask
);
1235 static ssize_t
flags_show(struct device
*dev
,
1236 struct device_attribute
*attr
, char *buf
)
1238 u16 flags
= to_nfit_memdev(dev
)->flags
;
1240 return sprintf(buf
, "%s%s%s%s%s\n",
1241 flags
& ACPI_NFIT_MEM_SAVE_FAILED
? "save_fail " : "",
1242 flags
& ACPI_NFIT_MEM_RESTORE_FAILED
? "restore_fail " : "",
1243 flags
& ACPI_NFIT_MEM_FLUSH_FAILED
? "flush_fail " : "",
1244 flags
& ACPI_NFIT_MEM_NOT_ARMED
? "not_armed " : "",
1245 flags
& ACPI_NFIT_MEM_HEALTH_OBSERVED
? "smart_event " : "");
1247 static DEVICE_ATTR_RO(flags
);
1249 static ssize_t
id_show(struct device
*dev
,
1250 struct device_attribute
*attr
, char *buf
)
1252 struct acpi_nfit_control_region
*dcr
= to_nfit_dcr(dev
);
1254 if (dcr
->valid_fields
& ACPI_NFIT_CONTROL_MFG_INFO_VALID
)
1255 return sprintf(buf
, "%04x-%02x-%04x-%08x\n",
1256 be16_to_cpu(dcr
->vendor_id
),
1257 dcr
->manufacturing_location
,
1258 be16_to_cpu(dcr
->manufacturing_date
),
1259 be32_to_cpu(dcr
->serial_number
));
1261 return sprintf(buf
, "%04x-%08x\n",
1262 be16_to_cpu(dcr
->vendor_id
),
1263 be32_to_cpu(dcr
->serial_number
));
1265 static DEVICE_ATTR_RO(id
);
1267 static struct attribute
*acpi_nfit_dimm_attributes
[] = {
1268 &dev_attr_handle
.attr
,
1269 &dev_attr_phys_id
.attr
,
1270 &dev_attr_vendor
.attr
,
1271 &dev_attr_device
.attr
,
1272 &dev_attr_rev_id
.attr
,
1273 &dev_attr_subsystem_vendor
.attr
,
1274 &dev_attr_subsystem_device
.attr
,
1275 &dev_attr_subsystem_rev_id
.attr
,
1276 &dev_attr_format
.attr
,
1277 &dev_attr_formats
.attr
,
1278 &dev_attr_format1
.attr
,
1279 &dev_attr_serial
.attr
,
1280 &dev_attr_flags
.attr
,
1282 &dev_attr_family
.attr
,
1283 &dev_attr_dsm_mask
.attr
,
1287 static umode_t
acpi_nfit_dimm_attr_visible(struct kobject
*kobj
,
1288 struct attribute
*a
, int n
)
1290 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
1291 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1293 if (!to_nfit_dcr(dev
))
1295 if (a
== &dev_attr_format1
.attr
&& num_nvdimm_formats(nvdimm
) <= 1)
1300 static struct attribute_group acpi_nfit_dimm_attribute_group
= {
1302 .attrs
= acpi_nfit_dimm_attributes
,
1303 .is_visible
= acpi_nfit_dimm_attr_visible
,
1306 static const struct attribute_group
*acpi_nfit_dimm_attribute_groups
[] = {
1307 &nvdimm_attribute_group
,
1308 &nd_device_attribute_group
,
1309 &acpi_nfit_dimm_attribute_group
,
1313 static struct nvdimm
*acpi_nfit_dimm_by_handle(struct acpi_nfit_desc
*acpi_desc
,
1316 struct nfit_mem
*nfit_mem
;
1318 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
)
1319 if (__to_nfit_memdev(nfit_mem
)->device_handle
== device_handle
)
1320 return nfit_mem
->nvdimm
;
1325 void __acpi_nvdimm_notify(struct device
*dev
, u32 event
)
1327 struct nfit_mem
*nfit_mem
;
1328 struct acpi_nfit_desc
*acpi_desc
;
1330 dev_dbg(dev
->parent
, "%s: %s: event: %d\n", dev_name(dev
), __func__
,
1333 if (event
!= NFIT_NOTIFY_DIMM_HEALTH
) {
1334 dev_dbg(dev
->parent
, "%s: unknown event: %d\n", dev_name(dev
),
1339 acpi_desc
= dev_get_drvdata(dev
->parent
);
1344 * If we successfully retrieved acpi_desc, then we know nfit_mem data
1347 nfit_mem
= dev_get_drvdata(dev
);
1348 if (nfit_mem
&& nfit_mem
->flags_attr
)
1349 sysfs_notify_dirent(nfit_mem
->flags_attr
);
1351 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify
);
1353 static void acpi_nvdimm_notify(acpi_handle handle
, u32 event
, void *data
)
1355 struct acpi_device
*adev
= data
;
1356 struct device
*dev
= &adev
->dev
;
1358 device_lock(dev
->parent
);
1359 __acpi_nvdimm_notify(dev
, event
);
1360 device_unlock(dev
->parent
);
1363 static int acpi_nfit_add_dimm(struct acpi_nfit_desc
*acpi_desc
,
1364 struct nfit_mem
*nfit_mem
, u32 device_handle
)
1366 struct acpi_device
*adev
, *adev_dimm
;
1367 struct device
*dev
= acpi_desc
->dev
;
1368 unsigned long dsm_mask
;
1372 /* nfit test assumes 1:1 relationship between commands and dsms */
1373 nfit_mem
->dsm_mask
= acpi_desc
->dimm_cmd_force_en
;
1374 nfit_mem
->family
= NVDIMM_FAMILY_INTEL
;
1375 adev
= to_acpi_dev(acpi_desc
);
1379 adev_dimm
= acpi_find_child_device(adev
, device_handle
, false);
1380 nfit_mem
->adev
= adev_dimm
;
1382 dev_err(dev
, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1384 return force_enable_dimms
? 0 : -ENODEV
;
1387 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm
->handle
,
1388 ACPI_DEVICE_NOTIFY
, acpi_nvdimm_notify
, adev_dimm
))) {
1389 dev_err(dev
, "%s: notification registration failed\n",
1390 dev_name(&adev_dimm
->dev
));
1395 * Until standardization materializes we need to consider 4
1396 * different command sets. Note, that checking for function0 (bit0)
1397 * tells us if any commands are reachable through this uuid.
1399 for (i
= NVDIMM_FAMILY_INTEL
; i
<= NVDIMM_FAMILY_MSFT
; i
++)
1400 if (acpi_check_dsm(adev_dimm
->handle
, to_nfit_uuid(i
), 1, 1))
1403 /* limit the supported commands to those that are publicly documented */
1404 nfit_mem
->family
= i
;
1405 if (nfit_mem
->family
== NVDIMM_FAMILY_INTEL
) {
1407 if (disable_vendor_specific
)
1408 dsm_mask
&= ~(1 << ND_CMD_VENDOR
);
1409 } else if (nfit_mem
->family
== NVDIMM_FAMILY_HPE1
) {
1410 dsm_mask
= 0x1c3c76;
1411 } else if (nfit_mem
->family
== NVDIMM_FAMILY_HPE2
) {
1413 if (disable_vendor_specific
)
1414 dsm_mask
&= ~(1 << 8);
1415 } else if (nfit_mem
->family
== NVDIMM_FAMILY_MSFT
) {
1416 dsm_mask
= 0xffffffff;
1418 dev_dbg(dev
, "unknown dimm command family\n");
1419 nfit_mem
->family
= -1;
1420 /* DSMs are optional, continue loading the driver... */
1424 uuid
= to_nfit_uuid(nfit_mem
->family
);
1425 for_each_set_bit(i
, &dsm_mask
, BITS_PER_LONG
)
1426 if (acpi_check_dsm(adev_dimm
->handle
, uuid
, 1, 1ULL << i
))
1427 set_bit(i
, &nfit_mem
->dsm_mask
);
1432 static void shutdown_dimm_notify(void *data
)
1434 struct acpi_nfit_desc
*acpi_desc
= data
;
1435 struct nfit_mem
*nfit_mem
;
1437 mutex_lock(&acpi_desc
->init_mutex
);
1439 * Clear out the nfit_mem->flags_attr and shut down dimm event
1442 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
) {
1443 struct acpi_device
*adev_dimm
= nfit_mem
->adev
;
1445 if (nfit_mem
->flags_attr
) {
1446 sysfs_put(nfit_mem
->flags_attr
);
1447 nfit_mem
->flags_attr
= NULL
;
1450 acpi_remove_notify_handler(adev_dimm
->handle
,
1451 ACPI_DEVICE_NOTIFY
, acpi_nvdimm_notify
);
1453 mutex_unlock(&acpi_desc
->init_mutex
);
1456 static int acpi_nfit_register_dimms(struct acpi_nfit_desc
*acpi_desc
)
1458 struct nfit_mem
*nfit_mem
;
1459 int dimm_count
= 0, rc
;
1460 struct nvdimm
*nvdimm
;
1462 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
) {
1463 struct acpi_nfit_flush_address
*flush
;
1464 unsigned long flags
= 0, cmd_mask
;
1468 device_handle
= __to_nfit_memdev(nfit_mem
)->device_handle
;
1469 nvdimm
= acpi_nfit_dimm_by_handle(acpi_desc
, device_handle
);
1475 if (nfit_mem
->bdw
&& nfit_mem
->memdev_pmem
)
1476 flags
|= NDD_ALIASING
;
1478 mem_flags
= __to_nfit_memdev(nfit_mem
)->flags
;
1479 if (mem_flags
& ACPI_NFIT_MEM_NOT_ARMED
)
1480 flags
|= NDD_UNARMED
;
1482 rc
= acpi_nfit_add_dimm(acpi_desc
, nfit_mem
, device_handle
);
1487 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
1488 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
1489 * userspace interface.
1491 cmd_mask
= 1UL << ND_CMD_CALL
;
1492 if (nfit_mem
->family
== NVDIMM_FAMILY_INTEL
)
1493 cmd_mask
|= nfit_mem
->dsm_mask
;
1495 flush
= nfit_mem
->nfit_flush
? nfit_mem
->nfit_flush
->flush
1497 nvdimm
= nvdimm_create(acpi_desc
->nvdimm_bus
, nfit_mem
,
1498 acpi_nfit_dimm_attribute_groups
,
1499 flags
, cmd_mask
, flush
? flush
->hint_count
: 0,
1500 nfit_mem
->flush_wpq
);
1504 nfit_mem
->nvdimm
= nvdimm
;
1507 if ((mem_flags
& ACPI_NFIT_MEM_FAILED_MASK
) == 0)
1510 dev_info(acpi_desc
->dev
, "%s flags:%s%s%s%s\n",
1511 nvdimm_name(nvdimm
),
1512 mem_flags
& ACPI_NFIT_MEM_SAVE_FAILED
? " save_fail" : "",
1513 mem_flags
& ACPI_NFIT_MEM_RESTORE_FAILED
? " restore_fail":"",
1514 mem_flags
& ACPI_NFIT_MEM_FLUSH_FAILED
? " flush_fail" : "",
1515 mem_flags
& ACPI_NFIT_MEM_NOT_ARMED
? " not_armed" : "");
1519 rc
= nvdimm_bus_check_dimm_count(acpi_desc
->nvdimm_bus
, dimm_count
);
1524 * Now that dimms are successfully registered, and async registration
1525 * is flushed, attempt to enable event notification.
1527 list_for_each_entry(nfit_mem
, &acpi_desc
->dimms
, list
) {
1528 struct kernfs_node
*nfit_kernfs
;
1530 nvdimm
= nfit_mem
->nvdimm
;
1531 nfit_kernfs
= sysfs_get_dirent(nvdimm_kobj(nvdimm
)->sd
, "nfit");
1533 nfit_mem
->flags_attr
= sysfs_get_dirent(nfit_kernfs
,
1535 sysfs_put(nfit_kernfs
);
1536 if (!nfit_mem
->flags_attr
)
1537 dev_warn(acpi_desc
->dev
, "%s: notifications disabled\n",
1538 nvdimm_name(nvdimm
));
1541 return devm_add_action_or_reset(acpi_desc
->dev
, shutdown_dimm_notify
,
1545 static void acpi_nfit_init_dsms(struct acpi_nfit_desc
*acpi_desc
)
1547 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1548 const u8
*uuid
= to_nfit_uuid(NFIT_DEV_BUS
);
1549 struct acpi_device
*adev
;
1552 nd_desc
->cmd_mask
= acpi_desc
->bus_cmd_force_en
;
1553 adev
= to_acpi_dev(acpi_desc
);
1557 for (i
= ND_CMD_ARS_CAP
; i
<= ND_CMD_CLEAR_ERROR
; i
++)
1558 if (acpi_check_dsm(adev
->handle
, uuid
, 1, 1ULL << i
))
1559 set_bit(i
, &nd_desc
->cmd_mask
);
1562 static ssize_t
range_index_show(struct device
*dev
,
1563 struct device_attribute
*attr
, char *buf
)
1565 struct nd_region
*nd_region
= to_nd_region(dev
);
1566 struct nfit_spa
*nfit_spa
= nd_region_provider_data(nd_region
);
1568 return sprintf(buf
, "%d\n", nfit_spa
->spa
->range_index
);
1570 static DEVICE_ATTR_RO(range_index
);
1572 static struct attribute
*acpi_nfit_region_attributes
[] = {
1573 &dev_attr_range_index
.attr
,
1577 static struct attribute_group acpi_nfit_region_attribute_group
= {
1579 .attrs
= acpi_nfit_region_attributes
,
1582 static const struct attribute_group
*acpi_nfit_region_attribute_groups
[] = {
1583 &nd_region_attribute_group
,
1584 &nd_mapping_attribute_group
,
1585 &nd_device_attribute_group
,
1586 &nd_numa_attribute_group
,
1587 &acpi_nfit_region_attribute_group
,
1591 /* enough info to uniquely specify an interleave set */
1592 struct nfit_set_info
{
1593 struct nfit_set_info_map
{
1600 static size_t sizeof_nfit_set_info(int num_mappings
)
1602 return sizeof(struct nfit_set_info
)
1603 + num_mappings
* sizeof(struct nfit_set_info_map
);
1606 static int cmp_map_compat(const void *m0
, const void *m1
)
1608 const struct nfit_set_info_map
*map0
= m0
;
1609 const struct nfit_set_info_map
*map1
= m1
;
1611 return memcmp(&map0
->region_offset
, &map1
->region_offset
,
1615 static int cmp_map(const void *m0
, const void *m1
)
1617 const struct nfit_set_info_map
*map0
= m0
;
1618 const struct nfit_set_info_map
*map1
= m1
;
1620 if (map0
->region_offset
< map1
->region_offset
)
1622 else if (map0
->region_offset
> map1
->region_offset
)
1627 /* Retrieve the nth entry referencing this spa */
1628 static struct acpi_nfit_memory_map
*memdev_from_spa(
1629 struct acpi_nfit_desc
*acpi_desc
, u16 range_index
, int n
)
1631 struct nfit_memdev
*nfit_memdev
;
1633 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
)
1634 if (nfit_memdev
->memdev
->range_index
== range_index
)
1636 return nfit_memdev
->memdev
;
1640 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc
*acpi_desc
,
1641 struct nd_region_desc
*ndr_desc
,
1642 struct acpi_nfit_system_address
*spa
)
1644 int i
, spa_type
= nfit_spa_type(spa
);
1645 struct device
*dev
= acpi_desc
->dev
;
1646 struct nd_interleave_set
*nd_set
;
1647 u16 nr
= ndr_desc
->num_mappings
;
1648 struct nfit_set_info
*info
;
1650 if (spa_type
== NFIT_SPA_PM
|| spa_type
== NFIT_SPA_VOLATILE
)
1655 nd_set
= devm_kzalloc(dev
, sizeof(*nd_set
), GFP_KERNEL
);
1659 info
= devm_kzalloc(dev
, sizeof_nfit_set_info(nr
), GFP_KERNEL
);
1662 for (i
= 0; i
< nr
; i
++) {
1663 struct nd_mapping_desc
*mapping
= &ndr_desc
->mapping
[i
];
1664 struct nfit_set_info_map
*map
= &info
->mapping
[i
];
1665 struct nvdimm
*nvdimm
= mapping
->nvdimm
;
1666 struct nfit_mem
*nfit_mem
= nvdimm_provider_data(nvdimm
);
1667 struct acpi_nfit_memory_map
*memdev
= memdev_from_spa(acpi_desc
,
1668 spa
->range_index
, i
);
1670 if (!memdev
|| !nfit_mem
->dcr
) {
1671 dev_err(dev
, "%s: failed to find DCR\n", __func__
);
1675 map
->region_offset
= memdev
->region_offset
;
1676 map
->serial_number
= nfit_mem
->dcr
->serial_number
;
1679 sort(&info
->mapping
[0], nr
, sizeof(struct nfit_set_info_map
),
1681 nd_set
->cookie
= nd_fletcher64(info
, sizeof_nfit_set_info(nr
), 0);
1683 /* support namespaces created with the wrong sort order */
1684 sort(&info
->mapping
[0], nr
, sizeof(struct nfit_set_info_map
),
1685 cmp_map_compat
, NULL
);
1686 nd_set
->altcookie
= nd_fletcher64(info
, sizeof_nfit_set_info(nr
), 0);
1688 ndr_desc
->nd_set
= nd_set
;
1689 devm_kfree(dev
, info
);
1694 static u64
to_interleave_offset(u64 offset
, struct nfit_blk_mmio
*mmio
)
1696 struct acpi_nfit_interleave
*idt
= mmio
->idt
;
1697 u32 sub_line_offset
, line_index
, line_offset
;
1698 u64 line_no
, table_skip_count
, table_offset
;
1700 line_no
= div_u64_rem(offset
, mmio
->line_size
, &sub_line_offset
);
1701 table_skip_count
= div_u64_rem(line_no
, mmio
->num_lines
, &line_index
);
1702 line_offset
= idt
->line_offset
[line_index
]
1704 table_offset
= table_skip_count
* mmio
->table_size
;
1706 return mmio
->base_offset
+ line_offset
+ table_offset
+ sub_line_offset
;
1709 static u32
read_blk_stat(struct nfit_blk
*nfit_blk
, unsigned int bw
)
1711 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[DCR
];
1712 u64 offset
= nfit_blk
->stat_offset
+ mmio
->size
* bw
;
1713 const u32 STATUS_MASK
= 0x80000037;
1715 if (mmio
->num_lines
)
1716 offset
= to_interleave_offset(offset
, mmio
);
1718 return readl(mmio
->addr
.base
+ offset
) & STATUS_MASK
;
1721 static void write_blk_ctl(struct nfit_blk
*nfit_blk
, unsigned int bw
,
1722 resource_size_t dpa
, unsigned int len
, unsigned int write
)
1725 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[DCR
];
1728 BCW_OFFSET_MASK
= (1ULL << 48)-1,
1730 BCW_LEN_MASK
= (1ULL << 8) - 1,
1734 cmd
= (dpa
>> L1_CACHE_SHIFT
) & BCW_OFFSET_MASK
;
1735 len
= len
>> L1_CACHE_SHIFT
;
1736 cmd
|= ((u64
) len
& BCW_LEN_MASK
) << BCW_LEN_SHIFT
;
1737 cmd
|= ((u64
) write
) << BCW_CMD_SHIFT
;
1739 offset
= nfit_blk
->cmd_offset
+ mmio
->size
* bw
;
1740 if (mmio
->num_lines
)
1741 offset
= to_interleave_offset(offset
, mmio
);
1743 writeq(cmd
, mmio
->addr
.base
+ offset
);
1744 nvdimm_flush(nfit_blk
->nd_region
);
1746 if (nfit_blk
->dimm_flags
& NFIT_BLK_DCR_LATCH
)
1747 readq(mmio
->addr
.base
+ offset
);
1750 static int acpi_nfit_blk_single_io(struct nfit_blk
*nfit_blk
,
1751 resource_size_t dpa
, void *iobuf
, size_t len
, int rw
,
1754 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[BDW
];
1755 unsigned int copied
= 0;
1759 base_offset
= nfit_blk
->bdw_offset
+ dpa
% L1_CACHE_BYTES
1760 + lane
* mmio
->size
;
1761 write_blk_ctl(nfit_blk
, lane
, dpa
, len
, rw
);
1766 if (mmio
->num_lines
) {
1769 offset
= to_interleave_offset(base_offset
+ copied
,
1771 div_u64_rem(offset
, mmio
->line_size
, &line_offset
);
1772 c
= min_t(size_t, len
, mmio
->line_size
- line_offset
);
1774 offset
= base_offset
+ nfit_blk
->bdw_offset
;
1779 memcpy_to_pmem(mmio
->addr
.aperture
+ offset
,
1782 if (nfit_blk
->dimm_flags
& NFIT_BLK_READ_FLUSH
)
1783 mmio_flush_range((void __force
*)
1784 mmio
->addr
.aperture
+ offset
, c
);
1786 memcpy_from_pmem(iobuf
+ copied
,
1787 mmio
->addr
.aperture
+ offset
, c
);
1795 nvdimm_flush(nfit_blk
->nd_region
);
1797 rc
= read_blk_stat(nfit_blk
, lane
) ? -EIO
: 0;
1801 static int acpi_nfit_blk_region_do_io(struct nd_blk_region
*ndbr
,
1802 resource_size_t dpa
, void *iobuf
, u64 len
, int rw
)
1804 struct nfit_blk
*nfit_blk
= nd_blk_region_provider_data(ndbr
);
1805 struct nfit_blk_mmio
*mmio
= &nfit_blk
->mmio
[BDW
];
1806 struct nd_region
*nd_region
= nfit_blk
->nd_region
;
1807 unsigned int lane
, copied
= 0;
1810 lane
= nd_region_acquire_lane(nd_region
);
1812 u64 c
= min(len
, mmio
->size
);
1814 rc
= acpi_nfit_blk_single_io(nfit_blk
, dpa
+ copied
,
1815 iobuf
+ copied
, c
, rw
, lane
);
1822 nd_region_release_lane(nd_region
, lane
);
1827 static int nfit_blk_init_interleave(struct nfit_blk_mmio
*mmio
,
1828 struct acpi_nfit_interleave
*idt
, u16 interleave_ways
)
1831 mmio
->num_lines
= idt
->line_count
;
1832 mmio
->line_size
= idt
->line_size
;
1833 if (interleave_ways
== 0)
1835 mmio
->table_size
= mmio
->num_lines
* interleave_ways
1842 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor
*nd_desc
,
1843 struct nvdimm
*nvdimm
, struct nfit_blk
*nfit_blk
)
1845 struct nd_cmd_dimm_flags flags
;
1848 memset(&flags
, 0, sizeof(flags
));
1849 rc
= nd_desc
->ndctl(nd_desc
, nvdimm
, ND_CMD_DIMM_FLAGS
, &flags
,
1850 sizeof(flags
), NULL
);
1852 if (rc
>= 0 && flags
.status
== 0)
1853 nfit_blk
->dimm_flags
= flags
.flags
;
1854 else if (rc
== -ENOTTY
) {
1855 /* fall back to a conservative default */
1856 nfit_blk
->dimm_flags
= NFIT_BLK_DCR_LATCH
| NFIT_BLK_READ_FLUSH
;
1864 static int acpi_nfit_blk_region_enable(struct nvdimm_bus
*nvdimm_bus
,
1867 struct nvdimm_bus_descriptor
*nd_desc
= to_nd_desc(nvdimm_bus
);
1868 struct nd_blk_region
*ndbr
= to_nd_blk_region(dev
);
1869 struct nfit_blk_mmio
*mmio
;
1870 struct nfit_blk
*nfit_blk
;
1871 struct nfit_mem
*nfit_mem
;
1872 struct nvdimm
*nvdimm
;
1875 nvdimm
= nd_blk_region_to_dimm(ndbr
);
1876 nfit_mem
= nvdimm_provider_data(nvdimm
);
1877 if (!nfit_mem
|| !nfit_mem
->dcr
|| !nfit_mem
->bdw
) {
1878 dev_dbg(dev
, "%s: missing%s%s%s\n", __func__
,
1879 nfit_mem
? "" : " nfit_mem",
1880 (nfit_mem
&& nfit_mem
->dcr
) ? "" : " dcr",
1881 (nfit_mem
&& nfit_mem
->bdw
) ? "" : " bdw");
1885 nfit_blk
= devm_kzalloc(dev
, sizeof(*nfit_blk
), GFP_KERNEL
);
1888 nd_blk_region_set_provider_data(ndbr
, nfit_blk
);
1889 nfit_blk
->nd_region
= to_nd_region(dev
);
1891 /* map block aperture memory */
1892 nfit_blk
->bdw_offset
= nfit_mem
->bdw
->offset
;
1893 mmio
= &nfit_blk
->mmio
[BDW
];
1894 mmio
->addr
.base
= devm_nvdimm_memremap(dev
, nfit_mem
->spa_bdw
->address
,
1895 nfit_mem
->spa_bdw
->length
, ARCH_MEMREMAP_PMEM
);
1896 if (!mmio
->addr
.base
) {
1897 dev_dbg(dev
, "%s: %s failed to map bdw\n", __func__
,
1898 nvdimm_name(nvdimm
));
1901 mmio
->size
= nfit_mem
->bdw
->size
;
1902 mmio
->base_offset
= nfit_mem
->memdev_bdw
->region_offset
;
1903 mmio
->idt
= nfit_mem
->idt_bdw
;
1904 mmio
->spa
= nfit_mem
->spa_bdw
;
1905 rc
= nfit_blk_init_interleave(mmio
, nfit_mem
->idt_bdw
,
1906 nfit_mem
->memdev_bdw
->interleave_ways
);
1908 dev_dbg(dev
, "%s: %s failed to init bdw interleave\n",
1909 __func__
, nvdimm_name(nvdimm
));
1913 /* map block control memory */
1914 nfit_blk
->cmd_offset
= nfit_mem
->dcr
->command_offset
;
1915 nfit_blk
->stat_offset
= nfit_mem
->dcr
->status_offset
;
1916 mmio
= &nfit_blk
->mmio
[DCR
];
1917 mmio
->addr
.base
= devm_nvdimm_ioremap(dev
, nfit_mem
->spa_dcr
->address
,
1918 nfit_mem
->spa_dcr
->length
);
1919 if (!mmio
->addr
.base
) {
1920 dev_dbg(dev
, "%s: %s failed to map dcr\n", __func__
,
1921 nvdimm_name(nvdimm
));
1924 mmio
->size
= nfit_mem
->dcr
->window_size
;
1925 mmio
->base_offset
= nfit_mem
->memdev_dcr
->region_offset
;
1926 mmio
->idt
= nfit_mem
->idt_dcr
;
1927 mmio
->spa
= nfit_mem
->spa_dcr
;
1928 rc
= nfit_blk_init_interleave(mmio
, nfit_mem
->idt_dcr
,
1929 nfit_mem
->memdev_dcr
->interleave_ways
);
1931 dev_dbg(dev
, "%s: %s failed to init dcr interleave\n",
1932 __func__
, nvdimm_name(nvdimm
));
1936 rc
= acpi_nfit_blk_get_flags(nd_desc
, nvdimm
, nfit_blk
);
1938 dev_dbg(dev
, "%s: %s failed get DIMM flags\n",
1939 __func__
, nvdimm_name(nvdimm
));
1943 if (nvdimm_has_flush(nfit_blk
->nd_region
) < 0)
1944 dev_warn(dev
, "unable to guarantee persistence of writes\n");
1946 if (mmio
->line_size
== 0)
1949 if ((u32
) nfit_blk
->cmd_offset
% mmio
->line_size
1950 + 8 > mmio
->line_size
) {
1951 dev_dbg(dev
, "cmd_offset crosses interleave boundary\n");
1953 } else if ((u32
) nfit_blk
->stat_offset
% mmio
->line_size
1954 + 8 > mmio
->line_size
) {
1955 dev_dbg(dev
, "stat_offset crosses interleave boundary\n");
1962 static int ars_get_cap(struct acpi_nfit_desc
*acpi_desc
,
1963 struct nd_cmd_ars_cap
*cmd
, struct nfit_spa
*nfit_spa
)
1965 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1966 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1969 cmd
->address
= spa
->address
;
1970 cmd
->length
= spa
->length
;
1971 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_CAP
, cmd
,
1972 sizeof(*cmd
), &cmd_rc
);
1978 static int ars_start(struct acpi_nfit_desc
*acpi_desc
, struct nfit_spa
*nfit_spa
)
1982 struct nd_cmd_ars_start ars_start
;
1983 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
1984 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
1986 memset(&ars_start
, 0, sizeof(ars_start
));
1987 ars_start
.address
= spa
->address
;
1988 ars_start
.length
= spa
->length
;
1989 if (nfit_spa_type(spa
) == NFIT_SPA_PM
)
1990 ars_start
.type
= ND_ARS_PERSISTENT
;
1991 else if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
)
1992 ars_start
.type
= ND_ARS_VOLATILE
;
1996 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_START
, &ars_start
,
1997 sizeof(ars_start
), &cmd_rc
);
2004 static int ars_continue(struct acpi_nfit_desc
*acpi_desc
)
2007 struct nd_cmd_ars_start ars_start
;
2008 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
2009 struct nd_cmd_ars_status
*ars_status
= acpi_desc
->ars_status
;
2011 memset(&ars_start
, 0, sizeof(ars_start
));
2012 ars_start
.address
= ars_status
->restart_address
;
2013 ars_start
.length
= ars_status
->restart_length
;
2014 ars_start
.type
= ars_status
->type
;
2015 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_START
, &ars_start
,
2016 sizeof(ars_start
), &cmd_rc
);
2022 static int ars_get_status(struct acpi_nfit_desc
*acpi_desc
)
2024 struct nvdimm_bus_descriptor
*nd_desc
= &acpi_desc
->nd_desc
;
2025 struct nd_cmd_ars_status
*ars_status
= acpi_desc
->ars_status
;
2028 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_STATUS
, ars_status
,
2029 acpi_desc
->ars_status_size
, &cmd_rc
);
2035 static int ars_status_process_records(struct acpi_nfit_desc
*acpi_desc
,
2036 struct nd_cmd_ars_status
*ars_status
)
2038 struct nvdimm_bus
*nvdimm_bus
= acpi_desc
->nvdimm_bus
;
2043 * First record starts at 44 byte offset from the start of the
2046 if (ars_status
->out_length
< 44)
2048 for (i
= 0; i
< ars_status
->num_records
; i
++) {
2049 /* only process full records */
2050 if (ars_status
->out_length
2051 < 44 + sizeof(struct nd_ars_record
) * (i
+ 1))
2053 rc
= nvdimm_bus_add_poison(nvdimm_bus
,
2054 ars_status
->records
[i
].err_address
,
2055 ars_status
->records
[i
].length
);
2059 if (i
< ars_status
->num_records
)
2060 dev_warn(acpi_desc
->dev
, "detected truncated ars results\n");
2065 static void acpi_nfit_remove_resource(void *data
)
2067 struct resource
*res
= data
;
2069 remove_resource(res
);
2072 static int acpi_nfit_insert_resource(struct acpi_nfit_desc
*acpi_desc
,
2073 struct nd_region_desc
*ndr_desc
)
2075 struct resource
*res
, *nd_res
= ndr_desc
->res
;
2078 /* No operation if the region is already registered as PMEM */
2079 is_pmem
= region_intersects(nd_res
->start
, resource_size(nd_res
),
2080 IORESOURCE_MEM
, IORES_DESC_PERSISTENT_MEMORY
);
2081 if (is_pmem
== REGION_INTERSECTS
)
2084 res
= devm_kzalloc(acpi_desc
->dev
, sizeof(*res
), GFP_KERNEL
);
2088 res
->name
= "Persistent Memory";
2089 res
->start
= nd_res
->start
;
2090 res
->end
= nd_res
->end
;
2091 res
->flags
= IORESOURCE_MEM
;
2092 res
->desc
= IORES_DESC_PERSISTENT_MEMORY
;
2094 ret
= insert_resource(&iomem_resource
, res
);
2098 ret
= devm_add_action_or_reset(acpi_desc
->dev
,
2099 acpi_nfit_remove_resource
,
2107 static int acpi_nfit_init_mapping(struct acpi_nfit_desc
*acpi_desc
,
2108 struct nd_mapping_desc
*mapping
, struct nd_region_desc
*ndr_desc
,
2109 struct acpi_nfit_memory_map
*memdev
,
2110 struct nfit_spa
*nfit_spa
)
2112 struct nvdimm
*nvdimm
= acpi_nfit_dimm_by_handle(acpi_desc
,
2113 memdev
->device_handle
);
2114 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
2115 struct nd_blk_region_desc
*ndbr_desc
;
2116 struct nfit_mem
*nfit_mem
;
2120 dev_err(acpi_desc
->dev
, "spa%d dimm: %#x not found\n",
2121 spa
->range_index
, memdev
->device_handle
);
2125 mapping
->nvdimm
= nvdimm
;
2126 switch (nfit_spa_type(spa
)) {
2128 case NFIT_SPA_VOLATILE
:
2129 mapping
->start
= memdev
->address
;
2130 mapping
->size
= memdev
->region_size
;
2133 nfit_mem
= nvdimm_provider_data(nvdimm
);
2134 if (!nfit_mem
|| !nfit_mem
->bdw
) {
2135 dev_dbg(acpi_desc
->dev
, "spa%d %s missing bdw\n",
2136 spa
->range_index
, nvdimm_name(nvdimm
));
2138 mapping
->size
= nfit_mem
->bdw
->capacity
;
2139 mapping
->start
= nfit_mem
->bdw
->start_address
;
2140 ndr_desc
->num_lanes
= nfit_mem
->bdw
->windows
;
2144 ndr_desc
->mapping
= mapping
;
2145 ndr_desc
->num_mappings
= blk_valid
;
2146 ndbr_desc
= to_blk_region_desc(ndr_desc
);
2147 ndbr_desc
->enable
= acpi_nfit_blk_region_enable
;
2148 ndbr_desc
->do_io
= acpi_desc
->blk_do_io
;
2149 nfit_spa
->nd_region
= nvdimm_blk_region_create(acpi_desc
->nvdimm_bus
,
2151 if (!nfit_spa
->nd_region
)
2159 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address
*spa
)
2161 return (nfit_spa_type(spa
) == NFIT_SPA_VDISK
||
2162 nfit_spa_type(spa
) == NFIT_SPA_VCD
||
2163 nfit_spa_type(spa
) == NFIT_SPA_PDISK
||
2164 nfit_spa_type(spa
) == NFIT_SPA_PCD
);
2167 static int acpi_nfit_register_region(struct acpi_nfit_desc
*acpi_desc
,
2168 struct nfit_spa
*nfit_spa
)
2170 static struct nd_mapping_desc mappings
[ND_MAX_MAPPINGS
];
2171 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
2172 struct nd_blk_region_desc ndbr_desc
;
2173 struct nd_region_desc
*ndr_desc
;
2174 struct nfit_memdev
*nfit_memdev
;
2175 struct nvdimm_bus
*nvdimm_bus
;
2176 struct resource res
;
2179 if (nfit_spa
->nd_region
)
2182 if (spa
->range_index
== 0 && !nfit_spa_is_virtual(spa
)) {
2183 dev_dbg(acpi_desc
->dev
, "%s: detected invalid spa index\n",
2188 memset(&res
, 0, sizeof(res
));
2189 memset(&mappings
, 0, sizeof(mappings
));
2190 memset(&ndbr_desc
, 0, sizeof(ndbr_desc
));
2191 res
.start
= spa
->address
;
2192 res
.end
= res
.start
+ spa
->length
- 1;
2193 ndr_desc
= &ndbr_desc
.ndr_desc
;
2194 ndr_desc
->res
= &res
;
2195 ndr_desc
->provider_data
= nfit_spa
;
2196 ndr_desc
->attr_groups
= acpi_nfit_region_attribute_groups
;
2197 if (spa
->flags
& ACPI_NFIT_PROXIMITY_VALID
)
2198 ndr_desc
->numa_node
= acpi_map_pxm_to_online_node(
2199 spa
->proximity_domain
);
2201 ndr_desc
->numa_node
= NUMA_NO_NODE
;
2203 list_for_each_entry(nfit_memdev
, &acpi_desc
->memdevs
, list
) {
2204 struct acpi_nfit_memory_map
*memdev
= nfit_memdev
->memdev
;
2205 struct nd_mapping_desc
*mapping
;
2207 if (memdev
->range_index
!= spa
->range_index
)
2209 if (count
>= ND_MAX_MAPPINGS
) {
2210 dev_err(acpi_desc
->dev
, "spa%d exceeds max mappings %d\n",
2211 spa
->range_index
, ND_MAX_MAPPINGS
);
2214 mapping
= &mappings
[count
++];
2215 rc
= acpi_nfit_init_mapping(acpi_desc
, mapping
, ndr_desc
,
2221 ndr_desc
->mapping
= mappings
;
2222 ndr_desc
->num_mappings
= count
;
2223 rc
= acpi_nfit_init_interleave_set(acpi_desc
, ndr_desc
, spa
);
2227 nvdimm_bus
= acpi_desc
->nvdimm_bus
;
2228 if (nfit_spa_type(spa
) == NFIT_SPA_PM
) {
2229 rc
= acpi_nfit_insert_resource(acpi_desc
, ndr_desc
);
2231 dev_warn(acpi_desc
->dev
,
2232 "failed to insert pmem resource to iomem: %d\n",
2237 nfit_spa
->nd_region
= nvdimm_pmem_region_create(nvdimm_bus
,
2239 if (!nfit_spa
->nd_region
)
2241 } else if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
) {
2242 nfit_spa
->nd_region
= nvdimm_volatile_region_create(nvdimm_bus
,
2244 if (!nfit_spa
->nd_region
)
2246 } else if (nfit_spa_is_virtual(spa
)) {
2247 nfit_spa
->nd_region
= nvdimm_pmem_region_create(nvdimm_bus
,
2249 if (!nfit_spa
->nd_region
)
2255 dev_err(acpi_desc
->dev
, "failed to register spa range %d\n",
2256 nfit_spa
->spa
->range_index
);
2260 static int ars_status_alloc(struct acpi_nfit_desc
*acpi_desc
,
2263 struct device
*dev
= acpi_desc
->dev
;
2264 struct nd_cmd_ars_status
*ars_status
;
2266 if (acpi_desc
->ars_status
&& acpi_desc
->ars_status_size
>= max_ars
) {
2267 memset(acpi_desc
->ars_status
, 0, acpi_desc
->ars_status_size
);
2271 if (acpi_desc
->ars_status
)
2272 devm_kfree(dev
, acpi_desc
->ars_status
);
2273 acpi_desc
->ars_status
= NULL
;
2274 ars_status
= devm_kzalloc(dev
, max_ars
, GFP_KERNEL
);
2277 acpi_desc
->ars_status
= ars_status
;
2278 acpi_desc
->ars_status_size
= max_ars
;
2282 static int acpi_nfit_query_poison(struct acpi_nfit_desc
*acpi_desc
,
2283 struct nfit_spa
*nfit_spa
)
2285 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
2288 if (!nfit_spa
->max_ars
) {
2289 struct nd_cmd_ars_cap ars_cap
;
2291 memset(&ars_cap
, 0, sizeof(ars_cap
));
2292 rc
= ars_get_cap(acpi_desc
, &ars_cap
, nfit_spa
);
2295 nfit_spa
->max_ars
= ars_cap
.max_ars_out
;
2296 nfit_spa
->clear_err_unit
= ars_cap
.clear_err_unit
;
2297 /* check that the supported scrub types match the spa type */
2298 if (nfit_spa_type(spa
) == NFIT_SPA_VOLATILE
&&
2299 ((ars_cap
.status
>> 16) & ND_ARS_VOLATILE
) == 0)
2301 else if (nfit_spa_type(spa
) == NFIT_SPA_PM
&&
2302 ((ars_cap
.status
>> 16) & ND_ARS_PERSISTENT
) == 0)
2306 if (ars_status_alloc(acpi_desc
, nfit_spa
->max_ars
))
2309 rc
= ars_get_status(acpi_desc
);
2310 if (rc
< 0 && rc
!= -ENOSPC
)
2313 if (ars_status_process_records(acpi_desc
, acpi_desc
->ars_status
))
2319 static void acpi_nfit_async_scrub(struct acpi_nfit_desc
*acpi_desc
,
2320 struct nfit_spa
*nfit_spa
)
2322 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
2323 unsigned int overflow_retry
= scrub_overflow_abort
;
2324 u64 init_ars_start
= 0, init_ars_len
= 0;
2325 struct device
*dev
= acpi_desc
->dev
;
2326 unsigned int tmo
= scrub_timeout
;
2329 if (!nfit_spa
->ars_required
|| !nfit_spa
->nd_region
)
2332 rc
= ars_start(acpi_desc
, nfit_spa
);
2334 * If we timed out the initial scan we'll still be busy here,
2335 * and will wait another timeout before giving up permanently.
2337 if (rc
< 0 && rc
!= -EBUSY
)
2341 u64 ars_start
, ars_len
;
2343 if (acpi_desc
->cancel
)
2345 rc
= acpi_nfit_query_poison(acpi_desc
, nfit_spa
);
2348 if (rc
== -EBUSY
&& !tmo
) {
2349 dev_warn(dev
, "range %d ars timeout, aborting\n",
2356 * Note, entries may be appended to the list
2357 * while the lock is dropped, but the workqueue
2358 * being active prevents entries being deleted /
2361 mutex_unlock(&acpi_desc
->init_mutex
);
2364 mutex_lock(&acpi_desc
->init_mutex
);
2368 /* we got some results, but there are more pending... */
2369 if (rc
== -ENOSPC
&& overflow_retry
--) {
2370 if (!init_ars_len
) {
2371 init_ars_len
= acpi_desc
->ars_status
->length
;
2372 init_ars_start
= acpi_desc
->ars_status
->address
;
2374 rc
= ars_continue(acpi_desc
);
2378 dev_warn(dev
, "range %d ars continuation failed\n",
2384 ars_start
= init_ars_start
;
2385 ars_len
= init_ars_len
;
2387 ars_start
= acpi_desc
->ars_status
->address
;
2388 ars_len
= acpi_desc
->ars_status
->length
;
2390 dev_dbg(dev
, "spa range: %d ars from %#llx + %#llx complete\n",
2391 spa
->range_index
, ars_start
, ars_len
);
2392 /* notify the region about new poison entries */
2393 nvdimm_region_notify(nfit_spa
->nd_region
,
2394 NVDIMM_REVALIDATE_POISON
);
2399 static void acpi_nfit_scrub(struct work_struct
*work
)
2402 u64 init_scrub_length
= 0;
2403 struct nfit_spa
*nfit_spa
;
2404 u64 init_scrub_address
= 0;
2405 bool init_ars_done
= false;
2406 struct acpi_nfit_desc
*acpi_desc
;
2407 unsigned int tmo
= scrub_timeout
;
2408 unsigned int overflow_retry
= scrub_overflow_abort
;
2410 acpi_desc
= container_of(work
, typeof(*acpi_desc
), work
);
2411 dev
= acpi_desc
->dev
;
2414 * We scrub in 2 phases. The first phase waits for any platform
2415 * firmware initiated scrubs to complete and then we go search for the
2416 * affected spa regions to mark them scanned. In the second phase we
2417 * initiate a directed scrub for every range that was not scrubbed in
2418 * phase 1. If we're called for a 'rescan', we harmlessly pass through
2419 * the first phase, but really only care about running phase 2, where
2420 * regions can be notified of new poison.
2423 /* process platform firmware initiated scrubs */
2425 mutex_lock(&acpi_desc
->init_mutex
);
2426 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
2427 struct nd_cmd_ars_status
*ars_status
;
2428 struct acpi_nfit_system_address
*spa
;
2429 u64 ars_start
, ars_len
;
2432 if (acpi_desc
->cancel
)
2435 if (nfit_spa
->nd_region
)
2438 if (init_ars_done
) {
2440 * No need to re-query, we're now just
2441 * reconciling all the ranges covered by the
2446 rc
= acpi_nfit_query_poison(acpi_desc
, nfit_spa
);
2448 if (rc
== -ENOTTY
) {
2449 /* no ars capability, just register spa and move on */
2450 acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2454 if (rc
== -EBUSY
&& !tmo
) {
2455 /* fallthrough to directed scrub in phase 2 */
2456 dev_warn(dev
, "timeout awaiting ars results, continuing...\n");
2458 } else if (rc
== -EBUSY
) {
2459 mutex_unlock(&acpi_desc
->init_mutex
);
2465 /* we got some results, but there are more pending... */
2466 if (rc
== -ENOSPC
&& overflow_retry
--) {
2467 ars_status
= acpi_desc
->ars_status
;
2469 * Record the original scrub range, so that we
2470 * can recall all the ranges impacted by the
2473 if (!init_scrub_length
) {
2474 init_scrub_length
= ars_status
->length
;
2475 init_scrub_address
= ars_status
->address
;
2477 rc
= ars_continue(acpi_desc
);
2479 mutex_unlock(&acpi_desc
->init_mutex
);
2486 * Initial scrub failed, we'll give it one more
2492 /* We got some final results, record completed ranges */
2493 ars_status
= acpi_desc
->ars_status
;
2494 if (init_scrub_length
) {
2495 ars_start
= init_scrub_address
;
2496 ars_len
= ars_start
+ init_scrub_length
;
2498 ars_start
= ars_status
->address
;
2499 ars_len
= ars_status
->length
;
2501 spa
= nfit_spa
->spa
;
2503 if (!init_ars_done
) {
2504 init_ars_done
= true;
2505 dev_dbg(dev
, "init scrub %#llx + %#llx complete\n",
2506 ars_start
, ars_len
);
2508 if (ars_start
<= spa
->address
&& ars_start
+ ars_len
2509 >= spa
->address
+ spa
->length
)
2510 acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2514 * For all the ranges not covered by an initial scrub we still
2515 * want to see if there are errors, but it's ok to discover them
2518 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
2520 * Flag all the ranges that still need scrubbing, but
2521 * register them now to make data available.
2523 if (!nfit_spa
->nd_region
) {
2524 nfit_spa
->ars_required
= 1;
2525 acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2529 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
)
2530 acpi_nfit_async_scrub(acpi_desc
, nfit_spa
);
2531 acpi_desc
->scrub_count
++;
2532 if (acpi_desc
->scrub_count_state
)
2533 sysfs_notify_dirent(acpi_desc
->scrub_count_state
);
2534 mutex_unlock(&acpi_desc
->init_mutex
);
2537 static int acpi_nfit_register_regions(struct acpi_nfit_desc
*acpi_desc
)
2539 struct nfit_spa
*nfit_spa
;
2542 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
)
2543 if (nfit_spa_type(nfit_spa
->spa
) == NFIT_SPA_DCR
) {
2544 /* BLK regions don't need to wait for ars results */
2545 rc
= acpi_nfit_register_region(acpi_desc
, nfit_spa
);
2550 queue_work(nfit_wq
, &acpi_desc
->work
);
2554 static int acpi_nfit_check_deletions(struct acpi_nfit_desc
*acpi_desc
,
2555 struct nfit_table_prev
*prev
)
2557 struct device
*dev
= acpi_desc
->dev
;
2559 if (!list_empty(&prev
->spas
) ||
2560 !list_empty(&prev
->memdevs
) ||
2561 !list_empty(&prev
->dcrs
) ||
2562 !list_empty(&prev
->bdws
) ||
2563 !list_empty(&prev
->idts
) ||
2564 !list_empty(&prev
->flushes
)) {
2565 dev_err(dev
, "new nfit deletes entries (unsupported)\n");
2571 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc
*acpi_desc
)
2573 struct device
*dev
= acpi_desc
->dev
;
2574 struct kernfs_node
*nfit
;
2575 struct device
*bus_dev
;
2577 if (!ars_supported(acpi_desc
->nvdimm_bus
))
2580 bus_dev
= to_nvdimm_bus_dev(acpi_desc
->nvdimm_bus
);
2581 nfit
= sysfs_get_dirent(bus_dev
->kobj
.sd
, "nfit");
2583 dev_err(dev
, "sysfs_get_dirent 'nfit' failed\n");
2586 acpi_desc
->scrub_count_state
= sysfs_get_dirent(nfit
, "scrub");
2588 if (!acpi_desc
->scrub_count_state
) {
2589 dev_err(dev
, "sysfs_get_dirent 'scrub' failed\n");
2596 static void acpi_nfit_destruct(void *data
)
2598 struct acpi_nfit_desc
*acpi_desc
= data
;
2599 struct device
*bus_dev
= to_nvdimm_bus_dev(acpi_desc
->nvdimm_bus
);
2602 * Destruct under acpi_desc_lock so that nfit_handle_mce does not
2605 mutex_lock(&acpi_desc_lock
);
2606 acpi_desc
->cancel
= 1;
2608 * Bounce the nvdimm bus lock to make sure any in-flight
2609 * acpi_nfit_ars_rescan() submissions have had a chance to
2610 * either submit or see ->cancel set.
2612 device_lock(bus_dev
);
2613 device_unlock(bus_dev
);
2615 flush_workqueue(nfit_wq
);
2616 if (acpi_desc
->scrub_count_state
)
2617 sysfs_put(acpi_desc
->scrub_count_state
);
2618 nvdimm_bus_unregister(acpi_desc
->nvdimm_bus
);
2619 acpi_desc
->nvdimm_bus
= NULL
;
2620 list_del(&acpi_desc
->list
);
2621 mutex_unlock(&acpi_desc_lock
);
2624 int acpi_nfit_init(struct acpi_nfit_desc
*acpi_desc
, void *data
, acpi_size sz
)
2626 struct device
*dev
= acpi_desc
->dev
;
2627 struct nfit_table_prev prev
;
2631 if (!acpi_desc
->nvdimm_bus
) {
2632 acpi_nfit_init_dsms(acpi_desc
);
2634 acpi_desc
->nvdimm_bus
= nvdimm_bus_register(dev
,
2635 &acpi_desc
->nd_desc
);
2636 if (!acpi_desc
->nvdimm_bus
)
2639 rc
= devm_add_action_or_reset(dev
, acpi_nfit_destruct
,
2644 rc
= acpi_nfit_desc_init_scrub_attr(acpi_desc
);
2648 /* register this acpi_desc for mce notifications */
2649 mutex_lock(&acpi_desc_lock
);
2650 list_add_tail(&acpi_desc
->list
, &acpi_descs
);
2651 mutex_unlock(&acpi_desc_lock
);
2654 mutex_lock(&acpi_desc
->init_mutex
);
2656 INIT_LIST_HEAD(&prev
.spas
);
2657 INIT_LIST_HEAD(&prev
.memdevs
);
2658 INIT_LIST_HEAD(&prev
.dcrs
);
2659 INIT_LIST_HEAD(&prev
.bdws
);
2660 INIT_LIST_HEAD(&prev
.idts
);
2661 INIT_LIST_HEAD(&prev
.flushes
);
2663 list_cut_position(&prev
.spas
, &acpi_desc
->spas
,
2664 acpi_desc
->spas
.prev
);
2665 list_cut_position(&prev
.memdevs
, &acpi_desc
->memdevs
,
2666 acpi_desc
->memdevs
.prev
);
2667 list_cut_position(&prev
.dcrs
, &acpi_desc
->dcrs
,
2668 acpi_desc
->dcrs
.prev
);
2669 list_cut_position(&prev
.bdws
, &acpi_desc
->bdws
,
2670 acpi_desc
->bdws
.prev
);
2671 list_cut_position(&prev
.idts
, &acpi_desc
->idts
,
2672 acpi_desc
->idts
.prev
);
2673 list_cut_position(&prev
.flushes
, &acpi_desc
->flushes
,
2674 acpi_desc
->flushes
.prev
);
2677 while (!IS_ERR_OR_NULL(data
))
2678 data
= add_table(acpi_desc
, &prev
, data
, end
);
2681 dev_dbg(dev
, "%s: nfit table parsing error: %ld\n", __func__
,
2687 rc
= acpi_nfit_check_deletions(acpi_desc
, &prev
);
2691 rc
= nfit_mem_init(acpi_desc
);
2695 rc
= acpi_nfit_register_dimms(acpi_desc
);
2699 rc
= acpi_nfit_register_regions(acpi_desc
);
2702 mutex_unlock(&acpi_desc
->init_mutex
);
2705 EXPORT_SYMBOL_GPL(acpi_nfit_init
);
2707 struct acpi_nfit_flush_work
{
2708 struct work_struct work
;
2709 struct completion cmp
;
2712 static void flush_probe(struct work_struct
*work
)
2714 struct acpi_nfit_flush_work
*flush
;
2716 flush
= container_of(work
, typeof(*flush
), work
);
2717 complete(&flush
->cmp
);
2720 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor
*nd_desc
)
2722 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
2723 struct device
*dev
= acpi_desc
->dev
;
2724 struct acpi_nfit_flush_work flush
;
2727 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
2732 * Scrub work could take 10s of seconds, userspace may give up so we
2733 * need to be interruptible while waiting.
2735 INIT_WORK_ONSTACK(&flush
.work
, flush_probe
);
2736 COMPLETION_INITIALIZER_ONSTACK(flush
.cmp
);
2737 queue_work(nfit_wq
, &flush
.work
);
2739 rc
= wait_for_completion_interruptible(&flush
.cmp
);
2740 cancel_work_sync(&flush
.work
);
2744 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor
*nd_desc
,
2745 struct nvdimm
*nvdimm
, unsigned int cmd
)
2747 struct acpi_nfit_desc
*acpi_desc
= to_acpi_nfit_desc(nd_desc
);
2751 if (cmd
!= ND_CMD_ARS_START
)
2755 * The kernel and userspace may race to initiate a scrub, but
2756 * the scrub thread is prepared to lose that initial race. It
2757 * just needs guarantees that any ars it initiates are not
2758 * interrupted by any intervening start reqeusts from userspace.
2760 if (work_busy(&acpi_desc
->work
))
2766 int acpi_nfit_ars_rescan(struct acpi_nfit_desc
*acpi_desc
)
2768 struct device
*dev
= acpi_desc
->dev
;
2769 struct nfit_spa
*nfit_spa
;
2771 if (work_busy(&acpi_desc
->work
))
2774 if (acpi_desc
->cancel
)
2777 mutex_lock(&acpi_desc
->init_mutex
);
2778 list_for_each_entry(nfit_spa
, &acpi_desc
->spas
, list
) {
2779 struct acpi_nfit_system_address
*spa
= nfit_spa
->spa
;
2781 if (nfit_spa_type(spa
) != NFIT_SPA_PM
)
2784 nfit_spa
->ars_required
= 1;
2786 queue_work(nfit_wq
, &acpi_desc
->work
);
2787 dev_dbg(dev
, "%s: ars_scan triggered\n", __func__
);
2788 mutex_unlock(&acpi_desc
->init_mutex
);
2793 void acpi_nfit_desc_init(struct acpi_nfit_desc
*acpi_desc
, struct device
*dev
)
2795 struct nvdimm_bus_descriptor
*nd_desc
;
2797 dev_set_drvdata(dev
, acpi_desc
);
2798 acpi_desc
->dev
= dev
;
2799 acpi_desc
->blk_do_io
= acpi_nfit_blk_region_do_io
;
2800 nd_desc
= &acpi_desc
->nd_desc
;
2801 nd_desc
->provider_name
= "ACPI.NFIT";
2802 nd_desc
->module
= THIS_MODULE
;
2803 nd_desc
->ndctl
= acpi_nfit_ctl
;
2804 nd_desc
->flush_probe
= acpi_nfit_flush_probe
;
2805 nd_desc
->clear_to_send
= acpi_nfit_clear_to_send
;
2806 nd_desc
->attr_groups
= acpi_nfit_attribute_groups
;
2808 INIT_LIST_HEAD(&acpi_desc
->spas
);
2809 INIT_LIST_HEAD(&acpi_desc
->dcrs
);
2810 INIT_LIST_HEAD(&acpi_desc
->bdws
);
2811 INIT_LIST_HEAD(&acpi_desc
->idts
);
2812 INIT_LIST_HEAD(&acpi_desc
->flushes
);
2813 INIT_LIST_HEAD(&acpi_desc
->memdevs
);
2814 INIT_LIST_HEAD(&acpi_desc
->dimms
);
2815 INIT_LIST_HEAD(&acpi_desc
->list
);
2816 mutex_init(&acpi_desc
->init_mutex
);
2817 INIT_WORK(&acpi_desc
->work
, acpi_nfit_scrub
);
2819 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init
);
2821 static int acpi_nfit_add(struct acpi_device
*adev
)
2823 struct acpi_buffer buf
= { ACPI_ALLOCATE_BUFFER
, NULL
};
2824 struct acpi_nfit_desc
*acpi_desc
;
2825 struct device
*dev
= &adev
->dev
;
2826 struct acpi_table_header
*tbl
;
2827 acpi_status status
= AE_OK
;
2831 status
= acpi_get_table(ACPI_SIG_NFIT
, 0, &tbl
);
2832 if (ACPI_FAILURE(status
)) {
2833 /* This is ok, we could have an nvdimm hotplugged later */
2834 dev_dbg(dev
, "failed to find NFIT at startup\n");
2839 acpi_desc
= devm_kzalloc(dev
, sizeof(*acpi_desc
), GFP_KERNEL
);
2842 acpi_nfit_desc_init(acpi_desc
, &adev
->dev
);
2844 /* Save the acpi header for exporting the revision via sysfs */
2845 acpi_desc
->acpi_header
= *tbl
;
2847 /* Evaluate _FIT and override with that if present */
2848 status
= acpi_evaluate_object(adev
->handle
, "_FIT", NULL
, &buf
);
2849 if (ACPI_SUCCESS(status
) && buf
.length
> 0) {
2850 union acpi_object
*obj
= buf
.pointer
;
2852 if (obj
->type
== ACPI_TYPE_BUFFER
)
2853 rc
= acpi_nfit_init(acpi_desc
, obj
->buffer
.pointer
,
2854 obj
->buffer
.length
);
2856 dev_dbg(dev
, "%s invalid type %d, ignoring _FIT\n",
2857 __func__
, (int) obj
->type
);
2860 /* skip over the lead-in header table */
2861 rc
= acpi_nfit_init(acpi_desc
, (void *) tbl
2862 + sizeof(struct acpi_table_nfit
),
2863 sz
- sizeof(struct acpi_table_nfit
));
2867 static int acpi_nfit_remove(struct acpi_device
*adev
)
2869 /* see acpi_nfit_destruct */
2873 void __acpi_nfit_notify(struct device
*dev
, acpi_handle handle
, u32 event
)
2875 struct acpi_nfit_desc
*acpi_desc
= dev_get_drvdata(dev
);
2876 struct acpi_buffer buf
= { ACPI_ALLOCATE_BUFFER
, NULL
};
2877 union acpi_object
*obj
;
2881 dev_dbg(dev
, "%s: event: %d\n", __func__
, event
);
2883 if (event
!= NFIT_NOTIFY_UPDATE
)
2887 /* dev->driver may be null if we're being removed */
2888 dev_dbg(dev
, "%s: no driver found for dev\n", __func__
);
2893 acpi_desc
= devm_kzalloc(dev
, sizeof(*acpi_desc
), GFP_KERNEL
);
2896 acpi_nfit_desc_init(acpi_desc
, dev
);
2899 * Finish previous registration before considering new
2902 flush_workqueue(nfit_wq
);
2906 status
= acpi_evaluate_object(handle
, "_FIT", NULL
, &buf
);
2907 if (ACPI_FAILURE(status
)) {
2908 dev_err(dev
, "failed to evaluate _FIT\n");
2913 if (obj
->type
== ACPI_TYPE_BUFFER
) {
2914 ret
= acpi_nfit_init(acpi_desc
, obj
->buffer
.pointer
,
2915 obj
->buffer
.length
);
2917 dev_err(dev
, "failed to merge updated NFIT\n");
2919 dev_err(dev
, "Invalid _FIT\n");
2922 EXPORT_SYMBOL_GPL(__acpi_nfit_notify
);
2924 static void acpi_nfit_notify(struct acpi_device
*adev
, u32 event
)
2926 device_lock(&adev
->dev
);
2927 __acpi_nfit_notify(&adev
->dev
, adev
->handle
, event
);
2928 device_unlock(&adev
->dev
);
2931 static const struct acpi_device_id acpi_nfit_ids
[] = {
2935 MODULE_DEVICE_TABLE(acpi
, acpi_nfit_ids
);
2937 static struct acpi_driver acpi_nfit_driver
= {
2938 .name
= KBUILD_MODNAME
,
2939 .ids
= acpi_nfit_ids
,
2941 .add
= acpi_nfit_add
,
2942 .remove
= acpi_nfit_remove
,
2943 .notify
= acpi_nfit_notify
,
2947 static __init
int nfit_init(void)
2949 BUILD_BUG_ON(sizeof(struct acpi_table_nfit
) != 40);
2950 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address
) != 56);
2951 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map
) != 48);
2952 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave
) != 20);
2953 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios
) != 9);
2954 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region
) != 80);
2955 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region
) != 40);
2957 acpi_str_to_uuid(UUID_VOLATILE_MEMORY
, nfit_uuid
[NFIT_SPA_VOLATILE
]);
2958 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY
, nfit_uuid
[NFIT_SPA_PM
]);
2959 acpi_str_to_uuid(UUID_CONTROL_REGION
, nfit_uuid
[NFIT_SPA_DCR
]);
2960 acpi_str_to_uuid(UUID_DATA_REGION
, nfit_uuid
[NFIT_SPA_BDW
]);
2961 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK
, nfit_uuid
[NFIT_SPA_VDISK
]);
2962 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD
, nfit_uuid
[NFIT_SPA_VCD
]);
2963 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK
, nfit_uuid
[NFIT_SPA_PDISK
]);
2964 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD
, nfit_uuid
[NFIT_SPA_PCD
]);
2965 acpi_str_to_uuid(UUID_NFIT_BUS
, nfit_uuid
[NFIT_DEV_BUS
]);
2966 acpi_str_to_uuid(UUID_NFIT_DIMM
, nfit_uuid
[NFIT_DEV_DIMM
]);
2967 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1
, nfit_uuid
[NFIT_DEV_DIMM_N_HPE1
]);
2968 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2
, nfit_uuid
[NFIT_DEV_DIMM_N_HPE2
]);
2969 acpi_str_to_uuid(UUID_NFIT_DIMM_N_MSFT
, nfit_uuid
[NFIT_DEV_DIMM_N_MSFT
]);
2971 nfit_wq
= create_singlethread_workqueue("nfit");
2975 nfit_mce_register();
2977 return acpi_bus_register_driver(&acpi_nfit_driver
);
2980 static __exit
void nfit_exit(void)
2982 nfit_mce_unregister();
2983 acpi_bus_unregister_driver(&acpi_nfit_driver
);
2984 destroy_workqueue(nfit_wq
);
2985 WARN_ON(!list_empty(&acpi_descs
));
2988 module_init(nfit_init
);
2989 module_exit(nfit_exit
);
2990 MODULE_LICENSE("GPL v2");
2991 MODULE_AUTHOR("Intel Corporation");