2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/device.h>
14 #include <linux/ndctl.h>
15 #include <linux/uuid.h>
16 #include <linux/slab.h>
23 static guid_t nvdimm_btt_guid
;
24 static guid_t nvdimm_btt2_guid
;
25 static guid_t nvdimm_pfn_guid
;
26 static guid_t nvdimm_dax_guid
;
28 static u32
best_seq(u32 a
, u32 b
)
30 a
&= NSINDEX_SEQ_MASK
;
31 b
&= NSINDEX_SEQ_MASK
;
37 else if (nd_inc_seq(a
) == b
)
43 unsigned sizeof_namespace_label(struct nvdimm_drvdata
*ndd
)
45 return ndd
->nslabel_size
;
48 size_t sizeof_namespace_index(struct nvdimm_drvdata
*ndd
)
52 if (ndd
->nsindex_size
)
53 return ndd
->nsindex_size
;
56 * The minimum index space is 512 bytes, with that amount of
57 * index we can describe ~1400 labels which is less than a byte
58 * of overhead per label. Round up to a byte of overhead per
59 * label and determine the size of the index region. Yes, this
60 * starts to waste space at larger config_sizes, but it's
61 * unlikely we'll ever see anything but 128K.
63 index_span
= ndd
->nsarea
.config_size
/ (sizeof_namespace_label(ndd
) + 1);
64 index_span
/= NSINDEX_ALIGN
* 2;
65 ndd
->nsindex_size
= index_span
* NSINDEX_ALIGN
;
67 return ndd
->nsindex_size
;
70 int nvdimm_num_label_slots(struct nvdimm_drvdata
*ndd
)
72 return ndd
->nsarea
.config_size
/ (sizeof_namespace_label(ndd
) + 1);
75 static int __nd_label_validate(struct nvdimm_drvdata
*ndd
)
78 * On media label format consists of two index blocks followed
79 * by an array of labels. None of these structures are ever
80 * updated in place. A sequence number tracks the current
81 * active index and the next one to write, while labels are
82 * written to free slots.
104 struct nd_namespace_index
*nsindex
[] = {
105 to_namespace_index(ndd
, 0),
106 to_namespace_index(ndd
, 1),
108 const int num_index
= ARRAY_SIZE(nsindex
);
109 struct device
*dev
= ndd
->dev
;
110 bool valid
[2] = { 0 };
111 int i
, num_valid
= 0;
114 for (i
= 0; i
< num_index
; i
++) {
116 u8 sig
[NSINDEX_SIG_LEN
];
117 u64 sum_save
, sum
, size
;
118 unsigned int version
, labelsize
;
120 memcpy(sig
, nsindex
[i
]->sig
, NSINDEX_SIG_LEN
);
121 if (memcmp(sig
, NSINDEX_SIGNATURE
, NSINDEX_SIG_LEN
) != 0) {
122 dev_dbg(dev
, "%s: nsindex%d signature invalid\n",
127 /* label sizes larger than 128 arrived with v1.2 */
128 version
= __le16_to_cpu(nsindex
[i
]->major
) * 100
129 + __le16_to_cpu(nsindex
[i
]->minor
);
131 labelsize
= 1 << (7 + nsindex
[i
]->labelsize
);
135 if (labelsize
!= sizeof_namespace_label(ndd
)) {
136 dev_dbg(dev
, "%s: nsindex%d labelsize %d invalid\n",
137 __func__
, i
, nsindex
[i
]->labelsize
);
141 sum_save
= __le64_to_cpu(nsindex
[i
]->checksum
);
142 nsindex
[i
]->checksum
= __cpu_to_le64(0);
143 sum
= nd_fletcher64(nsindex
[i
], sizeof_namespace_index(ndd
), 1);
144 nsindex
[i
]->checksum
= __cpu_to_le64(sum_save
);
145 if (sum
!= sum_save
) {
146 dev_dbg(dev
, "%s: nsindex%d checksum invalid\n",
151 seq
= __le32_to_cpu(nsindex
[i
]->seq
);
152 if ((seq
& NSINDEX_SEQ_MASK
) == 0) {
153 dev_dbg(dev
, "%s: nsindex%d sequence: %#x invalid\n",
158 /* sanity check the index against expected values */
159 if (__le64_to_cpu(nsindex
[i
]->myoff
)
160 != i
* sizeof_namespace_index(ndd
)) {
161 dev_dbg(dev
, "%s: nsindex%d myoff: %#llx invalid\n",
162 __func__
, i
, (unsigned long long)
163 __le64_to_cpu(nsindex
[i
]->myoff
));
166 if (__le64_to_cpu(nsindex
[i
]->otheroff
)
167 != (!i
) * sizeof_namespace_index(ndd
)) {
168 dev_dbg(dev
, "%s: nsindex%d otheroff: %#llx invalid\n",
169 __func__
, i
, (unsigned long long)
170 __le64_to_cpu(nsindex
[i
]->otheroff
));
174 size
= __le64_to_cpu(nsindex
[i
]->mysize
);
175 if (size
> sizeof_namespace_index(ndd
)
176 || size
< sizeof(struct nd_namespace_index
)) {
177 dev_dbg(dev
, "%s: nsindex%d mysize: %#llx invalid\n",
182 nslot
= __le32_to_cpu(nsindex
[i
]->nslot
);
183 if (nslot
* sizeof_namespace_label(ndd
)
184 + 2 * sizeof_namespace_index(ndd
)
185 > ndd
->nsarea
.config_size
) {
186 dev_dbg(dev
, "%s: nsindex%d nslot: %u invalid, config_size: %#x\n",
188 ndd
->nsarea
.config_size
);
199 for (i
= 0; i
< num_index
; i
++)
202 /* can't have num_valid > 0 but valid[] = { false, false } */
206 /* pick the best index... */
207 seq
= best_seq(__le32_to_cpu(nsindex
[0]->seq
),
208 __le32_to_cpu(nsindex
[1]->seq
));
209 if (seq
== (__le32_to_cpu(nsindex
[1]->seq
) & NSINDEX_SEQ_MASK
))
219 int nd_label_validate(struct nvdimm_drvdata
*ndd
)
222 * In order to probe for and validate namespace index blocks we
223 * need to know the size of the labels, and we can't trust the
224 * size of the labels until we validate the index blocks.
225 * Resolve this dependency loop by probing for known label
226 * sizes, but default to v1.2 256-byte namespace labels if
229 int label_size
[] = { 128, 256 };
232 for (i
= 0; i
< ARRAY_SIZE(label_size
); i
++) {
233 ndd
->nslabel_size
= label_size
[i
];
234 rc
= __nd_label_validate(ndd
);
242 void nd_label_copy(struct nvdimm_drvdata
*ndd
, struct nd_namespace_index
*dst
,
243 struct nd_namespace_index
*src
)
250 memcpy(dst
, src
, sizeof_namespace_index(ndd
));
253 static struct nd_namespace_label
*nd_label_base(struct nvdimm_drvdata
*ndd
)
255 void *base
= to_namespace_index(ndd
, 0);
257 return base
+ 2 * sizeof_namespace_index(ndd
);
260 static int to_slot(struct nvdimm_drvdata
*ndd
,
261 struct nd_namespace_label
*nd_label
)
263 unsigned long label
, base
;
265 label
= (unsigned long) nd_label
;
266 base
= (unsigned long) nd_label_base(ndd
);
268 return (label
- base
) / sizeof_namespace_label(ndd
);
271 static struct nd_namespace_label
*to_label(struct nvdimm_drvdata
*ndd
, int slot
)
273 unsigned long label
, base
;
275 base
= (unsigned long) nd_label_base(ndd
);
276 label
= base
+ sizeof_namespace_label(ndd
) * slot
;
278 return (struct nd_namespace_label
*) label
;
281 #define for_each_clear_bit_le(bit, addr, size) \
282 for ((bit) = find_next_zero_bit_le((addr), (size), 0); \
284 (bit) = find_next_zero_bit_le((addr), (size), (bit) + 1))
287 * preamble_index - common variable initialization for nd_label_* routines
288 * @ndd: dimm container for the relevant label set
289 * @idx: namespace_index index
290 * @nsindex_out: on return set to the currently active namespace index
291 * @free: on return set to the free label bitmap in the index
292 * @nslot: on return set to the number of slots in the label space
294 static bool preamble_index(struct nvdimm_drvdata
*ndd
, int idx
,
295 struct nd_namespace_index
**nsindex_out
,
296 unsigned long **free
, u32
*nslot
)
298 struct nd_namespace_index
*nsindex
;
300 nsindex
= to_namespace_index(ndd
, idx
);
304 *free
= (unsigned long *) nsindex
->free
;
305 *nslot
= __le32_to_cpu(nsindex
->nslot
);
306 *nsindex_out
= nsindex
;
311 char *nd_label_gen_id(struct nd_label_id
*label_id
, u8
*uuid
, u32 flags
)
313 if (!label_id
|| !uuid
)
315 snprintf(label_id
->id
, ND_LABEL_ID_SIZE
, "%s-%pUb",
316 flags
& NSLABEL_FLAG_LOCAL
? "blk" : "pmem", uuid
);
320 static bool preamble_current(struct nvdimm_drvdata
*ndd
,
321 struct nd_namespace_index
**nsindex
,
322 unsigned long **free
, u32
*nslot
)
324 return preamble_index(ndd
, ndd
->ns_current
, nsindex
,
328 static bool preamble_next(struct nvdimm_drvdata
*ndd
,
329 struct nd_namespace_index
**nsindex
,
330 unsigned long **free
, u32
*nslot
)
332 return preamble_index(ndd
, ndd
->ns_next
, nsindex
,
336 static bool slot_valid(struct nvdimm_drvdata
*ndd
,
337 struct nd_namespace_label
*nd_label
, u32 slot
)
339 /* check that we are written where we expect to be written */
340 if (slot
!= __le32_to_cpu(nd_label
->slot
))
343 /* check that DPA allocations are page aligned */
344 if ((__le64_to_cpu(nd_label
->dpa
)
345 | __le64_to_cpu(nd_label
->rawsize
)) % SZ_4K
)
349 if (namespace_label_has(ndd
, checksum
)) {
352 sum_save
= __le64_to_cpu(nd_label
->checksum
);
353 nd_label
->checksum
= __cpu_to_le64(0);
354 sum
= nd_fletcher64(nd_label
, sizeof_namespace_label(ndd
), 1);
355 nd_label
->checksum
= __cpu_to_le64(sum_save
);
356 if (sum
!= sum_save
) {
357 dev_dbg(ndd
->dev
, "%s fail checksum. slot: %d expect: %#llx\n",
358 __func__
, slot
, sum
);
366 int nd_label_reserve_dpa(struct nvdimm_drvdata
*ndd
)
368 struct nd_namespace_index
*nsindex
;
372 if (!preamble_current(ndd
, &nsindex
, &free
, &nslot
))
373 return 0; /* no label, nothing to reserve */
375 for_each_clear_bit_le(slot
, free
, nslot
) {
376 struct nd_namespace_label
*nd_label
;
377 struct nd_region
*nd_region
= NULL
;
378 u8 label_uuid
[NSLABEL_UUID_LEN
];
379 struct nd_label_id label_id
;
380 struct resource
*res
;
383 nd_label
= to_label(ndd
, slot
);
385 if (!slot_valid(ndd
, nd_label
, slot
))
388 memcpy(label_uuid
, nd_label
->uuid
, NSLABEL_UUID_LEN
);
389 flags
= __le32_to_cpu(nd_label
->flags
);
390 nd_label_gen_id(&label_id
, label_uuid
, flags
);
391 res
= nvdimm_allocate_dpa(ndd
, &label_id
,
392 __le64_to_cpu(nd_label
->dpa
),
393 __le64_to_cpu(nd_label
->rawsize
));
394 nd_dbg_dpa(nd_region
, ndd
, res
, "reserve\n");
402 int nd_label_active_count(struct nvdimm_drvdata
*ndd
)
404 struct nd_namespace_index
*nsindex
;
409 if (!preamble_current(ndd
, &nsindex
, &free
, &nslot
))
412 for_each_clear_bit_le(slot
, free
, nslot
) {
413 struct nd_namespace_label
*nd_label
;
415 nd_label
= to_label(ndd
, slot
);
417 if (!slot_valid(ndd
, nd_label
, slot
)) {
418 u32 label_slot
= __le32_to_cpu(nd_label
->slot
);
419 u64 size
= __le64_to_cpu(nd_label
->rawsize
);
420 u64 dpa
= __le64_to_cpu(nd_label
->dpa
);
423 "%s: slot%d invalid slot: %d dpa: %llx size: %llx\n",
424 __func__
, slot
, label_slot
, dpa
, size
);
432 struct nd_namespace_label
*nd_label_active(struct nvdimm_drvdata
*ndd
, int n
)
434 struct nd_namespace_index
*nsindex
;
438 if (!preamble_current(ndd
, &nsindex
, &free
, &nslot
))
441 for_each_clear_bit_le(slot
, free
, nslot
) {
442 struct nd_namespace_label
*nd_label
;
444 nd_label
= to_label(ndd
, slot
);
445 if (!slot_valid(ndd
, nd_label
, slot
))
449 return to_label(ndd
, slot
);
455 u32
nd_label_alloc_slot(struct nvdimm_drvdata
*ndd
)
457 struct nd_namespace_index
*nsindex
;
461 if (!preamble_next(ndd
, &nsindex
, &free
, &nslot
))
464 WARN_ON(!is_nvdimm_bus_locked(ndd
->dev
));
466 slot
= find_next_bit_le(free
, nslot
, 0);
470 clear_bit_le(slot
, free
);
475 bool nd_label_free_slot(struct nvdimm_drvdata
*ndd
, u32 slot
)
477 struct nd_namespace_index
*nsindex
;
481 if (!preamble_next(ndd
, &nsindex
, &free
, &nslot
))
484 WARN_ON(!is_nvdimm_bus_locked(ndd
->dev
));
487 return !test_and_set_bit_le(slot
, free
);
491 u32
nd_label_nfree(struct nvdimm_drvdata
*ndd
)
493 struct nd_namespace_index
*nsindex
;
497 WARN_ON(!is_nvdimm_bus_locked(ndd
->dev
));
499 if (!preamble_next(ndd
, &nsindex
, &free
, &nslot
))
500 return nvdimm_num_label_slots(ndd
);
502 return bitmap_weight(free
, nslot
);
505 static int nd_label_write_index(struct nvdimm_drvdata
*ndd
, int index
, u32 seq
,
508 struct nd_namespace_index
*nsindex
;
509 unsigned long offset
;
514 nsindex
= to_namespace_index(ndd
, index
);
515 if (flags
& ND_NSINDEX_INIT
)
516 nslot
= nvdimm_num_label_slots(ndd
);
518 nslot
= __le32_to_cpu(nsindex
->nslot
);
520 memcpy(nsindex
->sig
, NSINDEX_SIGNATURE
, NSINDEX_SIG_LEN
);
521 memset(&nsindex
->flags
, 0, 3);
522 nsindex
->labelsize
= sizeof_namespace_label(ndd
) >> 8;
523 nsindex
->seq
= __cpu_to_le32(seq
);
524 offset
= (unsigned long) nsindex
525 - (unsigned long) to_namespace_index(ndd
, 0);
526 nsindex
->myoff
= __cpu_to_le64(offset
);
527 nsindex
->mysize
= __cpu_to_le64(sizeof_namespace_index(ndd
));
528 offset
= (unsigned long) to_namespace_index(ndd
,
529 nd_label_next_nsindex(index
))
530 - (unsigned long) to_namespace_index(ndd
, 0);
531 nsindex
->otheroff
= __cpu_to_le64(offset
);
532 offset
= (unsigned long) nd_label_base(ndd
)
533 - (unsigned long) to_namespace_index(ndd
, 0);
534 nsindex
->labeloff
= __cpu_to_le64(offset
);
535 nsindex
->nslot
= __cpu_to_le32(nslot
);
536 nsindex
->major
= __cpu_to_le16(1);
537 if (sizeof_namespace_label(ndd
) < 256)
538 nsindex
->minor
= __cpu_to_le16(1);
540 nsindex
->minor
= __cpu_to_le16(2);
541 nsindex
->checksum
= __cpu_to_le64(0);
542 if (flags
& ND_NSINDEX_INIT
) {
543 unsigned long *free
= (unsigned long *) nsindex
->free
;
544 u32 nfree
= ALIGN(nslot
, BITS_PER_LONG
);
547 memset(nsindex
->free
, 0xff, nfree
/ 8);
548 for (i
= 0, last_bits
= nfree
- nslot
; i
< last_bits
; i
++)
549 clear_bit_le(nslot
+ i
, free
);
551 checksum
= nd_fletcher64(nsindex
, sizeof_namespace_index(ndd
), 1);
552 nsindex
->checksum
= __cpu_to_le64(checksum
);
553 rc
= nvdimm_set_config_data(ndd
, __le64_to_cpu(nsindex
->myoff
),
554 nsindex
, sizeof_namespace_index(ndd
));
558 if (flags
& ND_NSINDEX_INIT
)
561 /* copy the index we just wrote to the new 'next' */
562 WARN_ON(index
!= ndd
->ns_next
);
563 nd_label_copy(ndd
, to_current_namespace_index(ndd
), nsindex
);
564 ndd
->ns_current
= nd_label_next_nsindex(ndd
->ns_current
);
565 ndd
->ns_next
= nd_label_next_nsindex(ndd
->ns_next
);
566 WARN_ON(ndd
->ns_current
== ndd
->ns_next
);
571 static unsigned long nd_label_offset(struct nvdimm_drvdata
*ndd
,
572 struct nd_namespace_label
*nd_label
)
574 return (unsigned long) nd_label
575 - (unsigned long) to_namespace_index(ndd
, 0);
578 enum nvdimm_claim_class
to_nvdimm_cclass(guid_t
*guid
)
580 if (guid_equal(guid
, &nvdimm_btt_guid
))
581 return NVDIMM_CCLASS_BTT
;
582 else if (guid_equal(guid
, &nvdimm_btt2_guid
))
583 return NVDIMM_CCLASS_BTT2
;
584 else if (guid_equal(guid
, &nvdimm_pfn_guid
))
585 return NVDIMM_CCLASS_PFN
;
586 else if (guid_equal(guid
, &nvdimm_dax_guid
))
587 return NVDIMM_CCLASS_DAX
;
588 else if (guid_equal(guid
, &guid_null
))
589 return NVDIMM_CCLASS_NONE
;
591 return NVDIMM_CCLASS_UNKNOWN
;
594 static const guid_t
*to_abstraction_guid(enum nvdimm_claim_class claim_class
,
597 if (claim_class
== NVDIMM_CCLASS_BTT
)
598 return &nvdimm_btt_guid
;
599 else if (claim_class
== NVDIMM_CCLASS_BTT2
)
600 return &nvdimm_btt2_guid
;
601 else if (claim_class
== NVDIMM_CCLASS_PFN
)
602 return &nvdimm_pfn_guid
;
603 else if (claim_class
== NVDIMM_CCLASS_DAX
)
604 return &nvdimm_dax_guid
;
605 else if (claim_class
== NVDIMM_CCLASS_UNKNOWN
) {
607 * If we're modifying a namespace for which we don't
608 * know the claim_class, don't touch the existing guid.
615 static int __pmem_label_update(struct nd_region
*nd_region
,
616 struct nd_mapping
*nd_mapping
, struct nd_namespace_pmem
*nspm
,
619 struct nd_namespace_common
*ndns
= &nspm
->nsio
.common
;
620 struct nd_interleave_set
*nd_set
= nd_region
->nd_set
;
621 struct nvdimm_drvdata
*ndd
= to_ndd(nd_mapping
);
622 struct nd_label_ent
*label_ent
, *victim
= NULL
;
623 struct nd_namespace_label
*nd_label
;
624 struct nd_namespace_index
*nsindex
;
625 struct nd_label_id label_id
;
626 struct resource
*res
;
633 if (!preamble_next(ndd
, &nsindex
, &free
, &nslot
))
636 cookie
= nd_region_interleave_set_cookie(nd_region
, nsindex
);
637 nd_label_gen_id(&label_id
, nspm
->uuid
, 0);
638 for_each_dpa_resource(ndd
, res
)
639 if (strcmp(res
->name
, label_id
.id
) == 0)
647 /* allocate and write the label to the staging (next) index */
648 slot
= nd_label_alloc_slot(ndd
);
649 if (slot
== UINT_MAX
)
651 dev_dbg(ndd
->dev
, "%s: allocated: %d\n", __func__
, slot
);
653 nd_label
= to_label(ndd
, slot
);
654 memset(nd_label
, 0, sizeof_namespace_label(ndd
));
655 memcpy(nd_label
->uuid
, nspm
->uuid
, NSLABEL_UUID_LEN
);
657 memcpy(nd_label
->name
, nspm
->alt_name
, NSLABEL_NAME_LEN
);
658 nd_label
->flags
= __cpu_to_le32(NSLABEL_FLAG_UPDATING
);
659 nd_label
->nlabel
= __cpu_to_le16(nd_region
->ndr_mappings
);
660 nd_label
->position
= __cpu_to_le16(pos
);
661 nd_label
->isetcookie
= __cpu_to_le64(cookie
);
662 nd_label
->rawsize
= __cpu_to_le64(resource_size(res
));
663 nd_label
->lbasize
= __cpu_to_le64(nspm
->lbasize
);
664 nd_label
->dpa
= __cpu_to_le64(res
->start
);
665 nd_label
->slot
= __cpu_to_le32(slot
);
666 if (namespace_label_has(ndd
, type_guid
))
667 guid_copy(&nd_label
->type_guid
, &nd_set
->type_guid
);
668 if (namespace_label_has(ndd
, abstraction_guid
))
669 guid_copy(&nd_label
->abstraction_guid
,
670 to_abstraction_guid(ndns
->claim_class
,
671 &nd_label
->abstraction_guid
));
672 if (namespace_label_has(ndd
, checksum
)) {
675 nd_label
->checksum
= __cpu_to_le64(0);
676 sum
= nd_fletcher64(nd_label
, sizeof_namespace_label(ndd
), 1);
677 nd_label
->checksum
= __cpu_to_le64(sum
);
679 nd_dbg_dpa(nd_region
, ndd
, res
, "%s\n", __func__
);
682 offset
= nd_label_offset(ndd
, nd_label
);
683 rc
= nvdimm_set_config_data(ndd
, offset
, nd_label
,
684 sizeof_namespace_label(ndd
));
688 /* Garbage collect the previous label */
689 mutex_lock(&nd_mapping
->lock
);
690 list_for_each_entry(label_ent
, &nd_mapping
->labels
, list
) {
691 if (!label_ent
->label
)
693 if (memcmp(nspm
->uuid
, label_ent
->label
->uuid
,
694 NSLABEL_UUID_LEN
) != 0)
697 list_move_tail(&victim
->list
, &nd_mapping
->labels
);
701 dev_dbg(ndd
->dev
, "%s: free: %d\n", __func__
, slot
);
702 slot
= to_slot(ndd
, victim
->label
);
703 nd_label_free_slot(ndd
, slot
);
704 victim
->label
= NULL
;
708 rc
= nd_label_write_index(ndd
, ndd
->ns_next
,
709 nd_inc_seq(__le32_to_cpu(nsindex
->seq
)), 0);
711 list_for_each_entry(label_ent
, &nd_mapping
->labels
, list
)
712 if (!label_ent
->label
) {
713 label_ent
->label
= nd_label
;
717 dev_WARN_ONCE(&nspm
->nsio
.common
.dev
, nd_label
,
718 "failed to track label: %d\n",
719 to_slot(ndd
, nd_label
));
723 mutex_unlock(&nd_mapping
->lock
);
728 static bool is_old_resource(struct resource
*res
, struct resource
**list
, int n
)
732 if (res
->flags
& DPA_RESOURCE_ADJUSTED
)
734 for (i
= 0; i
< n
; i
++)
740 static struct resource
*to_resource(struct nvdimm_drvdata
*ndd
,
741 struct nd_namespace_label
*nd_label
)
743 struct resource
*res
;
745 for_each_dpa_resource(ndd
, res
) {
746 if (res
->start
!= __le64_to_cpu(nd_label
->dpa
))
748 if (resource_size(res
) != __le64_to_cpu(nd_label
->rawsize
))
757 * 1/ Account all the labels that can be freed after this update
758 * 2/ Allocate and write the label to the staging (next) index
759 * 3/ Record the resources in the namespace device
761 static int __blk_label_update(struct nd_region
*nd_region
,
762 struct nd_mapping
*nd_mapping
, struct nd_namespace_blk
*nsblk
,
765 int i
, alloc
, victims
, nfree
, old_num_resources
, nlabel
, rc
= -ENXIO
;
766 struct nd_interleave_set
*nd_set
= nd_region
->nd_set
;
767 struct nd_namespace_common
*ndns
= &nsblk
->common
;
768 struct nvdimm_drvdata
*ndd
= to_ndd(nd_mapping
);
769 struct nd_namespace_label
*nd_label
;
770 struct nd_label_ent
*label_ent
, *e
;
771 struct nd_namespace_index
*nsindex
;
772 unsigned long *free
, *victim_map
= NULL
;
773 struct resource
*res
, **old_res_list
;
774 struct nd_label_id label_id
;
775 u8 uuid
[NSLABEL_UUID_LEN
];
780 if (!preamble_next(ndd
, &nsindex
, &free
, &nslot
))
783 old_res_list
= nsblk
->res
;
784 nfree
= nd_label_nfree(ndd
);
785 old_num_resources
= nsblk
->num_resources
;
786 nd_label_gen_id(&label_id
, nsblk
->uuid
, NSLABEL_FLAG_LOCAL
);
789 * We need to loop over the old resources a few times, which seems a
790 * bit inefficient, but we need to know that we have the label
791 * space before we start mutating the tracking structures.
792 * Otherwise the recovery method of last resort for userspace is
793 * disable and re-enable the parent region.
796 for_each_dpa_resource(ndd
, res
) {
797 if (strcmp(res
->name
, label_id
.id
) != 0)
799 if (!is_old_resource(res
, old_res_list
, old_num_resources
))
804 if (old_num_resources
) {
805 /* convert old local-label-map to dimm-slot victim-map */
806 victim_map
= kcalloc(BITS_TO_LONGS(nslot
), sizeof(long),
811 /* mark unused labels for garbage collection */
812 for_each_clear_bit_le(slot
, free
, nslot
) {
813 nd_label
= to_label(ndd
, slot
);
814 memcpy(uuid
, nd_label
->uuid
, NSLABEL_UUID_LEN
);
815 if (memcmp(uuid
, nsblk
->uuid
, NSLABEL_UUID_LEN
) != 0)
817 res
= to_resource(ndd
, nd_label
);
818 if (res
&& is_old_resource(res
, old_res_list
,
821 slot
= to_slot(ndd
, nd_label
);
822 set_bit(slot
, victim_map
);
827 /* don't allow updates that consume the last label */
828 if (nfree
- alloc
< 0 || nfree
- alloc
+ victims
< 1) {
829 dev_info(&nsblk
->common
.dev
, "insufficient label space\n");
833 /* from here on we need to abort on error */
836 /* assign all resources to the namespace before writing the labels */
838 nsblk
->num_resources
= 0;
839 for_each_dpa_resource(ndd
, res
) {
840 if (strcmp(res
->name
, label_id
.id
) != 0)
842 if (!nsblk_add_resource(nd_region
, ndd
, nsblk
, res
->start
)) {
849 * Find the resource associated with the first label in the set
850 * per the v1.2 namespace specification.
852 for (i
= 0; i
< nsblk
->num_resources
; i
++) {
853 struct resource
*min
= nsblk
->res
[min_dpa_idx
];
856 if (res
->start
< min
->start
)
860 for (i
= 0; i
< nsblk
->num_resources
; i
++) {
864 if (is_old_resource(res
, old_res_list
, old_num_resources
))
865 continue; /* carry-over */
866 slot
= nd_label_alloc_slot(ndd
);
867 if (slot
== UINT_MAX
)
869 dev_dbg(ndd
->dev
, "%s: allocated: %d\n", __func__
, slot
);
871 nd_label
= to_label(ndd
, slot
);
872 memset(nd_label
, 0, sizeof_namespace_label(ndd
));
873 memcpy(nd_label
->uuid
, nsblk
->uuid
, NSLABEL_UUID_LEN
);
875 memcpy(nd_label
->name
, nsblk
->alt_name
,
877 nd_label
->flags
= __cpu_to_le32(NSLABEL_FLAG_LOCAL
);
880 * Use the presence of the type_guid as a flag to
881 * determine isetcookie usage and nlabel + position
882 * policy for blk-aperture namespaces.
884 if (namespace_label_has(ndd
, type_guid
)) {
885 if (i
== min_dpa_idx
) {
886 nd_label
->nlabel
= __cpu_to_le16(nsblk
->num_resources
);
887 nd_label
->position
= __cpu_to_le16(0);
889 nd_label
->nlabel
= __cpu_to_le16(0xffff);
890 nd_label
->position
= __cpu_to_le16(0xffff);
892 nd_label
->isetcookie
= __cpu_to_le64(nd_set
->cookie2
);
894 nd_label
->nlabel
= __cpu_to_le16(0); /* N/A */
895 nd_label
->position
= __cpu_to_le16(0); /* N/A */
896 nd_label
->isetcookie
= __cpu_to_le64(0); /* N/A */
899 nd_label
->dpa
= __cpu_to_le64(res
->start
);
900 nd_label
->rawsize
= __cpu_to_le64(resource_size(res
));
901 nd_label
->lbasize
= __cpu_to_le64(nsblk
->lbasize
);
902 nd_label
->slot
= __cpu_to_le32(slot
);
903 if (namespace_label_has(ndd
, type_guid
))
904 guid_copy(&nd_label
->type_guid
, &nd_set
->type_guid
);
905 if (namespace_label_has(ndd
, abstraction_guid
))
906 guid_copy(&nd_label
->abstraction_guid
,
907 to_abstraction_guid(ndns
->claim_class
,
908 &nd_label
->abstraction_guid
));
910 if (namespace_label_has(ndd
, checksum
)) {
913 nd_label
->checksum
= __cpu_to_le64(0);
914 sum
= nd_fletcher64(nd_label
,
915 sizeof_namespace_label(ndd
), 1);
916 nd_label
->checksum
= __cpu_to_le64(sum
);
920 offset
= nd_label_offset(ndd
, nd_label
);
921 rc
= nvdimm_set_config_data(ndd
, offset
, nd_label
,
922 sizeof_namespace_label(ndd
));
927 /* free up now unused slots in the new index */
928 for_each_set_bit(slot
, victim_map
, victim_map
? nslot
: 0) {
929 dev_dbg(ndd
->dev
, "%s: free: %d\n", __func__
, slot
);
930 nd_label_free_slot(ndd
, slot
);
934 rc
= nd_label_write_index(ndd
, ndd
->ns_next
,
935 nd_inc_seq(__le32_to_cpu(nsindex
->seq
)), 0);
940 * Now that the on-dimm labels are up to date, fix up the tracking
941 * entries in nd_mapping->labels
944 mutex_lock(&nd_mapping
->lock
);
945 list_for_each_entry_safe(label_ent
, e
, &nd_mapping
->labels
, list
) {
946 nd_label
= label_ent
->label
;
950 memcpy(uuid
, nd_label
->uuid
, NSLABEL_UUID_LEN
);
951 if (memcmp(uuid
, nsblk
->uuid
, NSLABEL_UUID_LEN
) != 0)
954 list_move(&label_ent
->list
, &list
);
955 label_ent
->label
= NULL
;
957 list_splice_tail_init(&list
, &nd_mapping
->labels
);
958 mutex_unlock(&nd_mapping
->lock
);
960 if (nlabel
+ nsblk
->num_resources
> num_labels
) {
962 * Bug, we can't end up with more resources than
970 mutex_lock(&nd_mapping
->lock
);
971 label_ent
= list_first_entry_or_null(&nd_mapping
->labels
,
972 typeof(*label_ent
), list
);
975 mutex_unlock(&nd_mapping
->lock
);
979 for_each_clear_bit_le(slot
, free
, nslot
) {
980 nd_label
= to_label(ndd
, slot
);
981 memcpy(uuid
, nd_label
->uuid
, NSLABEL_UUID_LEN
);
982 if (memcmp(uuid
, nsblk
->uuid
, NSLABEL_UUID_LEN
) != 0)
984 res
= to_resource(ndd
, nd_label
);
985 res
->flags
&= ~DPA_RESOURCE_ADJUSTED
;
986 dev_vdbg(&nsblk
->common
.dev
, "assign label slot: %d\n", slot
);
987 list_for_each_entry_from(label_ent
, &nd_mapping
->labels
, list
) {
988 if (label_ent
->label
)
990 label_ent
->label
= nd_label
;
995 dev_WARN(&nsblk
->common
.dev
,
996 "failed to track label slot%d\n", slot
);
998 mutex_unlock(&nd_mapping
->lock
);
1001 kfree(old_res_list
);
1007 * 1/ repair the allocated label bitmap in the index
1008 * 2/ restore the resource list
1010 nd_label_copy(ndd
, nsindex
, to_current_namespace_index(ndd
));
1012 nsblk
->res
= old_res_list
;
1013 nsblk
->num_resources
= old_num_resources
;
1014 old_res_list
= NULL
;
1018 static int init_labels(struct nd_mapping
*nd_mapping
, int num_labels
)
1020 int i
, old_num_labels
= 0;
1021 struct nd_label_ent
*label_ent
;
1022 struct nd_namespace_index
*nsindex
;
1023 struct nvdimm_drvdata
*ndd
= to_ndd(nd_mapping
);
1025 mutex_lock(&nd_mapping
->lock
);
1026 list_for_each_entry(label_ent
, &nd_mapping
->labels
, list
)
1028 mutex_unlock(&nd_mapping
->lock
);
1031 * We need to preserve all the old labels for the mapping so
1032 * they can be garbage collected after writing the new labels.
1034 for (i
= old_num_labels
; i
< num_labels
; i
++) {
1035 label_ent
= kzalloc(sizeof(*label_ent
), GFP_KERNEL
);
1038 mutex_lock(&nd_mapping
->lock
);
1039 list_add_tail(&label_ent
->list
, &nd_mapping
->labels
);
1040 mutex_unlock(&nd_mapping
->lock
);
1043 if (ndd
->ns_current
== -1 || ndd
->ns_next
== -1)
1046 return max(num_labels
, old_num_labels
);
1048 nsindex
= to_namespace_index(ndd
, 0);
1049 memset(nsindex
, 0, ndd
->nsarea
.config_size
);
1050 for (i
= 0; i
< 2; i
++) {
1051 int rc
= nd_label_write_index(ndd
, i
, i
*2, ND_NSINDEX_INIT
);
1057 ndd
->ns_current
= 0;
1059 return max(num_labels
, old_num_labels
);
1062 static int del_labels(struct nd_mapping
*nd_mapping
, u8
*uuid
)
1064 struct nvdimm_drvdata
*ndd
= to_ndd(nd_mapping
);
1065 struct nd_label_ent
*label_ent
, *e
;
1066 struct nd_namespace_index
*nsindex
;
1067 u8 label_uuid
[NSLABEL_UUID_LEN
];
1068 unsigned long *free
;
1076 /* no index || no labels == nothing to delete */
1077 if (!preamble_next(ndd
, &nsindex
, &free
, &nslot
))
1080 mutex_lock(&nd_mapping
->lock
);
1081 list_for_each_entry_safe(label_ent
, e
, &nd_mapping
->labels
, list
) {
1082 struct nd_namespace_label
*nd_label
= label_ent
->label
;
1087 memcpy(label_uuid
, nd_label
->uuid
, NSLABEL_UUID_LEN
);
1088 if (memcmp(label_uuid
, uuid
, NSLABEL_UUID_LEN
) != 0)
1091 slot
= to_slot(ndd
, nd_label
);
1092 nd_label_free_slot(ndd
, slot
);
1093 dev_dbg(ndd
->dev
, "%s: free: %d\n", __func__
, slot
);
1094 list_move_tail(&label_ent
->list
, &list
);
1095 label_ent
->label
= NULL
;
1097 list_splice_tail_init(&list
, &nd_mapping
->labels
);
1100 nd_mapping_free_labels(nd_mapping
);
1101 dev_dbg(ndd
->dev
, "%s: no more active labels\n", __func__
);
1103 mutex_unlock(&nd_mapping
->lock
);
1105 return nd_label_write_index(ndd
, ndd
->ns_next
,
1106 nd_inc_seq(__le32_to_cpu(nsindex
->seq
)), 0);
1109 int nd_pmem_namespace_label_update(struct nd_region
*nd_region
,
1110 struct nd_namespace_pmem
*nspm
, resource_size_t size
)
1114 for (i
= 0; i
< nd_region
->ndr_mappings
; i
++) {
1115 struct nd_mapping
*nd_mapping
= &nd_region
->mapping
[i
];
1116 struct nvdimm_drvdata
*ndd
= to_ndd(nd_mapping
);
1117 struct resource
*res
;
1121 rc
= del_labels(nd_mapping
, nspm
->uuid
);
1127 for_each_dpa_resource(ndd
, res
)
1128 if (strncmp(res
->name
, "pmem", 4) == 0)
1130 WARN_ON_ONCE(!count
);
1132 rc
= init_labels(nd_mapping
, count
);
1136 rc
= __pmem_label_update(nd_region
, nd_mapping
, nspm
, i
);
1144 int nd_blk_namespace_label_update(struct nd_region
*nd_region
,
1145 struct nd_namespace_blk
*nsblk
, resource_size_t size
)
1147 struct nd_mapping
*nd_mapping
= &nd_region
->mapping
[0];
1148 struct resource
*res
;
1152 return del_labels(nd_mapping
, nsblk
->uuid
);
1154 for_each_dpa_resource(to_ndd(nd_mapping
), res
)
1157 count
= init_labels(nd_mapping
, count
);
1161 return __blk_label_update(nd_region
, nd_mapping
, nsblk
, count
);
1164 int __init
nd_label_init(void)
1166 WARN_ON(guid_parse(NVDIMM_BTT_GUID
, &nvdimm_btt_guid
));
1167 WARN_ON(guid_parse(NVDIMM_BTT2_GUID
, &nvdimm_btt2_guid
));
1168 WARN_ON(guid_parse(NVDIMM_PFN_GUID
, &nvdimm_pfn_guid
));
1169 WARN_ON(guid_parse(NVDIMM_DAX_GUID
, &nvdimm_dax_guid
));