1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright 2019 Google LLC
7 * DOC: The Keyslot Manager
9 * Many devices with inline encryption support have a limited number of "slots"
10 * into which encryption contexts may be programmed, and requests can be tagged
11 * with a slot number to specify the key to use for en/decryption.
13 * As the number of slots is limited, and programming keys is expensive on
14 * many inline encryption hardware, we don't want to program the same key into
15 * multiple slots - if multiple requests are using the same key, we want to
16 * program just one slot with that key and use that slot for all requests.
18 * The keyslot manager manages these keyslots appropriately, and also acts as
19 * an abstraction between the inline encryption hardware and the upper layers.
21 * Lower layer devices will set up a keyslot manager in their request queue
22 * and tell it how to perform device specific operations like programming/
23 * evicting keys from keyslots.
25 * Upper layers will call blk_ksm_get_slot_for_key() to program a
26 * key into some slot in the inline encryption hardware.
29 #define pr_fmt(fmt) "blk-crypto: " fmt
31 #include <linux/keyslot-manager.h>
32 #include <linux/device.h>
33 #include <linux/atomic.h>
34 #include <linux/mutex.h>
35 #include <linux/pm_runtime.h>
36 #include <linux/wait.h>
37 #include <linux/blkdev.h>
39 struct blk_ksm_keyslot
{
41 struct list_head idle_slot_node
;
42 struct hlist_node hash_node
;
43 const struct blk_crypto_key
*key
;
44 struct blk_keyslot_manager
*ksm
;
47 static inline void blk_ksm_hw_enter(struct blk_keyslot_manager
*ksm
)
50 * Calling into the driver requires ksm->lock held and the device
51 * resumed. But we must resume the device first, since that can acquire
52 * and release ksm->lock via blk_ksm_reprogram_all_keys().
55 pm_runtime_get_sync(ksm
->dev
);
56 down_write(&ksm
->lock
);
59 static inline void blk_ksm_hw_exit(struct blk_keyslot_manager
*ksm
)
63 pm_runtime_put_sync(ksm
->dev
);
66 static inline bool blk_ksm_is_passthrough(struct blk_keyslot_manager
*ksm
)
68 return ksm
->num_slots
== 0;
72 * blk_ksm_init() - Initialize a keyslot manager
73 * @ksm: The keyslot_manager to initialize.
74 * @num_slots: The number of key slots to manage.
76 * Allocate memory for keyslots and initialize a keyslot manager. Called by
77 * e.g. storage drivers to set up a keyslot manager in their request_queue.
79 * Return: 0 on success, or else a negative error code.
81 int blk_ksm_init(struct blk_keyslot_manager
*ksm
, unsigned int num_slots
)
85 unsigned int slot_hashtable_size
;
87 memset(ksm
, 0, sizeof(*ksm
));
92 ksm
->slots
= kvcalloc(num_slots
, sizeof(ksm
->slots
[0]), GFP_KERNEL
);
96 ksm
->num_slots
= num_slots
;
98 init_rwsem(&ksm
->lock
);
100 init_waitqueue_head(&ksm
->idle_slots_wait_queue
);
101 INIT_LIST_HEAD(&ksm
->idle_slots
);
103 for (slot
= 0; slot
< num_slots
; slot
++) {
104 ksm
->slots
[slot
].ksm
= ksm
;
105 list_add_tail(&ksm
->slots
[slot
].idle_slot_node
,
109 spin_lock_init(&ksm
->idle_slots_lock
);
111 slot_hashtable_size
= roundup_pow_of_two(num_slots
);
113 * hash_ptr() assumes bits != 0, so ensure the hash table has at least 2
114 * buckets. This only makes a difference when there is only 1 keyslot.
116 if (slot_hashtable_size
< 2)
117 slot_hashtable_size
= 2;
119 ksm
->log_slot_ht_size
= ilog2(slot_hashtable_size
);
120 ksm
->slot_hashtable
= kvmalloc_array(slot_hashtable_size
,
121 sizeof(ksm
->slot_hashtable
[0]),
123 if (!ksm
->slot_hashtable
)
124 goto err_destroy_ksm
;
125 for (i
= 0; i
< slot_hashtable_size
; i
++)
126 INIT_HLIST_HEAD(&ksm
->slot_hashtable
[i
]);
131 blk_ksm_destroy(ksm
);
134 EXPORT_SYMBOL_GPL(blk_ksm_init
);
136 static void blk_ksm_destroy_callback(void *ksm
)
138 blk_ksm_destroy(ksm
);
142 * devm_blk_ksm_init() - Resource-managed blk_ksm_init()
143 * @dev: The device which owns the blk_keyslot_manager.
144 * @ksm: The blk_keyslot_manager to initialize.
145 * @num_slots: The number of key slots to manage.
147 * Like blk_ksm_init(), but causes blk_ksm_destroy() to be called automatically
150 * Return: 0 on success, or else a negative error code.
152 int devm_blk_ksm_init(struct device
*dev
, struct blk_keyslot_manager
*ksm
,
153 unsigned int num_slots
)
155 int err
= blk_ksm_init(ksm
, num_slots
);
160 return devm_add_action_or_reset(dev
, blk_ksm_destroy_callback
, ksm
);
162 EXPORT_SYMBOL_GPL(devm_blk_ksm_init
);
164 static inline struct hlist_head
*
165 blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager
*ksm
,
166 const struct blk_crypto_key
*key
)
168 return &ksm
->slot_hashtable
[hash_ptr(key
, ksm
->log_slot_ht_size
)];
171 static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot
*slot
)
173 struct blk_keyslot_manager
*ksm
= slot
->ksm
;
176 spin_lock_irqsave(&ksm
->idle_slots_lock
, flags
);
177 list_del(&slot
->idle_slot_node
);
178 spin_unlock_irqrestore(&ksm
->idle_slots_lock
, flags
);
181 static struct blk_ksm_keyslot
*blk_ksm_find_keyslot(
182 struct blk_keyslot_manager
*ksm
,
183 const struct blk_crypto_key
*key
)
185 const struct hlist_head
*head
= blk_ksm_hash_bucket_for_key(ksm
, key
);
186 struct blk_ksm_keyslot
*slotp
;
188 hlist_for_each_entry(slotp
, head
, hash_node
) {
189 if (slotp
->key
== key
)
195 static struct blk_ksm_keyslot
*blk_ksm_find_and_grab_keyslot(
196 struct blk_keyslot_manager
*ksm
,
197 const struct blk_crypto_key
*key
)
199 struct blk_ksm_keyslot
*slot
;
201 slot
= blk_ksm_find_keyslot(ksm
, key
);
204 if (atomic_inc_return(&slot
->slot_refs
) == 1) {
205 /* Took first reference to this slot; remove it from LRU list */
206 blk_ksm_remove_slot_from_lru_list(slot
);
211 unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot
*slot
)
213 return slot
- slot
->ksm
->slots
;
215 EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx
);
218 * blk_ksm_get_slot_for_key() - Program a key into a keyslot.
219 * @ksm: The keyslot manager to program the key into.
220 * @key: Pointer to the key object to program, including the raw key, crypto
221 * mode, and data unit size.
222 * @slot_ptr: A pointer to return the pointer of the allocated keyslot.
224 * Get a keyslot that's been programmed with the specified key. If one already
225 * exists, return it with incremented refcount. Otherwise, wait for a keyslot
226 * to become idle and program it.
228 * Context: Process context. Takes and releases ksm->lock.
229 * Return: BLK_STS_OK on success (and keyslot is set to the pointer of the
230 * allocated keyslot), or some other blk_status_t otherwise (and
231 * keyslot is set to NULL).
233 blk_status_t
blk_ksm_get_slot_for_key(struct blk_keyslot_manager
*ksm
,
234 const struct blk_crypto_key
*key
,
235 struct blk_ksm_keyslot
**slot_ptr
)
237 struct blk_ksm_keyslot
*slot
;
243 if (blk_ksm_is_passthrough(ksm
))
246 down_read(&ksm
->lock
);
247 slot
= blk_ksm_find_and_grab_keyslot(ksm
, key
);
253 blk_ksm_hw_enter(ksm
);
254 slot
= blk_ksm_find_and_grab_keyslot(ksm
, key
);
256 blk_ksm_hw_exit(ksm
);
261 * If we're here, that means there wasn't a slot that was
262 * already programmed with the key. So try to program it.
264 if (!list_empty(&ksm
->idle_slots
))
267 blk_ksm_hw_exit(ksm
);
268 wait_event(ksm
->idle_slots_wait_queue
,
269 !list_empty(&ksm
->idle_slots
));
272 slot
= list_first_entry(&ksm
->idle_slots
, struct blk_ksm_keyslot
,
274 slot_idx
= blk_ksm_get_slot_idx(slot
);
276 err
= ksm
->ksm_ll_ops
.keyslot_program(ksm
, key
, slot_idx
);
278 wake_up(&ksm
->idle_slots_wait_queue
);
279 blk_ksm_hw_exit(ksm
);
280 return errno_to_blk_status(err
);
283 /* Move this slot to the hash list for the new key. */
285 hlist_del(&slot
->hash_node
);
287 hlist_add_head(&slot
->hash_node
, blk_ksm_hash_bucket_for_key(ksm
, key
));
289 atomic_set(&slot
->slot_refs
, 1);
291 blk_ksm_remove_slot_from_lru_list(slot
);
293 blk_ksm_hw_exit(ksm
);
300 * blk_ksm_put_slot() - Release a reference to a slot
301 * @slot: The keyslot to release the reference of.
303 * Context: Any context.
305 void blk_ksm_put_slot(struct blk_ksm_keyslot
*slot
)
307 struct blk_keyslot_manager
*ksm
;
315 if (atomic_dec_and_lock_irqsave(&slot
->slot_refs
,
316 &ksm
->idle_slots_lock
, flags
)) {
317 list_add_tail(&slot
->idle_slot_node
, &ksm
->idle_slots
);
318 spin_unlock_irqrestore(&ksm
->idle_slots_lock
, flags
);
319 wake_up(&ksm
->idle_slots_wait_queue
);
324 * blk_ksm_crypto_cfg_supported() - Find out if a crypto configuration is
325 * supported by a ksm.
326 * @ksm: The keyslot manager to check
327 * @cfg: The crypto configuration to check for.
329 * Checks for crypto_mode/data unit size/dun bytes support.
331 * Return: Whether or not this ksm supports the specified crypto config.
333 bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager
*ksm
,
334 const struct blk_crypto_config
*cfg
)
338 if (!(ksm
->crypto_modes_supported
[cfg
->crypto_mode
] &
339 cfg
->data_unit_size
))
341 if (ksm
->max_dun_bytes_supported
< cfg
->dun_bytes
)
347 * blk_ksm_evict_key() - Evict a key from the lower layer device.
348 * @ksm: The keyslot manager to evict from
349 * @key: The key to evict
351 * Find the keyslot that the specified key was programmed into, and evict that
352 * slot from the lower layer device. The slot must not be in use by any
353 * in-flight IO when this function is called.
355 * Context: Process context. Takes and releases ksm->lock.
356 * Return: 0 on success or if there's no keyslot with the specified key, -EBUSY
357 * if the keyslot is still in use, or another -errno value on other
360 int blk_ksm_evict_key(struct blk_keyslot_manager
*ksm
,
361 const struct blk_crypto_key
*key
)
363 struct blk_ksm_keyslot
*slot
;
366 if (blk_ksm_is_passthrough(ksm
)) {
367 if (ksm
->ksm_ll_ops
.keyslot_evict
) {
368 blk_ksm_hw_enter(ksm
);
369 err
= ksm
->ksm_ll_ops
.keyslot_evict(ksm
, key
, -1);
370 blk_ksm_hw_exit(ksm
);
376 blk_ksm_hw_enter(ksm
);
377 slot
= blk_ksm_find_keyslot(ksm
, key
);
381 if (WARN_ON_ONCE(atomic_read(&slot
->slot_refs
) != 0)) {
385 err
= ksm
->ksm_ll_ops
.keyslot_evict(ksm
, key
,
386 blk_ksm_get_slot_idx(slot
));
390 hlist_del(&slot
->hash_node
);
394 blk_ksm_hw_exit(ksm
);
399 * blk_ksm_reprogram_all_keys() - Re-program all keyslots.
400 * @ksm: The keyslot manager
402 * Re-program all keyslots that are supposed to have a key programmed. This is
403 * intended only for use by drivers for hardware that loses its keys on reset.
405 * Context: Process context. Takes and releases ksm->lock.
407 void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager
*ksm
)
411 if (blk_ksm_is_passthrough(ksm
))
414 /* This is for device initialization, so don't resume the device */
415 down_write(&ksm
->lock
);
416 for (slot
= 0; slot
< ksm
->num_slots
; slot
++) {
417 const struct blk_crypto_key
*key
= ksm
->slots
[slot
].key
;
423 err
= ksm
->ksm_ll_ops
.keyslot_program(ksm
, key
, slot
);
426 up_write(&ksm
->lock
);
428 EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys
);
430 void blk_ksm_destroy(struct blk_keyslot_manager
*ksm
)
434 kvfree(ksm
->slot_hashtable
);
435 kvfree_sensitive(ksm
->slots
, sizeof(ksm
->slots
[0]) * ksm
->num_slots
);
436 memzero_explicit(ksm
, sizeof(*ksm
));
438 EXPORT_SYMBOL_GPL(blk_ksm_destroy
);
440 bool blk_ksm_register(struct blk_keyslot_manager
*ksm
, struct request_queue
*q
)
442 if (blk_integrity_queue_supports_integrity(q
)) {
443 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n");
449 EXPORT_SYMBOL_GPL(blk_ksm_register
);
451 void blk_ksm_unregister(struct request_queue
*q
)
457 * blk_ksm_intersect_modes() - restrict supported modes by child device
458 * @parent: The keyslot manager for parent device
459 * @child: The keyslot manager for child device, or NULL
461 * Clear any crypto mode support bits in @parent that aren't set in @child.
462 * If @child is NULL, then all parent bits are cleared.
464 * Only use this when setting up the keyslot manager for a layered device,
465 * before it's been exposed yet.
467 void blk_ksm_intersect_modes(struct blk_keyslot_manager
*parent
,
468 const struct blk_keyslot_manager
*child
)
473 parent
->max_dun_bytes_supported
=
474 min(parent
->max_dun_bytes_supported
,
475 child
->max_dun_bytes_supported
);
476 for (i
= 0; i
< ARRAY_SIZE(child
->crypto_modes_supported
);
478 parent
->crypto_modes_supported
[i
] &=
479 child
->crypto_modes_supported
[i
];
482 parent
->max_dun_bytes_supported
= 0;
483 memset(parent
->crypto_modes_supported
, 0,
484 sizeof(parent
->crypto_modes_supported
));
487 EXPORT_SYMBOL_GPL(blk_ksm_intersect_modes
);
490 * blk_ksm_is_superset() - Check if a KSM supports a superset of crypto modes
491 * and DUN bytes that another KSM supports. Here,
492 * "superset" refers to the mathematical meaning of the
493 * word - i.e. if two KSMs have the *same* capabilities,
494 * they *are* considered supersets of each other.
495 * @ksm_superset: The KSM that we want to verify is a superset
496 * @ksm_subset: The KSM that we want to verify is a subset
498 * Return: True if @ksm_superset supports a superset of the crypto modes and DUN
499 * bytes that @ksm_subset supports.
501 bool blk_ksm_is_superset(struct blk_keyslot_manager
*ksm_superset
,
502 struct blk_keyslot_manager
*ksm_subset
)
512 for (i
= 0; i
< ARRAY_SIZE(ksm_superset
->crypto_modes_supported
); i
++) {
513 if (ksm_subset
->crypto_modes_supported
[i
] &
514 (~ksm_superset
->crypto_modes_supported
[i
])) {
519 if (ksm_subset
->max_dun_bytes_supported
>
520 ksm_superset
->max_dun_bytes_supported
) {
526 EXPORT_SYMBOL_GPL(blk_ksm_is_superset
);
529 * blk_ksm_update_capabilities() - Update the restrictions of a KSM to those of
531 * @target_ksm: The KSM whose restrictions to update.
532 * @reference_ksm: The KSM to whose restrictions this function will update
533 * @target_ksm's restrictions to.
535 * Blk-crypto requires that crypto capabilities that were
536 * advertised when a bio was created continue to be supported by the
537 * device until that bio is ended. This is turn means that a device cannot
538 * shrink its advertised crypto capabilities without any explicit
539 * synchronization with upper layers. So if there's no such explicit
540 * synchronization, @reference_ksm must support all the crypto capabilities that
542 * (i.e. we need blk_ksm_is_superset(@reference_ksm, @target_ksm) == true).
544 * Note also that as long as the crypto capabilities are being expanded, the
545 * order of updates becoming visible is not important because it's alright
546 * for blk-crypto to see stale values - they only cause blk-crypto to
547 * believe that a crypto capability isn't supported when it actually is (which
548 * might result in blk-crypto-fallback being used if available, or the bio being
551 void blk_ksm_update_capabilities(struct blk_keyslot_manager
*target_ksm
,
552 struct blk_keyslot_manager
*reference_ksm
)
554 memcpy(target_ksm
->crypto_modes_supported
,
555 reference_ksm
->crypto_modes_supported
,
556 sizeof(target_ksm
->crypto_modes_supported
));
558 target_ksm
->max_dun_bytes_supported
=
559 reference_ksm
->max_dun_bytes_supported
;
561 EXPORT_SYMBOL_GPL(blk_ksm_update_capabilities
);
564 * blk_ksm_init_passthrough() - Init a passthrough keyslot manager
565 * @ksm: The keyslot manager to init
567 * Initialize a passthrough keyslot manager.
568 * Called by e.g. storage drivers to set up a keyslot manager in their
569 * request_queue, when the storage driver wants to manage its keys by itself.
570 * This is useful for inline encryption hardware that doesn't have the concept
571 * of keyslots, and for layered devices.
573 void blk_ksm_init_passthrough(struct blk_keyslot_manager
*ksm
)
575 memset(ksm
, 0, sizeof(*ksm
));
576 init_rwsem(&ksm
->lock
);
578 EXPORT_SYMBOL_GPL(blk_ksm_init_passthrough
);