1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright 2019 Google LLC
7 * DOC: The Keyslot Manager
9 * Many devices with inline encryption support have a limited number of "slots"
10 * into which encryption contexts may be programmed, and requests can be tagged
11 * with a slot number to specify the key to use for en/decryption.
13 * As the number of slots is limited, and programming keys is expensive on
14 * many inline encryption hardware, we don't want to program the same key into
15 * multiple slots - if multiple requests are using the same key, we want to
16 * program just one slot with that key and use that slot for all requests.
18 * The keyslot manager manages these keyslots appropriately, and also acts as
19 * an abstraction between the inline encryption hardware and the upper layers.
21 * Lower layer devices will set up a keyslot manager in their request queue
22 * and tell it how to perform device specific operations like programming/
23 * evicting keys from keyslots.
25 * Upper layers will call blk_ksm_get_slot_for_key() to program a
26 * key into some slot in the inline encryption hardware.
29 #define pr_fmt(fmt) "blk-crypto: " fmt
31 #include <linux/keyslot-manager.h>
32 #include <linux/atomic.h>
33 #include <linux/mutex.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/wait.h>
36 #include <linux/blkdev.h>
38 struct blk_ksm_keyslot
{
40 struct list_head idle_slot_node
;
41 struct hlist_node hash_node
;
42 const struct blk_crypto_key
*key
;
43 struct blk_keyslot_manager
*ksm
;
46 static inline void blk_ksm_hw_enter(struct blk_keyslot_manager
*ksm
)
49 * Calling into the driver requires ksm->lock held and the device
50 * resumed. But we must resume the device first, since that can acquire
51 * and release ksm->lock via blk_ksm_reprogram_all_keys().
54 pm_runtime_get_sync(ksm
->dev
);
55 down_write(&ksm
->lock
);
58 static inline void blk_ksm_hw_exit(struct blk_keyslot_manager
*ksm
)
62 pm_runtime_put_sync(ksm
->dev
);
66 * blk_ksm_init() - Initialize a keyslot manager
67 * @ksm: The keyslot_manager to initialize.
68 * @num_slots: The number of key slots to manage.
70 * Allocate memory for keyslots and initialize a keyslot manager. Called by
71 * e.g. storage drivers to set up a keyslot manager in their request_queue.
73 * Return: 0 on success, or else a negative error code.
75 int blk_ksm_init(struct blk_keyslot_manager
*ksm
, unsigned int num_slots
)
79 unsigned int slot_hashtable_size
;
81 memset(ksm
, 0, sizeof(*ksm
));
86 ksm
->slots
= kvcalloc(num_slots
, sizeof(ksm
->slots
[0]), GFP_KERNEL
);
90 ksm
->num_slots
= num_slots
;
92 init_rwsem(&ksm
->lock
);
94 init_waitqueue_head(&ksm
->idle_slots_wait_queue
);
95 INIT_LIST_HEAD(&ksm
->idle_slots
);
97 for (slot
= 0; slot
< num_slots
; slot
++) {
98 ksm
->slots
[slot
].ksm
= ksm
;
99 list_add_tail(&ksm
->slots
[slot
].idle_slot_node
,
103 spin_lock_init(&ksm
->idle_slots_lock
);
105 slot_hashtable_size
= roundup_pow_of_two(num_slots
);
106 ksm
->log_slot_ht_size
= ilog2(slot_hashtable_size
);
107 ksm
->slot_hashtable
= kvmalloc_array(slot_hashtable_size
,
108 sizeof(ksm
->slot_hashtable
[0]),
110 if (!ksm
->slot_hashtable
)
111 goto err_destroy_ksm
;
112 for (i
= 0; i
< slot_hashtable_size
; i
++)
113 INIT_HLIST_HEAD(&ksm
->slot_hashtable
[i
]);
118 blk_ksm_destroy(ksm
);
121 EXPORT_SYMBOL_GPL(blk_ksm_init
);
123 static inline struct hlist_head
*
124 blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager
*ksm
,
125 const struct blk_crypto_key
*key
)
127 return &ksm
->slot_hashtable
[hash_ptr(key
, ksm
->log_slot_ht_size
)];
130 static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot
*slot
)
132 struct blk_keyslot_manager
*ksm
= slot
->ksm
;
135 spin_lock_irqsave(&ksm
->idle_slots_lock
, flags
);
136 list_del(&slot
->idle_slot_node
);
137 spin_unlock_irqrestore(&ksm
->idle_slots_lock
, flags
);
140 static struct blk_ksm_keyslot
*blk_ksm_find_keyslot(
141 struct blk_keyslot_manager
*ksm
,
142 const struct blk_crypto_key
*key
)
144 const struct hlist_head
*head
= blk_ksm_hash_bucket_for_key(ksm
, key
);
145 struct blk_ksm_keyslot
*slotp
;
147 hlist_for_each_entry(slotp
, head
, hash_node
) {
148 if (slotp
->key
== key
)
154 static struct blk_ksm_keyslot
*blk_ksm_find_and_grab_keyslot(
155 struct blk_keyslot_manager
*ksm
,
156 const struct blk_crypto_key
*key
)
158 struct blk_ksm_keyslot
*slot
;
160 slot
= blk_ksm_find_keyslot(ksm
, key
);
163 if (atomic_inc_return(&slot
->slot_refs
) == 1) {
164 /* Took first reference to this slot; remove it from LRU list */
165 blk_ksm_remove_slot_from_lru_list(slot
);
170 unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot
*slot
)
172 return slot
- slot
->ksm
->slots
;
174 EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx
);
177 * blk_ksm_get_slot_for_key() - Program a key into a keyslot.
178 * @ksm: The keyslot manager to program the key into.
179 * @key: Pointer to the key object to program, including the raw key, crypto
180 * mode, and data unit size.
181 * @slot_ptr: A pointer to return the pointer of the allocated keyslot.
183 * Get a keyslot that's been programmed with the specified key. If one already
184 * exists, return it with incremented refcount. Otherwise, wait for a keyslot
185 * to become idle and program it.
187 * Context: Process context. Takes and releases ksm->lock.
188 * Return: BLK_STS_OK on success (and keyslot is set to the pointer of the
189 * allocated keyslot), or some other blk_status_t otherwise (and
190 * keyslot is set to NULL).
192 blk_status_t
blk_ksm_get_slot_for_key(struct blk_keyslot_manager
*ksm
,
193 const struct blk_crypto_key
*key
,
194 struct blk_ksm_keyslot
**slot_ptr
)
196 struct blk_ksm_keyslot
*slot
;
201 down_read(&ksm
->lock
);
202 slot
= blk_ksm_find_and_grab_keyslot(ksm
, key
);
208 blk_ksm_hw_enter(ksm
);
209 slot
= blk_ksm_find_and_grab_keyslot(ksm
, key
);
211 blk_ksm_hw_exit(ksm
);
216 * If we're here, that means there wasn't a slot that was
217 * already programmed with the key. So try to program it.
219 if (!list_empty(&ksm
->idle_slots
))
222 blk_ksm_hw_exit(ksm
);
223 wait_event(ksm
->idle_slots_wait_queue
,
224 !list_empty(&ksm
->idle_slots
));
227 slot
= list_first_entry(&ksm
->idle_slots
, struct blk_ksm_keyslot
,
229 slot_idx
= blk_ksm_get_slot_idx(slot
);
231 err
= ksm
->ksm_ll_ops
.keyslot_program(ksm
, key
, slot_idx
);
233 wake_up(&ksm
->idle_slots_wait_queue
);
234 blk_ksm_hw_exit(ksm
);
235 return errno_to_blk_status(err
);
238 /* Move this slot to the hash list for the new key. */
240 hlist_del(&slot
->hash_node
);
242 hlist_add_head(&slot
->hash_node
, blk_ksm_hash_bucket_for_key(ksm
, key
));
244 atomic_set(&slot
->slot_refs
, 1);
246 blk_ksm_remove_slot_from_lru_list(slot
);
248 blk_ksm_hw_exit(ksm
);
255 * blk_ksm_put_slot() - Release a reference to a slot
256 * @slot: The keyslot to release the reference of.
258 * Context: Any context.
260 void blk_ksm_put_slot(struct blk_ksm_keyslot
*slot
)
262 struct blk_keyslot_manager
*ksm
;
270 if (atomic_dec_and_lock_irqsave(&slot
->slot_refs
,
271 &ksm
->idle_slots_lock
, flags
)) {
272 list_add_tail(&slot
->idle_slot_node
, &ksm
->idle_slots
);
273 spin_unlock_irqrestore(&ksm
->idle_slots_lock
, flags
);
274 wake_up(&ksm
->idle_slots_wait_queue
);
279 * blk_ksm_crypto_cfg_supported() - Find out if a crypto configuration is
280 * supported by a ksm.
281 * @ksm: The keyslot manager to check
282 * @cfg: The crypto configuration to check for.
284 * Checks for crypto_mode/data unit size/dun bytes support.
286 * Return: Whether or not this ksm supports the specified crypto config.
288 bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager
*ksm
,
289 const struct blk_crypto_config
*cfg
)
293 if (!(ksm
->crypto_modes_supported
[cfg
->crypto_mode
] &
294 cfg
->data_unit_size
))
296 if (ksm
->max_dun_bytes_supported
< cfg
->dun_bytes
)
302 * blk_ksm_evict_key() - Evict a key from the lower layer device.
303 * @ksm: The keyslot manager to evict from
304 * @key: The key to evict
306 * Find the keyslot that the specified key was programmed into, and evict that
307 * slot from the lower layer device. The slot must not be in use by any
308 * in-flight IO when this function is called.
310 * Context: Process context. Takes and releases ksm->lock.
311 * Return: 0 on success or if there's no keyslot with the specified key, -EBUSY
312 * if the keyslot is still in use, or another -errno value on other
315 int blk_ksm_evict_key(struct blk_keyslot_manager
*ksm
,
316 const struct blk_crypto_key
*key
)
318 struct blk_ksm_keyslot
*slot
;
321 blk_ksm_hw_enter(ksm
);
322 slot
= blk_ksm_find_keyslot(ksm
, key
);
326 if (WARN_ON_ONCE(atomic_read(&slot
->slot_refs
) != 0)) {
330 err
= ksm
->ksm_ll_ops
.keyslot_evict(ksm
, key
,
331 blk_ksm_get_slot_idx(slot
));
335 hlist_del(&slot
->hash_node
);
339 blk_ksm_hw_exit(ksm
);
344 * blk_ksm_reprogram_all_keys() - Re-program all keyslots.
345 * @ksm: The keyslot manager
347 * Re-program all keyslots that are supposed to have a key programmed. This is
348 * intended only for use by drivers for hardware that loses its keys on reset.
350 * Context: Process context. Takes and releases ksm->lock.
352 void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager
*ksm
)
356 /* This is for device initialization, so don't resume the device */
357 down_write(&ksm
->lock
);
358 for (slot
= 0; slot
< ksm
->num_slots
; slot
++) {
359 const struct blk_crypto_key
*key
= ksm
->slots
[slot
].key
;
365 err
= ksm
->ksm_ll_ops
.keyslot_program(ksm
, key
, slot
);
368 up_write(&ksm
->lock
);
370 EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys
);
372 void blk_ksm_destroy(struct blk_keyslot_manager
*ksm
)
376 kvfree(ksm
->slot_hashtable
);
377 kvfree_sensitive(ksm
->slots
, sizeof(ksm
->slots
[0]) * ksm
->num_slots
);
378 memzero_explicit(ksm
, sizeof(*ksm
));
380 EXPORT_SYMBOL_GPL(blk_ksm_destroy
);
382 bool blk_ksm_register(struct blk_keyslot_manager
*ksm
, struct request_queue
*q
)
384 if (blk_integrity_queue_supports_integrity(q
)) {
385 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n");
391 EXPORT_SYMBOL_GPL(blk_ksm_register
);
393 void blk_ksm_unregister(struct request_queue
*q
)