1 // SPDX-License-Identifier: GPL-2.0
3 * Hardware spinlock framework
5 * Copyright (C) 2010 Texas Instruments Incorporated - http://www.ti.com
7 * Contact: Ohad Ben-Cohen <ohad@wizery.com>
10 #define pr_fmt(fmt) "%s: " fmt, __func__
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/spinlock.h>
15 #include <linux/types.h>
16 #include <linux/err.h>
17 #include <linux/jiffies.h>
18 #include <linux/radix-tree.h>
19 #include <linux/hwspinlock.h>
20 #include <linux/pm_runtime.h>
21 #include <linux/mutex.h>
24 #include "hwspinlock_internal.h"
27 #define HWSPINLOCK_UNUSED (0) /* tags an hwspinlock as unused */
30 * A radix tree is used to maintain the available hwspinlock instances.
31 * The tree associates hwspinlock pointers with their integer key id,
32 * and provides easy-to-use API which makes the hwspinlock core code simple
35 * Radix trees are quick on lookups, and reasonably efficient in terms of
36 * storage, especially with high density usages such as this framework
37 * requires (a continuous range of integer keys, beginning with zero, is
38 * used as the ID's of the hwspinlock instances).
40 * The radix tree API supports tagging items in the tree, which this
41 * framework uses to mark unused hwspinlock instances (see the
42 * HWSPINLOCK_UNUSED tag above). As a result, the process of querying the
43 * tree, looking for an unused hwspinlock instance, is now reduced to a
44 * single radix tree API call.
46 static RADIX_TREE(hwspinlock_tree
, GFP_KERNEL
);
49 * Synchronization of access to the tree is achieved using this mutex,
50 * as the radix-tree API requires that users provide all synchronisation.
51 * A mutex is needed because we're using non-atomic radix tree allocations.
53 static DEFINE_MUTEX(hwspinlock_tree_lock
);
57 * __hwspin_trylock() - attempt to lock a specific hwspinlock
58 * @hwlock: an hwspinlock which we want to trylock
59 * @mode: controls whether local interrupts are disabled or not
60 * @flags: a pointer where the caller's interrupt state will be saved at (if
63 * This function attempts to lock an hwspinlock, and will immediately
64 * fail if the hwspinlock is already taken.
66 * Caution: If the mode is HWLOCK_RAW, that means user must protect the routine
67 * of getting hardware lock with mutex or spinlock. Since in some scenarios,
68 * user need some time-consuming or sleepable operations under the hardware
69 * lock, they need one sleepable lock (like mutex) to protect the operations.
71 * If the mode is not HWLOCK_RAW, upon a successful return from this function,
72 * preemption (and possibly interrupts) is disabled, so the caller must not
73 * sleep, and is advised to release the hwspinlock as soon as possible. This is
74 * required in order to minimize remote cores polling on the hardware
77 * The user decides whether local interrupts are disabled or not, and if yes,
78 * whether he wants their previous state to be saved. It is up to the user
79 * to choose the appropriate @mode of operation, exactly the same way users
80 * should decide between spin_trylock, spin_trylock_irq and
81 * spin_trylock_irqsave.
83 * Returns 0 if we successfully locked the hwspinlock or -EBUSY if
84 * the hwspinlock was already taken.
85 * This function will never sleep.
87 int __hwspin_trylock(struct hwspinlock
*hwlock
, int mode
, unsigned long *flags
)
92 BUG_ON(!flags
&& mode
== HWLOCK_IRQSTATE
);
95 * This spin_lock{_irq, _irqsave} serves three purposes:
97 * 1. Disable preemption, in order to minimize the period of time
98 * in which the hwspinlock is taken. This is important in order
99 * to minimize the possible polling on the hardware interconnect
100 * by a remote user of this lock.
101 * 2. Make the hwspinlock SMP-safe (so we can take it from
102 * additional contexts on the local host).
103 * 3. Ensure that in_atomic/might_sleep checks catch potential
104 * problems with hwspinlock usage (e.g. scheduler checks like
105 * 'scheduling while atomic' etc.)
108 case HWLOCK_IRQSTATE
:
109 ret
= spin_trylock_irqsave(&hwlock
->lock
, *flags
);
112 ret
= spin_trylock_irq(&hwlock
->lock
);
118 ret
= spin_trylock(&hwlock
->lock
);
122 /* is lock already taken by another context on the local cpu ? */
126 /* try to take the hwspinlock device */
127 ret
= hwlock
->bank
->ops
->trylock(hwlock
);
129 /* if hwlock is already taken, undo spin_trylock_* and exit */
132 case HWLOCK_IRQSTATE
:
133 spin_unlock_irqrestore(&hwlock
->lock
, *flags
);
136 spin_unlock_irq(&hwlock
->lock
);
142 spin_unlock(&hwlock
->lock
);
150 * We can be sure the other core's memory operations
151 * are observable to us only _after_ we successfully take
152 * the hwspinlock, and we must make sure that subsequent memory
153 * operations (both reads and writes) will not be reordered before
154 * we actually took the hwspinlock.
156 * Note: the implicit memory barrier of the spinlock above is too
157 * early, so we need this additional explicit memory barrier.
163 EXPORT_SYMBOL_GPL(__hwspin_trylock
);
166 * __hwspin_lock_timeout() - lock an hwspinlock with timeout limit
167 * @hwlock: the hwspinlock to be locked
168 * @timeout: timeout value in msecs
169 * @mode: mode which controls whether local interrupts are disabled or not
170 * @flags: a pointer to where the caller's interrupt state will be saved at (if
173 * This function locks the given @hwlock. If the @hwlock
174 * is already taken, the function will busy loop waiting for it to
175 * be released, but give up after @timeout msecs have elapsed.
177 * Caution: If the mode is HWLOCK_RAW, that means user must protect the routine
178 * of getting hardware lock with mutex or spinlock. Since in some scenarios,
179 * user need some time-consuming or sleepable operations under the hardware
180 * lock, they need one sleepable lock (like mutex) to protect the operations.
182 * If the mode is not HWLOCK_RAW, upon a successful return from this function,
183 * preemption is disabled (and possibly local interrupts, too), so the caller
184 * must not sleep, and is advised to release the hwspinlock as soon as possible.
185 * This is required in order to minimize remote cores polling on the
186 * hardware interconnect.
188 * The user decides whether local interrupts are disabled or not, and if yes,
189 * whether he wants their previous state to be saved. It is up to the user
190 * to choose the appropriate @mode of operation, exactly the same way users
191 * should decide between spin_lock, spin_lock_irq and spin_lock_irqsave.
193 * Returns 0 when the @hwlock was successfully taken, and an appropriate
194 * error code otherwise (most notably -ETIMEDOUT if the @hwlock is still
195 * busy after @timeout msecs). The function will never sleep.
197 int __hwspin_lock_timeout(struct hwspinlock
*hwlock
, unsigned int to
,
198 int mode
, unsigned long *flags
)
201 unsigned long expire
;
203 expire
= msecs_to_jiffies(to
) + jiffies
;
206 /* Try to take the hwspinlock */
207 ret
= __hwspin_trylock(hwlock
, mode
, flags
);
212 * The lock is already taken, let's check if the user wants
215 if (time_is_before_eq_jiffies(expire
))
219 * Allow platform-specific relax handlers to prevent
220 * hogging the interconnect (no sleeping, though)
222 if (hwlock
->bank
->ops
->relax
)
223 hwlock
->bank
->ops
->relax(hwlock
);
228 EXPORT_SYMBOL_GPL(__hwspin_lock_timeout
);
231 * __hwspin_unlock() - unlock a specific hwspinlock
232 * @hwlock: a previously-acquired hwspinlock which we want to unlock
233 * @mode: controls whether local interrupts needs to be restored or not
234 * @flags: previous caller's interrupt state to restore (if requested)
236 * This function will unlock a specific hwspinlock, enable preemption and
237 * (possibly) enable interrupts or restore their previous state.
238 * @hwlock must be already locked before calling this function: it is a bug
239 * to call unlock on a @hwlock that is already unlocked.
241 * The user decides whether local interrupts should be enabled or not, and
242 * if yes, whether he wants their previous state to be restored. It is up
243 * to the user to choose the appropriate @mode of operation, exactly the
244 * same way users decide between spin_unlock, spin_unlock_irq and
245 * spin_unlock_irqrestore.
247 * The function will never sleep.
249 void __hwspin_unlock(struct hwspinlock
*hwlock
, int mode
, unsigned long *flags
)
252 BUG_ON(!flags
&& mode
== HWLOCK_IRQSTATE
);
255 * We must make sure that memory operations (both reads and writes),
256 * done before unlocking the hwspinlock, will not be reordered
257 * after the lock is released.
259 * That's the purpose of this explicit memory barrier.
261 * Note: the memory barrier induced by the spin_unlock below is too
262 * late; the other core is going to access memory soon after it will
263 * take the hwspinlock, and by then we want to be sure our memory
264 * operations are already observable.
268 hwlock
->bank
->ops
->unlock(hwlock
);
270 /* Undo the spin_trylock{_irq, _irqsave} called while locking */
272 case HWLOCK_IRQSTATE
:
273 spin_unlock_irqrestore(&hwlock
->lock
, *flags
);
276 spin_unlock_irq(&hwlock
->lock
);
282 spin_unlock(&hwlock
->lock
);
286 EXPORT_SYMBOL_GPL(__hwspin_unlock
);
289 * of_hwspin_lock_simple_xlate - translate hwlock_spec to return a lock id
290 * @bank: the hwspinlock device bank
291 * @hwlock_spec: hwlock specifier as found in the device tree
293 * This is a simple translation function, suitable for hwspinlock platform
294 * drivers that only has a lock specifier length of 1.
296 * Returns a relative index of the lock within a specified bank on success,
297 * or -EINVAL on invalid specifier cell count.
300 of_hwspin_lock_simple_xlate(const struct of_phandle_args
*hwlock_spec
)
302 if (WARN_ON(hwlock_spec
->args_count
!= 1))
305 return hwlock_spec
->args
[0];
309 * of_hwspin_lock_get_id() - get lock id for an OF phandle-based specific lock
310 * @np: device node from which to request the specific hwlock
311 * @index: index of the hwlock in the list of values
313 * This function provides a means for DT users of the hwspinlock module to
314 * get the global lock id of a specific hwspinlock using the phandle of the
315 * hwspinlock device, so that it can be requested using the normal
316 * hwspin_lock_request_specific() API.
318 * Returns the global lock id number on success, -EPROBE_DEFER if the hwspinlock
319 * device is not yet registered, -EINVAL on invalid args specifier value or an
320 * appropriate error as returned from the OF parsing of the DT client node.
322 int of_hwspin_lock_get_id(struct device_node
*np
, int index
)
324 struct of_phandle_args args
;
325 struct hwspinlock
*hwlock
;
326 struct radix_tree_iter iter
;
331 ret
= of_parse_phandle_with_args(np
, "hwlocks", "#hwlock-cells", index
,
336 /* Find the hwspinlock device: we need its base_id */
339 radix_tree_for_each_slot(slot
, &hwspinlock_tree
, &iter
, 0) {
340 hwlock
= radix_tree_deref_slot(slot
);
341 if (unlikely(!hwlock
))
343 if (radix_tree_deref_retry(hwlock
)) {
344 slot
= radix_tree_iter_retry(&iter
);
348 if (hwlock
->bank
->dev
->of_node
== args
.np
) {
357 id
= of_hwspin_lock_simple_xlate(&args
);
358 if (id
< 0 || id
>= hwlock
->bank
->num_locks
) {
362 id
+= hwlock
->bank
->base_id
;
365 of_node_put(args
.np
);
366 return ret
? ret
: id
;
368 EXPORT_SYMBOL_GPL(of_hwspin_lock_get_id
);
371 * of_hwspin_lock_get_id_byname() - get lock id for an specified hwlock name
372 * @np: device node from which to request the specific hwlock
375 * This function provides a means for DT users of the hwspinlock module to
376 * get the global lock id of a specific hwspinlock using the specified name of
377 * the hwspinlock device, so that it can be requested using the normal
378 * hwspin_lock_request_specific() API.
380 * Returns the global lock id number on success, -EPROBE_DEFER if the hwspinlock
381 * device is not yet registered, -EINVAL on invalid args specifier value or an
382 * appropriate error as returned from the OF parsing of the DT client node.
384 int of_hwspin_lock_get_id_byname(struct device_node
*np
, const char *name
)
391 index
= of_property_match_string(np
, "hwlock-names", name
);
395 return of_hwspin_lock_get_id(np
, index
);
397 EXPORT_SYMBOL_GPL(of_hwspin_lock_get_id_byname
);
399 static int hwspin_lock_register_single(struct hwspinlock
*hwlock
, int id
)
401 struct hwspinlock
*tmp
;
404 mutex_lock(&hwspinlock_tree_lock
);
406 ret
= radix_tree_insert(&hwspinlock_tree
, id
, hwlock
);
409 pr_err("hwspinlock id %d already exists!\n", id
);
413 /* mark this hwspinlock as available */
414 tmp
= radix_tree_tag_set(&hwspinlock_tree
, id
, HWSPINLOCK_UNUSED
);
416 /* self-sanity check which should never fail */
417 WARN_ON(tmp
!= hwlock
);
420 mutex_unlock(&hwspinlock_tree_lock
);
424 static struct hwspinlock
*hwspin_lock_unregister_single(unsigned int id
)
426 struct hwspinlock
*hwlock
= NULL
;
429 mutex_lock(&hwspinlock_tree_lock
);
431 /* make sure the hwspinlock is not in use (tag is set) */
432 ret
= radix_tree_tag_get(&hwspinlock_tree
, id
, HWSPINLOCK_UNUSED
);
434 pr_err("hwspinlock %d still in use (or not present)\n", id
);
438 hwlock
= radix_tree_delete(&hwspinlock_tree
, id
);
440 pr_err("failed to delete hwspinlock %d\n", id
);
445 mutex_unlock(&hwspinlock_tree_lock
);
450 * hwspin_lock_register() - register a new hw spinlock device
451 * @bank: the hwspinlock device, which usually provides numerous hw locks
452 * @dev: the backing device
453 * @ops: hwspinlock handlers for this device
454 * @base_id: id of the first hardware spinlock in this bank
455 * @num_locks: number of hwspinlocks provided by this device
457 * This function should be called from the underlying platform-specific
458 * implementation, to register a new hwspinlock device instance.
460 * Should be called from a process context (might sleep)
462 * Returns 0 on success, or an appropriate error code on failure
464 int hwspin_lock_register(struct hwspinlock_device
*bank
, struct device
*dev
,
465 const struct hwspinlock_ops
*ops
, int base_id
, int num_locks
)
467 struct hwspinlock
*hwlock
;
470 if (!bank
|| !ops
|| !dev
|| !num_locks
|| !ops
->trylock
||
472 pr_err("invalid parameters\n");
478 bank
->base_id
= base_id
;
479 bank
->num_locks
= num_locks
;
481 for (i
= 0; i
< num_locks
; i
++) {
482 hwlock
= &bank
->lock
[i
];
484 spin_lock_init(&hwlock
->lock
);
487 ret
= hwspin_lock_register_single(hwlock
, base_id
+ i
);
496 hwspin_lock_unregister_single(base_id
+ i
);
499 EXPORT_SYMBOL_GPL(hwspin_lock_register
);
502 * hwspin_lock_unregister() - unregister an hw spinlock device
503 * @bank: the hwspinlock device, which usually provides numerous hw locks
505 * This function should be called from the underlying platform-specific
506 * implementation, to unregister an existing (and unused) hwspinlock.
508 * Should be called from a process context (might sleep)
510 * Returns 0 on success, or an appropriate error code on failure
512 int hwspin_lock_unregister(struct hwspinlock_device
*bank
)
514 struct hwspinlock
*hwlock
, *tmp
;
517 for (i
= 0; i
< bank
->num_locks
; i
++) {
518 hwlock
= &bank
->lock
[i
];
520 tmp
= hwspin_lock_unregister_single(bank
->base_id
+ i
);
524 /* self-sanity check that should never fail */
525 WARN_ON(tmp
!= hwlock
);
530 EXPORT_SYMBOL_GPL(hwspin_lock_unregister
);
533 * __hwspin_lock_request() - tag an hwspinlock as used and power it up
535 * This is an internal function that prepares an hwspinlock instance
536 * before it is given to the user. The function assumes that
537 * hwspinlock_tree_lock is taken.
539 * Returns 0 or positive to indicate success, and a negative value to
540 * indicate an error (with the appropriate error code)
542 static int __hwspin_lock_request(struct hwspinlock
*hwlock
)
544 struct device
*dev
= hwlock
->bank
->dev
;
545 struct hwspinlock
*tmp
;
548 /* prevent underlying implementation from being removed */
549 if (!try_module_get(dev
->driver
->owner
)) {
550 dev_err(dev
, "%s: can't get owner\n", __func__
);
554 /* notify PM core that power is now needed */
555 ret
= pm_runtime_get_sync(dev
);
557 dev_err(dev
, "%s: can't power on device\n", __func__
);
558 pm_runtime_put_noidle(dev
);
559 module_put(dev
->driver
->owner
);
563 /* mark hwspinlock as used, should not fail */
564 tmp
= radix_tree_tag_clear(&hwspinlock_tree
, hwlock_to_id(hwlock
),
567 /* self-sanity check that should never fail */
568 WARN_ON(tmp
!= hwlock
);
574 * hwspin_lock_get_id() - retrieve id number of a given hwspinlock
575 * @hwlock: a valid hwspinlock instance
577 * Returns the id number of a given @hwlock, or -EINVAL if @hwlock is invalid.
579 int hwspin_lock_get_id(struct hwspinlock
*hwlock
)
582 pr_err("invalid hwlock\n");
586 return hwlock_to_id(hwlock
);
588 EXPORT_SYMBOL_GPL(hwspin_lock_get_id
);
591 * hwspin_lock_request() - request an hwspinlock
593 * This function should be called by users of the hwspinlock device,
594 * in order to dynamically assign them an unused hwspinlock.
595 * Usually the user of this lock will then have to communicate the lock's id
596 * to the remote core before it can be used for synchronization (to get the
597 * id of a given hwlock, use hwspin_lock_get_id()).
599 * Should be called from a process context (might sleep)
601 * Returns the address of the assigned hwspinlock, or NULL on error
603 struct hwspinlock
*hwspin_lock_request(void)
605 struct hwspinlock
*hwlock
;
608 mutex_lock(&hwspinlock_tree_lock
);
610 /* look for an unused lock */
611 ret
= radix_tree_gang_lookup_tag(&hwspinlock_tree
, (void **)&hwlock
,
612 0, 1, HWSPINLOCK_UNUSED
);
614 pr_warn("a free hwspinlock is not available\n");
619 /* sanity check that should never fail */
622 /* mark as used and power up */
623 ret
= __hwspin_lock_request(hwlock
);
628 mutex_unlock(&hwspinlock_tree_lock
);
631 EXPORT_SYMBOL_GPL(hwspin_lock_request
);
634 * hwspin_lock_request_specific() - request for a specific hwspinlock
635 * @id: index of the specific hwspinlock that is requested
637 * This function should be called by users of the hwspinlock module,
638 * in order to assign them a specific hwspinlock.
639 * Usually early board code will be calling this function in order to
640 * reserve specific hwspinlock ids for predefined purposes.
642 * Should be called from a process context (might sleep)
644 * Returns the address of the assigned hwspinlock, or NULL on error
646 struct hwspinlock
*hwspin_lock_request_specific(unsigned int id
)
648 struct hwspinlock
*hwlock
;
651 mutex_lock(&hwspinlock_tree_lock
);
653 /* make sure this hwspinlock exists */
654 hwlock
= radix_tree_lookup(&hwspinlock_tree
, id
);
656 pr_warn("hwspinlock %u does not exist\n", id
);
660 /* sanity check (this shouldn't happen) */
661 WARN_ON(hwlock_to_id(hwlock
) != id
);
663 /* make sure this hwspinlock is unused */
664 ret
= radix_tree_tag_get(&hwspinlock_tree
, id
, HWSPINLOCK_UNUSED
);
666 pr_warn("hwspinlock %u is already in use\n", id
);
671 /* mark as used and power up */
672 ret
= __hwspin_lock_request(hwlock
);
677 mutex_unlock(&hwspinlock_tree_lock
);
680 EXPORT_SYMBOL_GPL(hwspin_lock_request_specific
);
683 * hwspin_lock_free() - free a specific hwspinlock
684 * @hwlock: the specific hwspinlock to free
686 * This function mark @hwlock as free again.
687 * Should only be called with an @hwlock that was retrieved from
688 * an earlier call to omap_hwspin_lock_request{_specific}.
690 * Should be called from a process context (might sleep)
692 * Returns 0 on success, or an appropriate error code on failure
694 int hwspin_lock_free(struct hwspinlock
*hwlock
)
697 struct hwspinlock
*tmp
;
701 pr_err("invalid hwlock\n");
705 dev
= hwlock
->bank
->dev
;
706 mutex_lock(&hwspinlock_tree_lock
);
708 /* make sure the hwspinlock is used */
709 ret
= radix_tree_tag_get(&hwspinlock_tree
, hwlock_to_id(hwlock
),
712 dev_err(dev
, "%s: hwlock is already free\n", __func__
);
718 /* notify the underlying device that power is not needed */
719 ret
= pm_runtime_put(dev
);
723 /* mark this hwspinlock as available */
724 tmp
= radix_tree_tag_set(&hwspinlock_tree
, hwlock_to_id(hwlock
),
727 /* sanity check (this shouldn't happen) */
728 WARN_ON(tmp
!= hwlock
);
730 module_put(dev
->driver
->owner
);
733 mutex_unlock(&hwspinlock_tree_lock
);
736 EXPORT_SYMBOL_GPL(hwspin_lock_free
);
738 MODULE_LICENSE("GPL v2");
739 MODULE_DESCRIPTION("Hardware spinlock interface");
740 MODULE_AUTHOR("Ohad Ben-Cohen <ohad@wizery.com>");